######################################################################## # Copyright (C) 2010-2020 VMware, Inc. # All Rights Reserved ######################################################################## """This module contains the HostImage class, used for applying and extracting an image profile from the host. """ import datetime import logging import os import time from operator import attrgetter from vmware.runcommand import runcommand, RunCommandError from .ImageProfile import AcceptanceChecker from .Installer import BootBankInstaller, LiveImageInstaller, LockerInstaller from . import Bulletin from . import Database from . import Errors from . import Downloader from . import Vib from . import VibCollection from .Utils import LockFile, Ramdisk, HostInfo from .Utils.XmlUtils import _utctzinfo from .Utils.Misc import byteToStr try: import spicy HAVE_SPICY = True except ImportError: HAVE_SPICY = False # Please don't rearrange the sequence of the following # DEFAULT_INSTALLERS tuple DEFAULT_INSTALLERS = (LiveImageInstaller.LiveImageInstaller, BootBankInstaller.BootBankInstaller, LockerInstaller.LockerInstaller,) INSTALLERTYPE_TO_NAME = {'boot' : 'BootBankInstaller', 'live' : 'LiveImageInstaller', 'locker': 'LockerInstaller'} ADVCFG = '/sbin/esxcfg-advcfg' VMKVOB = '/usr/lib/vmware/vob/bin/vmkvob' LOCKFILE = '/var/run/esximg.pid' # # This is the default acceptance level that gets returned if the configuration # cannot be found. DEFAULT_HOST_ACCEPTANCE = Vib.ArFileVib.ACCEPTANCE_PARTNER log = logging.getLogger('HostImage') class HostImage(object): """The HostImage class enables the basic operation of applying an image profile to an ESX host via the Stage and Remediate methods. It also provides a way to obtain the current VIB inventory and current image profile. Class Variables: * IMGSTATE_FRESH_BOOT - The host is local booted and VisorFS VIBs are the same as /bootbank VIBs (no VIBs have been updated); or the host is PXE booted * IMGSTATE_LIVE_UPDATED - VIBs have been live installed or removed but no reboot-required VIBs have been installed since the last boot. The VIBs in VisorFS are diff- erent from the /bootbank VIBs, but should be the same as /altbootbank VIBs. * IMGSTATE_BOOTBANK_UPDATED - VIBs requiring a reboot have been installed, so that the VIBs in /altbootbank are diff- erent than the VIBs in VisorFS. Once a host enters this state, live installs are not permitted until after a reboot. Attributes: * installers - A dict of supported Installer classes. * imgstate - Current ESX image state, one of the IMGSTATE_* values * livevibs - A VibCollection instance of live VIBs * livestagedvibs - A VibCollection instance of VIBs staged for LiveImageInstaller. * bootbankupdatedvibs - A VibCollection of VIBs, which are installed into altbootbank and will be live after reboot. * bootbankstagedvibs - A VibCollection of VIBs, which are in altbootbank, but altbootbank is in staged state. """ IMGSTATE_UNKNOWN = -1 IMGSTATE_FRESH_BOOT = 0 IMGSTATE_LIVE_UPDATED = 1 IMGSTATE_BOOTBANK_UPDATED = 2 DB_AUTO = 0 DB_VISORFS = 1 DB_BOOTBANK = 2 DB_CURBOOTBANK = 3 DBS = (DB_AUTO, DB_VISORFS, DB_BOOTBANK, DB_CURBOOTBANK) # Ramdisk for vib download. # GetContainerId() retrieves simulator name to generate unique ramdisk name # and path for each simulator environment. It returns an empty string if not # in simulator environment. VIB_DOWNLOAD_NAME = HostInfo.GetContainerId() + 'vibdownload' VIB_DOWNLOAD = os.path.join(os.path.sep, 'tmp', VIB_DOWNLOAD_NAME) # Audit events and messages AUDIT_START_EVENTID = 'system.update.start' AUDIT_END_EVENTID = 'system.update.end' AUDIT_NOTE_NOSIG = 'The user has specified to bypass signature verification.' AUDIT_NOTE_NOSIG_IGNORED = 'The user has specified to bypass signature ' \ 'verification, but the request has been ' \ 'ignored since SecureBoot is enabled.' # VIB signature verification result, lower number is more benign. SIG_VERIFY_PASSED = 0 SIG_VERIFY_LOCKER = 1 SIG_VERIFY_MISSING = 2 SIG_VERIFY_UNKNOWN = 3 SIG_VERIFY_ERROR = 4 def __init__(self, installerclasses=DEFAULT_INSTALLERS, initInstallers=True): """HostImage class constructor. Initialize all the installers. Note, on PXE systems the BootbankInstaller and LockerInstaller will not initialize and will not be present. Parameters: * installerclasses - a list or iterable of class objects inherited from Installer.Installer * intInstallers - whether to initiate all installers, use False with a transaction that would call _getLock() at the start. TODO: Exceptions: SetupError - if no installer classes could be initialized """ self._lock = LockFile.LockFile(LOCKFILE) self._hosthw = None self._installerClasses = installerclasses if initInstallers: self._initInstallers() else: log.debug('Deferring initiating installers') def _initInstallers(self): """Initiate the installer instances. """ self.installers = dict() # in DEFAULT_INSTALLERS tuple, LockerInstaller is arranged # next to BootBankInstaller in sequence, if for some reason # the BootBankInstaller is not initiated, we don't need to # initiate LockerInstaller. for cls in self._installerClasses: try: installer = cls() self.installers[installer.installertype] = installer except Exception as e: if hasattr(e, 'msg'): log.info('Installer %s was not initiated - reason: %s', str(cls), e.msg) else: log.info('Installer %s was not initiated - reason: %s', str(cls), str(e)) # Host requires both bootbank and locker installers in stateful # env or none of them in stateless env. But Simulators run # neither completely stateful nor completely stateless - they # require locker partition to allow scan to complete, but do not # require all tardisks to be in bootbank and altbootbank. if 'BootBankInstaller' in str(cls) and \ not HostInfo.HostOSIsSimulator(): break log.info('Installers initiated are %s', str(self.installers)) def _getImgState(self): if 'boot' not in self.installers: if 'live' not in self.installers: return self.IMGSTATE_UNKNOWN # Probably PXE # XXX: if we support fresh install, there might be a drift return self.IMGSTATE_FRESH_BOOT elif 'live' not in self.installers: # bootbank exists but not live installer, probably offline # State depends on whether /altbootbank image has been updated bootinst = self.installers['boot'] if bootinst.bootbankstate & bootinst.BOOTBANK_STATE_UPDATED: return self.IMGSTATE_BOOTBANK_UPDATED else: return self.IMGSTATE_FRESH_BOOT else: # Both bootbank and live images exist. bootinst = self.installers['boot'] liveinst = self.installers['live'] # # Altbootbank contains a valid, newer image than bootbank. # If altbootbank image is the same as the live image, then # the live image has been updated as well. If they are different, # then altbootbank is newer than the live image. It is not possible # for the live image to be newer than /altbootbank once /altbootbank # is updated, we prevent that scenario. if bootinst.bootbankstate & bootinst.BOOTBANK_STATE_UPDATED: if liveinst.database.vibIDs == bootinst.database.vibIDs: return self.IMGSTATE_LIVE_UPDATED else: return self.IMGSTATE_BOOTBANK_UPDATED # Altbootbank contains an older image than /bootbank. # Most likely this is a fresh boot, so the live image is the same as # the bootbank image. # It is possible that a live install happened, but for some reason the # altbootbank was not updated. In this case, we still return live # updated, but if a user reboots, they will lose their live image changes. elif bootinst.database.vibIDs == liveinst.database.vibIDs: return self.IMGSTATE_FRESH_BOOT else: log.info('Live image has been updated but /altbootbank image has ' 'not. A reboot will disgard live changes.') # Logging VIB diff of the databases to help debugging bootinstunique = list(bootinst.database.vibIDs - liveinst.database.vibIDs) liveinstunique = list(liveinst.database.vibIDs - bootinst.database.vibIDs) log.debug('Live image has unique VIB %s, bootbank has unique VIB ' '%s' % (liveinstunique, bootinstunique)) return self.IMGSTATE_LIVE_UPDATED imgstate = property(_getImgState) def _getDatabase(self, database = DB_AUTO): if database not in self.DBS: raise ValueError("Database value of %s not valid" % (database)) # # Pick database automatically. Pick from bootbank database if # bootbank-only remediation has occured; otherwise from live database # if database == self.DB_AUTO: if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED: database = self.DB_BOOTBANK elif 'live' in self.installers: database = self.DB_VISORFS else: database = self.DB_CURBOOTBANK if database == self.DB_BOOTBANK and 'boot' in self.installers: return self.installers['boot'].database elif database == self.DB_VISORFS and 'live' in self.installers: return self.installers['live'].database elif database == self.DB_CURBOOTBANK and 'boot' in self.installers: return self.installers['boot'].bootbank.db else: # When the intended DB is unavailable, we may have ran into an issue. # Stopping is not necessary for all transactions, here we return an # empty database without creating it in visorfs. log.warn('Intended database location %d is not available, database ' 'read issue suspected. Current host image state is %d.' % (database, self.imgstate)) visordbdir = os.path.join('/', LiveImageInstaller.LiveImage.DB_DIR) return Database.Database(visordbdir, dbcreate=False) def _verifyVibSignatureAndPayloads(self, vib, imgprofile, skiplive=False): """Verifies VIB signature and checksums on the payloads of a given vib. The checksums in the vib metadata are compared with the checksums of the modules actually installed. Caller of this method needs to wrap with _getLock() and _freeLock() when verifying VIBs of interest. Parameters: * vib - The vib whose payloads need to be verified * imgprofile - The image profile to use * skiplive - Skip payload verification for live image Exceptions: A sub-type of VibSignatureError - Error in verifying VIB signature ChecksumVerificationError - Error in verifying the checksums InstallationError - Error in the installer """ vib.VerifyAcceptanceLevel() for payload in vib.payloads: # if the payload is not in this image profile, it is assumed # to be a locker vib if payload.name in imgprofile.vibstates[vib.id].payloads: payload.localname = \ imgprofile.vibstates[vib.id].payloads[payload.name] for installer in self._GetOrderedInstallers(): installertype = installer.installertype if installertype == 'live' or installertype == 'boot': if not skiplive or installertype != 'live': installer.VerifyPayloadChecksum(vib.id, payload) else: # Skip the locker installer assert installertype == 'locker' def GetComponents(self, database=DB_AUTO): """Obtains the collection of components installed on the ESX host. If the host is PXE-booted or in the 'Fresh Boot' and 'Live Image Updated' states, then the bulletin inventory will be taken from the VisorFS database. If the host is in the 'Boot Image Updated' state, then the VIB inventory is taken from the /altbootbank database. Components from locker will be added to the return value. Parameters: * database - Override the automatic database selection algorithm and read the inventory from a specific database. Should be one of the DB_* class variables. Returns: A ComponentCollection instance, or None if no database was found Exceptions: ValueError - Illegal database value passed DatabaseFormatError - Database files not formatted correctly DatabaseIOError - Database could not be read ComponentFormatError - One or more files were not a valid Bulletin xml. """ allbulletins = Bulletin.BulletinCollection() allbulletins += self._getDatabase(database).bulletins # Locker components are only kept in locker installer if 'locker' in self.installers: allbulletins += self.installers['locker'].database.bulletins return Bulletin.ComponentCollection(allbulletins, True) def GetInventory(self, database = DB_AUTO): """Obtains the collection of VIBs installed on the ESX host. If the host is PXE-booted or in the 'Fresh Boot' and 'Live Image Updated' states, then the VIB inventory will be taken from the VisorFS database. If the host is in the 'Boot Image Updated' state, then the VIB inventory is taken from the /altbootbank database. If database is DB_AUTO, VIBs from locker will be added to the return value. Parameters: * database - Override the automatic database selection algorithm and read the inventory from a specific database. Should be one of the DB_* class variables. Returns: A VibCollection instance, or None if no database was found Exceptions: ValueError - Illegal database value passed DatabaseFormatError - Database files not formatted correctly DatabaseIOError - Database could not be read """ allvibs = VibCollection.VibCollection() allvibs += self._getDatabase(database).vibs # Locker VIBs are only kept in locker installer if 'locker' in self.installers: allvibs += self.installers['locker'].database.vibs return allvibs def GetProfile(self, database = DB_AUTO): """Obtains the last image profile applied to the ESX host. If the host is PXE-booted or in the 'Fresh Boot' and 'Live Image Updated' states, then the image profile will be taken from the VisorFS database. If the host is in the 'Boot Image Updated' state, then the image profile is taken from the /altbootbank database. Locker VIBs are added back to image profile because they are saved separately in product locker. Parameters: * database - Override the automatic database selection algorithm and read the profile from a specific database. Should be one of the DB_* class variables. Returns: An ImageProfile instance, or None if no image profile is in the database. Exceptions: ValueError - Illegal database value passed DatabaseFormatError - Database files not formatted correctly DatabaseIOError - Database could not be read """ db = self._getDatabase(database) profile = None if db.profile: profile = db.profile.Copy() profile.creationtime = db.profile.creationtime profile._modifiedtime = db.profile._modifiedtime # Adding back locker VIBs and components as live/boot installer # doesn't have locker database contents. # Modify the profile._modifiedtime to the greater of # db.profile._modifiedtime or the latest of the vib installdate # in the locker database. if profile and 'locker' in self.installers: lockerDb = self.installers['locker'].database lockerComps = Bulletin.ComponentCollection(lockerDb.bulletins, True) prevprofilemodifiedtime = profile._modifiedtime for vib in lockerDb.vibs.values(): profile.AddVib(vib) if prevprofilemodifiedtime < vib.installdate: prevprofilemodifiedtime = vib.installdate for comp in lockerComps.IterComponents(): try: profile.AddComponent(comp) except KeyError as e: # After an upgrade from 7.0 pre-U1, if the bootbank image # database was last written by legacy code, e.g. when another # esxcli command was called after upgrade or vib command was used # for the upgrade, we will find that the locker component is # already in the image profile. log.warning('Cannot add locker component: %s. The database was ' 'likely created by legacy code. Another image apply ' 'should resolve this warning.', e) profile._modifiedtime = prevprofilemodifiedtime # Update acceptance level to reflect system config. # Always retrieve the acceptance level each time since it may have # changed in the background. hostAcl = self.GetHostAcceptance() if profile and hostAcl in Vib.ArFileVib.ACCEPTANCE_LEVELS: profile.acceptancelevel = hostAcl return profile @property def livevibs(self): vibs = VibCollection.VibCollection() if 'live' in self.installers: vibs.update(self.GetInventory(self.DB_VISORFS)) return vibs @property def livestagedvibs(self): vibs = VibCollection.VibCollection() if 'live' in self.installers: livevibids = set(self.livevibs.keys()) db = self.installers['live'].stagedatabase if db is not None: for k, v in db.vibs.items(): if k not in livevibids: vibs.AddVib(v) return vibs @property def bootbankupdatedvibs(self): vibs = VibCollection.VibCollection() if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED: # Altbootbank has the latest ImageProfile, GetInventory will return # vibs from latest DB by default. vibs.update(self.GetInventory()) return vibs @property def bootbankstagedvibs(self): vibs = VibCollection.VibCollection() if 'boot' in self.installers: db = self.installers['boot'].stagedatabase if db is not None: vibs.update(db.vibs) return vibs def GetHostHwPlatform(self): if self._hosthw is None: self._hosthw = list() vendor, model = HostInfo.GetBiosVendorModel() self._hosthw.append(Vib.HwPlatform(vendor, model)) # translate OEM Strings into vendor contraints # PR1086517 for vendor in HostInfo.GetBiosOEMStrings(): self._hosthw.append(Vib.HwPlatform(vendor, model='')) return self._hosthw def GetHostAcceptance(self): """Returns the host acceptance level setting. If there is no setting or if the configured value is invalid, then a default acceptance level will be returned. Parameters: None Returns: One of the Vib.ArFileVib.ACCEPTANCE_LEVELS settings. Raises: ValueError -- Illegal value retrieved """ return self._getHostAcceptance() def SetHostAcceptance(self, newlevel): """Sets the host acceptance level to a new level. Validates that the new level still works with the current image profile, ie that no VIBs in the current image has a lower level than the new level. To prevent the acceptance level from changing while an image is being updated, locking will be used. Parameters: * newlevel - the new acceptance level, should be one of the constants Vib.ArFileVib.ACCEPTANCE_LEVELS Returns: None Raises: LockError - Image is being updated, it's not safe to change the level AcceptanceConfigError - new level invalidates the image profile, or unable to run esxcfg-advcfg ValueError - "newlevel" is not a valid acceptance level """ if newlevel not in Vib.ArFileVib.ACCEPTANCE_LEVELS: raise ValueError("Invalid acceptance level of '%s' was passed in" % (newlevel)) if newlevel == Vib.ArFileVib.ACCEPTANCE_COMMUNITY and \ HostInfo.IsHostSecureBooted(): log.info("Secure Boot enabled: Installation of %s " "VIBs is not allowed." % (Vib.ArFileVib.ACCEPTANCE_COMMUNITY)) msg = "Secure Boot enabled: Cannot change acceptance level to %s."\ % (newlevel) raise Errors.AcceptanceConfigError(msg) self._getLock() try: prof = self.GetProfile() oldlevel = prof.acceptancelevel log.info("Attempting to change the host acceptance level from %s to %s" % (oldlevel, newlevel)) prof.acceptancelevel = newlevel prof.vibs = self.GetInventory() # Focus on only acceptance level problems # TODO: Disable extensibility rule checks problems = prof.Validate(nodeps=True, noconflicts=True, allowobsoletes=True, allowfileconflicts=True) if problems: badvibs = [getattr(p, 'vibid', '') for p in problems] msg = "Unable to set acceptance level of %s due to installed VIBs "\ "%s having a lower acceptance level." \ % (newlevel, ', '.join(badvibs)) raise Errors.AcceptanceConfigError(msg) self._setHostAcceptance(newlevel) # See #bora/apps/addvob/addvob.c for the vob format string. self.SendVob("hostacceptance.changed", oldlevel, newlevel) finally: self._freeLock() def _GetOrderedInstallers(self): return sorted(self.installers.values(), key=attrgetter('priority')) def _verify_and_write_payload(self, installer, vib, payload, sourcefp): """Write payload to installer staging area and verify checksum.""" dfp = installer.OpenPayloadFile(vib.id, payload, write=True, read=False) if not dfp: log.debug('Payload %s from vib %s skipped by %s' % (payload.name, vib.id, installer.__class__.__name__)) return # LiveImageInstaller requires decompressed payloads (tardisks) decompress = (installer.installertype == 'live') try: Vib.copyPayloadFileObj(payload, sourcefp, dfp, decompress=decompress, checkdigest=True) finally: dfp.close() def _find_path_in_deploy_dir(self, deploydir, filename): """Find absolute path of a file in deploy directory.""" upperpath = os.path.join(deploydir, filename.upper()) lowerpath = os.path.join(deploydir, filename.lower()) if os.path.exists(upperpath): return upperpath elif os.path.exists(lowerpath): return lowerpath else: return None def _stage_from_deploy_dir(self, imgprofile, adds, installer, deploydir): """Stage payloads of VIBs from deploy format folder to installer.""" for vibid in adds: vib = imgprofile.vibs[vibid] # Validate vib signature and schema vib.VerifyAcceptanceLevel() # Update installer database try: if installer.installertype in ['boot', 'live']: installer.UpdateVibDatabase(vib) except Exception as e: # continue even if we fail to update the vib database. pass # Only support payloads for bootbank or locker installer SUPPORTED_PAYLOADS = set(list(BootBankInstaller.BootBankInstaller. \ SUPPORTED_PAYLOADS) + \ list(LockerInstaller.LockerInstaller. \ SUPPORTED_PAYLOADS)) for payload in vib.payloads: if not payload.localname: raise Errors.InstallationError(None, [vibid], 'No local name available for ' 'payload %s' % payload.name) # Keep source localname and update new localname srcname = payload.localname payload.localname = imgprofile.vibstates[vibid]. \ payloads[payload.name] # Skip unsupported payloads if not payload.payloadtype in SUPPORTED_PAYLOADS: continue sourcepath = self._find_path_in_deploy_dir(deploydir, srcname) if not sourcepath: raise Errors.InstallationError(None, [vibid], 'Failed to locate payload file ' '%s in directory %s' % (payload.localname, deploydir)) with open(sourcepath, 'rb') as sourcefp: self._verify_and_write_payload(installer, vib, payload, sourcefp) # Add install date for newly installed vib if vib.installdate is None: vib.installdate = datetime.datetime.now(_utctzinfo) def _download_and_stage(self, imgprofile, vibid, installer, checkacceptance = True, schema = None): """Download a vib, verify and stage the payloads.""" arvib = src = None vib = imgprofile.vibs[vibid] log.info('Attempting to download VIB %s', vib.name) download_path = self.VIB_DOWNLOAD + '/' + vib.id + '.vib' for url in vib.remotelocations: try: d = Downloader.Downloader(url, download_path) actual_path = d.Get() src = open(actual_path, 'rb') arvib = Vib.ArFileVib.FromFile(src, schema) break except Exception as e: if src: src.close() src = None log.info('Unable to download from %s, error [%s]. Trying next ' 'url...', url, str(e)) # remove downloaded data if there is any if os.path.isfile(download_path): os.unlink(download_path) continue if not arvib: urls = ', '.join(vib.remotelocations) raise Errors.VibDownloadError(urls, download_path, "Unable to download VIB from any of the URLs %s" % urls) # Verify descriptor matches metadata: try: arvib.MergeVib(vib) if checkacceptance: arvib.VerifyAcceptanceLevel() except Exception as e: src.close() raise Errors.InstallationError(e, [arvib.id], str(e)) # The vib metadata maintained by the installer does not contain # information that is not part of the image profile metadata. # New information like the original vib descriptor and the signature # information must be updated in that database. try: # Locker installer does not support signature verification yet if installer.installertype == 'boot' or installer.installertype == 'live': installer.UpdateVibDatabase(arvib) except Exception as e: # continue even if we fail to update the vib database. pass # TODO: # - verify hardware platform try: # In the locker parition, there could be extra files like vSAN traces. # Make sure the parition has enough free space to install the VIB. # This check has to be late because we need the arctual VIB file to # calculate space usage. As a result, old locker VIB has already been # removed. User has to retry the transaction, or install the old # VIB again after clean up the partition. if installer.installertype == 'locker': installer._CheckDiskSpaceForVib(arvib) # Loop through each payload, open dest payload file, and start buffer # copy. Compute digest. for payload, sourcefp in arvib.IterPayloads(): if payload.name in imgprofile.vibstates[vibid].payloads: vibstate = imgprofile.vibstates[vibid] payload.localname = vibstate.payloads[payload.name] else: payload.localname = None self._verify_and_write_payload(installer, arvib, payload, sourcefp) except Exception as e: raise Errors.InstallationError(e, [arvib.id], str(e)) finally: src.close() # Explicitly check for installdate is None. If installdate is already # set, we do not want to reset it. if vib.installdate is None: vib.installdate = datetime.datetime.now(_utctzinfo) # If the vib was downloaded from HTTP, remove it if os.path.isfile(download_path): os.unlink(download_path) def _get_tardisk_payload(self, imgprofile, vibid, vib): """Stage compressed payloads of a VIB for stagebootbank from uncompressed tardisks in /tardisks. """ vibstate = imgprofile.vibstates[vibid] for pl in vib.payloads: if pl.payloadtype not in BootBankInstaller.\ BootBankInstaller.SUPPORTED_PAYLOADS: continue plname = pl.name srcfile = None if plname in vibstate.payloads: srcname = vibstate.payloads[plname] srcfile = os.path.join('/tardisks', srcname) destfile = os.path.join(BootBankInstaller.\ BootBankInstaller.STAGEBOOTBANK, srcname) if srcfile is not None and os.path.isfile(srcfile): log.info('Compressing %s to %s' % (srcfile, destfile)) with open(srcfile, 'rb') as srcfObj: with open(destfile, 'wb') as destfObj: Vib.copyPayloadFileObj(pl, srcfObj, destfObj, compress=True) else: msg = 'Cannot locate source for payload %s of '\ 'VIB %s ' % (plname, vibid) if HostInfo.HostOSIsSimulator(): log.info('HostSimulator: %s' % (msg)) else: raise Errors.InstallationError(None, [vibid], msg) def _stage_with_url(self, imgprofile, adds, installer, checkacceptance, vibschema): """Stage VIBs with URLs in vib/profile transaction""" # Use the largest VIB size to create download ramdisk maxVibSize = 0 for vibid in adds: vib = imgprofile.vibs[vibid] vibPackedSize = 0 for payload in vib.payloads: vibPackedSize += payload.size vibPackedSize = int(vibPackedSize / 1024 / 1024) + 1 if vibPackedSize > maxVibSize: maxVibSize = vibPackedSize # Add a buffer for descriptor and signature maxVibSize += 5 Downloader.Downloader.setEsxupdateFirewallRule('true') Ramdisk.CreateRamdisk(maxVibSize, self.VIB_DOWNLOAD_NAME, self.VIB_DOWNLOAD) try: for vibid in adds: vib = imgprofile.vibs[vibid] if not vib.remotelocations: # for vib installed after the host boots up, # get vib payloads from /tardisks. # This is only applicable to BootBankInstaller since # for LiveImageInstaller and LockerInstaller # new vibs always have vib.remotelocations. log.info("Constructing payloads from tardisks for vib %s" % vibid) self._get_tardisk_payload(imgprofile, vibid, vib) else: self._download_and_stage(imgprofile, vibid, installer, checkacceptance, vibschema) finally: Downloader.Downloader.setEsxupdateFirewallRule('false') Ramdisk.RemoveRamdisk(self.VIB_DOWNLOAD_NAME, self.VIB_DOWNLOAD) def Stage(self, imgprofile, nosigcheck = False, forcebootbank = False, dryrun = False, vibschema = None, stageonly = False, checkacceptance = True, deploydir=None): """Stage prepares or 'stages' a new ESX image for application based on the specified image profile. If the current image or last staged image is composed of the same VIBs as the passed in image profile, then the Stage method will have no work to do. Maintenance Mode is not required during a stage operation. Stage also checks for any hwplatform dependencies. The VIBs are downloaded and a new ESXi image is prepared. The StartTransaction, OpenPayloadFile, and CompleteStage methods of *Installer subclasses actually carry out the work. This Stage method is responsible for coordinating between different Installers, VIB signature validation, and driving each Installer to install successive VIB payloads. The databases will be locked to prevent another process from calling Stage() at the same time. Stage manages the sourcing of VIBs, either initiating downloads or sourcing them from previous Installers for chained installation. TBD: Cycling may not be practical. We want to skip LiveImageInstaller once the image state becomes BOOTBANK_UPDATED or forcebootbank is True. Parameters: * imgprofile - The ImageProfile instance to stage. It must have a populated vibs attribute. * nosigcheck - Boolean, skips signature validation if True * forcebootbank - Force an install using BootbankInstaller even if the LiveImageInstaller could be used * dryrun - Dry run only, report what will be changed but do not make changes to the system. * vibschema - A file path to an XML schema for validation of the VIB descriptor. Set to None to disable validation. * checkacceptance - If True (the default), VIB acceptance levels will be validated. A failed validation raises an exception. * deploydir - Directory which contain image in deploy format, such as ISO and PXE. When supplied, database and payloads will be fetched from the directory. Returns: None Exceptions: InstallationError LockingError - Unable to lock database for writing. Most likely another thread has invoked Stage(). VibDownloadError - error during download of a VIB VibFormatError - improper VIB or descriptor structure VibSignatureError - invalid VIB signature DatabaseFormatError - Database files not formatted correctly DatabaseIOError - Database could not be read HardwareError - VIB hwplatform dependency could not be satisfied """ assert len(imgprofile.vibs) >= len(imgprofile.vibIDs) log.debug('Staging image profile [%s]' % (imgprofile.name)) log.debug('VIBs in image profile: %s' % (', '.join(imgprofile.vibIDs))) # Check potential installation problems before hand # - current bootbank bootstate must be 0 # - installation size must fit in bootbank partition if 'boot' in self.installers: self.installers['boot'].PreInstCheck(imgprofile) # hwplatform requirement check hwmatchproblems = list() hosthws = self.GetHostHwPlatform() for imghw in imgprofile.GetHwPlatforms(): for hw in hosthws: prob = imghw.MatchProblem(hw) if prob is None: # We have a match, forget any other mismatches hwmatchproblems = list() break else: hwmatchproblems.append(prob) else: continue break # if we break out of the inner loop, break out of the outer too if len(hwmatchproblems) > 0: reasons = ["BIOS %s %s is required but host BIOS %s is %s" \ % (p[0], p[1], p[0], p[2]) for p in hwmatchproblems] msg = "Host doesn't meet image profile '%s' hardware requirements:" \ "\n%s" % (imgprofile.name, '\n'.join(reasons)) log.error(msg) # See #bora/apps/addvob/addvob.c for the vob format string. self.SendVob("install.invalidhardware", imgprofile.name, \ '\n'.join(reasons)) raise Errors.HardwareError(msg) # Send VOB warning if installing image profile with nosigcheck or # checkacceptance is False if nosigcheck or not checkacceptance: reasons = list() if nosigcheck: reasons.append("signature validation disabled") if not checkacceptance: reasons.append("acceptance level checking disabled") msg = "SECURITY ALERT: Installing image profile '%s' with %s." \ % (imgprofile.name, " and ".join(reasons)) log.warn(msg) # See #bora/apps/addvob/addvob.c for the vob format string. self.SendVob("install.securityalert", imgprofile.name, \ " and ".join(reasons)) try: # # Cycle through each installer. Installers are ordered by its # priority. Current design is to run live installer first # installerstate = {'unsupported' : list(), 'nochange' : list(), 'finished' : list()} for installer in self._GetOrderedInstallers(): # Skip bootbank stage if live staged installertype = installer.installertype if installertype == 'boot' and \ 'live' in installerstate['finished']: log.debug("Already staged to live image, skipping " "BootbankInstaller.") installerstate['finished'].append(installertype) continue try: adds, removes, staged = installer.StartTransaction(imgprofile, imgstate = self.imgstate, forcebootbank = forcebootbank, stageonly = stageonly, preparedest = not dryrun) except EnvironmentError as e: raise Errors.InstallationError(e, None, "Failed to start installation: %s" % str(e)) # (None, None, False) is returned if installer could not handle the # transaction if adds is None and removes is None: log.info('%s is not supported, skipping.' % (installer.__class__.__name__)) installerstate['unsupported'].append(installertype) continue if not adds and not removes: log.info('Nothing for %s to do, skipping.' % (installer.__class__.__name__)) installerstate['nochange'].append(installertype) continue log.debug(' --- Stage: %s adding [%s], removing [%s]' % ( installer.__class__.__name__, ', '.join(adds), ', '.join(removes))) if not (staged or dryrun): if deploydir: # for upgrade using deploy format self._stage_from_deploy_dir(imgprofile, adds, installer, deploydir) else: # for tranasction with vib/profile url self._stage_with_url(imgprofile, adds, installer, checkacceptance, vibschema) installer.CompleteStage() installerstate['finished'].append(installertype) if len(installerstate['unsupported']) == len(self.installers): msgs = list() for installer in self.installers.values(): for problem in installer.problems: msgs.append(str(problem)) if 'boot' not in self.installers: raise Errors.StatelessError( 'The transaction is not supported: %s' % (' '.join(msgs))) else: raise Errors.InstallationError(None, None, 'The transaction is not supported: %s' % (' '.join(msgs))) if dryrun: installers = [INSTALLERTYPE_TO_NAME[inst] for inst in installerstate['finished']] msg = ('Dryrun only, host not changed. ' 'The following installers will be applied: [%s]' % (', '.join(installers))) # If only boot bank installer is applied, then # live image is not changed - reboot is required if 'boot' in installerstate['finished'] and \ 'live' not in installerstate['finished']: raise Errors.NeedsRebootResult(msg) else: raise Errors.NormalExit(msg) except Errors.NormalExit: raise except Exception as e: msg = "Could not stage image profile '%s': %s" % (imgprofile.name, e) # See #bora/apps/addvob/addvob.c for the vob format string. self.SendVob("install.stage.error", imgprofile.name, e) # cleanup installers for installer in self._GetOrderedInstallers(): # locker install is done in-place, should not cleanup if installer.installertype in ['live', 'boot']: installer.Cleanup() raise def Remediate(self, imgprofile, checkmaintmode = True): """Remediate applies a previously staged image. The exact behavior depends on the *Installer subclasses carrying out the work. Maintenance Mode is checked if any VIBs require it. The databases will be locked for writing; therefore only one process may invoke Remediate() at a time. Stage must be called before Remediate(). If not, Remediate may not do anything. Parameters: * imgprofile - The ImageProfile that has been staged. * checkmaintmode - Boolean, check that the ESX host is in maintenance mode if the VIBs require it. Note that for bootbank-only installs, Remediate is a NOP, so maintenance mode is not required. Returns: None Exceptions: InstallationError LockingError - Unable to lock database for writing. Most likely another thread has invoked Remediate(). MaintenanceModeError - Maintenance mode was required but the host is not in maintenance mode. HostNotChanged - If host is not changed. NeedsRebootResult - Remediation is bootbank only install, reboot is required. """ finishedinstallers = set() try: for installer in self._GetOrderedInstallers(): try: if 'live' in finishedinstallers \ and installer.installertype == 'boot': installer.CacheNewImage(self.installers['live']) else: installer.Remediate(checkmaintmode) except Errors.HostNotChanged as e: continue except Exception as e: log.debug('installer %s failed: %s. Clean up the installation.', installer.__class__.__name__, e) raise finishedinstallers.add(installer.installertype) except Errors.NormalExit: raise except Exception as e: # See #bora/apps/addvob/addvob.c for the vob format string. self.SendVob("install.error", e) raise finally: # no more action for installers, perform cleanup for installer in self._GetOrderedInstallers(): # locker install is done in-place, should not cleanup if installer.installertype in ['live', 'boot']: installer.Cleanup() log.debug('Host is remediated by installer: %s' % ( ', '.join(finishedinstallers))) # The image profile in the live/bootbank/locker installers may need to be # updated even if there is no VIB change in the respective installer. # 1) In a locker/bootbank installers, update description and component # metadata. # 2) In a transaction with no VIB change at all, base image, addon and # component metadata may have changed. for installer in self._GetOrderedInstallers(): # VIB change in installer, no need to check metadata update. if installer.installertype in finishedinstallers: continue # If the host is already pending reboot, modify altbootbank and locker # database only. Otherwise both current bootbank and visorfs databases # will be updated. if (installer.installertype == 'live' and self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED): continue log.debug('Check for image profile metadata update in installer ' '%s', installer.installertype) installer.UpdateImageProfile(imgprofile) if len(finishedinstallers) == 0: if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED: # return NeedsRebootResult when host is pending reboot raise Errors.NeedsRebootResult('Host is not changed. ' 'Reboot is pending from previous transaction.') else: raise Errors.HostNotChanged('Host is not changed.') elif 'live' not in finishedinstallers \ and 'boot' in finishedinstallers: raise Errors.NeedsRebootResult() elif 'live' in finishedinstallers \ and 'boot' not in finishedinstallers: raise Errors.LiveInstallOnlyResult() def GetVibPayloadMap(self): """Returns a map of installed VIB ID to local file names of payloads in the VIB. """ payloads = dict() imgProfile = self.GetProfile() for vibId, vibState in imgProfile.vibstates.items(): payloads[vibId] = sorted(vibState.payloads.values()) return payloads def GetCompSigVerifyMap(self, rebootingImage=False): """Verifies component VIB signatures and payload checksums, returns a map from component IDs to component info like name, version, vendor, acceptance level and a verification result as a map from VIB IDs to its verification result returned from GetVibSigVerifyMap(). """ res = dict() comps = self.GetComponents() vibResult = self.GetVibSigVerifyMap(rebootingImage) if comps and vibResult: # When there are components on host and rebootingImage, if set, is # used properly. for comp in comps.IterComponents(): compRes = dict(name=comp.compNameStr, version=comp.compVersionStr, vendor=comp.vendor, vibs=dict()) # Acceptance level of a component is the minimum level of the VIBs. minAccLevel = Vib.ArFileVib.ACCEPTANCE_CERTIFIED for vibId in comp.vibids: vibInfo = vibResult[vibId] accLevel = vibInfo['acceptance_level'] if (AcceptanceChecker.TRUST_ORDER[accLevel] < AcceptanceChecker.TRUST_ORDER[minAccLevel]): minAccLevel = accLevel compRes['vibs'][vibId] = vibInfo['verification_result'] compRes['acceptance_level'] = minAccLevel res[comp.id] = compRes return res def GetVibSigVerifyMap(self, rebootingImage=False): """Verifies VIB signature and payload checksums, returns a map from VIB IDs to VIB info like name, version, vendor, acceptance level and a verification result as a tuple of one of SIG_VERIFY_* states and an error message for SIG_VERIFY_ERROR. Parameter: rebootingImage - when set, check VIBs in the altbootbank image. """ self._getLock() try: res = dict() if (rebootingImage and self.imgstate in (self.IMGSTATE_LIVE_UPDATED, self.IMGSTATE_BOOTBANK_UPDATED)): # Specified rebooting image and /altbootbank is actually updated. database = self.DB_BOOTBANK else: # Verifying live image or /bootbank will be the rebooting image. database = self.DB_VISORFS imgProfile = self.GetProfile(database=database) # Do not use vibs of image profile in case an VIB ID is mismatching # due to corruption/tampering, which leads to misreporting. vibs = self.GetInventory(database=database) for vib in vibs.values(): accLevel = getattr(vib, 'acceptancelevel', 'None') vibRes = (self.SIG_VERIFY_UNKNOWN, None) if vib.vibtype == Vib.ArFileVib.TYPE_LOCKER: vibRes = (self.SIG_VERIFY_LOCKER, None) elif isinstance(vib, Vib.ArFileVib): try: self._verifyVibSignatureAndPayloads(vib, imgProfile, rebootingImage) vibRes = (self.SIG_VERIFY_PASSED, None) except Errors.VibSigMissingError: vibRes = (self.SIG_VERIFY_MISSING, None) except (Errors.VibFormatError, Errors.VibValidationError, Errors.VibSignatureError, Errors.ChecksumVerificationError) as e: vibRes = (self.SIG_VERIFY_ERROR, str(e)) res[vib.id] = dict(name=vib.name, version=vib.versionstr, vendor=vib.vendor, acceptance_level=accLevel, verification_result=vibRes) finally: self._freeLock() return res @staticmethod def SendVob(eventid, *args): """Sends a VOB (VmKernel Observation) event. VOB events allow important host events to be easily tracked in various clients. This method is designed to send VOBs related to esximage. Parameters: * eventid - The VOB ID relative to vob.user.esximage, and (more importantly) the event type ID relative to esx.problem.esximage of the event that will be posted to hostd. That "esx.problem.esximage." + eventid string will then be associated with a KB article. * args - The arguments associated with the vob. See #bora/apps/addvob/addvob.c for details. """ # Avoid import esx module globally due to confusion to vmotion test # lib in testesx. import esx.vob vob = esx.vob.createVob("vob.user.esximage." + eventid, *args) try: vob.send() except Exception as e: # Not logging the arguments, as syslog has length limits. log.error('Failed to send vob %s: %s' % (eventid, e)) @staticmethod def SendConsoleMsg(msg): """Sends msg to console. The message will be displayed in red on informational console to warn user about potential issues. Parameters: * msg - A string describing the warning. """ if HAVE_SPICY: spicy.set_sysalert(msg) @staticmethod def SendAuditEvent(eventID, note, error, adds, removes): """Send an system audit event. Read bora/lib/public/vmwAudit.h on system.update.start and system.update.end events. Parameters: eventID - AUDIT_START_EVENTID or AUDIT_END_EVENTID. note - String. In a success case, supply extra information to be included in the reason field. Must be defined as an AUDIT_* variable. error - An Exception object. In a failure case, message of this object is used to form the reason field. adds - VIB IDs that are being added to the image profile. removes - VIB IDs that are being removed from the image profile. """ import pwd import socket # From RFC 5424: # https://tools.ietf.org/html/rfc5424 PRIORITY_INFO = 6 FACILITY_AUDIT = 13 # VMware IANA number: # https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers VMWARE_IANA_NUM = 6876 AUDIT_LOG_SOCKET = '/dev/auditlog' AUDIT_LOG_FORMAT = '%(facility)u\0%(priority)u\0%(timeStamp)s\0' \ 'esxupdate\0%(pid)u\0' \ '[%(eventID)s@%(originator)u subject="%(userName)s" ' \ 'object="" result="%(result)s" reason="%(reason)s" ' \ 'vib="%(vib)s"]\0\0' if error: # Fomulate error reason. Try to use msg for esxupdate errors and # replace the escaped chars and new lines. msg = error.msg if hasattr(error, 'msg') and error.msg \ else str(error) replaces = [('\n', ','), ('"', ''), ('\'', ''), ('[', '{'), (']', '}')] for old, new in replaces: msg = msg.replace(old, new) else: # Use the extra note for a success msg = note if note else '' for i in range(4): s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) try: s.connect(AUDIT_LOG_SOCKET) utcNow = datetime.datetime.utcnow() isoTimeStamp = utcNow.strftime('%Y-%m-%dT%H:%M:%S.') + \ "%03dZ" % (utcNow.microsecond // 1000) msgArgs = { 'facility': FACILITY_AUDIT, 'priority': PRIORITY_INFO, 'timeStamp': isoTimeStamp, 'pid': os.getpid(), 'eventID': eventID, 'originator': VMWARE_IANA_NUM, 'userName': pwd.getpwuid(os.geteuid()).pw_name, 'result': 'failure' if error else 'success', 'reason': msg, 'vib': 'add: {%s} remove: {%s}' \ % (','.join(adds), ','.join(removes)), } s.send((AUDIT_LOG_FORMAT % msgArgs).encode()) except OSError as e: log.warn('Failed to send audit event, try #%d: %s' % (i + 1, str(e))) time.sleep(2) else: break finally: s.close() @staticmethod def _getHostAcceptance(): # # If there is an error or the configured value does not make sense, # we return a default. try: _, out = runcommand([ADVCFG, '-U', 'host-acceptance-level', '-G'], timeout=30.0) except RunCommandError as e: msg = 'Unable to execute %s: %s' % (os.path.basename(ADVCFG), str(e)) log.error(msg) raise Errors.AcceptanceGetError(msg) hostaccept = byteToStr(out).strip() if hostaccept in Vib.ArFileVib.ACCEPTANCE_LEVELS: return hostaccept elif hostaccept: log.error("Illegal acceptance level '%s' obtained from " "configuration" % (hostaccept)) return hostaccept else: log.error("No host acceptance level is configured") return "" @staticmethod def _setHostAcceptance(acceptance): assert acceptance in Vib.ArFileVib.ACCEPTANCE_LEVELS try: res, out = runcommand([ADVCFG, '-U', 'host-acceptance-level', '-S', acceptance]) except RunCommandError as e: msg = 'Unable to execute %s: %s' % (os.path.basename(ADVCFG), str(e)) log.error(msg) raise Errors.AcceptanceConfigError(msg) if res != 0: msg = ('%s exited with non-zero status %d, output: %s' % (os.path.basename(ADVCFG), res, out)) log.error(msg) raise Errors.AcceptanceConfigError(msg) def _getLock(self): """Lock esximage exclusive file lock. Should be called with write transactions at entry points. Most of such transactions happen in Transaction with a few exception in HostImage. """ # We will try to acquire lock for one min with 5 sec interval retry retries = 13 delay = 5 for i in range(retries): try: self._lock.Lock() break except LockFile.LockFileError as e: log.warn('Failed to acquire lock: %s', str(e)) if i < retries - 1: time.sleep(delay) else: raise Errors.LockingError("Another process is updating the ESX " "image. Please try again later.") # Refresh the installers to make sure we load the latest state. # Even without a single locking error the state might have been just # refreshed before this method is called. self._initInstallers() def _freeLock(self): """Free esximage exclusive file lock. """ try: self._lock.Unlock() except Exception as e: raise Errors.LockingError("Unable to free lock: %s" % (str(e)))