######################################################################## # Copyright (C) 2009-2020 VMWare, Inc. # All Rights Reserved ######################################################################## # # Bulletin.py # __all__ = ['Bulletin', 'BulletinCollection'] import datetime import logging import os import operator import shutil import collections from . import Errors from . import VibCollection from . import Version from .ComponentScanner import ComponentScanner, ComponentScanProblem from .Utils import XmlUtils etree = XmlUtils.FindElementTree() SCHEMADIR = XmlUtils.GetSchemaDir() log = logging.getLogger('Bulletin') def getDefaultBulletinFileName(bulletin): """Default naming function for bulletins/components. """ return bulletin.id + '.xml' def getDatabaseComponentFileName(comp): """Naming function for components in database, hash is used on the ID to shorten the length. """ return comp.compNameStr + '-' + str(hash(comp.id)) + '.xml' class InvalidRelationToken(Exception): """Exception class that is used to signify a bad relation token. i.e. There is no >, >=, <. <=, in the relation """ pass class ContentBody(object): """Represents the tag in a notification bulletin. Attributes: * html - HTML text content. * text - Plain text content. """ def __init__(self, html="", text=""): self.html = html self.text = text def __cmp__(self, other): compare = lambda x, y: (x > y) - (x < y) return compare((self.html, self.text), (other.html, other.text)) __lt__ = lambda self, other: self.__cmp__(other) < 0 __le__ = lambda self, other: self.__cmp__(other) <= 0 __eq__ = lambda self, other: self.__cmp__(other) == 0 __ne__ = lambda self, other: self.__cmp__(other) != 0 __ge__ = lambda self, other: self.__cmp__(other) >= 0 __gt__ = lambda self, other: self.__cmp__(other) > 0 @classmethod def FromXml(cls, xml): if etree.iselement(xml): node = xml else: try: node = etree.fromstring(xml) except Exception as e: msg = "Error parsing XML data: %s." % e raise Errors.BulletinFormatError(msg) # Note: This implies that the HTML tags are XML-encoded. I.e., we do not # just handle the element content as CDATA. html = (node.findtext("htmlData") or "").strip() text = (node.findtext("defaultText") or "").strip() return cls(html, text) def ToXml(self): root = etree.Element("contentBody") if self.html: etree.SubElement(root, "htmlData").text = self.html if self.text: etree.SubElement(root, "defaultText").text = self.text return root class RecallResolution(object): """Represents the tag in a notification bulletin. Attributes: * recallfixid - The recall ID. * bulletins - A set containing IDs of fixed bulletins. """ def __init__(self, recallfixid, bulletins=None): self.recallfixid = recallfixid self.bulletins = bulletins is not None and bulletins or set() def __cmp__(self, other): compare = lambda x, y: (x > y) - (x < y) return compare((self.recallfixid, self.bulletins), (other.recallfixid, other.bulletins)) __lt__ = lambda self, other: self.__cmp__(other) < 0 __le__ = lambda self, other: self.__cmp__(other) <= 0 __eq__ = lambda self, other: self.__cmp__(other) == 0 __ne__ = lambda self, other: self.__cmp__(other) != 0 __ge__ = lambda self, other: self.__cmp__(other) >= 0 __gt__ = lambda self, other: self.__cmp__(other) > 0 @classmethod def FromXml(cls, xml): if etree.iselement(xml): node = xml else: try: node = etree.fromstring(xml) except Exception as e: msg = "Error parsing XML data: %s." % e raise Errors.BulletinFormatError(msg) recallfixid = (node.findtext("recallFixID") or "").strip() if not recallfixid: raise Errors.BulletinFormatError("Invalid recall fix ID.") bulletins = set() for bulletinid in node.findall("bulletinIDList/bulletinID"): newid = (bulletinid.text or '').strip() if newid: bulletins.add(newid) return cls(recallfixid, bulletins) def ToXml(self): root = etree.Element("recallResolution") etree.SubElement(root, "recallFixID").text = self.recallfixid if self.bulletins: node = etree.SubElement(root, "bulletinIDList") for bulletinid in self.bulletins: etree.SubElement(node, "bulletinID").text = bulletinid return root class RecallResolutionList(object): """Represents the tag in a notification bulletin. Attributes: * recallid - The ID of the recall. * resolutions - A list of RecallResolution objects. """ def __init__(self, recallid, resolutions = None): self.recallid = recallid self.resolutions = resolutions is not None and resolutions or list() def __cmp__(self, other): compare = lambda x, y: (x > y) - (x < y) return compare((self.recallid, self.resolutions), (other.recallid, other.resolutions)) __lt__ = lambda self, other: self.__cmp__(other) < 0 __le__ = lambda self, other: self.__cmp__(other) <= 0 __eq__ = lambda self, other: self.__cmp__(other) == 0 __ne__ = lambda self, other: self.__cmp__(other) != 0 __ge__ = lambda self, other: self.__cmp__(other) >= 0 __gt__ = lambda self, other: self.__cmp__(other) > 0 @classmethod def FromXml(cls, xml): if etree.iselement(xml): node = xml else: try: node = etree.fromstring(xml) except Exception as e: msg = "Error parsing XML data: %s." % e raise Errors.BulletinFormatError(msg) recallid = (node.findtext("recallID") or "").strip() if not recallid: raise Errors.BulletinFormatError("Invalid recall ID.") resolutions = [RecallResolution.FromXml(x) for x in node.findall("recallResolution")] return cls(recallid, resolutions) def ToXml(self): root = etree.Element("resolvedRecalls") etree.SubElement(root, "recallID").text = self.recallid for resolution in self.resolutions: root.append(resolution.ToXml()) return root class Bulletin(object): """A Bulletin defines a set of Vib packages for a patch, update, or ESX extension. Class Variables: * NOTIFICATION_RECALL * NOTIFICATION_RECALLFIX * NOTIFICATION_INFO * NOTIFICATION_TYPES * RELEASE_PATCH * RELEASE_ROLLUP * RELEASE_UPDATE * RELEASE_EXTENSION * RELEASE_NOTIFICATION * RELEASE_UPGRADE * SEVERITY_CRITICAL * SEVERITY_SECURITY * SEVERITY_GENERAL Attributes: * id - A string specifying the unique bulletin ID. * vendor - A string specifying the vendor/publisher. * summary - The abbreviated (single-line) bulletin summary text. * severity - A string specifying the bulletin's severity. * urgency - A string specifying the bulletin's "urgency." * category - A string specifying the bulletin's category. * releasetype - A string specifying the release type. * componentnamespec - A dict specifying the component 'name' and a human friendly 'uiString' which describes the name. * componentversionspec - A dict specifying the component 'version' and a human friendly 'uiString' which describes the version. * description - The (multi-line) bulletin description text. * kburl - A URL to a knowledgebase article related to the bulletin. * contact - Contact information for the bulletin's publisher. * releasedate - An integer or float value giving the bulletin's release date/time. May be None if release date is unknown. * platforms - A list of tuples, each of which holds a (version, locale, productLineID) tuple. * vibids - A set of VIB IDs with no corresponding VIB object. These would generally be from a XML element, and are meant to be referenced later in order to properly establish keys in the Bulletin with proper VIB objects as values. * configSchemaVibs - A list of IDs of VIBs that have a config schema. """ NOTIFICATION_RECALL = 'recall' NOTIFICATION_RECALLFIX = 'recallfix' NOTIFICATION_INFO = 'info' NOTIFICATION_TYPES = (NOTIFICATION_RECALL, NOTIFICATION_RECALLFIX, NOTIFICATION_INFO) RELEASE_PATCH = 'patch' RELEASE_ROLLUP = 'rollup' RELEASE_UPDATE = 'update' RELEASE_EXTENSION = 'extension' RELEASE_NOTIFICATION = 'notification' RELEASE_UPGRADE = 'upgrade' RELEASE_TYPES = (RELEASE_PATCH, RELEASE_ROLLUP, RELEASE_UPDATE, RELEASE_EXTENSION, RELEASE_NOTIFICATION, RELEASE_UPGRADE) SEVERITY_CRITICAL = 'critical' SEVERITY_SECURITY = 'security' SEVERITY_GENERAL = 'general' SEVERITY_TYPES = (SEVERITY_GENERAL, SEVERITY_SECURITY, SEVERITY_CRITICAL) URGENCY_CRITICAL = 'critical' URGENCY_IMPORTANT = 'important' URGENCY_MODERATE = 'moderate' URGENCY_LOW = 'low' URGENCY_GENERAL = 'general' URGENCY_TYPES = (URGENCY_CRITICAL, URGENCY_IMPORTANT, URGENCY_MODERATE, URGENCY_LOW, URGENCY_GENERAL) CATEGORY_SECURITY = 'security' CATEGORY_BUGFIX = 'bugfix' CATEGORY_ENHANCEMENT = 'enhancement' CATEGORY_RECALL = 'recall' CATEGORY_RECALLFIX = 'recallfix' CATEGORY_INFO = 'info' CATEGORY_MISC = 'misc' CATEGORY_GENERAL = 'general' CATEGORY_TYPES = (CATEGORY_SECURITY, CATEGORY_BUGFIX, CATEGORY_ENHANCEMENT, CATEGORY_RECALL, CATEGORY_RECALLFIX, CATEGORY_INFO, CATEGORY_MISC, CATEGORY_GENERAL) SCHEMA_FILE = 'bulletin-xml.rng' def __init__(self, id, **kwargs): """Class constructor. Parameters: * id - A string giving the unique ID of the Bulletin. * kwargs - A list of keyword arguments used to populate the object's attributes. Returns: A new Bulletin instance. """ if not id: raise Errors.BulletinFormatError("id parameter cannot be None") self._id = id tz = XmlUtils.UtcInfo() now = datetime.datetime.now(tz=tz) self.vendor = kwargs.pop('vendor', '') self.summary = kwargs.pop('summary', '') self.severity = kwargs.pop('severity', '') self.urgency = kwargs.pop('urgency', '') self.category = kwargs.pop('category', '') self.releasetype = kwargs.pop('releasetype', '') self.componentnamespec = kwargs.pop('componentnamespec', dict()) self.componentversionspec = kwargs.pop('componentversionspec', dict()) self.description = kwargs.pop('description', '') self.kburl = kwargs.pop('kburl', '') self.contact = kwargs.pop('contact', '') self.releasedate = kwargs.pop('releasedate', now) self.platforms = kwargs.pop('platforms', list()) self.vibids = kwargs.pop('vibids', set()) self.configSchemaVibs = kwargs.pop('configSchemaVibs', set()) # These are specific to notification bulletins. self.recalledvibs = kwargs.pop('recalledvibs', set()) self.recalledbulletins = kwargs.pop('recalledbulletins', set()) self.contentbody = kwargs.pop('contentbody', None) self.resolvedrecalls = kwargs.pop('resolvedrecalls', None) if kwargs: badkws = ', '.join("'%s'" % kw for kw in kwargs) raise TypeError("Unrecognized keyword argument(s): %s." % badkws) __repr__ = lambda self: self.__str__() __hash__ = lambda self: hash(self._id) id = property(lambda self: self._id) # Component name/version string properties. @property def compNameStr(self): if self.componentnamespec: return self.componentnamespec['name'] return '' @property def compNameUiStr(self): if self.componentnamespec: return self.componentnamespec['uistring'] return '' @property def compVersionStr(self): if self.componentversionspec: return self.componentversionspec['version'].versionstring return '' @property def compVersion(self): if self.componentversionspec: return self.componentversionspec['version'] return None @property def compVersionUiStr(self): if self.componentversionspec: return self.componentversionspec['uistring'] return '' @property def isComponent(self): """Returns if this bulletin is a component. """ return self.compNameStr and self.compVersionStr @property def hasConfigSchema(self): """Returns if this bulletin contains at least one VIB with config schema. """ return bool(self.configSchemaVibs) # needed for id in set(Bulletin) test def __eq__(self, other): return self._id == str(other) def __str__(self): return etree.tostring(self.ToXml()).decode() def __add__(self, other): """Merge this bulletin with another to form a new object consisting of the attributes and VIB list from the newer bulletin. Parameters: * other - another Bulletin instance. Returns: A new Bulletin instance. Raises: * RuntimeError - If attempting to add bulletins with different IDs, or attempting to add an object that is not a Bulletin object. * BulletinFormatError - If attempting to add two bulletins with the same ID and release date, but which are not identical. """ if not isinstance(other, self.__class__): msg = "Operation not supported for type %s." % other.__class__.__name__ raise RuntimeError(msg) if self.id != other.id: raise RuntimeError("Cannot merge bulletins with different IDs.") metaattrs = ('vendor', 'summary', 'severity', 'urgency', 'category', 'releasetype', 'componentnamespec', 'componentversionspec', 'description', 'kburl', 'contact', 'releasedate', 'platforms', 'vibids', 'recalledvibs', 'recalledbulletins', 'contentbody', 'resolvedrecalls') if self.releasedate > other.releasedate: newer = self elif self.releasedate < other.releasedate: newer = other else: for attr in metaattrs: if getattr(self, attr) != getattr(other, attr): msg = ("Duplicate definitions of bulletin %s with unequal " "attributes." % self.id) raise Errors.BulletinFormatError(msg) if self.vibids != other.vibids: msg = ("Duplicate definitions of bulletin %s with unequal VIB " "lists." % self.id) raise Errors.BulletinFormatError(msg) # Doesn't really matter which one is 'newer' at this point... newer = self ret = Bulletin(self.id) for attr in metaattrs: setattr(ret, attr, getattr(newer, attr)) return ret @classmethod def _XmlToKwargs(cls, xml): kwargs = {} for tag in ('kbUrl', 'kburl'): tagval = (xml.findtext(tag) or "").strip() if tagval is not "": kwargs[tag.lower()] = tagval break for tag in ('id', 'vendor', 'summary', 'severity', 'urgency', 'description', 'contact'): kwargs[tag.lower()] = (xml.findtext(tag) or "").strip() # VUM's spec uses CamelCase for valid enumeration values in the schema, # but then uses lowercase in the examples. So just convert to lowercase. # https://wiki/SYSIMAGE:Patch_recall_notification_bulletin_XML_schema for tag in ('category', 'releaseType'): kwargs[tag.lower()] = (xml.findtext(tag) or "").strip().lower() rd = (xml.findtext("releaseDate") or "").strip() if rd: try: kwargs['releasedate'] = XmlUtils.ParseXsdDateTime(rd) except Exception as e: bullid = kwargs.pop('id', 'unkown') msg = 'Bulletin %s has invalid releaseDate: %s' % (bullid, e) raise Errors.BulletinFormatError(msg) else: #Set release date if it is not in the input now = datetime.datetime.now(tz=XmlUtils.UtcInfo()) kwargs['releasedate'] = now kwargs['componentnamespec'] = {} if xml.find('componentNameSpec') is not None: compName = xml.find('componentNameSpec').attrib if len(compName): kwargs['componentnamespec']['name'] = compName.get('name', None) kwargs['componentnamespec']['uistring'] = compName.get( 'uiString', '') try: kwargs['componentversionspec'] = {} if xml.find('componentVersionSpec') is not None: compVersion = xml.find('componentVersionSpec').attrib if len(compVersion): verStr = compVersion.get('version', None) kwargs['componentversionspec']['version'] = \ Version.VibVersion.fromstring(verStr) if verStr else None kwargs['componentversionspec']['uistring'] = compVersion.get( 'uiString', '') except ValueError as e: bullid = kwargs.pop('id', 'unknown') msg = 'Bulletin %s has invalid component version: %s' % (bullid, e) raise Errors.BulletinFormatError(msg) cls._checkComponentNameVersion(kwargs['id'], kwargs['componentnamespec'], kwargs['componentversionspec']) kwargs['platforms'] = list() for platform in xml.findall('platforms/softwarePlatform'): kwargs['platforms'].append((platform.get('version', ''), platform.get('locale', ''), platform.get('productLineID', ''))) kwargs['vibids'] = set() for vibid in xml.findall('vibList/vibID') + xml.findall('vibList/vib/vibID'): newid = (vibid.text or '').strip() if newid: kwargs['vibids'].add(newid) configSchemaVibs = set() for configSchemaVib in xml.findall('configSchemaVibs/vibID'): configSchemaVibs.add(configSchemaVib.text) kwargs['configSchemaVibs'] = configSchemaVibs if kwargs['releasetype'] == cls.RELEASE_NOTIFICATION: if kwargs['category'] == cls.NOTIFICATION_RECALL: recalledvibs = set() for vibid in xml.findall('recalledVibList/vibID'): newid = (vibid.text or '').strip() if newid: recalledvibs.add(newid) if recalledvibs: kwargs['recalledvibs'] = recalledvibs recalledbulletins = set() for bulletinid in xml.findall('recalledBulletinList/bulletinID'): newid = (bulletinid.text or '').strip() if newid: recalledbulletins.add(newid) if recalledbulletins: kwargs['recalledbulletins'] = recalledbulletins elif kwargs['category'] == cls.NOTIFICATION_RECALLFIX: for node in xml.findall('resolvedRecalls'): kwargs['resolvedrecalls'] = RecallResolutionList.FromXml(node) node = xml.find('contentBody') if node is not None: kwargs['contentbody'] = ContentBody.FromXml(node) return kwargs @classmethod def _checkComponentNameVersion(cls, cId, componentNameSpec, componentVersionSpec): """This method performs 3 checks: (1) For Bulletin, check if any one of componentNameSpec or componentVersionSpec is missing when the other is present. For Component, both of them should be present. (2) When componentNameSpec is present, both name and name UI string are present. (3) When componentVersionSpec is present, both version and UI string and present. Parameters: * cId - The bulletin/component ID * componentNameSpec - A dict specifying name of the component * componentVersionSpec - A dict specifying version of the component Exceptions: * BulletinFormatError - when cls is Bulletin and any one of the 3 checks fails. * ComponentFormatError - when cls is Component and any one of the 3 checks fails. """ # This method is dual-used, proper exception will be raised. if cls is Bulletin: objType = 'Bulletin' exType = Errors.BulletinFormatError else: objType = 'Component' exType = Errors.ComponentFormatError # Additional check for component that both two specs must present. if not componentNameSpec or not componentVersionSpec: msg = ('%s %s: componentNameSpec and componentNameSpec are not ' 'both present.' % (objType, cId)) raise exType(msg) if componentNameSpec: if not 'name' in componentNameSpec or not componentNameSpec['name']: msg = ('%s %s: name attribute in componentNameSpec is ' 'either missing or the value is empty' % (objType, cId)) raise exType(msg) if (not 'uistring' in componentNameSpec or not componentNameSpec['uistring']): msg = ('%s %s: uistring attribute in componentNameSpec is ' 'either missing or the value is empty' % (objType, cId)) raise exType(msg) if not componentVersionSpec: msg = '%s %s: Missing componentVersionSpec' % (objType, cId) raise exType(msg) if componentVersionSpec: if (not 'version' in componentVersionSpec or not componentVersionSpec['version']): msg = ('%s %s: version attribute in componentVersionSpec' ' is either missing or the value is empty' % (objType, cId)) raise exType(msg) if (not 'uistring' in componentVersionSpec or not componentVersionSpec['uistring']): msg = ('%s %s: uistring attribute in componentVersionSpec is ' 'either missing or the value is empty' % (objType, cId)) raise exType(msg) if not componentNameSpec: msg = '%s %s: Missing componentNameSpec' % (objType, cId) raise exType(msg) @classmethod def FromXml(cls, xml, **kwargs): """Creates a Bulletin instance from XML. Parameters: * xml - Must be either an instance of ElementTree, or a string of XML-formatted data. * kwargs - Initialize constructor arguments from keywords. Primarily useful to provide default or required arguments when XML data is from a template. Returns: A new Bulletin object. Exceptions: * BulletinFormatError - If the given xml is not a valid XML, or does not contain required elements or attributes. """ if etree.iselement(xml): node = xml else: try: node = etree.fromstring(xml) except Exception as e: msg = "Error parsing XML data: %s." % e raise Errors.BulletinFormatError(msg) kwargs.update(cls._XmlToKwargs(node)) bullid = kwargs.pop('id', '') return cls(bullid, **kwargs) def ToXml(self, vibs=None): """Serializes the object to XML Parameters: * vibs - A VibCollection object. If given, will write bulletin with detailed VIB info. If no vibs is passed, write bulletin with vibID only for vibList. Returns: An ElementTree.Element object Exceptions: * BulletinBuildError - If the VIB id can not be resolved in the given vibs VibCollection. """ self._checkComponentNameVersion(self.id, self.componentnamespec, self.componentversionspec) root = etree.Element('bulletin') for tag in ('id', 'vendor', 'summary', 'severity', 'category', 'urgency', 'releaseType', 'description', 'kbUrl', 'contact'): elem = etree.SubElement(root, tag).text = str(getattr(self, tag.lower())) if self.componentnamespec and self.componentversionspec: elem = etree.SubElement(root, 'componentNameSpec', name=self.componentnamespec.get('name'), uiString=self.componentnamespec.get('uistring')) elem = etree.SubElement(root, 'componentVersionSpec', version=str(self.componentversionspec.get('version')), uiString=self.componentversionspec.get('uistring')) etree.SubElement(root, 'releaseDate').text = self.releasedate.isoformat() platforms = etree.SubElement(root, 'platforms') for ver, locale, product in self.platforms: etree.SubElement(platforms, 'softwarePlatform', version=ver, locale=locale, productLineID=product) viblist = etree.SubElement(root, 'vibList') for vibid in self.vibids: if vibs is None: etree.SubElement(viblist, 'vibID').text = vibid else: try: vib = vibs[vibid] except KeyError: msg = 'Can not resolve VIB id %s from VibCollection %s' % (vibid, vibs) raise Errors.BulletinBuildError(msg) viblist.append(vib.toxml(target=vib.XML_DEST_META)) if self.configSchemaVibs: configSchemaVibs = etree.SubElement(root, 'configSchemaVibs') for vibID in self.configSchemaVibs: etree.SubElement(configSchemaVibs, 'vibID').text = vibID if self.releasetype == self.RELEASE_NOTIFICATION: if self.category == self.NOTIFICATION_RECALL: if self.recalledvibs: elem = etree.SubElement(root, 'recalledVibList') for vibid in self.recalledvibs: etree.SubElement(elem, 'vibID').text = vibid if self.recalledbulletins: elem = etree.SubElement(root, 'recalledBulletinList') for bulletinid in self.recalledbulletins: etree.SubElement(elem, 'bulletinID').text = bulletinid if self.contentbody is not None: root.append(self.contentbody.ToXml()) if self.category == self.NOTIFICATION_RECALLFIX: root.append(self.resolvedrecalls.ToXml()) return root def CheckSchema(self): if self.severity not in self.SEVERITY_TYPES: msg = 'Unrecognized value "%s", severity must be one of %s.' % ( self.severity, self.SEVERITY_TYPES) raise Errors.BulletinValidationError(msg) if self.category not in self.CATEGORY_TYPES: msg = 'Unrecognized value "%s", category must be one of %s.' % ( self.category, self.CATEGORY_TYPES) raise Errors.BulletinValidationError(msg) if self.urgency not in self.URGENCY_TYPES: msg = 'Unrecognized value "%s", urgency must be one of %s.' % ( self.urgency, self.URGENCY_TYPES) raise Errors.BulletinValidationError(msg) if self.releasetype not in self.RELEASE_TYPES: msg = 'Unrecognized value "%s", releasetype must be one of %s.' % ( self.releasetype, self.RELEASE_TYPES) raise Errors.BulletinValidationError(msg) schema = os.path.join(SCHEMADIR, self.SCHEMA_FILE) try: schemaobj = XmlUtils.GetSchemaObj(schema) except XmlUtils.ValidationError as e: msg = "Unable to obtain Bulletin schema: %s" % e raise Errors.BulletinValidationError(msg) result = XmlUtils.ValidateXml(self.ToXml(), schemaobj) if not result: msg = ("Bulletin (%s) XML data failed schema validation." " Errors: %s" % (self._id, result.errorstrings)) raise Errors.BulletinValidationError(msg) def PopulateConfigSchemaVibs(self, allVibs): """Populates configSchemaVibs attributes with alll VIBs of the bulletin available. """ configSchemaVibs = set() vibs = self.GetVibCollection(allVibs) for vib in vibs.values(): if vib.hasConfigSchema: configSchemaVibs.add(vib.id) self.configSchemaVibs = configSchemaVibs def requiresVibs(self): """Checks if the bulletin requires vibs associated with it "info" and "recall" bulletins does not have associated vibs. "recallFix" ones deliver vibs for fixing the recall. Parameters: None Returns: Boolean Exceptions: None """ return not (self.releasetype == self.RELEASE_NOTIFICATION and not self.category == self.NOTIFICATION_RECALLFIX) def GetVibCollection(self, allVibs): """Get the Vibs that are part of this bulletin. """ missing = list(self.vibids - set(allVibs.keys())) if missing: msg = ('Cannot find VIB(s) %s in the given VIB collection' % ', '.join(missing)) raise Errors.MissingVibError(missing, msg) vibsDict = {vibId: allVibs[vibId] for vibId in self.vibids} return VibCollection.VibCollection(vibsDict) class BulletinCollection(dict): """This class represents a collection of Bulletin objects and provides methods and properties for modifying the collection. """ def __iadd__(self, other): """Merge this collection with another collection. Parameters: * other - another BulletinCollection instance. """ for b in other.values(): self.AddBulletin(b) return self def __add__(self, other): """Merge this collection with another to form a new collection consisting of the union of Bulletins from both. Parameters: * other - another BulletinCollection instance. Returns: A new BulletinCollection instance. """ new = BulletinCollection(self) new.update(self) for b in other.values(): new.AddBulletin(b) return new def AddBulletin(self, bulletin): """Add a Bulletin instance to the collection. Parameters: * bulletin - An Bulletin instance. """ bullid = bulletin.id if bullid in self and id(bulletin) != id(self[bullid]): self[bullid] += bulletin else: self[bullid] = bulletin def AddBulletinFromXml(self, xml): """Add a Bulletin instance based on the xml data. Parameters: * xml - An instance of ElementTree or an XML string Exceptions: * BulletinFormatError """ b = Bulletin.FromXml(xml) self.AddBulletin(b) def AddBulletinsFromXml(self, xml): """Add multiple bulletins from an XML file. Parameters: * xml = An instance of ElementTree or an XML string. Exceptions: * BulletinFormatError """ if etree.iselement(xml): node = xml else: try: node = etree.fromstring(xml) except Exception as e: msg = "Error parsing XML data: %s." % e raise Errors.BulletinFormatError(msg) for b in node.findall("bulletin"): self.AddBulletinFromXml(b) def FromDirectory(self, path, ignoreinvalidfiles=False): """Populate this BulletinCollection instance from a directory of Bulletin xml files. This method may replace existing Bulletins in the collection. Parameters: * path - A string specifying a directory name. * ignoreinvalidfiles - If True, causes the method to silently ignore BulletinFormatError exceptions. Useful if a directory may contain both Bulletin xml content and other content. Returns: None Exceptions: * BulletinIOError - The specified directory does not exist or cannot be read, or one or more files could not be read. * BulletinFormatError - One or more files were not a valid Bulletin xml. """ if not os.path.exists(path): msg = 'BulletinCollection directory %s does not exist.' % (path) raise Errors.BulletinIOError(msg) elif not os.path.isdir(path): msg = 'BulletinCollection path %s is not a directory.' % (path) raise Errors.BulletinIOError(msg) for root, _, files in os.walk(path, topdown=True): for name in files: filepath = os.path.join(root, name) try: with open(filepath) as f: c = f.read() self.AddBulletinFromXml(c) except Errors.BulletinFormatError as e: if not ignoreinvalidfiles: msg = 'Failed to add file %s to BulletinCollection: %s' % ( filepath, e) raise Errors.BulletinFormatError(msg) except EnvironmentError as e: msg = 'Failed to add Bulletin from file %s: %s' % (filepath, e) raise Errors.BulletinIOError(msg) def ToDirectory(self, path, namingfunc=None): """Write Bulletin XML in the BulletinCollection to a directory. If the directory exists, the content of the directory will be clobbered. Parameters: * path - A string specifying a directory name. * namingfunc - A function that names an individual XML file, by default getDefaultBulletinFileName(). Return: None Exceptions: * BulletinIOError - The specified directory is not a directory or cannot create an empty directory """ try: if os.path.isdir(path): shutil.rmtree(path) os.makedirs(path) except EnvironmentError as e: msg = 'Could not create dir %s for BulletinCollection: %s' % (path, e) raise Errors.BulletinIOError(msg) if not os.path.isdir(path): msg = 'Failed to write BulletinCollection, %s is not a directory.' % path raise Errors.BulletinIOError(msg) if namingfunc is None: namingfunc = getDefaultBulletinFileName for b in self.values(): filepath = os.path.join(path, namingfunc(b)) try: xml = b.ToXml() with open(filepath, 'wb') as f: f.write(etree.tostring(xml)) except EnvironmentError as e: msg = 'Failed to write Bulletin xml to %s: %s' % (filepath, e) raise Errors.BulletinIOError(msg) def GetBulletinsFromVibIds(self, vibIds): """From a given lis of VIB IDs, create a new BulletinCollection that maps to the vibs in it. We can claim that a bulletin is present IFF all of it's 'vibids' are subset of the input vibsIds. Parameters: * self: We use 'self' as the source for all the bulletins. * vibs: List of input vibIds. Returns: * New Bulletin collection which is smaller subset of the 'self'. Raises: None """ newDict = {_id: bulletin for _id, bulletin in self.items() if bulletin.vibids.issubset(set(vibIds))} return BulletinCollection(newDict) def GetVibCollection(self, allVibs): """Get the VIBs that are part of the bulletins in this collection. Parameters: * allVibs: All the relevant VIBs which needs be used for the calculation. We get the VIB object from this collection. Usually 'allVibs' is a bigger set of vibs and it points to all the vibs in the depot. Returns: * A VibCollection of bulletin VIBs. Raises: VibMissingError: If a VIB is not found in allVibs. """ vibs = VibCollection.VibCollection() for bul in self.values(): vibs += bul.GetVibCollection(allVibs) return vibs class Component(Bulletin): """A component defines a logical collection of VIB packages for ESXi installation and update. It is the smallest unit of software used in the Personality Manager. """ # The following attributes are equal if two components are equal ATTRS_TO_VERIFY = ('id', 'vendor', 'platforms', 'vibids', 'componentnamespec', 'componentversionspec') # The following attributes are updated to reflect newest values # depending on its releasedate. ATTRS_TO_COPY = ('summary', 'severity', 'urgency', 'category', 'releasetype', 'description', 'kburl', 'contact', 'releasedate', 'configSchemaVibs') SCHEMA_FILE = 'component-xml.rng' def __init__(self, **kwargs): self.componentnamespec = kwargs.get('componentnamespec', {}) self.componentversionspec = kwargs.get('componentversionspec', {}) if not 'id' in kwargs: # Generate an ID from component name/version. bulId = '%s_%s' % (self.compNameStr, self.compVersionStr) else: bulId = kwargs.pop('id') if not self.componentnamespec or not self.componentversionspec: raise Errors.ComponentFormatError('Failed to create component: ' 'missing componentnamespec and/or componentversionspec.') try: self._checkComponentNameVersion(bulId, self.componentnamespec, self.componentversionspec) except Errors.BulletinFormatError as e: # Fix-up the exception message of the bulletin error and raise # a component one. msg = str(e).replace('Bulletin ', 'Component ') raise Errors.ComponentFormatError(msg) super(Component, self).__init__(bulId, **kwargs) def __eq__(self, other): """Compare two components. Two components are equal when attributes in ATTRS_TO_VERIFY match. """ for attr in self.ATTRS_TO_VERIFY: old = getattr(self, attr) new = getattr(other, attr) if old != new: return False return True def __add__(self, other): """Merge this component with other to form a new object depending on their releasedates. """ # Two components are merged only when ATTRS_TO_VERIFY are equal. if self != other: raise ValueError("Cannot merge unequal components.") # Attributes in added component is always latest depending # on their releasedate. if self.releasedate > other.releasedate: newer = self elif self.releasedate < other.releasedate: newer = other else: for attr in self.ATTRS_TO_COPY: if getattr(self, attr) != getattr(other, attr): log.error("Duplicate definitions of component %s:%s with unequal" " attribute, %s" % (self.componentnamespec['name'], self.componentversionspec['version'], attr)) newer = other kwargs = {} for attr in self.ATTRS_TO_VERIFY: kwargs[attr] = getattr(newer, attr) for attr in self.ATTRS_TO_COPY: kwargs[attr] = getattr(newer, attr) ret = self.__class__(**kwargs) return ret def Validate(self, compVibs=None): """Validate component against its schema and relations of VIBS in the component. Parameters: * compVibs - VibCollection object with VIBs that correspond to this component. """ schema = os.path.join(SCHEMADIR, self.SCHEMA_FILE) try: schemaobj = XmlUtils.GetSchemaObj(schema) except XmlUtils.ValidationError as e: msg = "Unable to obtain Component schema: %s" % e raise Errors.ComponentValidationError(msg) result = XmlUtils.ValidateXml(self.ToXml(), schemaobj) if not result: msg = ("Component (%s) XML data failed schema validation." " Errors: %s" % (self.id, result.errorstrings)) raise Errors.ComponentValidationError(msg) # Check for self-conflict and self-obsolete if compVibs: collection = ComponentCollection() collection.AddComponent(self) problems = collection.Validate(compVibs) reltypes = (ComponentScanProblem.TYPE_SELFCONFLICT, ComponentScanProblem.TYPE_SELFOBSOLETE) componentProblems = [p.msg for p in problems.values() if p.reltype in reltypes] if componentProblems: raise Errors.ComponentValidationError('Component %s contains' ' following VIB relation' ' issues: %s' % (self.id, ','.join(componentProblems))) @classmethod def FromBulletin(cls, bulletin): """Generates component from bulletin. """ try: kwargs = {} for attr in cls.ATTRS_TO_VERIFY: kwargs[attr] = getattr(bulletin, attr) for attr in cls.ATTRS_TO_COPY: kwargs[attr] = getattr(bulletin, attr) return cls(**kwargs) except Exception as e: msg = "%s (from bulletin %s)" % (e, bulletin.id) raise Errors.ComponentFormatError(msg) class ComponentCollection(collections.defaultdict): """This class represents a collection of Component objects and implements methods for modifying the component collection. """ def __iadd__(self, other): """Merge this collection with another collection. Parameters: * other - another ComponentCollection instance. """ for comp in other.IterComponents(): self.AddComponent(comp) return self def __add__(self, other): """Merge this collection with another to form a new collection containing of the union of Components from both. Parameters: * other - another ComponentCollection instance. Returns: A new ComponentCollection instance. """ new = ComponentCollection() # AddComponent() is needed to cache name/version of components. for comp in self.IterComponents(): new.AddComponent(comp) for comp in other.IterComponents(): new.AddComponent(comp) return new def __init__(self, bulletinCollection={}, ignoreNonComponents=False): super(ComponentCollection, self).__init__() # Mapping from component ID to component name/version tuple. self._idToNameVersion = dict() if not bulletinCollection: return nonComponents = [] for bulId, bull in bulletinCollection.items(): if isinstance(bull, Bulletin) and bull.isComponent: comp = Component.FromBulletin(bull) name = comp.compNameStr version = comp.compVersionStr self.setdefault(name, dict())[version] = comp self._idToNameVersion[bulId] = (name, version) else: nonComponents.append(bulId) if not ignoreNonComponents and nonComponents: msg = ('Failed to create ComponentCollection: ' 'Bulletins %s are not components.' % ', '.join(nonComponents)) raise Errors.ComponentFormatError(msg) def __copy__(self): """Copy constructor. """ copy = self.__class__() copy += self return copy copy = __copy__ def _getNameVersionFromInput(self, *args): """Returns component name and version from 1-2 string input, in format id, name, (name,version) or name:version. """ if len(args) == 1: if ':' in args[0]: # Colon spec. name, version = args[0].split(':') elif args[0] in self._idToNameVersion: # Known component ID. name, version = self._idToNameVersion[args[0]] else: # Treat as component name. name, version = args[0], None elif len(args) == 2: name, version = args else: raise ValueError('Expected 1-2 input, got %u' % len(args)) return name, version def HasComponent(self, *args): """Checks if the collection has component corresponding to given component name and (optionally) version, a colon spec in format name:version, or component's ID. Usage: HasComponent(id) HasComponent(name:version) HasComponent(name) HasComponent(name, version) Returns: True if the collection has the component, else False. """ try: name, version = self._getNameVersionFromInput(*args) if version is not None: return name in self and version in self[name] else: return name in self except ValueError: # Invalid number of input. raise ValueError('%s: invalid component argument' % str(args)) def GetComponent(self, *args): """Gets the component corresponding to the given component name and (optionally) version, a colon spec in format name:version, or component's ID. When only name is given, there must be only one component in the collection with the name, otherwise GetComponents() should be used. Usage: GetComponent(id) GetComponent(name:version) GetComponent(name) GetComponent(name, version) Returns: Component corresponding to given name and version. Raises: KeyError - there is no component matches the id, name, (name,version) or name:version input. ValueError - there is more than one component with the name. """ name, version = self._getNameVersionFromInput(*args) if not name in self: raise KeyError(str(args)) if version is None: if len(self[name]) == 1: return list(self[name].values())[0] else: raise ValueError('Expected 1 component, found %u' % len(self[name])) else: if version in self[name]: return self[name][version] else: raise KeyError('[%s,%s]' % (name, version)) def IterComponents(self): """Iterator of all components in the collection. """ for versionDict in self.values(): for comp in versionDict.values(): yield comp def GetComponents(self, name=None): """Returns a list of components in the collection, optionally filtered by a component name. Raises: KeyError - component with name is not found. """ if name and name not in self: raise KeyError(name) versionDicts = self.values() if name is None else [self[name]] return [comp for versionDict in versionDicts for comp in versionDict.values()] def GetComponentIds(self): """Returns IDs of the component in the collection """ return [comp.id for comp in self.IterComponents()] def GetComponentNameIds(self, compIds=None): """Returns a list of name IDs (name:version) for components in the collection, optionally filtered by a list of component IDs. """ compIds = compIds if compIds else list(self._idToNameVersion.keys()) nameIds = list() for compId in compIds: if not compId in self._idToNameVersion: raise ValueError('Component with ID %s does not exist' % compId) name, version = self._idToNameVersion[compId] nameIds.append('%s:%s' % (name, version)) return nameIds def GetComponentsFromVibIds(self, vibIds): """From a given list of VIB IDs, create a new ComponentCollection that maps to these VIBs. A component will be added to the collection IFF its vibids is a subset of the input IDs. """ comps = self.__class__() for comp in self.IterComponents(): if comp.vibids.issubset(set(vibIds)): comps.AddComponent(comp) return comps def AddComponent(self, comp, replace=False): """Adds a component in the collection. Parameters: * comp - Component to be added to the collection. * replace - Replace component(s) with the same name in the collection. If True, there will be only one version of a component in the collection. Raises: ComponentFormatError - comp is not a component. BulletinFormatError - comp cannot be merged with the component of the same name/version in the collection. """ if not comp.isComponent: msg = "%s is not a component" % comp.id raise Errors.ComponentFormatError(msg) name = comp.compNameStr version = comp.compVersionStr if name in self and self[name]: if replace: for c in self[name].values(): del self._idToNameVersion[c.id] self[name] = dict() self[name][version] = comp else: if version in self[name]: self[name][version] += comp else: self[name][version] = comp else: self.setdefault(name, dict())[version] = comp self._idToNameVersion[comp.id] = (name, version) def RemoveComponent(self, *args): """Removes a component corresponding to the given component name and (optionally) version, a colon spec in format name:version, or component's ID. When only name is given, there must be only one component in the collection with the name. Usage: RemoveComponent(id) RemoveComponent(name:version) RemoveComponent(name) RemoveComponent(name, version) Raises: KeyError - there is no component matches the id, name, (name,version) or name:version input. ValueError - there is more than one component with the name. """ name, version = self._getNameVersionFromInput(*args) if name is not None and not name in self: raise KeyError(str(args)) if version is None: if len(self[name]) == 1: del self._idToNameVersion[list(self[name].values())[0].id] del self[name] else: raise ValueError('Expected 1 component, found %u' % len(self[name])) else: if version in self[name]: del self._idToNameVersion[self[name][version].id] del self[name][version] if len(self[name]) == 0: # Remove the component name as well. del self[name] else: raise KeyError('[%s,%s]' % (name, version)) def GetBulletinCollection(self): """Gets the Bulletin Collection from Component Collection. Returns: Bulletin Collection """ bc = BulletinCollection() for _, comp in self.items(): for _, bull in comp.items(): bc.AddBulletin(bull) return bc def GetVibCollection(self, allVibs): """Get the VIBs that are part of the components in this collection. Returns: * A VibCollection with component VIBs. Raises: VibMissingError: If a VIB is not found in allVibs. """ vibs = VibCollection.VibCollection() for verDict in self.values(): for comp in verDict.values(): vibs += comp.GetVibCollection(allVibs) return vibs def Validate(self, allVibs, effectiveComps=None): """Validates the component collection object. Parameters: * allVibs - All the vibs which is used for validation. It is a collection of all vibs in the depot. * effectiveComps - ComponentCollection object which contains effective components. When effectiveComps are provided Validation is performed on these. This component collection will help offer resolutions. Returns: * Returns ComponentScanner.ValidateResult object which is a list of problems and possible resolutions. """ scanner = ComponentScanner(self, self.GetVibCollection(allVibs), effectiveComps) return scanner.Validate() def AddComponentFromXml(self, xml): """Add a Bulletin instance based on the xml data. Parameters: * xml - An instance of ElementTree or an XML string Exceptions: * BulletinFormatError """ b = Bulletin.FromXml(xml) self.AddComponent(b) def FromDirectory(self, path, ignoreinvalidfiles=False): """Populate this ComponentCollection instance from a directory of xml files. This method may replace existing components in the collection. Parameters: * path - A string specifying a directory name. * ignoreinvalidfiles - If True, causes the method to silently ignore ComponentFormatError exceptions. Useful if a directory contains both component xml content and other content. Returs: None Exceptions: * ComponentIOError - The specified directory does not exist or cannot be read, or one or more files could not be read. * ComponentFormatError - One or more files were not a valid Bulletin xml. """ if not os.path.exists(path): msg = 'ComponentCollection directory %s does not exist.' % path raise Errors.ComponentIOError(msg) elif not os.path.isdir(path): msg = 'ComponentCollection path %s is not a directory.' % path raise Errors.ComponentIOError(msg) for root, _, files in os.walk(path, topdown=True): for name in files: filepath = os.path.join(root, name) try: with open(filepath) as f: c = f.read() b = Bulletin.FromXml(c) self.AddComponent(b) except Errors.BulletinFormatError as e: if not ignoreinvalidfiles: msg = ('Failed to add file %s to the collection: %s' % (filepath, e)) raise Errors.ComponentFormatError(msg) except EnvironmentError as e: msg = 'Failed to add component from file %s: %s' % (filepath, e) raise Errors.ComponentIOError(msg) def ToDirectory(self, path, namingfunc=None): """Write Component XML in the ComponentCollection to a directory. If the directory exists, the content of the directory will be clobbered. Parameters: * path - A string specifying a directory name. * namingfunc - A function that names an individual XML file, by default getDefaultBulletinFileName(). Return: None Exceptions: * ComponentIOError - The specified directory is not a directory or cannot create an empty directory.""" try: if os.path.isdir(path): shutil.rmtree(path) os.makedirs(path) except EnvironmentError as e: msg = 'Could not create dir %s for ComponentCollection: %s' % (path, e) raise Errors.ComponentIOError(msg) if namingfunc is None: namingfunc = getDefaultBulletinFileName for component in self.IterComponents(): filepath = os.path.join(path, namingfunc(component)) try: xml = component.ToXml() with open(filepath, 'wb') as f: f.write(etree.tostring(xml)) except EnvironmentError as e: msg = 'Failed to write Component xml to %s: %s' % (filepath, e) raise Errors.ComponentIOError(msg) class ComponentRelation(object): """Describes a <=, <, >=, > relationship between two components. """ def __init__(self, name, op, version): """Class constructor. Parameters: * name - The name of the component. * operator - An operator describing whether the relation matches only particular versions. Must be one of "<", "<=", ">=" or ">". * version - An instance of Version.VibVersion used for comparison based on the relation operator. If this parameter is specified, version must also be specified. """ OPERATOR_DICT = {'>=': operator.ge, '>': operator.gt, '<=': operator.le, '<': operator.lt} if op not in OPERATOR_DICT.keys(): raise InvalidRelationToken('Invalid token in relation:' ' %s. Valid tokens include %s.' % (op, ",".join(OPERATOR_DICT.keys()))) self.name = name self.versionCheck = lambda x: OPERATOR_DICT[op]( Version.VibVersion.fromstring(x), Version.VibVersion.fromstring(version)) def Validate(self, component): """Validates that a component that meets a certain name and versioning relations. Parameters: * component - The component to name and version check """ componentName = component.componentnamespec['name'] componentVersion = component.componentversionspec['version'].versionstring return self.name == componentName and self.versionCheck(componentVersion)