########################################################################
# Copyright (C) 2010-2020 VMWare, Inc.
# All Rights Reserved
########################################################################

"""This module defines classes for working with Image Profiles.
"""

import datetime
import logging
import os
import shutil
import fnmatch
import copy
import sys

if sys.version_info[0] >= 3:
   from urllib.parse import quote
else:
   from urllib import quote

from . import AcceptanceLevels
from . import Bulletin
from . import Errors
from . import Manifest
from . import ReleaseCollection
from . import Vib
from . import VibCollection
from . import Version
from .ComponentScanner import ComponentScanner, ComponentScanProblem
from .Scan import ScanResult
from .Utils import PathUtils, XmlUtils
from .Utils.Misc import isString

# Use the XmlUtils module to find ElementTree.
etree = XmlUtils.FindElementTree()

SCHEMADIR = XmlUtils.GetSchemaDir()
RULE_KEY_HWVENDOR = "Vendor"
RULE_CONFLICTING_VENDORS = "<Conflicting Vendors>"

log = logging.getLogger('imageprofile')

# Map for the component name and its blacklisted VIBs.
blackListMap = {'ESXi' : ['VMware_locker_tools'],
                'ESXi-VM-Tools' : ['VMware_locker_tools']}

def _InBlackList(vibid, blackList):
   """ Check whether a VIB in a component is blacklisted. The
       blacklisted VIB is considered as installed.
   """
   for prefix in blackList:
      if prefix in vibid:
         return True
   return False

def _GetComponentVersion(component):
   version = ''
   if component.componentversionspec:
      verObj = component.componentversionspec['version']
      if verObj:
         version = verObj.versionstring
   return version

class VibState(object):
   """Holds information about VIB local installation information.

      Attributes:
         * id          - VIB ID
         * boot        - This VIB is required at boot-time.
         * payloads    - A dictionary. Each key is a payload name, the value is the
                         local file name for the payload.
         * installdate - A DateTime instance, the time the VIB is installed.
   """
   def __init__(self, vibid, payloads=None, installdate=None, boot=True):
      if not vibid:
         raise ValueError('vibid cannot be empty for VibState')
      self.id = vibid
      if payloads:
         self.payloads = payloads
      else:
         self.payloads = {}
      self.installdate = installdate
      self.boot = boot

   def ToXml(self, toDB = False):
      """Serializes the object to XML.

         Parameters:
            * toDB - If True, payload and installdate info will be added.
                     Otherwise, the only sub-element is vib-id.
      """
      xml = etree.Element('vib')
      etree.SubElement(xml, 'vib-id').text = self.id
      if not self.boot:
         etree.SubElement(xml, "boot").text = 'false'

      # Vib installation information is only needed for database.
      if toDB:
         elem = etree.SubElement(xml, 'payloads')
         for plname, localname in self.payloads.items():
            plelem = etree.SubElement(elem, 'payload')
            plelem.text = localname
            plelem.set('payload-name', plname)
         if self.installdate is not None:
            etree.SubElement(xml, "installdate").text = \
                              self.installdate.isoformat()

      return xml

   @classmethod
   def FromXml(cls, xml):
      """Constructs a VibState instance with given XML node.

         Parameters:
            * xml - Either a string or an ElementTree instance containing
                    VibState XML

         Returns: A new VibState object

         Exceptions:
            * ValueError - If the given XML is not valid XML, or invalid
                           elements or attributes.
      """
      if not etree.iselement(xml):
         try:
            xml = etree.fromstring(xml)
         except Exception as e:
            raise ValueError('Could not parse VibState XML data: %s.' % e)

      kwargs = {}
      kwargs['vibid'] = xml.findtext('vib-id')

      text = xml.findtext('boot') or "true"
      try:
         kwargs['boot'] = XmlUtils.ParseXsdBoolean(text)
      except Exception as e:
         raise Errors.ProfileFormatError(None, "failed to parse "
                                         "VIB boot attribute: %s" % e)

      # installdate and payloads are optional. They will only be available for
      # esximage database
      text = xml.findtext('installdate')
      if text:
         try:
            kwargs['installdate'] = XmlUtils.ParseXsdDateTime(text)
         except Exception as e:
            raise ValueError('VibState has invalid %s: %s' % (
               'installdate', e))

      kwargs['payloads'] = {}
      for elem in xml.findall('payloads/payload'):
         if elem.text is not None:
            localname = elem.text.strip()
         else:
            localname = ''
         payloadname = elem.get('payload-name')
         if not payloadname:
            raise ValueError("VibState payload '%s' has missing "
                  "'payload-name'" % (localname))
         if payloadname in kwargs['payloads']:
            raise ValueError("VibState payload '%s' has duplicated "
                  "entries" % (payloadname))
         kwargs['payloads'][payloadname] = localname

      return cls(**kwargs)

class AcceptanceChecker(object):
   TRUST_ORDER = {Vib.ArFileVib.ACCEPTANCE_COMMUNITY : 0,
                  Vib.ArFileVib.ACCEPTANCE_PARTNER   : 5,
                  Vib.ArFileVib.ACCEPTANCE_ACCEPTED  : 10,
                  Vib.ArFileVib.ACCEPTANCE_CERTIFIED : 15,
                  # Temporary change for compatibility:
                  "unsigned": 0, "signed": 5}

   def __init__(self, hostlevel):
      if hostlevel not in self.TRUST_ORDER:
         raise ValueError("'hostlelvel' must be one of %s" %
                          list(self.TRUST_ORDER.keys()))
      self._hostlevel = hostlevel
      levelvalue = self.TRUST_ORDER[hostlevel]
      self._acceptedlevels = tuple([k for k, v in self.TRUST_ORDER.items() if \
                                    v >= levelvalue])

   def Check(self, vibacceptancelevel):
      if vibacceptancelevel not in self.TRUST_ORDER:
         msg = "Unrecognized acceptance level '%s'" %(vibacceptancelevel)
         raise Errors.VibFormatError(msg)

      if vibacceptancelevel not in self._acceptedlevels:
         msg = ("VIB acceptance level '%s' is not acceptable for profile "
                "acceptance level '%s'" % (vibacceptancelevel, self._hostlevel))
         return [msg]

      return []

class ImageProfile(object):
   PROFILE_SCHEMA = os.path.join(SCHEMADIR, 'imageprofile.rng')

   """Class for managing a single Image Profile, including adding
      and removing VIBs, and performing many validation functions.

      The tuple of (name, creator) attributes from an Image Profile should
      be unique within any given system.

      Attributes:
         * profileID    - A read-only string uniquely identifying an image profile.
                          Created at construction time and cannot be modified
                          thereafter.
         * name         - A friendly string identifying the profile to end users
         * creator      - A string identifying the organization or person who
                          created or modified this profile
         * creationtime - A DateTime instance representing the time the profile
                          was created.  Initialized at construction time to the
                          current UTC time.
         * modifiedtime - A DateTime instance, updated to the last time one of the
                          class attributes was modified or methods that modify
                          the imageprofile was called.
         * serialno     - An integer that gets increased every time the profile
                          is modified.  Starts at 0.  Used for comparisons.
         * acceptancelevel - One of the Vib.ArFileVIB.ACCEPTANCE_* values
         * description  - A detailed string description of the image profile
         * readonly     - Boolean, if True, then the profile cannot be modified
         * statelessready - True if every VIB in the image profile has its
                          statelessready attribute set to True
         * vibIDs       - Set of VIB IDs (strings) that make up the image
                          profile. Correspond to vibstates.
         * rules        - A generated attribute, a list of (key, value) pairs
                          representing constraints to be passed to the Rules Engine.
                          Each value is a list of strings.
                          Generated when read.
         * vibs         - A VibCollection, supposed to correspond with vibIDs.
                          VIB in this collection will be added to profile
                          VibCollection if the VIB is also in 'vibIDs'.
                          This collection will not be converted to XML in
                          ToXml().
         * vibstates    - A dictionary, vibID -> VibState.
         * componentIDs - A set of IDs of components.
                          Must match with components to perform component
                          lookup.
         * components   - A ComponentCollection, components in this image
                          profile.
         * bulletinIDs  - A set of bulletin IDs representing 'components',
                          supposed to correspond with bulletins to perform
                          component lookup.
                          Deprecated: legacy support only.
         * bulletins    - A BulletinCollection representing 'components'.
                          This collection will not be converted to XML in
                          ToXml().
                          Deprecated: legacy support only.
         * baseimageID  - ID of the base image in this image profile.
         * baseimage    - The base image object, set at runtime and not
                          converted to XML.
         * addonID      - ID of the addon in this image profile.
         * addon        - The addon object, set at runtime and not converted
                          to XML.
         * manifestIDs  - IDs of the manifests in this image profile.
         * manifests    - A ManifestCollection that contains the manifest
                          objects in the image profile, set at runtime and not
                          converted to XML.
         * solutionIDs  - IDs of solutions that are in this image profile.
         * solutions    - A SolutionCollection that contains the solution
                          objects, set at runtime and not converted to XML.
         * reservedComponentIDs
                        - The IDs of components are in base image, addon
                          and/or manifest but not installed.
         * reservedComponents
                        - The components are in base image, addon and/or
                          manifest but not installed.
   """
   def __init__(self, name, creator, creationtime=None, profileID=None,
                serialno=0, modifiedtime=None,
                acceptancelevel=Vib.ArFileVib.ACCEPTANCE_PARTNER,
                description="", readonly=True, vibIDs=None, vibs=None,
                vibstates={},
                # TODO remove bulletins from constructor
                bulletinIDs=None, bulletins=None,
                componentIDs=None, components=None,
                baseimageID=None, baseimage=None,
                addonID=None, addon=None,
                solutionIDs=None, solutions=None,
                manifestIDs=None, manifests=None,
                reservedComponentIDs=None, reservedComponents=None):
      """ImageProfile class constructor"""
      self.name = name
      self.creator = creator
      self.creationtime = creationtime
      self._setacceptance(acceptancelevel, modifytime=False)
      if not creationtime:
         tz = XmlUtils.UtcInfo()
         self.creationtime = datetime.datetime.now(tz)
      if not modifiedtime:
         self._modifiedtime = self.creationtime
      else:
         self._modifiedtime = modifiedtime
      self.description = description
      self.readonly = readonly
      self._id = profileID
      self._baseimageID = baseimageID
      self._addonID = addonID
      self._reservedComponentIDs = \
         reservedComponentIDs if reservedComponentIDs else []

      if not profileID:
         self._id = self._getID()

      self._serialno = serialno

      self.vibstates = {}
      self.vibs = VibCollection.VibCollection()
      if vibIDs:
         # Check duplicated vibIDs
         if len(vibIDs) != len(set(vibIDs)):
            msg = ("Duplicated entries in 'vibIDs' (%s) for ImageProfile "
                   "'%s'" % (vibIDs, name))
            raise ValueError(msg)

         for vibid in vibIDs:
            if vibid in vibstates:
               self.vibstates[vibid] = vibstates[vibid]
            else:
               self.vibstates[vibid] = VibState(vibid)

            if vibs is not None:
               if vibid in vibs:
                  self.vibs.AddVib(vibs[vibid])

      self.addon = addon
      self.baseimage = baseimage

      self._manifestIDs = manifestIDs or set()
      self.manifests = ReleaseCollection.ManifestCollection()
      if manifestIDs and manifests:
         for manifestID in manifestIDs:
            self.manifests.AddManifest(manifests.GetManifest(manifestID),
                                       replace=True)

      self._solutionIDs = solutionIDs or set()
      self.solutions = ReleaseCollection.SolutionCollection()
      if solutionIDs and solutions:
         for solutionID in solutionIDs:
            try:
               self.solutions[solutionID] = solutions[solutionID]
            except KeyError:
               raise KeyError('Solution with releaseID %s is not found '
                              'in the collection' % solutionID)

      # Add each reserved component so the collection can be modified.
      self.reservedComponents = Bulletin.ComponentCollection()
      if reservedComponentIDs and reservedComponents:
         for comp in reservedComponents.GetComponents():
            if (comp.compNameStr, comp.compVersionStr) in reservedComponentIDs:
               self.reservedComponents.AddComponent(comp)

      # Both componentIDs and components must present to provide full component
      # information. In practice, components object are added later after
      # FromXml() produces the image profile with only componentIDs.
      self._components = Bulletin.ComponentCollection()
      self._componentIDs = set()

      if componentIDs and bulletinIDs:
         raise ValueError('componentIDs cannot be used with bulletinIDs')
      if components and bulletins:
         raise ValueError('components cannot be used with bulletins')

      if componentIDs:
         self._componentIDs = componentIDs
         if components:
            for comp in components.IterComponents():
               if comp.id in componentIDs:
                  self._components.AddComponent(comp)
      elif bulletinIDs:
         log.debug('Use of bulletinIDs in the constructor is being deprecated.')
         self._componentIDs = bulletinIDs
         if bulletins:
            for bulID in bulletinIDs:
               if bulID in bulletins:
                  self._components.AddComponent(bulletins[bulID])

   def Copy(self):
      """Copy constructor -- creates a new ImageProfile instance with the same
         name, creator, description, acceptancelevel, readonly status;
         The vibIDs will be the same.  The vibs collection will be a new
         collection with the same instances of Vib's --- thus Adds and
         Removes in the copied Image Profile will not impact the original.
         Components and release unit IDs will be copies of the original.
         The creationtime and modifiedtime will be set to the current time.
         VibStates will be deep copied so that running GenerateVFatNames
         on the copy will not change the original.
         Also the copied Image Profile will contain a new profileID.
      """
      # Assumes that any class inheriting from this class will accept the same
      # keyword arguments
      return self.__class__(name=self.name, creator=self.creator,
                            acceptancelevel=self.acceptancelevel,
                            description=self.description,
                            readonly=self.readonly,
                            vibIDs=self.vibIDs,
                            vibs=self.vibs,
                            vibstates=copy.deepcopy(self.vibstates),
                            componentIDs=self.componentIDs.copy(),
                            components=self.components,
                            baseimageID=self.baseimageID,
                            baseimage=self.baseimage,
                            addonID=self.addonID,
                            addon=self.addon,
                            solutionIDs=self.solutionIDs.copy(),
                            solutions=self.solutions,
                            manifestIDs=self.manifestIDs.copy(),
                            manifests=self.manifests,
                            # copy() of list is available only in Python 3.
                            reservedComponentIDs=\
                               self.reservedComponentIDs[:],
                            reservedComponents=self.reservedComponents)

   __copy__ = Copy

   @property
   def components(self):
      """Components in the image profile.
         Read-only, should be set with SetComponents() together with IDs,
         or filled with PopulateComponents().
      """
      return self._components

   @property
   def componentIDs(self):
      """IDs of components in the image profile.
         Read-only, should be set in the consturctor, or with SetComponents()
         together with components objects.
      """
      return self._componentIDs

   @property
   def bulletins(self):
      """Returns components of this image profile in a BulletinCollection.
      """
      return self._components.GetBulletinCollection()

   @bulletins.setter
   def bulletins(self, bulletins):
      log.debug('Setter of bulletins is being deprecated.')
      self._components = Bulletin.ComponentCollection(bulletins)

   @property
   def bulletinIDs(self):
      """Returns IDs of bulletins in this image profile.
         Deprecated: legacy support only.
      """
      log.debug('Use of bulletinIDs is being deprecated.')
      return self.componentIDs

   @bulletinIDs.setter
   def bulletinIDs(self, bulletinIDs):
      log.debug('Setter of bulletinIDs is being deprecated.')
      self._componentIDs = bulletinIDs

   def _getID(self):
      """Use uuid to assign a unique ID for the image profile.
         The ID must not change once created.
      """
      import uuid
      return uuid.uuid1().hex

   @property
   def profileID(self):
      return self._id

   def _setacceptance(self, acceptancelevel, modifytime=True):
      if acceptancelevel not in Vib.ArFileVib.ACCEPTANCE_LEVELS:
         raise ValueError("Invalid acceptance level value '%s'"
                          % acceptancelevel)
      #
      # TODO: Check if acceptance level was raised and if this makes any VIBs
      # invalid.  If so, throw AcceptanceLevelError.
      #
      self._acceptancelevel = acceptancelevel
      if modifytime:
         self._updatemodifiedtime()

   def _SetBaseImageID(self, imageid):
      self._baseimageID = imageid

   def _SetAddonID(self, addonid):
      self._addonID = addonid

   def _SetSolutionIDs(self, solutionIds):
      self._solutionIDs = solutionIds

   def _SetManifestIDs(self, manifestids):
      self._manifestIDs = set(manifestids)

   def HasSameInventory(self, other):
      """Compare with another image profile regarding inventory, i.e. VIB IDs,
         component IDs, Base Image ID, Addon ID, Manifest ID and Solution IDs.
      """
      return (self.vibIDs == other.vibIDs and
              self.componentIDs == other.componentIDs and
              self.baseimageID == other.baseimageID and
              self.addonID == other.addonID and
              self.manifestIDs == other.manifestIDs and
              self.solutionIDs == other.solutionIDs)

   def GetOrphanVibs(self):
      """Get a collection of VIBs that do not belong to components.
      """
      compVibIDs = set([v for c in self._components.IterComponents()
                        for v in c.vibids])
      orphanVibPairs = [(vId, vib) for vId, vib in self.vibs.items()
                        if vId not in compVibIDs]
      return VibCollection.VibCollection(orphanVibPairs)

   def GetFinalComponents(self, allComps):
      """Get components that are part of this image profile.
         Parameters:
            allComps - a ComponentCollection object that has all reference
                       components.
      """
      newComps = Bulletin.ComponentCollection()
      for comp in allComps.IterComponents():
         if comp.vibids.issubset(self.vibIDs):
            newComps.AddComponent(comp)
      return newComps

   def GetReservedComponentIDs(self):
      """ Get the components not listed in the components list but in
          base image, addon or manifest. """
      reservedCIDs = set()
      profComps = self._components
      if self.baseimage:
         reservedCIDs.update(
            set(self.baseimage.CollectReservedComponents(profComps)))
      if self.addon:
         reservedCIDs.update(
            set(self.addon.CollectReservedComponents(profComps)))
      if self.manifests:
         for manifest in self.manifests.values():
            reservedCIDs.update(
               set(manifest.CollectReservedComponents(profComps)))
      self._reservedComponentIDs = list(reservedCIDs)
      return self._reservedComponentIDs

   def GetKnownComponents(self):
      """Get all components known in the image profile, this includes
         active and reserved components.
      """
      return self._components + self.reservedComponents

   def GetReservedVibs(self, allVibs):
      """Generate reserved VIBs as a VibCollection.
      """
      reservedVibs = VibCollection.VibCollection()
      missing = []
      for comp in self.reservedComponents.IterComponents():
         for vibid in comp.vibids:
            try:
               reservedVibs.AddVib(allVibs[vibid])
            except KeyError:
               missing.append(vibid)

      if missing:
         raise Errors.MissingVibError(missing,
                              'Missing reserved VIBs %s' % ', '.join(missing))

      return reservedVibs

   modifiedtime = property(lambda self: self._modifiedtime)
   serialno = property(lambda self: self._serialno)
   acceptancelevel = property(lambda self: self._acceptancelevel,
                              _setacceptance)
   vibIDs = property(lambda self: set(self.vibstates.keys()))
   baseimageID = property(lambda self: self._baseimageID, _SetBaseImageID)
   addonID = property(lambda self: self._addonID, _SetAddonID)
   solutionIDs = property(lambda self: self._solutionIDs, _SetSolutionIDs)
   manifestIDs = property(lambda self: self._manifestIDs, _SetManifestIDs)
   reservedComponentIDs = property(GetReservedComponentIDs)

   def _updatemodifiedtime(self):
      tz = XmlUtils.UtcInfo()
      self._modifiedtime = datetime.datetime.now(tz)
      self._serialno += 1

   @classmethod
   def FromXml(cls, xml, validate=False, schema=PROFILE_SCHEMA):
      """Constructs an ImageProfile instance given an XML node.
         Parameters:
            * xml      - Either a string or an ElementTree instance containing
                         image profile XML
            * validate - If True, XML will be validated against a schema. If
                         False, no validation will be done. Defaults to
                         True.
            * schema   - A file path giving the location of an image profile
                         schema.
          Returns:
            An ImageProfile instance
         Exceptions:
            ProfileFormatError  - error parsing the profile XML
            ProfileValidationError - if validate was True, an error occurred
                                     during schema validation
      """
      def getTextSet(xml, path):
         """Parse a list of elements and return a set containing their text.
         """
         return set([e.text for e in xml.findall(path) if e.text])

      if not etree.iselement(xml):
         try:
            xml = etree.fromstring(xml)
         except Exception as e:
            raise Errors.ProfileFormatError(None,
               "Could not parse profile XML data: %s." % e)

      if validate:
         try:
            schemaobj = XmlUtils.GetSchemaObj(schema)
         except Exception as e:
            raise Errors.ProfileValidationError(None, str(e))

         res = XmlUtils.ValidateXml(xml, schemaobj)
         name = xml.findtext("name") or ''
         if not res:
            msg = "Profile (%s) XML data failed schema validation. Errors: %s" \
                  % (name, res.errorstrings)
            raise Errors.ProfileValidationError(None, msg)

      kwargs = {}
      for tag in ("name", "creator", "description", "acceptancelevel",
                  "profileID"):
         kwargs[tag] = (xml.findtext(tag) or "").strip()

      for tag in ('baseimageID', 'addonID'):
         kwargs[tag] = xml.findtext(tag)

      text = (xml.findtext("serialno") or "0").strip()
      kwargs['serialno'] = int(text)

      if not kwargs['name']:
         raise Errors.ProfileFormatError(None, "Missing profile name.")
      if not kwargs['creator']:
         raise Errors.ProfileFormatError(None, "Missing profile creator.")

      for tag in ("creationtime", "modifiedtime"):
         text = (xml.findtext(tag) or "").strip()
         if text:
            try:
               kwargs[tag] = XmlUtils.ParseXsdDateTime(text)
            except Exception as e:
               msg = "Profile '%s' has invalid %s: %s." % (kwargs['name'],
                                                           tag, str(e))
               raise Errors.ProfileFormatError(kwargs['name'], msg)

      text = xml.findtext("readonly") or "false"
      try:
         kwargs["readonly"] = XmlUtils.ParseXsdBoolean(text)
      except Exception as e:
         raise Errors.ProfileFormatError(kwargs['name'], "Could not parse "
            "profile readonly: %s" % str(e))

      kwargs["vibIDs"] = set()
      kwargs["vibstates"] = {}
      for elem in xml.findall("viblist/vib"):
         vibstate = VibState.FromXml(elem)
         if vibstate.id in kwargs["vibIDs"]:
            msg = "Profile %s has duplicated vib-ids in viblist (%s)" %  (
                  kwargs["name"], vibstate.id)
            raise Errors.ProfileFormatError(msg)
         kwargs["vibIDs"].add((vibstate.id).strip())
         kwargs["vibstates"][vibstate.id] = vibstate

      kwargs["componentIDs"] = getTextSet(xml, "bulletinlist/bulletin-id")
      kwargs["manifestIDs"] = getTextSet(xml, "manifestlist/manifest-id")
      kwargs["solutionIDs"] = getTextSet(xml, "solutionlist/solution-id")

      kwargs["reservedComponentIDs"] = {}
      for elem in xml.findall("reservedComponentList/component"):
         attr = elem.attrib
         kwargs["reservedComponentIDs"][attr['name']] = attr['version']

      return cls(**kwargs)

   def ToXml(self, toDB=False):
      """Serializes this Image Profile instance out to XML.
         Parameters:
            * toDB - If True, VIB local states information will be saved.
         Returns:
            An ElementTree instance
         Exceptions:
            None
      """
      def addIdList(xml, idSet, listTag, itemTag):
         """Given a set of IDs, form a list of itemTag under listTag in the XML.
         """
         if idSet:
            listElem = etree.SubElement(xml, listTag)
            for item in idSet:
               elem = etree.Element(itemTag)
               elem.text = item
               listElem.append(elem)

      xml = etree.Element("imageprofile")
      etree.SubElement(xml, "name").text = self.name
      etree.SubElement(xml, "creator").text = self.creator
      etree.SubElement(xml, "profileID").text = self.profileID
      etree.SubElement(xml, "creationtime").text = self.creationtime.isoformat()
      etree.SubElement(xml, "modifiedtime").text = self.modifiedtime.isoformat()
      etree.SubElement(xml, "serialno").text = str(self.serialno)
      if self.description:
         etree.SubElement(xml, "description").text = self.description
      elem = etree.SubElement(xml, "readonly")
      elem.text = self.readonly and "true" or "false"
      elem = etree.SubElement(xml, "viblist")
      for vibid in self.vibIDs:
         if self.vibstates[vibid].boot or not toDB:
            elem.append(self.vibstates[vibid].ToXml(toDB))
      etree.SubElement(xml, "acceptancelevel").text = self.acceptancelevel

      # Write component IDs included in this image profile.
      # For backward compatibility to 7.0 GA we have to output to
      # bulletinlist/bulletin-id.
      addIdList(xml, self.componentIDs, "bulletinlist", "bulletin-id")

      if self.baseimageID:
         etree.SubElement(xml, "baseimageID").text = self.baseimageID

      if self.addonID:
         if not self.baseimageID:
            raise ValueError('Should have base image when an addon exists')
         etree.SubElement(xml, "addonID").text = self.addonID

      if self.manifestIDs and not self.baseimageID:
         raise ValueError('Should have base image when a manifest exists')
      addIdList(xml, self.manifestIDs, "manifestlist", "manifest-id")

      addIdList(xml, self.solutionIDs, "solutionlist", "solution-id")

      reservedCIDs = self.GetReservedComponentIDs()
      if reservedCIDs:
         rCompElem = etree.SubElement(xml, "reservedComponentList")
         for name, version in reservedCIDs:
            elem = etree.SubElement(rCompElem, 'component',
                                    version=version, name=name)

      return xml

   def ToXmlString(self, toDB=False):
      return etree.tostring(self.ToXml(toDB))

   ATTRS_TO_COPY = ("name", "creator", "description",
                    "readonly", "vibIDs", "vibs", "acceptancelevel",
                    "profileID", "serialno", "componentIDs", "components",
                    "baseimageID", "baseimage", "addonID", "addon",
                    "solutionIDs", "solutions", "manifestIDs", "manifests",
                    "reservedComponentIDs", "reservedComponents")

   def __add__(self, other):
      """Merges this image profile with another image profile.
         The attributes from the image profile with the newer serialno
         are taken.
         Parameters:
            * other - the ImageProfile object to add
         Returns:
            A new instance of ImageProfile.
         Exceptions:
            ValueError - if an add is attempted between profiles with different
                         profileIDs, or an add is attempted between
                         profiles with the same serialno, but different
                         attributes.
      """
      if other.profileID != self.profileID:
         raise ValueError("Image profiles does not have equal id: "
                          "'%s' != '%s'" % (self.profileID, other.profileID))
      if other.serialno > self.serialno:
         newer = other
      elif self.serialno > other.serialno:
         newer = self
      else:
         # See if the attributes are the same.   If not, give up.
         for attr in self.ATTRS_TO_COPY:
            if getattr(self, attr) != getattr(other, attr):
               raise ValueError("Image profiles %s (id: %s) and %s (id: %s) "
                                "have unequal values of the '%s' attribute: "
                                "'%s' != '%s'" % (self.name, self.profileID,
                                other.name, other.profileID, attr,
                                getattr(self, attr), getattr(other, attr)))
         newer = self
      kwargs = dict()
      for attr in self.ATTRS_TO_COPY:
         kwargs[attr] = getattr(newer, attr)
      return ImageProfile(**kwargs)

   def __eq__(self, other):
      return (self.name == other.name and
              self.creator == other.creator and
              self.acceptancelevel == other.acceptancelevel and
              self.description == other.description and
              self.vibIDs == other.vibIDs and
              self.componentIDs == other.componentIDs and
              self.baseimageID == other.baseimageID and
              self.addonID == other.addonID and
              self.manifestIDs == other.manifestIDs and
              self.solutionIDs == other.solutionIDs)

   def __ne__(self, other):
      return not self.__eq__(other)

   def __hash__(self):
      return hash('%s%s%s%s%s' % (self.name, self.creator,
         self.acceptancelevel, self.description, sorted(self.vibIDs)))

   def __str__(self):
      try:
         return etree.tostring(self.ToXml(), pretty_print=True).decode()
      except Exception:
         return etree.tostring(self.ToXml()).decode()

   def AddVib(self, vib, replace=False, boot=True):
      """Adds a VIB to this image profile, optionally replacing
         existing VIBs if appropriate.  The vibIDs attribute will
         gain the vibID from vib, and the vibs collection will
         gain the vib instance passed along.

         Parameters:
            * vib     - A Vib.BaseVib-compatible instance
            * replace - If False, vib will be added to the image profile
                        vibIDs and vibs attributes without further
                        validation.
                        If True, vib will replace existing vibs in this
                        image profile if vib obsoletes existing vibs, or
                        if an existing VIB obsoletes vib (downgrade).
            * boot    - True if the VIB is required at boot time.
         Exceptions:
            KeyError  - vib already exists in this profile.  Duplicate VIBs
                        are not allowed.
      """
      if vib.id in self.vibIDs:
         msg = '%s is already in ImageProfile "%s"' % (vib.id, self.name)
         raise KeyError(msg)
      else:
         log.info('Adding VIB %s to ImageProfile %s' % (vib.id, self.name))
         self.vibs.AddVib(vib)
         self.vibstates[vib.id] = VibState(vib.id, boot=boot)
         if replace:
            result = self.vibs.Scan()
            obsoleted = result.vibs[vib.id].replaces
            if obsoleted:
               log.debug('VIBs %s are replaced by VIB %s, removing them from '
                        'ImageProfile %s' % (obsoleted, vib.id, self.name))

            obsoletedby = result.vibs[vib.id].replacedBy
            if obsoletedby:
               log.debug('VIBs %s replacing VIB %s, removing them from '
                        'ImageProfile %s' % (obsoletedby, vib.id, self.name))

            for vibid in obsoleted | obsoletedby:
               self.RemoveVib(vibid)

         self._updatemodifiedtime()

   def AddVibs(self, vibs, replace = False):
      """Adds one or more VIBs to this image profile, optionally replacing
         existing VIBs if appropriate.  Similar to calling AddVib repeatedly,
         but it will not change the image profile if any of the VIBs already
         exist in the profile, or if they obsolete each other.

         Parameters:
            * vibs    - A VibCollection
            * replace - If False, vibs will be added to the image profile
                        vibIDs and vibs attributes without further
                        validation.
                        If True, vibs will replace existing vibs in this
                        image profile if vibs obsoletes existing vibs, or
                        if an existing VIB obsoletes vib (downgrade).
                        However, if any of the vibs passed along obsolete
                        each other, then we will error out.
         Exceptions:
            KeyError  - At least one vib already exists in this profile.
                        Duplicate VIBs are not allowed.
            ValueError - replace is True and some of the VIBs in the vibs
                        parameter obsolete each other.
      """
      duplicates = set(vibs.keys()) & self.vibIDs
      if len(duplicates):
         msg = '%s is already in ImageProfile "%s"' % (
            ', '.join(list(duplicates)), self.name)
         raise KeyError(msg)

      # Check if any VIBs obsolete each other
      if replace:
         result = vibs.Scan()
         for vid in result.vibs:
            obsoleted = result.vibs[vid].replaces
            if obsoleted:
               raise ValueError('VIB %s obsoletes VIBs %s in the input to '
                                'AddVibs -- cannot continue' % (vid,
                                ', '.join(list(obsoleted)) ))

      for vib in vibs.values():
         self.AddVib(vib, replace)

   def AddComponent(self, comp):
      """Adds a Component to this image profile.
         Use PopulateComponents() to fill components when componentIDs is
         populated.
      """
      if comp.id in self.componentIDs:
         msg = 'Component %s is already in ImageProfile "%s"' \
               % (comp.id, self.name)
         raise KeyError(msg)

      log.debug('Adding Component %s to ImageProfile %s', comp.id, self.name)
      self._components.AddComponent(comp)
      self._componentIDs.add(comp.id)

      nameVerPair = (comp.compNameStr, comp.compVersionStr)
      if nameVerPair in self._reservedComponentIDs:
         # When adding a previously reserved component, it will no longer be
         # reserved.
         log.debug('Removing reserved Component %s in ImageProfile %s',
                   comp.id, self.name)
         self._reservedComponentIDs.remove(nameVerPair)
         try:
            self.reservedComponents.RemoveComponent(comp.id)
         except KeyError:
            pass

   def PopulateWithDatabase(self, database):
      """Populate inventory from a database.
      """
      self.PopulateVibs(database.vibs)
      self.PopulateComponents(bulletins=database.bulletins)
      if self.addonID is not None:
         self.PopulateAddon(database.addons)
      if self.baseimageID is not None:
         self.PopulateBaseImage(database.baseimages)
      if self.manifestIDs:
         self.PopulateManifests(database.manifests)
      self._PopulateSolutions(database.solutions)
      self._PopulateReservedComponentsFromDB(database)

   def PopulateWithDepots(self, depots):
      """Populate inventory with a DepotCollection.
      """
      self.PopulateVibs(depots.vibs)
      self.PopulateComponents(bulletins=depots.bulletins)
      if self.addonID is not None:
         self.PopulateAddon(depots.addons)
      if self.baseimageID is not None:
         self.PopulateBaseImage(depots.baseimages)
      if self.manifestIDs:
         self.PopulateManifests(depots.manifests)
      self._PopulateSolutions(depots.solutions)

      allComps = Bulletin.ComponentCollection(depots.bulletins,
                                              ignoreNonComponents=True)
      self.PopulateReservedComponents(allComps)

   def PopulateVibs(self, vibs):
      """When a profile is created using FromXml() function,
         only vibIDs attribute is populated, but vibs set is not,
         and stays empty. In many cases the vibIDs and vibs need
         to be consistent.
         This function is used to pupulate the profile.vibs attribute
         from a VibCollection provided as a parameter

         Parameters:
            * vibs    - A VibCollection
      """
      for id in self.vibIDs:
         if id in vibs:
            if id not in self.vibs:
               vib = vibs[id]
               self.vibs.AddVib(vib)

   def PopulateComponents(self, components=None, bulletins=None):
      """Populate components according to componentIDs from either a
         ComponentCollection or a BulletinCollection.
         Parameter:
            * bulletins - A BulletinCollection
            * components - A ComponentCollection
      """
      if ((components is not None and bulletins is not None) or
          (components is None and bulletins is None)):
         raise ValueError('One of components and bulletins must be provided')
      if components:
         for cId in self.componentIDs:
            if (components.HasComponent(cId) and
                not self._components.HasComponent(cId)):
               self._components.AddComponent(components.GetComponent(cId))
      else:
         for cId in self.componentIDs:
            if cId in bulletins and not self._components.HasComponent(cId):
               self._components.AddComponent(bulletins[cId])

   def PopulateAddon(self, addons):
      """ Populate the addon object from self.addonID and the provided addon
          collection.

          Exception:
             KeyError: when self.addonID not found in addon collection.
      """
      try:
         self.addon = addons[self.addonID]
      except KeyError:
         raise KeyError('Addon %s not exist in database/depots' % self.addonID)

   def PopulateManifests(self, manifests):
      """ Populate the manifest object from self.manifestIDs and the provided
          manifest collection.

          Exception:
             KeyError: when an ID from self.manifestIDs not found in manifest
                       collection.
      """
      self.manifests.clear()
      if self.manifestIDs:
         for manifestID in self.manifestIDs:
            self.manifests.AddManifest(manifests.GetManifest(manifestID),
                                       replace=True)

   def PopulateBaseImage(self, baseimages):
      """ Populate the base image object from self.baseimageID and the provided
          base image collection.

          Exception:
             KeyError: when self.baseimageID not found in base image collection.
      """
      try:
         self.baseimage = baseimages[self.baseimageID]
      except KeyError:
         raise KeyError('Base image %s not exist in database/depots' %
                        self.baseimageID)

   def _PopulateSolutions(self, solutions):
      """ Populate solutions from a solution collection.

          Exception:
             KeyError: when ID of a solution is not found in the collection.
      """
      self.solutions.clear()
      if self.solutionIDs:
         for solutionID in self.solutionIDs:
            try:
               self.solutions[solutionID] = solutions[solutionID]
            except KeyError:
               raise KeyError('Solution with ID %s is not found in the '
                              'collection' % solutionID)

   def PopulateReservedComponents(self, allComps):
      """ Populate the reserved components from the provided component
          collection.

          Exception:
             MissingComponentError : When some of components not found.
      """
      reservedCIDs = self.GetReservedComponentIDs()
      self.reservedComponents.clear()
      if reservedCIDs:
         missing = []
         for name, version in self._reservedComponentIDs:
            try:
               comp = allComps.GetComponent(name, version)
            except KeyError:
               missing.append('%s_%s' % (name, version))
         if missing:
            raise Errors.MissingComponentError('Missing reserved components %s'
                                               % ','.join(missing))

         self.reservedComponents.clear()
         for name, version in self._reservedComponentIDs:
            comp = allComps.GetComponent(name, version)
            self.reservedComponents.AddComponent(comp)

   def _PopulateReservedComponentsFromDB(self, database):
      allComps = Bulletin.ComponentCollection(database.bulletins,
                                              ignoreNonComponents=True)
      allComps += database.reservedComponents
      self.PopulateReservedComponents(allComps)

   def RemoveVib(self, vib):
      """Removes a VIB from the image profile.
         Parameters:
            * vib - either a string VIB ID or a Vib instance representing
                    the VIB to remove from the profile.
         Exceptions:
            KeyError - vib ID is not part of the image profile (not in vibIDs)
      """
      if isString(vib):
         vibid = vib
      else:
         vibid = vib.id

      if vibid in self.vibstates:
         log.debug('VIB %s is being removed from ImageProfile %s' % (vibid,
            self.name))
         del self.vibstates[vibid]
      else:
         msg = 'Cannot remove VIB %s; it is not in the ImageProfile' % (vibid)
         raise KeyError(msg)

      if vibid in self.vibs:
         del self.vibs[vibid]

      self._updatemodifiedtime()

   def AddSolution(self, solution, replace=False):
      """Add a solution to the image profile. When there is an existing solution
         with the same name, it will be replaced if replace is set.
         Raises:
            ValueError: the solution of the same name is already present and
                        replace is not set.
      """
      name = solution.nameSpec.name
      for s in self.solutions.values():
         if s.nameSpec.name == name:
            if replace:
               del self.solutions[s.releaseID]
               self._solutionIDs.remove(s.releaseID)
               break
            else:
               raise ValueError('Solution with name %s (version %s) already '
                                'exist in the image profile.'
                                % (name, str(s.versionSpec.version)))
      self._solutionIDs.add(solution.releaseID)
      self.solutions.AddSolution(solution)

   def RemoveSolution(self, solution):
      """Remove a solution from the image profile.
         Raises:
            KeyError: the solution is not found.
      """
      if not solution.releaseID in self.solutionIDs:
         raise KeyError('Solution %s is not present in the image profile.'
                        % solution.releaseID)

      self._solutionIDs.remove(solution.releaseID)
      if solution.releaseID in self.solutions.keys():
         del self.solutions[solution.releaseID]

   def _getAllKnownComps(self, refComps):
      """Get all known components, including active, preserve and reference
         components.
      """
      allComps = self.GetKnownComponents()
      if refComps is not None:
         allComps += refComps
      return allComps

   def _syncVibs(self, orphanVibs=None, refComps=None, refVibs=None):
      """Update VIBs after a component change.
         Reserved component needs to be re-calculated as there can be new
         additions.
      """
      allVibs = VibCollection.VibCollection()
      allVibs += self.vibs
      if refVibs:
         allVibs += refVibs

      allComps = self._getAllKnownComps(refComps)
      newVibs = self._components.GetVibCollection(allVibs)

      # Add back old orphan VIBs that are not obsoleted by the new additions.
      if orphanVibs:
         tmpNewVibs = newVibs + orphanVibs
         vibScanRes = tmpNewVibs.Scan()
         for vibId, vib in orphanVibs.items():
            replacedBy = vibScanRes.vibs[vibId].replacedBy
            if replacedBy:
               log.debug('Dropping orphan VIB %s as it is replaced by %s',
                         vibId, replacedBy)
            else:
               newVibs.AddVib(vib)

      adds = set(newVibs.keys()) - set(self.vibs.keys())
      removes = set(self.vibs.keys()) - set(newVibs.keys())

      # Formal adds/removes will update the vibstates properly.
      for vibId in removes:
         self.RemoveVib(vibId)
      for vibId in adds:
         self.AddVib(newVibs[vibId], replace=True)

      self.PopulateReservedComponents(allComps)

   def ReserveComponent(self, compId):
      """Reserve a component by moving it from components to
         reservedComponents.
      """
      if compId not in self.componentIDs:
         raise ValueError('Component %s is not part of the image profile'
                          % compId)
      if not self.components.HasComponent(compId):
         raise ValueError('Cannot reserve component %s: this image profile has '
                          'not been populated' % compId)

      comp = self.components.GetComponent(compId)
      nameVerPair = (comp.compNameStr, comp.compVersionStr)
      if nameVerPair in self._reservedComponentIDs:
         raise ValueError('Component (%s, %s) is already reserved'
                          % nameVerPair)

      self._componentIDs.remove(compId)
      self._reservedComponentIDs.append(nameVerPair)
      self._components.RemoveComponent(compId)
      self.reservedComponents.AddComponent(comp)

   def SetComponents(self, comps):
      """Updates components and componentIDs.
      """
      self._components = comps
      self._componentIDs = set(comps.GetComponentIds())

   def SyncComponents(self, refComps=None):
      """Update components and reserved components after a VIB change.
         Parameters:
            refComps - reference component collection.
      """
      allComps = self._getAllKnownComps(refComps)
      self.SetComponents(self.GetFinalComponents(allComps))
      self.PopulateReservedComponents(allComps)

   def RemoveComponents(self, rmComps):
      """Removes components from the image profile and refreshes inventory.
         Parameters:
            * rmComps - a ComponentCollection object that contains components
                        to remove.
      """
      # Get current orphan VIBs before messing with the components.
      orphanVibs = self.GetOrphanVibs()

      newComps = self.components.copy()
      for comp in rmComps.IterComponents():
         newComps.RemoveComponent(comp.compNameStr, comp.compVersionStr)

      self.SetComponents(newComps)

      # Sync VIBs according to component removal. Provide components that
      # were just removed as candidates for reserved components.
      self._syncVibs(orphanVibs=orphanVibs, refComps=rmComps)

   def AddComponents(self, addComps, refVibs, replace=False):
      """Adds components to the image profile. When adding, new components are
         validated against self/partial obsolete, and fully obsoleted old
         components are removed.
         Parameters:
            * addComps - a ComponentCollection object that contains components
                         to add.
            * refVibs - a VibCollection that contains VIB metadata for the
                        additional components.
            * replace - if set, existing components that obsolete the new
                        components will be replaced by the new components;
                        otherwise only components that update the system
                        will be installed.
      """
      # Get current orphan VIBs before messing with the components.
      orphanVibs = self.GetOrphanVibs()

      # To be used for final components.
      newComps = self.components.copy()
      oldCompIds = self.componentIDs

      # Figuring out which components are obsoleted and remove them.
      # We will inspect new components and discard those being obsoleted,
      # such as lower versions of a component. And when merging the remaining
      # components, remove obsoleted ones from current components.

      allVibs = self.vibs + refVibs if refVibs else self.vibs
      addCompIds = set(addComps.GetComponentIds())
      allComps = newComps + addComps
      problems = ComponentScanner(allComps, allVibs).Validate()

      # Lists for partial and self obsoletes, such cases will raise
      # exceptions.
      partObsoletes = list()
      selfObsoletes = list()
      # Two map that handles both direction of all obsolete relations.
      # This helps us figure out what new components to add and what
      # old components to remove.
      # Map from the obsoleted component to the obsoleter(s) in a set,
      # either one of them needs to be in the newly added components
      # for the problem to be relevant.
      obsoletedComps = dict()
      # Map from obsoleter to the obsoleted in a set, only new components
      # register in this map.
      newCompObsolete = dict()
      for p in problems.values():
         if (not p.reltype in (ComponentScanProblem.TYPE_OBSOLETES,
                               ComponentScanProblem.TYPE_SELFOBSOLETE)):
            continue
         if p.comp in addCompIds or p.replacesComp in addCompIds:
            if p.reltype == ComponentScanProblem.TYPE_OBSOLETES:
               if not p.isFullObsolete:
                  partObsoletes.append((p.comp, p.replacesComp))
               else:
                  obsoletedComps.setdefault(p.replacesComp, set()).add(p.comp)
                  if p.comp in addCompIds:
                     newCompObsolete.setdefault(p.comp, set()).add(
                                                               p.replacesComp)
            elif p.reltype == ComponentScanProblem.TYPE_SELFOBSOLETE:
               selfObsoletes.append(p.comp)

      if partObsoletes:
         raise ValueError('Partial obsolete is found in the following '
                          'component pairs: %s'
                          % ' '.join([', '.join([c1, c2])
                                      for c1, c2 in partObsoletes]))
      if selfObsoletes:
         raise ValueError('Components %s obsolete themselves and cannot be '
                          'added' % ', '.join(selfObsoletes))

      # Discard new components that are obsoleted within themselves.
      # This is always done regardless of replace or not.
      obsoletedCompIds = set(obsoletedComps.keys())
      tempRmIds = set()
      for compId in addCompIds & obsoletedCompIds:
         obsoleteSet = obsoletedComps[compId] & addCompIds
         if obsoleteSet:
            log.debug('Component %s is obsoleted by %s, remove it from the '
                      'the add list', compId, ', '.join(obsoleteSet))
            tempRmIds.add(compId)
      addCompIds -= tempRmIds

      if replace:
         # With replace we will add all missing surviving components forcefully,
         # which means removing existing components that obsolete them or
         # obsoleted by them.
         finalAddCompIds = addCompIds - oldCompIds
         rmCompIds = set()
         for compId in addCompIds & obsoletedCompIds:
            obsoleteSet = obsoletedComps[compId] & oldCompIds
            if obsoleteSet:
               log.debug('Remove components %s that obsolete %s',
                         ', '.join(obsoleteSet), compId)
               rmCompIds.update(obsoleteSet)
         for compId, obsoletes in newCompObsolete.items():
            obsoletedSet = obsoletes & (oldCompIds - rmCompIds)
            if obsoletedSet:
               log.debug('Remove components %s obsoleted by %s',
                         ', '.join(obsoletedSet), compId)
               rmCompIds.update(obsoletedSet)
      else:
         # Loop each new component that is not decided to be added to
         # check if it can be added, i.e. no old component obsolete it.
         # This process will end only when no component is being removed
         # to make sure an obsoleted old component would not obsolete a new
         # component.
         finalAddCompIds = set()
         rmCompIds = set()
         oldCompRemoved = True
         while oldCompRemoved:
            oldCompRemoved = False
            for compId in addCompIds - finalAddCompIds:
               obsoleteSet = (obsoletedComps.get(compId, set()) &
                              (oldCompIds - rmCompIds))
               if not obsoleteSet:
                  # New component not obsoleted will be added.
                  finalAddCompIds.add(compId)
                  oldObsoleteSet = (newCompObsolete.get(compId, set()) &
                                    (oldCompIds - rmCompIds))
                  if oldObsoleteSet:
                     # New component to obsolete old components.
                     log.debug('Component %s obsolete %s, remove them from '
                               'the image profile', compId,
                               ', '.join(oldObsoleteSet))
                     rmCompIds.update(oldObsoleteSet)
                     oldCompRemoved = True

      # Now actually do the adds and removes, keep removed components
      # for reference to calculate reserved components.
      refComps = Bulletin.ComponentCollection()
      log.debug('Components to add: %s, components to remove: %s.',
                ', '.join(finalAddCompIds), ', '.join(rmCompIds))
      for compId in finalAddCompIds:
         newComps.AddComponent(addComps.GetComponent(compId))
      for compId in rmCompIds:
         refComps.AddComponent(newComps.GetComponent(compId))
         newComps.RemoveComponent(compId)

      self.SetComponents(newComps)

      # Sync VIBs and calculate reserved components according to the component
      # changes. New VIBs are looked up in the reference collection.
      self._syncVibs(orphanVibs=orphanVibs, refComps=refComps, refVibs=refVibs)

   def RemoveSolutions(self, names, ignoreNotFound=False):
      """Removes solutions and associated components.
         Parameters:
            names - a list/set of names of solutions to remove.
            ignoreNotFound - when not set, raise SolutionNotFound error when
                             one or more of the solution names is not present.
         Returns:
            A set of solution names that are being removed.
      """
      hostSols = {s.nameSpec.name: s for s in self.solutions.values()}

      missingSols = set(names) - set(hostSols.keys())
      if missingSols:
         solNames = sorted(missingSols)
         msg = 'Solutions %s to be removed are not found' % ', '.join(solNames)
         log.error(msg)
         if not ignoreNotFound:
            # TODO: create a new error. Here the arguments are reported in VAPI,
            # and have to refer to a single solution.
            raise Errors.SolutionNotFound(solNames[0], '', msg)

      rmSols = set(names) & set(hostSols.keys())
      rmComps = Bulletin.ComponentCollection()
      for name in rmSols:
         solution = hostSols[name]
         for compList in solution.MatchComponents(self.components).values():
            for comp in compList:
               rmComps.AddComponent(comp)
         del self.solutions[solution.releaseID]
         self._solutionIDs.remove(solution.releaseID)

      if rmComps:
         # It is possible that there is no solution component installed if
         # an explicit component remove was used.
         self.RemoveComponents(rmComps)

      return rmSols

   def DiffComponents(self, other):
      """Compares the components in this image profile to another image profile,
         returning the differences.
         Parameters:
            * other - The ImageProfile instance to compare against
         Returns:
            (onlyInSelf, onlyInOther) - A tuple: a list of the component IDs
                  that are only present in this image profile, and a list of
                  component IDs that are only present in the other image
                  profile.
      """
      return (list(self.componentIDs - other.componentIDs),
              list(other.componentIDs - self.componentIDs))

   def Diff(self, other):
      """Compares the VIB lists from this image profile to another profile,
         returning the differences.
         Parameters:
            * other - The ImageProfile instance to compare against
         Returns:
            (onlyinself, onlyinother) - A tuple; first a list of the VIB IDs
                      only present in this image profile, and second a list of
                      VIB IDs only present in the other profile.
         Exceptions:
            None
      """
      return (list(set(self.vibIDs) - set(other.vibIDs)),
         list(set(other.vibIDs) - set(self.vibIDs)))

   def ScanVibs(self, vibs):
      """Scans a VibCollection against this image profile, returning sets of
         VIBs that are newer, older, overlapping, and not part of the profile.
         Parameters:
            * vibs    - The VibCollection to scan this profile against
         Returns:
            A tuple (updates, downgrades, new, existing); updates are the set of
            VIB IDs not in this profile which update a VIB in the profile;
            downgrades are the VIB IDs that downgrade any VIB in this profile;
            new are VIB IDs which do not update or downgrade VIBs in the profile
            and existing are the VIBs from vibs which already exist in this
            profile.
      """
      allvibs = VibCollection.VibCollection()
      allvibs += vibs
      allvibs += self.vibs
      scanner = allvibs.Scan()
      updates = scanner.GetUpdatesSet(self.vibs)
      dgrades = scanner.GetDowngradesSet(self.vibs)
      existing = set(vibs.keys()) & self.vibIDs
      new = set(vibs.keys()) - updates - dgrades - existing
      return (updates, dgrades, new, existing)

   def Validate(self, depotvibs=None, nodeps=False,
                noconflicts=False, allowobsoletes=False,
                noacceptance=False, allowfileconflicts=False,
                noextrules=False, checkCompDeps=False):
      """Validates the image profile, returning a list of Problems.
         VIB/component dependencies and conflicts, obsolescence, file conflicts
         and overlays, acceptance levels, and VIB extensibility rules
         are all checked. Also checked is that an image profile must
         have a minimum of one boot kernel.

         Parameters:
            * depotvibs      - a VibCollection to search through and return
                               suggestions for MissingDependency problems.
                               If None, no suggestions will be returned.
            * nodeps         - Boolean, disable checking of VIBs Depends
                               constraints.  If False, every VIBs Depends
                               must be satisfied by another VIB in the profile,
                               or else a MissingDependency problem is reported.
                               If True, no MissingDependency problems will
                               be added.
            * noconflicts    - Boolean, disable checking of VIBs Conflicts,
                               including medadata conflicts and hwplatform
                               conflicts.
                               If False, any profile VIBs satisfying any other
                               profile VIBs Conflicts constraints will result
                               in a ConflictingVIB problem being added; and when
                               a group of VIBs contain conflicting hwplatform
                               requirements, a HwPlatformConflict problem will
                               be added.
                               If True, no ConflictingVIB or HwPlatformConflict
                               problems are returned.
            * allowobsoletes - Boolean, disable checking of VIB obsolescence.
                               If False, no two VIBs in an image profile may
                               obsolete each other, otherwise ObsoletedVIB
                               problem will be added.
                               If True, no ObsoletedVIB problems will be added.
            * noacceptance   - Boolean, disable checking of acceptance levels.
                               If False, no VIB in an image profile may have
                               an acceptance level lower than the profile's
                               acceptance level, and if so, UnAcceptedVIB
                               problems will be added for each problem VIB.
                               If True, no UnAcceptedVIB problems are added.
            * allowfileconflicts - Boolean, disable checking of file conflicts.
            * noextrules     - Boolean, disable checking of extensibility rules.
                               If False, a VIB violating the check of extensi-
                               bility rules will have an ExtensibilityRule-
                               Violation problem added.
            * checkCompDeps  - Boolean, when set, raise dependency issues with
                               component info instead of VIB info.
         Returns:
            A list of Problem instances, or the empty list [] if no problems
            were found.
         Exceptions:
            VibFormatError:   The vib acceptance level value is invalid.
      """
      profilevibs = self.vibIDs
      objvibs = set(self.vibs.keys())
      diffVibs = set(profilevibs) - objvibs
      assert not diffVibs, \
            'VIB objects not found for VIB IDs %s listed in the profile' % \
            ', '.join(list(diffVibs))

      # XXX ideally, we only need to pull required VIBs if a
      # requirement is not satisfied within the ImageProfile, but it
      # will need to build a partial dependency map. For now just
      # link all the dependencies and leave it for optimization.

      if depotvibs is not None:
         referencevibs = set(depotvibs.keys())
         allvibs = self.vibs + depotvibs
      else:
         referencevibs = set()
         allvibs = self.vibs

      problems = set()
      hasbootkernel = False

      if not noacceptance:
         acceptancechecker = AcceptanceChecker(self.acceptancelevel)

      # two {filepath -> vibid} maps, the first one is for regular VIBs and
      # the second one is for overlay VIBs
      filevibmaps = [dict(), dict()]
      result = allvibs.Scan()

      if checkCompDeps:
         orphanVibs = self.GetOrphanVibs()
         if orphanVibs:
            # With orphan VIBs, ComponentScanner will not work properly, namely
            # when these VIBs are providing to components and when they have
            # conflict or unmet dependency.
            # The simplest fix is to fallback to VIB scanning, which trades
            # user-friendliness for correctness.
            log.debug('Falling back to VIB validation with orphan VIBs %s '
                      'present.', ', '.join(orphanVibs.keys()))
            checkCompDeps = False
         else:
            # Scan for component dependency issues, sort them into types and
            # add problems that are not filtered out by the options.
            relTypes = set()
            if not nodeps:
               relTypes.add(ComponentScanProblem.TYPE_DEPENDS)
            if not noconflicts:
               relTypes.add(ComponentScanProblem.TYPE_CONFLICT)
               relTypes.add(ComponentScanProblem.TYPE_SELFCONFLICT)
            if not allowobsoletes:
               # Both obsolete issues should not show up if components are added
               # using AddComponents() properly, adding them for the sake of
               # catching all issues.
               # Specifically, self-obsolete can end up in Scanner failing to
               # locate obsoleted VIB.
               relTypes.add(ComponentScanProblem.TYPE_OBSOLETES)
               relTypes.add(ComponentScanProblem.TYPE_SELFOBSOLETE)

            if relTypes:
               compIssues = ComponentScanner(self.components,
                                             self.vibs).Validate()
               for issue in compIssues.values():
                  if issue.reltype in relTypes:
                     problems.add(ComponentDependencyProblem(issue))

      # Add esximage library version (and in the future perhaps other external
      # deps) to the "VIBs" being provided by the profile
      esximgcomps = result.GetResultsByType(ScanResult.TYPE_ESXIMGLIB)
      providingvibs = profilevibs | set(esximgcomps.keys())

      for vibid in profilevibs:
         scanres = result.vibs[vibid]

         # Dependency problems
         if not nodeps and not checkCompDeps:
            for relation, vibids in scanres.depends.items():
               profileprovides = vibids & providingvibs
               if not profileprovides:
                  referenceprovides = vibids & referencevibs
                  problems.add(MissingDependency(scanres.id, relation,
                                                 referenceprovides))

         # VIB conflict and HwPlatform conflict problems
         if not noconflicts:
            profileconflicts = scanres.conflicts & providingvibs
            if profileconflicts and not checkCompDeps:
               problems.add(ConflictingVIB(profileconflicts | set([vibid,])))
            try:
               hwPlatforms = self.GetHwPlatforms()
            except Errors.HardwareError as e:
               problems.add(HwPlatformConflict(self.name, str(e)))

         # Obsolete problems
         if not allowobsoletes and not checkCompDeps:
            profilereplacedby = scanres.replacedBy & profilevibs
            if profilereplacedby:
               problems.add(ObsoletedVIB(vibid, profilereplacedby))

         vib = self.vibs[vibid]
         # acceptance level check
         if not noacceptance:
            problem = acceptancechecker.Check(vib.acceptancelevel)
            if problem:
               problems.add(UnAcceptedVIB(vibid, vib.acceptancelevel,
                  self.acceptancelevel))

         # check boot kernel module
         if not hasbootkernel:
            for pl in vib.payloads:
               if pl.payloadtype == pl.TYPE_BOOT:
                  hasbootkernel = True
                  break

         # file path overlapping calculation
         if not allowfileconflicts:
            for filepath in vib.filelist:
               if filepath == '' or filepath.endswith('/'):
                  continue

               filepath = PathUtils.CustomNormPath('/' + filepath)
               ind = vib.overlay and 1 or 0
               if filepath in filevibmaps[ind]:
                  filevibmaps[ind][filepath].add(vibid)
               else:
                  filevibmaps[ind][filepath] = set([vibid])

         if not noextrules:
            try:
               policy = AcceptanceLevels.GetPolicy(vib.acceptancelevel)
            except Exception as e:
               log.exception(e)
               log.info("Unable to obtain acceptance level policy: Skipping "
                        "extensibility rule validation for VIB %s" % (vibid))
               continue
            try:
               policy.VerifySchema(vib)
            except Errors.VibValidationError as e:
               problems.add(ExtensibilityRuleViolation(vibid, e.errors))

      # file conflict detection
      if not allowfileconflicts:
         for group, filevibmap in enumerate(filevibmaps):
            groupname = group and 'overlay' or 'non-overlay'
            for filepath, vibids in filevibmap.items():
               if len(vibids) > 1:
                  problems.add(FileConflict(vibids, filepath, groupname))

      # boot kernel check
      if not hasbootkernel:
         problems.add(ProfileTooShort(self.name, self.vibIDs))

      # Check ESX version to make sure base ESX VIB is included.
      esxVer = None
      try:
         esxVer = self.GetEsxVersion()
      except ValueError as e:
         problems.add(MissingBaseEsx(self.name, str(e)))

      if esxVer and self.baseimage:
         # Check Base Image version matches ESX version.
         baseImageVer = self.baseimage.versionSpec.version.versionstring
         if baseImageVer != esxVer:
            problems.add(MismatchBaseImage(self.name, baseImageVer, esxVer))

      # sort problems with priority
      return sorted(problems, key=lambda x: x.priority)

   def GetEsxVersion(self, rawversion = False):
      """Return the highest version string providing 'esx-version', which will
         be used to create boot.cfg.
         Exceptions:
            ValueError - If there is no VIB providing 'esx-version'
      """
      versions = list()
      for vibid in self.vibIDs:
         for prov in self.vibs[vibid].provides:
            if prov.name == 'esx-version':
               versions.append(prov.version)

      if versions:
         if rawversion:
            return max(versions)
         else:
            return str(max(versions))
      else:
         raise ValueError("There is no VIB providing 'esx-version'.")

   def IsSecureBootReady(self):
      """Check if the image profile is secure boot ready.
      """
      try:
         version = self.GetEsxVersion(rawversion=True)
         if version < Version.VibVersion('6.1.0', '0.0.000000'):
            savesig = False
         else:
            savesig = True
         return savesig
      except ValueError as e:
         log.info("Could not verify if this image is secure boot ready")
         return False

   def  GenerateVFATNames(self, reference = None, strictVFatName=False):
      """Generate non-conflicting 8.3 VFAT name for payloads in the image
         profile.

         Reference is a image profile with valid non-conflict 8.3 VFAT name for
         the payloads. If a VIB is in reference image profile, VFAT
         names from reference image profile are used.
         Otherwise, the algorithm will try to preserve explicitly specified
         vfatname. If the specified vfatname is already taken, the payload is
         treated as without specified vfatname.
         For all the payloads  without vfatname, the vfatname is
         constructed as 'payload-name.Txx', where 'T' is the first letter of the
         payloadtype, xx is the sequential number in hex format.
            Parameters:
               reference - An instance of ImageProfile, vfatname will be used to
                           populate payload vfatname in this profile.
               strictVFatName - Flag to enforce strict 8.3 vfatName validation.
            Exceptions:
               ValueError - If more than 256 payloads have been mapped to one
                            name space, or a payload in reference image profile
                            doesn't have vfatname, or two payloads in reference
                            image profile share the same vfatname.
      """
      def ValidVFATName(vfatName, strict=False):
         eightThree = vfatName.split('.')

         if len(eightThree) > 2:
            return False

         # PR 790728: autodeploy is using another way to name payload local filename
         # (<payload name>.<hash>) to satisfy the larger namespace. So disable the
         # check on the length of file name and extension, because it is no longer true.
         # PR 1165755: When stateless booted host image is cached, 8.3 VFat Name is required.
         if strict:
            if len(eightThree[0]) > 8:
               return False
            if len(eightThree) == 2:
               if len(eightThree[1]) > 3:
                  return False
         return True

      if reference is None:
         selfonly = self.vibIDs
         keeps = set()
      else:
         selfonly, othersonly = self.Diff(reference)
         keeps = set(self.vibIDs) - set(selfonly)

      payloadcounts = {}
      uniquenames = set()

      # retrieve name from reference profile
      for vibid in sorted(keeps):
         for p in self.vibs[vibid].payloads:
            if p.name in reference.vibstates[vibid].payloads:
               localname = reference.vibstates[vibid].payloads[p.name]
               if localname:
                  if localname not in uniquenames:
                     uniquenames.add(localname)
                  else:
                     #this should not happen if reference is good
                     msg = ('localname %s occurs more than once in reference '
                            'profile %s' % (localname, reference.name))
                     raise ValueError(msg)
               self.vibstates[vibid].payloads[p.name] = localname
            else:
               self.vibstates[vibid].payloads[p.name] = ''

      # try to honor a vfat name that's been pre-assigned.
      for vibid in sorted(selfonly):
         for p in self.vibs[vibid].payloads:
            vfatname = p.vfatname.lower()
            if vfatname and vfatname not in uniquenames:
               uniquenames.add(vfatname)
               self.vibstates[vibid].payloads[p.name] = vfatname
               continue
            self.vibstates[vibid].payloads[p.name] = ''

      # automatically generate local name
      for vibid in sorted(self.vibIDs):
         vib = self.vibs[vibid]
         state = self.vibstates[vibid]
         for p in vib.payloads:
            if not state.payloads[p.name] or \
               not ValidVFATName(state.payloads[p.name], strictVFatName):

               shortname = p.name[:8].replace('-', '_').lower()
               if shortname not in payloadcounts:
                  payloadcounts[shortname] = 0

               start = payloadcounts[shortname]
               vn = None
               for seq in range(start, 256):
                  hexv = '%x' % (seq)
                  vn = '%.8s.%s%s' % (shortname, p.payloadtype[0], hexv.zfill(2))
                  if vn not in uniquenames:
                     payloadcounts[shortname] = seq + 1
                     break
               if vn is not None and vn not in uniquenames:
                  state.payloads[p.name] = vn
                  uniquenames.add(vn)
               else:
                  # This should be very rare - need more than 256 payloads to be
                  # mapped into the name space
                  msg = ('Failed to generate unique vfatname for payload %s in '
                         'VIB %s.' % (vibid, p.name))
                  raise ValueError(msg)

   def SanityCheck(self):
      """Check the sanity of this image profile.

      Currently just checks to make sure the payload names are unique.
      """
      uniquenames = set()
      for state in self.vibstates.values():
         for vfatname in state.payloads.values():
            if vfatname in uniquenames:
               raise ValueError('vfatname "%s" is not unique' % vfatname)
            uniquenames.add(vfatname)

   def getExtraVibs(self):
      """Return the list of extra VIBs, i.e. the vibs that are part of this
      image profile, but that are not required at boot time.
      """
      return [(vibId, vib) for vibId, vib in self.vibs.items()
              if not self.vibstates[vibId].boot]

   def GetBootOrder(self, payload_types = None, vib_types = None,
                    customorder = None):
      """Returns the boot order of all boot-loadable modules as needed
         by bootloader config files such as boot.cfg / pxelinux.cfg.
         The boot order is determined as follows:
         1) All VIB payloads with type KERNEL come first, and are ordered
            in ascending order of the payload "bootorder" attribute
         2) Next comes all VGZ and TGZ payloads of non-overlay VIBs,
            in the same order as the VIBs appear in the profile's vibIDs
            list
         3) Finally comes all VGZ and TGZ payloads of overlay VIBs,
            in the same order as the VIBs appear in the profile's vibIDs
            list
         NOTE: must call GenerateVFATNames first to create local file name
         Parameters:
            * payload_types - List of Vib.Payload.TYPE_*. Only payloads of
                              these types will be included. If None, all types
                              are included.
            * vib_types     - List of Vib.BaseVib.TYPE_*. Only payloads from
                              these VIBs will be included. If None, payloads
                              from all VIBs are included.
            * customorder   - List of VIB ID of the VIBs in the imageprofile.
                              The payloads within 'overlay' and 'non-overlay'
                              group will be ordered according to ordering of
                              this list.
         Returns:
            A list of (vibID, payload) tuples, where vibID is the ID of
            the VIB containing the payload, and payload is an instance
            of Vib.Payload, containing the payload name.
      """
      kmodules = []
      rmodules = []
      omodules = []

      vibids = [vid for vid in self.vibIDs
                if (self.vibstates[vid].boot and
                    (vib_types is None or self.vibs[vid].vibtype in vib_types))]

      if customorder is not None:
         vibids = [vid for vid in customorder if vid in vibids]
      else:
         vibids.sort()

      for vibid in vibids:
         for p in self.vibs[vibid].payloads:
            if payload_types is not None and p.payloadtype not in payload_types:
               continue

            if p.payloadtype == p.TYPE_BOOT:
               p.localname = self.vibstates[vibid].payloads[p.name]
               kmodules.append((vibid, p))
            elif p.payloadtype in (p.TYPE_VGZ, p.TYPE_TGZ,
                                   p.TYPE_INSTALLER_VGZ):
               p.localname = self.vibstates[vibid].payloads[p.name]
               if self.vibs[vibid].overlay:
                  omodules.append((vibid, p))
               else:
                  # PR 763356: need to make sure sys payload is loaded before
                  # other vgz payloads.
                  # There is currently no ordering supported in vgz payloads.
                  # As it is only a special case for sys payload, we can assign a
                  # special value to its bootorder (originally unused field). Then
                  # we put the payload to head when see this special bootorder.
                  # Note that this change is only internal, and does not affect the
                  # existing interfaces. Adding support for ordering of vgz payloads
                  # impacts the existing interfaces. So we will reconsider it in OP
                  # if such requirement becomes common.
                  if p.bootorder == Vib.Payload.BOOT_ORDER_VGZ_BASE:
                     rmodules.insert(0, (vibid, p))
                  else:
                     rmodules.append((vibid, p))

      # order boot modules
      kmodules.sort(key=lambda item: item[1].bootorder)
      return kmodules + rmodules + omodules

   def GetHwPlatforms(self):
      """Returns the union of the hardware platforms for all VIBs in this image
         profile.  The union is defined as the set of hwplatform's common to
         all VIBs that contain at least one hwplatform, where common means having
         the same vendor field.
         If the vendor is the same but the models are different, then discard
         the platform.
         If the vendor is the same but one hwplatform has no model but the other
         one has a model, then set the model to be stricter.
         Raises:
            HardwareError - if no common platform exist for the image profile
         Returns:
            A set of HwPlatform instances
      """
      def _getVendorModels(hwplatforms):
         vm = dict()
         for hwp in hwplatforms:
            vm.setdefault(hwp.vendor, set())
            if hwp.model:
               vm[hwp.vendor].add(hwp.model)
         return vm

      vendormodels = dict()
      hwVibs = []
      for vibid in self.vibIDs:
         if vibid in self.vibs and len(self.vibs[vibid].hwplatforms) > 0:
            hwVibs.append(vibid)
            if len(vendormodels) == 0:
               vendormodels = _getVendorModels(self.vibs[vibid].hwplatforms)
            else:
               rm_list = []
               # Weed out existing common platforms not in the new VIB
               vibvm = _getVendorModels(self.vibs[vibid].hwplatforms)
               for vendor in vendormodels.keys():
                  if vendor not in vibvm:
                     rm_list.append(vendor)
                  elif len(vibvm[vendor]) > 0:
                     if len(vendormodels[vendor]) == 0:
                        vendormodels[vendor] = vibvm[vendor]
                     else:
                        vendormodels[vendor] &= vibvm[vendor]
                        if len(vendormodels[vendor]) == 0:
                           # models don't intersect at all, get rid of platform
                           rm_list.append(vendor)
               for vendor in rm_list:
                  del vendormodels[vendor]

               # If there are no more common vendors, we can stop looking
               if len(vendormodels) == 0:
                  break

      # If there were VIBs with hwplatforms and no common vendor/models,
      # report conflict
      if len(hwVibs) > 0 and len(vendormodels) == 0:
         msg = 'VIBs %s do not have any common hardware platform' % hwVibs
         raise Errors.HardwareError(msg)
      else:
         platforms = set()
         for vendor in vendormodels:
            if len(vendormodels[vendor]) == 0:
               platforms.add(Vib.HwPlatform(vendor))
            else:
               for model in vendormodels[vendor]:
                  platforms.add(Vib.HwPlatform(vendor, model))
         return platforms

   def GetRules(self):
      """Returns a set of Rule Engine rules describing any constraints
         for mapping the image profile to hosts.
         Currently the only rule is hardware SMBIOS vendor, specified by
         any <hwplatform> tags in VIB metadata.
         When an image profile contains conflicting hwplatform requirements,
         a special vendor is returned so the image profile will not map any
         host. However, a proper validation on the image profile should be
         done, which will flag this problem out.
         Returns:
            A list of (key, value) pairs.
         Exceptions:
            None
      """
      vendors = list()
      rules = list()
      try:
         hwPlatforms = self.GetHwPlatforms()
         for hwPlatform in hwPlatforms:
            vendors.append(hwPlatform.vendor)
      except Errors.HardwareError as e:
         log.error('Image profile %s is not applicable to any host: %s'
                   % (self.name, str(e)))
         vendors = [RULE_CONFLICTING_VENDORS]

      if vendors:
         rules.append((RULE_KEY_HWVENDOR, vendors))

      return rules

   rules = property(GetRules)

   def GetStatelessReady(self):
      """Determines if this image profile is ready for stateless deployment.
         Returns:
            True if every VIB in this image profile has its stateless ready
            attribute set to True also.
      """
      for vibid in self.vibIDs:
         if vibid in self.vibs:
            if not self.vibs[vibid].statelessready:
               return False
         else:
            raise KeyError("No VIB object found for VIB ID %s in image profile [%s]" % (
               vibid, self.name))

      return True

   statelessready = property(GetStatelessReady)

   def GetLowestAcceptanceLevel(self):
      """Determines the lowest Acceptance Level from the VIB list.
         Order of levels is dictated by AcceptanceChecker.TRUST_ORDER.
         Returns:
            A string representing one of the Acceptance Levels defined in
            Vib.ArFileVib.ACCEPTANCE_LEVELS

            None if no VIBs exist in self.vibs
      """
      levels = {}
      to = AcceptanceChecker.TRUST_ORDER
      if not self.vibs:
         return None
      for vib in self.vibs.values():
         levels[to[vib.acceptancelevel]] = vib.acceptancelevel
      return levels[sorted(levels.keys())[0]]

   def _IsFullyInstalled(self, comp):
       """ For personality manager. Check whether the VIBs for a component
           are all installed (exist in image profile).
       """
       try:
          name = comp.compNameStr
          blackList = blackListMap[name]

          for vibid in comp.vibids:
             if not (vibid in self.vibIDs or _InBlackList(vibid, blackList)):
                return False
       except:
          pass

       return True

   def GetComponentInfo(self, componentName):
      """ For personality manager. Return the following info:
             version, vendor, summary, and description
          for the specified component (bulletin).
          Relies on bulletins attribute populated with bulletin objects
          read from the database.
      """
      for comp in self.components.IterComponents():
         if comp.compNameStr == componentName:
            if self._IsFullyInstalled(comp):
               version = _GetComponentVersion(comp)
               return {'version'     : version,
                       'vendor'      : comp.vendor,
                       'summary'     : comp.summary,
                       'description' : comp.description}
            break
      return None

   def ListComponentSummaries(self):
      """ For personality manager. Return a list of summary with
             name, vendor, and version.
          Relies on self.components already populated with database/depot.
      """
      resultList = []
      for comp in self.components.IterComponents():
         if self._IsFullyInstalled(comp) and comp.componentnamespec:
            version = _GetComponentVersion(comp)
            resultList.append(
               {'component' : comp.compNameStr,
                'version' : version,
                'vendor'  : comp.vendor,
                'display_version': comp.compVersionUiStr,
                'display_name': comp.compNameUiStr,
                'release_type': comp.releasetype})
      return resultList

   def ToSoftwareSpec(self):
      """ For personality manager. Converts this Image Profile into
            a Software Specification.
      """
      components = self.ListComponentSummaries()
      softwareSpec = {'esx': None, 'components': {}}

      for comp in components:
         version = comp['version']
         name = comp['component']
         if name == 'ESXi':
            softwareSpec['esx'] = version
         else:
            softwareSpec['components'][name] = version

      return softwareSpec

   def GetHardwareSupportInfo(self):
      """Get hardware support package information, returns a tuple of:
         1) A dict of hardware support manager names that map to HSP component
            names and then actual installed versions.
         2) Name-version dict of all components that are listed as part of
            the HSPs, i.e. expected component name and versions.
         3) A set of all HSP remove component names.
      """
      hspInfo = dict()
      hspComps = dict()
      hspRmCompNames = set()

      if not self.manifests:
         return hspInfo, hspComps, hspRmCompNames

      hostComps = self.components
      for manifest in self.manifests.values():
         hspCompInfo = dict()
         for compName in manifest.components.keys():
            if hostComps.HasComponent(compName):
               hspCompInfo[compName] = \
                  hostComps.GetComponent(compName).compVersionStr
         hsi = manifest.hardwareSupportInfo
         hspInfo[hsi.manager.name] = hspCompInfo
         hspComps.update(manifest.components)
         hspRmCompNames.update(set(manifest.removedComponents))

      return hspInfo, hspComps, hspRmCompNames

   def GetSolutionInfo(self):
      """Returns a tuple of:
         1) A dict indexed by solution release IDs, each value is a list
            of components that belong to the solution.
         2) Name-version dict of all components that map to solutions in the
            image profile.
      """
      solCompInfo = dict()
      allSolComps = dict()

      if not self.solutions:
         return solCompInfo, allSolComps

      hostComps = self.components
      for solution in self.solutions.values():
         solCompDict = dict()
         solComps = list()
         matchedComps = solution.MatchComponents(hostComps)
         if matchedComps:
            for comps in matchedComps.values():
               assert len(comps) == 1
               solCompDict[comps[0].compNameStr] = comps[0].compVersionStr
               solComps.append(comps[0])
         solCompInfo[solution.releaseID] = solComps
         allSolComps.update(solCompDict)

      return solCompInfo, allSolComps

   def GetComponentSourceInfo(self):
      """Returns component name and version indexed by their sources:
         base image, addon, hardware support, solution or user.
      """
      from .ImageManager.Constants import (SOURCE_BASEIMAGE, SOURCE_ADDON,
                                           SOURCE_HSP, SOURCE_SOLUTION,
                                           SOURCE_USER)
      compDict = {c.compNameStr: c.compVersionStr
                  for c in self.bulletins.values()}
      biComps = self.baseimage.components if self.baseimage else dict()
      if self.addon:
         addonComps = self.addon.components
         addonRmCompNames = set(self.addon.removedComponents)
      else:
         addonComps, addonRmCompNames = dict(), set()
      _, hspComps, hspRmCompNames = self.GetHardwareSupportInfo()
      _, solComps = self.GetSolutionInfo()
      # Rules for determining the source of a component must be in the reversed
      # order of effective component calculation in SoftwareSpecMgr to sync with
      # its behavior. Current rules:
      # 1) Start with user component
      # 2) A solution has it -> solution.
      # 3) An HSP has it -> hardware support
      # 4) The addon has it and is not removed by HSP -> addon
      # 5) The base image has it and is not removed by addon -> base image
      # In theory a component can appear multiple times in different sources,
      # the first source in the end will take precedent.
      sources = dict()
      for n, v in compDict.items():
         sources[n] = SOURCE_USER
         if solComps.get(n, None) == v:
            sources[n] = SOURCE_SOLUTION
         if hspComps.get(n, None) == v:
            sources[n] = SOURCE_HSP
         if addonComps.get(n, None) == v and not n in hspRmCompNames:
            sources[n] = SOURCE_ADDON
         if biComps.get(n, None) == v and not n in addonRmCompNames:
            sources[n] = SOURCE_BASEIMAGE
      return sources

   def GetCompsDowngradeInfo(self, newProfile):
      """Get information on component downgrades from this image profile to the
         spcified one.
         Returns a dictionary containing all downgrades indexed by names; each
         downgrade is a tuple of source version, destination version,
         source, destination and a flag indicating if both versions of
         components have a config schema.
      """
      info = dict()
      curSource = self.GetComponentSourceInfo()
      newSource = newProfile.GetComponentSourceInfo()
      for newComp in newProfile.components.IterComponents():
         name = newComp.compNameStr
         if not self.components.HasComponent(name):
            continue
         curComp = self.components.GetComponent(name)
         if newComp.compVersion >= curComp.compVersion:
            continue
         info[name] = (curComp.compVersionStr,
                       newComp.compVersionStr,
                       curSource[name],
                       newSource[name],
                       curComp.hasConfigSchema and newComp.hasConfigSchema)
      return info

   def GetOrphanVibsDowngradeInfo(self, newProfile):
      """Get information on orphan VIB downgrades from this image profile to the
         specified one.
         Returns a dictionary containing a map from VIB names to tuples that
         each has the source version, destination version, component name of the
         VIB in the given image profile (if present), and a flag indicating if
         both versions of VIBs have a config schema.
      """
      info = dict()
      orphanVibs = self.GetOrphanVibs()

      if not orphanVibs:
         return info

      # Map from VIB name to object
      curVibMap = {v.name: v for v in orphanVibs.values()}
      curVibNames = set(curVibMap.keys())
      newVibMap = dict()
      newVibIds = set()
      for vib in newProfile.vibs.values():
         if vib.name in curVibNames:
            newVibMap[vib.name] = vib
            newVibIds.add(vib.id)

      # New VIB ID to associated component name
      vibIdToCompName = {v: c.compNameStr
                         for c in newProfile.components.IterComponents()
                         for v in c.vibids if v in newVibIds}

      for name, curVib in curVibMap.items():
         newVib = newVibMap.get(name, None)
         if newVib:
            if newVib.version < curVib.version:
               info[name] = (curVib.versionstr, newVib.versionstr,
                             vibIdToCompName.get(newVib.id, None),
                             curVib.hasConfigSchema and newVib.hasConfigSchema)
      return info

   def GetReleaseUnitComps(self):
      """Get pairs of release unit object and its installed components in a
         component collection.
      """
      relUnits = []
      if self.baseimage:
         relUnits.append(self.baseimage)
      if self.addon:
         relUnits.append(self.addon)
      relUnits.extend(self.manifests.values())

      pairs = []
      for relUnit in relUnits:
         comps = Bulletin.ComponentCollection()
         for name, ver in relUnit.components.items():
            # This does not guarantee a component is placed only once with
            # one release unit when multiple have the same version.
            # Mapping from component to source is provided by
            # GetComponentSourceInfo().
            try:
               comp = self.components.GetComponent(name, ver)
               comps.AddComponent(comp)
            except KeyError:
               pass
         pairs.append((relUnit, comps))

      for sol in self.solutions.values():
         comps = Bulletin.ComponentCollection()
         for name, compList in sol.MatchComponents(self.components).items():
            for comp in compList:
               comps.AddComponent(comp)
         pairs.append((sol, comps))

      return pairs


   def GetAllReleaseUnits(self):
      """ Collect all release units in this image profile.

          Return a set of (type, ID).
      """
      releaseUnits = set()

      if self.baseimageID:
         releaseUnits.add(('baseimage', self.baseimageID))

      if self.addonID:
         releaseUnits.add(('addon', self.addonID))

      if self.solutionIDs:
         for solID in self.solutionIDs:
            releaseUnits.add(('solution', solID))

      if self.manifestIDs:
         for mID in self.manifestIDs:
            if not mID.startswith(Manifest.FIRMWARE_ONLY_PREFIX):
               releaseUnits.add(('manifest', mID))
      return releaseUnits


class Problem(object):
   """A base class representing a ImageProfile validation problem."""
   def __eq__(self, other):
      return str(self) == str(other)

   def __ne__(self, other):
      return str(self) != str(other)

   def __hash__(self):
      return str(self).__hash__()

class DependencyProblem(Problem):
   """A base class representing a dependency (depends, replaces, or conflicts)
      that could not be satisfied.
   """
   priority = 3

class MissingDependency(DependencyProblem):
   """A class representing a depends that could not be satisfied within the VIBs
      in the ImageProfile. Additional VIBs from depot will be listed if they
      can satisfy the depends."""
   def __init__(self, vibid, constraint, providedby):
      self.vibid = vibid
      self.constraint = constraint
      self.providedby = providedby

   def __str__(self):
      msg = "VIB %s requires %s, but the requirement cannot be satisfied " \
            "within the ImageProfile." % (self.vibid, self.constraint)
      if self.providedby:
         msg = "%s However, additional VIB(s) %s from depot can satisfy " \
               "this requirement." % (msg, ', '.join(self.providedby))
      return msg

class ConflictingVIB(DependencyProblem):
   """A class representing a conflict among VIBs within the ImageProfile."""
   def __init__(self, vibids):
      self.vibids = vibids

   def __str__(self):
      return "VIBs (%s) are conflicting with each other" % (
            ', '.join(sorted(self.vibids)))

class ObsoletedVIB(DependencyProblem):
   """A class representing a dependency that could not be satisfied due to
      being obsoleted by other VIBs."""
   def __init__(self, vibid, newervibs):
      self.vibid = vibid
      self.newervibs = newervibs

   def __str__(self):
      return "VIB %s is obsoleted by %s" % (self.vibid, self.newervibs)

class ComponentDependencyProblem(DependencyProblem):
   """A catch-all wrapper class for a component dependency problem, including
      unmet dependency, conflict, obsolete and self-obsolete/conflict.
      This class channels the ComponentScanProblem object into the existing
      validation process.
   """
   def __init__(self, problem):
      self.problem = problem

   def __str__(self):
      return self.problem.msg

class AcceptanceProblem(Problem):
   """A base class representing acceptance problem"""
   priority = 2

class UnAcceptedVIB(AcceptanceProblem):
   """A class representing the acceptance level of a VIB is not compliant with
      the acceptance level of the ImageProfile."""
   def __init__(self, vibid, vibacceptlevel, profileacceptlevel):
      self.vibid = vibid
      self.vibacceptlevel = vibacceptlevel
      self.profileacceptlevel = profileacceptlevel

   def __str__(self):
      return "VIB %s's acceptance level is %s, which is not compliant with " \
             "the ImageProfile acceptance level %s" % (self.vibid,
                   self.vibacceptlevel, self.profileacceptlevel)

class VibValidationProblem(Problem):
   """A base class representing vib validation problem"""
   priority = 1

class ExtensibilityRuleViolation(VibValidationProblem):
   """A class representing a VIB breaking extensibility rule checks"""
   def __init__(self, vibid, errors):
      self.vibid = vibid
      self.errors = errors

   def __str__(self):
      return "VIB %s violates extensibility rule checks: %s" \
         % (self.vibid, self.errors)

class ProfileValidationProblem(Problem):
   """A base class representing profile validation problem"""
   priority = 0

class FileConflict(ProfileValidationProblem):
   """A class representing file conflicting within non-overlay or overlay group.
   """
   def __init__(self, vibids, filepath, groupname='non-overlay'):
      self.vibids = vibids
      self.filepath = filepath
      self.groupname = groupname

   def __str__(self):
      return "File path of '%s' is claimed by multiple %s VIBs: %s" % (
            self.filepath, self.groupname, self.vibids)

class ProfileTooShort(ProfileValidationProblem):
   """No VIB in the ImageProfile provides boot kernel"""
   def __init__(self, profilename, vibids):
      self.profilename = profilename
      self.vibids = vibids

   def __str__(self):
      return 'At least one boot kernel is required, but none is found for ' \
             'ImageProfile %s with VIBs: %s' % (self.profilename, self.vibids)

class MissingBaseEsx(ProfileValidationProblem):
   """No VIB in the ImageProfile provides esx-version"""
   def __init__(self, profilename, msg):
      self.profilename = profilename
      self.msg = msg

   def __str__(self):
      return 'Base ESX package not found in profile %s: %s' % (self.profilename,
            self.msg)

class HwPlatformConflict(ProfileValidationProblem):
   """Hardware platform conflict in the image profile"""
   def __init__(self, profilename, msg):
      self.profilename = profilename
      self.msg = msg

   def __str__(self):
      return 'ImageProfile %s contains conflicting hardware platforms: %s' \
             % (self.profilename, self.msg)

class MismatchBaseImage(ProfileValidationProblem):
   """The base image has a different version than esx-version.
   """
   def __init__(self, profileName, baseImageVer, esxBaseVer):
      self.profileName = profileName
      self.baseImageVer = baseImageVer
      self.esxBaseVer = esxBaseVer

   def __str__(self):
      return ('Base Image version %s does not match the base ESXi package '
              'version %s in ImageProfile %s. Use a depot instead of VIB URLs '
              'to perform ESXi patching.'
              % (self.baseImageVer, self.esxBaseVer, self.profileName))

class ImageProfileCollection(dict):
   """ImageProfileCollection holds a collection of Image Profiles,
      keyed by the profileID.
   """
   def __init__(self):
      pass

   def __iadd__(self, other):
      """Merges this collection with another collection.
         Parameters:
            * other - An instance of ImageProfileCollection.
      """
      for p in other.values():
         self.AddProfile(p)
      return self

   def __add__(self, other):
      """Merges this collection with another collection. The resulting
         collection will be a new object referencing the union of profiles
         from self and other.  If an image profile from self has the same
         profileID key as a profile from other, but the contents are
         different, the merge will be stopped.

         Parameters:
            * other - An instance of ImageProfileCollection.
         Returns:
            A new ImageProfileCollection instance.
         Raises:
            MergeError -
      """
      new = ImageProfileCollection()
      new.update(self)
      for p in other.values():
         new.AddProfile(p)
      return new

   def AddProfile(self, profile):
      """Add an image profile object to this collection. The exact image profile
         object is not guaranteed to be added. If the same image profile
         (same profileID) is already in the collection, the profile is
         updated using the ImageProfile.__add__ method.

         Parameters:
            * profile - A ImageProfile object to add.
         Returns: The added or updated image profile object in the collection
      """
      #
      # Something to thing about:
      #  If an image profile in the collection has its readonly attribute
      #  set to True, should we disallow updating it?
      #  Will there be some situation, such as published profiles, which may
      #  need updating?
      #
      profid = profile.profileID
      if profid in self and id(self[profid]) != id(profile):
         self[profid] += profile
      else:
         self[profid] = profile

      return self[profid]

   def AddProfileFromXml(self, xml, validate = True,
                         schema = ImageProfile.PROFILE_SCHEMA):
      """Add an image profile to this collection. The image profile object is
         created from xml.

         Parameters:
            * xml - A string or an ElementTree instance containing image profile
                    XML.
            * validate - If True, XML will be validated against a schema.
            * schema   - A file path of an image profile schema.
         Exceptions:
            ProfileFormatError -
            ProfileValidationError -
      """
      return self.AddProfile(ImageProfile.FromXml(xml, validate, schema))

   def FindProfiles(self, name=None, creator=None,
                    acceptance=None, glob=False):
      """Finds all of the image profiles in this collection that
         match by certain criteria.  If multiple criteria are specified,
         they will be used together for filtering (AND).
         Parameters:
            * name     - If defined, match only profiles with that name.
                         If None, do not restrict match by name
            * creator  - If defined, match only profiles with given creator
            * acceptance - If defined, match only profiles with the given
                         acceptance level.  Should be one of Vib.ArFileVib.
                         ACCEPTANCE_LEVELS.* values.
            * glob     - If True, allow '*' wildcard globbing.
         Returns:
            A new ImageProfileCollection.
      """
      def matchfunc(profile, attr, matchval):
         if matchval:
            return getattr(profile, attr, None) == matchval
         else:
            return True

      def matchfuncglob(profile, attr, matchval):
         if matchval:
            return fnmatch.fnmatchcase(getattr(profile, attr, ''), matchval)
         else:
            return True

      if glob:
         matchf = matchfuncglob
      else:
         matchf = matchfunc

      profiles = self.__class__()
      for profile in self.values():
         if matchf(profile, 'name', name) and        \
            matchf(profile, 'creator', creator) and  \
            matchf(profile, 'acceptancelevel', acceptance):
            profiles.AddProfile(profile)

      return profiles

   def FromDirectory(self, path, ignoreinvalidfiles = False,
                     validate = False, schema = ImageProfile.PROFILE_SCHEMA):
      """Populate this ImageProfileCollection instance from a directory of
         image profile XML files.  This method may replace existing image
         profiles in the collection.  Subdirectories will be searched.

         Parameters:
            * path               - A string specifying a directory name.
            * ignoreinvalidfiles - If True, causes the method to silently
                                   ignore ProfileFormatError exceptions. Useful
                                   if a directory may contain both profile
                                   content and other content.
            * validate - If True, XML will be validated against a schema. If
                         False, no validation will be done. Defaults to
                         True.
            * schema   - A file path giving the location of an image profile
                         schema.
         Returns:
            None
         Raises:
            * ProfileFormatError - One or more files could not be parsed as
                                   image profile XML files.
            * ProfileIOError     - Path not found, or error reading from file
            * ProfileValidationError - if XML schema validation of profiles
                                   fails
      """
      if not os.path.exists(path):
         msg = 'ImageProfileCollection directory %s does not exist.' % (path)
         raise Errors.ProfileIOError(msg)
      elif not os.path.isdir(path):
         msg = 'ImageProfileCollection path %s is not a directory.' % (path)
         raise Errors.ProfileIOError(msg)

      for root, dirs, files in os.walk(path, topdown=True):
         for name in files:
            filepath = os.path.join(root, name)
            try:
               fp = open(filepath, 'r')
            except Exception as e:
               raise Errors.ProfileIOError("Could not open %s: %s" %
                                           (filepath, e))

            try:
               xml = etree.parse(fp).getroot()
               fp.close()
            except EnvironmentError as e:
               fp.close()
               raise Errors.ProfileIOError("Could not read from %s: %s" % (
                                           filepath, str(e)))
            except Exception as e:
               fp.close()
               if ignoreinvalidfiles:
                  continue
               else:
                  raise Errors.ProfileFormatError("Error parsing image profile "
                     "XML at %s: %s" % (filepath, str(e)))

            try:
               self.AddProfile(ImageProfile.FromXml(xml, validate, schema))
            except (Errors.ProfileFormatError,
                    Errors.ProfileValidationError) as e:
               if not ignoreinvalidfiles:
                  raise

   def ToDirectory(self, path, namingfunc=None, toDB=False):
      """Serialises all imgae profiles in this collection to XML files in
         a directory. If the directory exists, the content of the directory
         will be clobbered.
            Parameters:
               * path       - A string specifying a directory name.
               * namingfunc - A function pointer, return a short string
                              with an Image Profile object as the only input
                              and the string
                              will be used as the file name of the descriptor.
            Return: None
            Exceptions:
               * ProfileIOError - The specified directory is not a directory or
                                  cannot create an empty directory
      """
      try:
         if os.path.isdir(path):
             shutil.rmtree(path)
         os.makedirs(path)
      except EnvironmentError as e:
         msg = 'Could not create dir %s: %s' % (path, e)
         raise Errors.ProfileIOError(msg)

      if not os.path.isdir(path):
         msg = 'Failed to write image profiles, %s is not a directory' % (path)
         raise Errors.ProfileIOError(msg)

      if namingfunc is None:
         namingfunc = self.FilenameForProfile

      for profile in self.values():
         name = namingfunc(profile)
         filepath = os.path.join(path, name)
         try:
            pnode = profile.ToXml(toDB=toDB)
            with open(filepath, 'wb') as f:
               f.write(etree.tostring(pnode))
         except EnvironmentError as e:
            msg = 'Failed to write image profile to %s: %s' % (filepath, e)
            raise Errors.ProfileIOError(msg)

   @classmethod
   def FilenameForProfile(self, profile):
      """Returns a unique filename for writing out image profile XML files
         based on the profile name and creator.
      """
      return quote(profile.name, '')[:50] + \
         str(hash((profile.name, profile.creator)))