# -*- coding: utf-8 -*- # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # # Written by Petr Šabata # Luboš Kocman """Generic component build functions.""" # TODO: Query the PDC to find what modules satisfy the build dependencies and # their tag names. # TODO: Ensure the RPM %dist tag is set according to the policy. import koji from abc import ABCMeta, abstractmethod from kobo.shortcuts import run # TODO: read defaults from rida's config KOJI_DEFAULT_GROUPS = { 'build': [ 'bash', 'bzip2', 'coreutils', 'cpio', 'diffutils', 'fedora-release', 'findutils', 'gawk', 'gcc', 'gcc-c++', 'grep', 'gzip', 'info', 'make', 'patch', 'redhat-rpm-config', 'rpm-build', 'sed', 'shadow-utils', 'tar', 'unzip', 'util-linux', 'which', 'xz', ], 'srpm-build': [ 'bash', 'fedora-release', 'fedpkg-minimal', 'gnupg2', 'redhat-rpm-config', 'rpm-build', 'shadow-utils', ] } class GenericBuilder: """External Api for builders""" __metaclass__ = ABCMeta backend = "generic" @abstractmethod def buildroot_prep(self): """ preps buildroot """ raise NotImplementedError() @abstractmethod def buildroot_resume(self): """ resumes buildroot (alternative to prep) """ raise NotImplementedError() @abstractmethod def buildroot_add_dependency(self, dependencies): """ :param dependencies: a list off modules which we build-depend on adds dependencies 'another module(s)' into buildroot """ raise NotImplementedError() @abstractmethod def buildroot_add_artifacts(self, artifacts): """ :param artifacts: list of artifacts to be available in buildroot add artifacts into buildroot, can be used to override buildroot macros """ raise NotImplementedError() @abstractmethod def buildroot_ready(self, artifact=None): """ :param artifact=None: wait for specific artifact to be present waits for buildroot to be ready and contain given artifact """ raise NotImplementedError() @abstractmethod def build(self, artifact_name, source): """ :param artifact_name : name of what are we building (used for whitelists) :param source : a scmurl to repository with receipt (e.g. spec) """ raise NotImplementedError() class Builder: """Wrapper class""" def __new__(cls, module, backend, config): """ :param module : a module string e.g. 'testmodule-1.0' :param backend: a string representing backend e.g. 'koji' :param config: instance of rida.config.Config """ if backend == "koji": return KojiModuleBuilder(module=module, config=config) else: raise ValueError("Builder backend='%s' not recognized" % backend) class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ backend = "koji" def __init__(self, module, config): """ :param koji_profile: koji profile to be used """ self.module_str = module self.__prep = False self._koji_profile_name = config.koji_profile self.koji_module = koji.get_profile_module(self._koji_profile_name) opts = {} krbservice = getattr(self.koji_module.config, "krbservice", None) if krbservice: opts["krbservice"] = krbservice self.koji_session = koji.ClientSession(self.koji_module.config.server, opts=opts) if self.koji_module.config.authtype == "kerberos": keytab = getattr(self.koji_module.config, "keytab", None) principal = getattr(self.koji_module.config, "principal", None) if keytab and principal: self.koji_session.krb_login(principal=principal, keytab=keytab, proxyuser=None) else: self.koji_session.krb_login() elif self.koji_module.config.authtype == "ssl": self.koji_session.ssl_login(self.koji_module.config.cert, None, self.koji_module.serverca, proxyuser=None) self.arches = config.koji_arches self.module_tag = self._get_module_tag_name() self.module_build_tag = self._get_module_build_tag_name() self.module_target = self._get_module_target_name() def buildroot_resume(self): # XXX: experimental """ Resume existing buildroot. Sets __prep=True """ chktag = self.koji_session.getTag(self._get_module_tag_name()) if not chktag: raise SystemError("Tag %s doesn't exist" % self._get_module_tag_name()) chkbuildtag = self.koji_session.getTag(self._get_module_build_tag_name()) if not chkbuildtag: raise SystemError("Build Tag %s doesn't exist" % self._get_module_build_tag_name()) chktarget = self.koji_session.getBuildTarget(self._get_module_target_name()) if not chktarget: raise SystemError("Target %s doesn't exist" % self._get_module_target_name()) self.module_tag = chktag self.module_build_tag = chkbuildtag self.module_target = chktarget self.__prep = True def buildroot_prep(self): """ :param module_deps_tags: a tag names of our build requires :param module_deps_tags: a tag names of our build requires """ self.module_tag = self._koji_create_tag(self._get_module_tag_name(), perm="admin") # returns tag obj self.module_build_tag = self._koji_create_tag(self._get_module_build_tag_name(), self.arches, perm="admin") groups = KOJI_DEFAULT_GROUPS # TODO: read from config if groups: self._koji_add_groups_to_tag(self.module_build_tag, groups) self.module_target = self._koji_add_target(self._get_module_target_name(), self.module_build_tag, self.module_tag) self.__prep = True def buildroot_add_dependency(self, dependencies): tags = [self._get_tag(d)['name'] for d in dependencies] self._koji_add_many_tag_inheritance(self.module_build_tag, tags) def buildroot_add_artifacts(self, artifacts): # TODO: import /usr/bin/koji's TaskWatcher() for nvr in artifacts: self.koji_session.tagBuild(self.module_build_tag, nvr, force=True) def buildroot_ready(self, artifact=None): # XXX: steal code from /usr/bin/koji cmd = "koji -p %s wait-repo %s " % (self._koji_profile_name, self.module_build_tag['name']) if artifact: cmd += " --build %s" % artifact print ("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name'])) run(cmd) # wait till repo is current def build(self, artifact_name, source): """ :param source : scmurl to spec repository :return koji taskid """ if not self.__prep: raise RuntimeError("Buildroot is not prep-ed") if '://' not in source: raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) self._koji_whitelist_packages([artifact_name,]) task_id = self.koji_session.build(source, self.module_target['name']) print("Building %s (taskid=%s)." % (source, task_id)) return task_id def _get_tag(self, tag, strict=True): if isinstance(tag, dict): tag = tag['name'] taginfo = self.koji_session.getTag(tag) if not taginfo: if strict: raise SystemError("Unknown tag: %s" % tag) return taginfo def _koji_add_many_tag_inheritance(self, tag_name, parent_tags): tag = self._get_tag(tag_name) inheritanceData = [] priority = 0 for parent in parent_tags: # We expect that they're sorted parent = self._get_tag(parent) parent_data = {} parent_data['parent_id'] = parent['id'] parent_data['priority'] = priority parent_data['maxdepth'] = None parent_data['intransitive'] = False parent_data['noconfig'] = False parent_data['pkg_filter'] = '' inheritanceData.append(parent_data) priority += 10 self.koji_session.setInheritanceData(tag['id'], inheritanceData) def _koji_add_groups_to_tag(self, dest_tag, groups=None): """ :param build_tag_name :param groups: A dict {'group' : [package, ...]} """ if groups and not isinstance(groups, dict): raise ValueError("Expected dict {'group' : [str(package1), ...]") dest_tag = self._get_tag(dest_tag)['name'] groups = dict([(p['name'], p['group_id']) for p in self.koji_session.getTagGroups(dest_tag, inherit=False)]) for group, packages in groups.iteritems(): group_id = groups.get(group, None) if group_id is not None: print("Group %s already exists for tag %s" % (group, dest_tag)) return 1 self.koji_session.groupListAdd(dest_tag, group) for pkg in packages: self.koji_session.groupPackageListAdd(dest_tag, group, pkg) def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None): chktag = self.koji_session.getTag(tag_name) if chktag and fail_if_exists: raise SystemError("Tag %s already exist" % tag_name) elif chktag: return self._get_tag(self.module_tag) else: opts = {} if arches: if not isinstance(arches, list): raise ValueError("Expected list or None on input got %s" % type(arches)) opts['arches'] = " ".join(arches) self.koji_session.createTag(tag_name, **opts) if perm: self._lock_tag(tag_name, perm) return self._get_tag(tag_name) def _get_module_target_name(self): return self.module_str def _get_module_tag_name(self): return self.module_str def _get_module_build_tag_name(self): return "%s-build" % self._get_module_tag_name() def _get_component_owner(self, package): user = self.koji_session.getLoggedInUser()['name'] return user def _koji_whitelist_packages(self, packages): # This will help with potential resubmiting of failed builds pkglist = dict([(p['package_name'], p['package_id']) for p in self.koji_session.listPackages(tagID=self.module_tag['id'])]) to_add = [] for package in packages: package_id = pkglist.get(package, None) if not package_id is None: print ("Package %s already exists in tag %s" % (package, self.module_tag['name'])) continue to_add.append(package) for package in to_add: owner = self._get_component_owner(package) if not self.koji_session.getUser(owner): raise ValueError("Unknown user %s" % owner) self.koji_session.packageListAdd(self.module_tag['name'], package, owner) def _koji_add_target(self, name, build_tag, dest_tag): build_tag = self._get_tag(build_tag) dest_tag = self._get_tag(dest_tag) barches = build_tag.get("arches", None) assert barches, "Build tag %s has no arches defined" % build_tag['name'] self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name']) return self.koji_session.getBuildTarget(name) def _lock_tag(self, tag, perm="admin"): taginfo = self._get_tag(tag) if taginfo['locked']: raise SystemError("Tag %s: master lock already set" % taginfo['name']) perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) if perm not in perm_ids.keys(): raise ValueError("Unknown permissions %s" % perm) perm_id = perm_ids[perm] self.koji_session.editTag2(taginfo['id'], perm=perm_id)