From a0e33c164984b289ff2f67dec27bae534a767fa0 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 8 Jul 2016 07:58:15 +0000 Subject: [PATCH 1/3] Add Builder, GenericBuilder, KojiBuilder classes * change config to reflect koji_config path and koji_profile Signed-off-by: Lubos Kocman --- rida.conf | 3 +- rida/builder.py | 337 +++++++++++++++++++++++++++++++++++++++++++++++- rida/config.py | 36 +++++- 3 files changed, 364 insertions(+), 12 deletions(-) diff --git a/rida.conf b/rida.conf index b5ba4c23..0bfbd085 100644 --- a/rida.conf +++ b/rida.conf @@ -1,7 +1,8 @@ [DEFAULT] system = koji messaging = fedmsg -koji = http://koji.stg.fedoraproject.org/kojihub +koji_config = /etc/rida/koji.conf +koji_profile = stage db = sqlite:///rida.db pdc = http://pdc.stg.fedoraproject.org/ scmurls = ["git://pkgs.stg.fedoraproject.org/modules/"] diff --git a/rida/builder.py b/rida/builder.py index d58e0f98..70c6438d 100644 --- a/rida/builder.py +++ b/rida/builder.py @@ -22,13 +22,342 @@ # SOFTWARE. # # Written by Petr Šabata +# Luboš Kocman """Generic component build functions.""" -# TODO: Create the relevant koji tags and targets. # TODO: Query the PDC to find what modules satisfy the build dependencies and # their tag names. -# TODO: Set tag inheritance for the created tag, using the build dependencies' -# tags. # TODO: Ensure the RPM %dist tag is set according to the policy. -# TODO: Build the module components in the prepared tag. + +import koji +from abc import ABCMeta, abstractmethod +from kobo.shortcuts import run + +# TODO: read defaults from rida's config +KOJI_DEFAULT_GROUPS = { + 'build': [ + 'bash', + 'bzip2', + 'coreutils', + 'cpio', + 'diffutils', + 'fedora-release', + 'findutils', + 'gawk', + 'gcc', + 'gcc-c++', + 'grep', + 'gzip', + 'info', + 'make', + 'patch', + 'redhat-rpm-config', + 'rpm-build', + 'sed', + 'shadow-utils', + 'tar', + 'unzip', + 'util-linux', + 'which', + 'xz', + ], + 'srpm-build': [ + 'bash', + 'fedora-release', + 'fedpkg-minimal', + 'gnupg2', + 'redhat-rpm-config', + 'rpm-build', + 'shadow-utils', + ] +} + +class GenericBuilder: + """External Api for builders""" + __metaclass__ = ABCMeta + + backend = "generic" + + @abstractmethod + def buildroot_prep(self): + """ + preps buildroot + """ + raise NotImplementedError() + + @abstractmethod + def buildroot_resume(self): + """ + resumes buildroot (alternative to prep) + """ + raise NotImplementedError() + + @abstractmethod + def buildroot_add_dependency(self, dependencies): + """ + :param dependencies: a list off modules which we build-depend on + adds dependencies 'another module(s)' into buildroot + """ + raise NotImplementedError() + + @abstractmethod + def buildroot_add_artifacts(self, artifacts): + """ + :param artifacts: list of artifacts to be available in buildroot + add artifacts into buildroot, can be used to override buildroot macros + """ + raise NotImplementedError() + + @abstractmethod + def buildroot_ready(self, artifact=None): + """ + :param artifact=None: wait for specific artifact to be present + waits for buildroot to be ready and contain given artifact + """ + raise NotImplementedError() + + @abstractmethod + def build(self, artifact_name, source): + """ + :param artifact_name : name of what are we building (used for whitelists) + :param source : a scmurl to repository with receipt (e.g. spec) + """ + raise NotImplementedError() + +class Builder: + """Wrapper class""" + + def __new__(cls, module, backend, config): + """ + :param module : a module string e.g. 'testmodule-1.0' + :param backend: a string representing backend e.g. 'koji' + :param config: instance of rida.config.Config + """ + + if backend == "koji": + return KojiModuleBuild(module=module, config=config) + else: + raise ValueError("Builder backend='%s' not recognized" % backend) + +class KojiModuleBuild(GenericBuilder): + """ Koji specific builder class """ + + backend = "koji" + + def __init__(self, module, config): + """ + :param koji_profile: koji profile to be used + """ + self.module_str = module + self.__prep = False + self._koji_profile_name = config.koji_profile + self.koji_module = koji.get_profile_module(self._koji_profile_name) + opts = {} + + krbservice = getattr(self.koji_module.config, "krbservice", None) + if krbservice: + opts["krbservice"] = krbservice + + self.koji_session = koji.ClientSession(self.koji_module.config.server, opts=opts) + + if self.koji_module.config.authtype == "kerberos": + keytab = getattr(self.koji_module.config, "keytab", None) + principal = getattr(self.koji_module.config, "principal", None) + if keytab and principal: + self.koji_session.krb_login(principal=principal, keytab=keytab, proxyuser=None) + else: + self.koji_session.krb_login() + + elif self.koji_module.config.authtype == "ssl": + self.koji_session.ssl_login(self.koji_module.config.cert, None, self.koji_module.serverca, proxyuser=None) + + self.arches = config.koji_arches + + self.module_tag = self._get_module_tag_name() + self.module_build_tag = self._get_module_build_tag_name() + self.module_target = self._get_module_target_name() + + def buildroot_resume(self): # XXX: experimental + """ + Resume existing buildroot. Sets __prep=True + """ + chktag = self.koji_session.getTag(self._get_module_tag_name()) + if not chktag: + raise SystemError("Tag %s doesn't exist" % self._get_module_tag_name()) + chkbuildtag = self.koji_session.getTag(self._get_module_build_tag_name()) + if not chkbuildtag: + raise SystemError("Build Tag %s doesn't exist" % self._get_module_build_tag_name()) + chktarget = self.koji_session.getBuildTarget(self._get_module_target_name()) + if not chktarget: + raise SystemError("Target %s doesn't exist" % self._get_module_target_name()) + self.module_tag = chktag + self.module_build_tag = chkbuildtag + self.module_target = chktarget + self.__prep = True + + def buildroot_prep(self): + """ + :param module_deps_tags: a tag names of our build requires + :param module_deps_tags: a tag names of our build requires + """ + self.module_tag = self._koji_create_tag(self._get_module_tag_name(), perm="admin") # returns tag obj + self.module_build_tag = self._koji_create_tag(self._get_module_build_tag_name(), self.arches, perm="admin") + + groups = KOJI_DEFAULT_GROUPS # TODO: read from config + if groups: + self._koji_add_groups_to_tag(self.module_build_tag, groups) + + self.module_target = self._koji_add_target(self._get_module_target_name(), self.module_build_tag, self.module_tag) + self.__prep = True + + def buildroot_add_dependency(self, dependencies): + tags = [self._get_tag(d)['name'] for d in dependencies] + self._koji_add_many_tag_inheritance(self.module_build_tag, tags) + + def buildroot_add_artifacts(self, artifacts): + # TODO: import /usr/bin/koji's TaskWatcher() + for nvr in artifacts: + self.koji_session.tagBuild(self.module_build_tag, nvr, force=True) + + def buildroot_ready(self, artifact=None): + # XXX: steal code from /usr/bin/koji + cmd = "koji -p %s wait-repo %s " % (self._koji_profile_name, self.module_build_tag['name']) + if artifact: + cmd += " --build %s" % artifact + print ("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name'])) + run(cmd) # wait till repo is current + + def build(self, artifact_name, source): + """ + :param source : scmurl to spec repository + :return koji taskid + """ + if not self.__prep: + raise RuntimeError("Buildroot is not prep-ed") + + if '://' not in source: + raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source) + self._koji_whitelist_packages([artifact_name,]) + task_id = self.koji_session.build(source, self.module_target['name']) + print("Building %s (taskid=%s)." % (source, task_id)) + return task_id + + def _get_tag(self, tag, strict=True): + if isinstance(tag, dict): + tag = tag['name'] + taginfo = self.koji_session.getTag(tag) + if not taginfo: + if strict: + raise SystemError("Unknown tag: %s" % tag) + return taginfo + + def _koji_add_many_tag_inheritance(self, tag_name, parent_tags): + tag = self._get_tag(tag_name) + + inheritanceData = [] + priority = 0 + for parent in parent_tags: # We expect that they're sorted + parent = self._get_tag(parent) + parent_data = {} + parent_data['parent_id'] = parent['id'] + parent_data['priority'] = priority + parent_data['maxdepth'] = None + parent_data['intransitive'] = False + parent_data['noconfig'] = False + parent_data['pkg_filter'] = '' + inheritanceData.append(parent_data) + priority += 10 + + self.koji_session.setInheritanceData(tag['id'], inheritanceData) + + def _koji_add_groups_to_tag(self, dest_tag, groups=None): + """ + :param build_tag_name + :param groups: A dict {'group' : [package, ...]} + """ + + if groups and not isinstance(groups, dict): + raise ValueError("Expected dict {'group' : [str(package1), ...]") + + dest_tag = self._get_tag(dest_tag)['name'] + groups = dict([(p['name'], p['group_id']) for p in self.koji_session.getTagGroups(dest_tag, inherit=False)]) + for group, packages in groups.iteritems(): + group_id = groups.get(group, None) + if group_id is not None: + print "Group %s already exists for tag %s" % (group, dest_tag) + return 1 + self.koji_session.groupListAdd(dest_tag, group) + for pkg in packages: + self.koji_session.groupPackageListAdd(dest_tag, group, pkg) + + + def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None): + chktag = self.koji_session.getTag(tag_name) + if chktag and fail_if_exists: + raise SystemError("Tag %s already exist" % tag_name) + + elif chktag: + return self._get_tag(self.module_tag) + + else: + opts = {} + if arches: + if not isinstance(arches, list): + raise ValueError("Expected list or None on input got %s" % type(arches)) + opts['arches'] = " ".join(arches) + + self.koji_session.createTag(tag_name, **opts) + if perm: + self._lock_tag(tag_name, perm) + return self._get_tag(tag_name) + + def _get_module_target_name(self): + return self.module_str + + def _get_module_tag_name(self): + return self.module_str + + def _get_module_build_tag_name(self): + return "%s-build" % self._get_module_tag_name() + + def _get_component_owner(self, package): + user = self.koji_session.getLoggedInUser()['name'] + return user + + def _koji_whitelist_packages(self, packages): + # This will help with potential resubmiting of failed builds + pkglist = dict([(p['package_name'], p['package_id']) for p in self.koji_session.listPackages(tagID=self.module_tag['id'])]) + to_add = [] + for package in packages: + package_id = pkglist.get(package, None) + if not package_id is None: + print ("Package %s already exists in tag %s" % (package, self.module_tag['name'])) + continue + to_add.append(package) + + for package in to_add: + owner = self._get_component_owner(package) + if not self.koji_session.getUser(owner): + raise ValueError("Unknown user %s" % owner) + + self.koji_session.packageListAdd(self.module_tag['name'], package, owner) + + def _koji_add_target(self, name, build_tag, dest_tag): + build_tag = self._get_tag(build_tag) + dest_tag = self._get_tag(dest_tag) + + barches = build_tag.get("arches", None) + assert barches, "Build tag %s has no arches defined" % build_tag['name'] + self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name']) + return self.koji_session.getBuildTarget(name) + + def _lock_tag(self, tag, perm="admin"): + taginfo = self._get_tag(tag) + if taginfo['locked']: + raise SystemError("Tag %s: master lock already set" % taginfo['name']) + perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()]) + if perm not in perm_ids.keys(): + raise ValueError("Unknown permissions %s" % perm) + perm_id = perm_ids[perm] + self.koji_session.editTag2(taginfo['id'], perm=perm_id) diff --git a/rida/config.py b/rida/config.py index 9dbd4554..6e227682 100644 --- a/rida/config.py +++ b/rida/config.py @@ -51,7 +51,8 @@ def from_file(filename=None): conf.system = default.get("system") conf.messaging = default.get("messaging") conf.pdc = default.get("pdc") - conf.koji = default.get("koji") + conf.koji_config = default.get("koji_config") + conf.koji_profile = default.get("koji_profile") conf.scmurls = json.loads(default.get("scmurls")) conf.rpms_default_repository = default.get("rpms_default_repository") conf.rpms_allow_repository = default.getboolean("rpms_allow_repository") @@ -77,7 +78,9 @@ class Config(object): self._messaging = "" self._db = "" self._pdc = "" - self._koji = "" + self._koji_config = None + self._koji_profile = None + self._koji_arches = None self._rpms_default_repository = "" self._rpms_allow_repository = False self._rpms_default_cache = "" @@ -132,13 +135,32 @@ class Config(object): self._pdc = str(s) @property - def koji(self): + def koji_config(self): """Koji URL.""" - return self._koji + return self._koji_config - @koji.setter - def koji(self, s): - self._koji = str(s) + @koji_config.setter + def koji_config(self, s): + self._koji_config = str(s) + + + @property + def koji_profile(self): + """Koji URL.""" + return self._koji_profile + + @koji_profile.setter + def koji_profile(self, s): + self._koji_profile = str(s) + + @property + def koji_arches(self): + """Koji architectures.""" + return self._koji_arches + + @koji_arches.setter + def koji_arches(self, s): + self._koji_arches = list(s) @property def scmurls(self): From f39be75bd82c2492c1c0bd29d67af895e55cdc9e Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 8 Jul 2016 07:59:35 +0000 Subject: [PATCH 2/3] Add example test-buildroot.py Signed-off-by: Lubos Kocman --- test-buildroot.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 test-buildroot.py diff --git a/test-buildroot.py b/test-buildroot.py new file mode 100644 index 00000000..c007b6e3 --- /dev/null +++ b/test-buildroot.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +""" A little script to test buildroot creation. """ + +from rida.builder import KojiModuleBuild, Builder +from rida.config import Config + + +cfg = Config() +cfg.koji_profile = "koji" +cfg.koji_config = "/etc/rida/koji.conf" +cfg.koji_arches = ["x86_64", "i686"] + +#mb = KojiModuleBuild(module="testmodule-1.0", config=cfg) # or By using Builder +mb = Builder(module="testmodule-1.0", backend="koji", config=cfg) + +resume = True + +if not resume: + mb.buildroot_prep() + mb.buildroot_add_dependency(["f24"]) + mb.buildroot_ready() + task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9") + mb.buildroot_add_artifacts(["fedora-release-24-2", ]) # just example with disttag macro + mb.buildroot_ready(artifact="fedora-release-24-2") +else: + mb.buildroot_resume() + +task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9") From 74703cce82cebd72b50407d9b2f02228eac5fb22 Mon Sep 17 00:00:00 2001 From: Lubos Kocman Date: Fri, 8 Jul 2016 10:18:19 +0200 Subject: [PATCH 3/3] Add koji and kobo requirements Signed-off-by: Lubos Kocman --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 113da4a9..a29127f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ sqlalchemy fedmsg modulemd pyOpenSSL +kobo +koji