modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-09 08:13:21 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #10 builder.py: Add Builder, GenericBuilder, KojiModuleBuilder classes

Browse Source
This commit is contained in:
Petr Šabata
2016-07-08 12:32:34 +00:00
parent 2b62d67387 74703cce82
commit cb68402c48
5 changed files with 394 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@@ -3,3 +3,5 @@ sqlalchemy
fedmsg
modulemd
pyOpenSSL
kobo
koji

3
rida.conf
View File

@@ -1,7 +1,8 @@
[DEFAULT]
system = koji
messaging = fedmsg
koji = http://koji.stg.fedoraproject.org/kojihub
koji_config = /etc/rida/koji.conf
koji_profile = stage
db = sqlite:///rida.db
pdc = http://pdc.stg.fedoraproject.org/
scmurls = ["git://pkgs.stg.fedoraproject.org/modules/"]

337
rida/builder.py
View File

@@ -22,13 +22,342 @@
# SOFTWARE.
#
# Written by Petr Šabata <contyk@redhat.com>
# Luboš Kocman <lkocman@redhat.com>
"""Generic component build functions."""
# TODO: Create the relevant koji tags and targets.
# TODO: Query the PDC to find what modules satisfy the build dependencies and
# their tag names.
# TODO: Set tag inheritance for the created tag, using the build dependencies'
# tags.
# TODO: Ensure the RPM %dist tag is set according to the policy.
# TODO: Build the module components in the prepared tag.
import koji
from abc import ABCMeta, abstractmethod
from kobo.shortcuts import run
# TODO: read defaults from rida's config
KOJI_DEFAULT_GROUPS = {
'build': [
'bash',
'bzip2',
'coreutils',
'cpio',
'diffutils',
'fedora-release',
'findutils',
'gawk',
'gcc',
'gcc-c++',
'grep',
'gzip',
'info',
'make',
'patch',
'redhat-rpm-config',
'rpm-build',
'sed',
'shadow-utils',
'tar',
'unzip',
'util-linux',
'which',
'xz',
],
'srpm-build': [
'bash',
'fedora-release',
'fedpkg-minimal',
'gnupg2',
'redhat-rpm-config',
'rpm-build',
'shadow-utils',
]
}
class GenericBuilder:
"""External Api for builders"""
__metaclass__ = ABCMeta
backend = "generic"
@abstractmethod
def buildroot_prep(self):
"""
preps buildroot
"""
raise NotImplementedError()
@abstractmethod
def buildroot_resume(self):
"""
resumes buildroot (alternative to prep)
"""
raise NotImplementedError()
@abstractmethod
def buildroot_add_dependency(self, dependencies):
"""
:param dependencies: a list off modules which we build-depend on
adds dependencies 'another module(s)' into buildroot
"""
raise NotImplementedError()
@abstractmethod
def buildroot_add_artifacts(self, artifacts):
"""
:param artifacts: list of artifacts to be available in buildroot
add artifacts into buildroot, can be used to override buildroot macros
"""
raise NotImplementedError()
@abstractmethod
def buildroot_ready(self, artifact=None):
"""
:param artifact=None: wait for specific artifact to be present
waits for buildroot to be ready and contain given artifact
"""
raise NotImplementedError()
@abstractmethod
def build(self, artifact_name, source):
"""
:param artifact_name : name of what are we building (used for whitelists)
:param source : a scmurl to repository with receipt (e.g. spec)
"""
raise NotImplementedError()
class Builder:
"""Wrapper class"""
def __new__(cls, module, backend, config):
"""
:param module : a module string e.g. 'testmodule-1.0'
:param backend: a string representing backend e.g. 'koji'
:param config: instance of rida.config.Config
"""
if backend == "koji":
return KojiModuleBuild(module=module, config=config)
else:
raise ValueError("Builder backend='%s' not recognized" % backend)
class KojiModuleBuild(GenericBuilder):
""" Koji specific builder class """
backend = "koji"
def __init__(self, module, config):
"""
:param koji_profile: koji profile to be used
"""
self.module_str = module
self.__prep = False
self._koji_profile_name = config.koji_profile
self.koji_module = koji.get_profile_module(self._koji_profile_name)
opts = {}
krbservice = getattr(self.koji_module.config, "krbservice", None)
if krbservice:
opts["krbservice"] = krbservice
self.koji_session = koji.ClientSession(self.koji_module.config.server, opts=opts)
if self.koji_module.config.authtype == "kerberos":
keytab = getattr(self.koji_module.config, "keytab", None)
principal = getattr(self.koji_module.config, "principal", None)
if keytab and principal:
self.koji_session.krb_login(principal=principal, keytab=keytab, proxyuser=None)
else:
self.koji_session.krb_login()
elif self.koji_module.config.authtype == "ssl":
self.koji_session.ssl_login(self.koji_module.config.cert, None, self.koji_module.serverca, proxyuser=None)
self.arches = config.koji_arches
self.module_tag = self._get_module_tag_name()
self.module_build_tag = self._get_module_build_tag_name()
self.module_target = self._get_module_target_name()
def buildroot_resume(self): # XXX: experimental
"""
Resume existing buildroot. Sets __prep=True
"""
chktag = self.koji_session.getTag(self._get_module_tag_name())
if not chktag:
raise SystemError("Tag %s doesn't exist" % self._get_module_tag_name())
chkbuildtag = self.koji_session.getTag(self._get_module_build_tag_name())
if not chkbuildtag:
raise SystemError("Build Tag %s doesn't exist" % self._get_module_build_tag_name())
chktarget = self.koji_session.getBuildTarget(self._get_module_target_name())
if not chktarget:
raise SystemError("Target %s doesn't exist" % self._get_module_target_name())
self.module_tag = chktag
self.module_build_tag = chkbuildtag
self.module_target = chktarget
self.__prep = True
def buildroot_prep(self):
"""
:param module_deps_tags: a tag names of our build requires
:param module_deps_tags: a tag names of our build requires
"""
self.module_tag = self._koji_create_tag(self._get_module_tag_name(), perm="admin") # returns tag obj
self.module_build_tag = self._koji_create_tag(self._get_module_build_tag_name(), self.arches, perm="admin")
groups = KOJI_DEFAULT_GROUPS # TODO: read from config
if groups:
self._koji_add_groups_to_tag(self.module_build_tag, groups)
self.module_target = self._koji_add_target(self._get_module_target_name(), self.module_build_tag, self.module_tag)
self.__prep = True
def buildroot_add_dependency(self, dependencies):
tags = [self._get_tag(d)['name'] for d in dependencies]
self._koji_add_many_tag_inheritance(self.module_build_tag, tags)
def buildroot_add_artifacts(self, artifacts):
# TODO: import /usr/bin/koji's TaskWatcher()
for nvr in artifacts:
self.koji_session.tagBuild(self.module_build_tag, nvr, force=True)
def buildroot_ready(self, artifact=None):
# XXX: steal code from /usr/bin/koji
cmd = "koji -p %s wait-repo %s " % (self._koji_profile_name, self.module_build_tag['name'])
if artifact:
cmd += " --build %s" % artifact
print ("Waiting for buildroot(%s) to be ready" % (self.module_build_tag['name']))
run(cmd) # wait till repo is current
def build(self, artifact_name, source):
"""
:param source : scmurl to spec repository
:return koji taskid
"""
if not self.__prep:
raise RuntimeError("Buildroot is not prep-ed")
if '://' not in source:
raise NotImplementedError("Only scm url is currently supported, got source='%s'" % source)
self._koji_whitelist_packages([artifact_name,])
task_id = self.koji_session.build(source, self.module_target['name'])
print("Building %s (taskid=%s)." % (source, task_id))
return task_id
def _get_tag(self, tag, strict=True):
if isinstance(tag, dict):
tag = tag['name']
taginfo = self.koji_session.getTag(tag)
if not taginfo:
if strict:
raise SystemError("Unknown tag: %s" % tag)
return taginfo
def _koji_add_many_tag_inheritance(self, tag_name, parent_tags):
tag = self._get_tag(tag_name)
inheritanceData = []
priority = 0
for parent in parent_tags: # We expect that they're sorted
parent = self._get_tag(parent)
parent_data = {}
parent_data['parent_id'] = parent['id']
parent_data['priority'] = priority
parent_data['maxdepth'] = None
parent_data['intransitive'] = False
parent_data['noconfig'] = False
parent_data['pkg_filter'] = ''
inheritanceData.append(parent_data)
priority += 10
self.koji_session.setInheritanceData(tag['id'], inheritanceData)
def _koji_add_groups_to_tag(self, dest_tag, groups=None):
"""
:param build_tag_name
:param groups: A dict {'group' : [package, ...]}
"""
if groups and not isinstance(groups, dict):
raise ValueError("Expected dict {'group' : [str(package1), ...]")
dest_tag = self._get_tag(dest_tag)['name']
groups = dict([(p['name'], p['group_id']) for p in self.koji_session.getTagGroups(dest_tag, inherit=False)])
for group, packages in groups.iteritems():
group_id = groups.get(group, None)
if group_id is not None:
print "Group %s already exists for tag %s" % (group, dest_tag)
return 1
self.koji_session.groupListAdd(dest_tag, group)
for pkg in packages:
self.koji_session.groupPackageListAdd(dest_tag, group, pkg)
def _koji_create_tag(self, tag_name, arches=None, fail_if_exists=True, perm=None):
chktag = self.koji_session.getTag(tag_name)
if chktag and fail_if_exists:
raise SystemError("Tag %s already exist" % tag_name)
elif chktag:
return self._get_tag(self.module_tag)
else:
opts = {}
if arches:
if not isinstance(arches, list):
raise ValueError("Expected list or None on input got %s" % type(arches))
opts['arches'] = " ".join(arches)
self.koji_session.createTag(tag_name, **opts)
if perm:
self._lock_tag(tag_name, perm)
return self._get_tag(tag_name)
def _get_module_target_name(self):
return self.module_str
def _get_module_tag_name(self):
return self.module_str
def _get_module_build_tag_name(self):
return "%s-build" % self._get_module_tag_name()
def _get_component_owner(self, package):
user = self.koji_session.getLoggedInUser()['name']
return user
def _koji_whitelist_packages(self, packages):
# This will help with potential resubmiting of failed builds
pkglist = dict([(p['package_name'], p['package_id']) for p in self.koji_session.listPackages(tagID=self.module_tag['id'])])
to_add = []
for package in packages:
package_id = pkglist.get(package, None)
if not package_id is None:
print ("Package %s already exists in tag %s" % (package, self.module_tag['name']))
continue
to_add.append(package)
for package in to_add:
owner = self._get_component_owner(package)
if not self.koji_session.getUser(owner):
raise ValueError("Unknown user %s" % owner)
self.koji_session.packageListAdd(self.module_tag['name'], package, owner)
def _koji_add_target(self, name, build_tag, dest_tag):
build_tag = self._get_tag(build_tag)
dest_tag = self._get_tag(dest_tag)
barches = build_tag.get("arches", None)
assert barches, "Build tag %s has no arches defined" % build_tag['name']
self.koji_session.createBuildTarget(name, build_tag['name'], dest_tag['name'])
return self.koji_session.getBuildTarget(name)
def _lock_tag(self, tag, perm="admin"):
taginfo = self._get_tag(tag)
if taginfo['locked']:
raise SystemError("Tag %s: master lock already set" % taginfo['name'])
perm_ids = dict([(p['name'], p['id']) for p in self.koji_session.getAllPerms()])
if perm not in perm_ids.keys():
raise ValueError("Unknown permissions %s" % perm)
perm_id = perm_ids[perm]
self.koji_session.editTag2(taginfo['id'], perm=perm_id)

36
rida/config.py
View File

@@ -51,7 +51,8 @@ def from_file(filename=None):
conf.system = default.get("system")
conf.messaging = default.get("messaging")
conf.pdc = default.get("pdc")
conf.koji = default.get("koji")
conf.koji_config = default.get("koji_config")
conf.koji_profile = default.get("koji_profile")
conf.scmurls = json.loads(default.get("scmurls"))
conf.rpms_default_repository = default.get("rpms_default_repository")
conf.rpms_allow_repository = default.getboolean("rpms_allow_repository")
@@ -78,7 +79,9 @@ class Config(object):
self._messaging = ""
self._db = ""
self._pdc = ""
self._koji = ""
self._koji_config = None
self._koji_profile = None
self._koji_arches = None
self._rpms_default_repository = ""
self._rpms_allow_repository = False
self._rpms_default_cache = ""
@@ -134,13 +137,32 @@ class Config(object):
self._pdc = str(s)
@property
def koji(self):
def koji_config(self):
"""Koji URL."""
return self._koji
return self._koji_config
@koji.setter
def koji(self, s):
self._koji = str(s)
@koji_config.setter
def koji_config(self, s):
self._koji_config = str(s)
@property
def koji_profile(self):
"""Koji URL."""
return self._koji_profile
@koji_profile.setter
def koji_profile(self, s):
self._koji_profile = str(s)
@property
def koji_arches(self):
"""Koji architectures."""
return self._koji_arches
@koji_arches.setter
def koji_arches(self, s):
self._koji_arches = list(s)
@property
def scmurls(self):

28
test-buildroot.py Normal file
View File

@@ -0,0 +1,28 @@
#!/usr/bin/env python3
""" A little script to test buildroot creation. """
from rida.builder import KojiModuleBuild, Builder
from rida.config import Config
cfg = Config()
cfg.koji_profile = "koji"
cfg.koji_config = "/etc/rida/koji.conf"
cfg.koji_arches = ["x86_64", "i686"]
#mb = KojiModuleBuild(module="testmodule-1.0", config=cfg) # or By using Builder
mb = Builder(module="testmodule-1.0", backend="koji", config=cfg)
resume = True
if not resume:
mb.buildroot_prep()
mb.buildroot_add_dependency(["f24"])
mb.buildroot_ready()
task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")
mb.buildroot_add_artifacts(["fedora-release-24-2", ]) # just example with disttag macro
mb.buildroot_ready(artifact="fedora-release-24-2")
else:
mb.buildroot_resume()
task_id = mb.build(artifact_name="fedora-release", source="git://pkgs.fedoraproject.org/rpms/fedora-release?#b1d65f349dca2f597b278a4aad9e41fb0aa96fc9")