From dda47acab7c8ea1b670aa09c93af9f8591220a40 Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Tue, 10 Jan 2017 14:29:54 +0100 Subject: [PATCH] Fix #276 - Use PDC to resolve buildroot components instead of MBS database --- module_build_service/builder.py | 19 ++++++ module_build_service/pdc.py | 67 +++++++++++++++++++ .../scheduler/handlers/components.py | 14 +--- .../scheduler/handlers/modules.py | 26 ++----- .../scheduler/handlers/repos.py | 13 +--- tests/__init__.py | 24 +++++++ tests/test_build/test_build.py | 13 +++- 7 files changed, 130 insertions(+), 46 deletions(-) diff --git a/module_build_service/builder.py b/module_build_service/builder.py index e1179c0f..2023b220 100644 --- a/module_build_service/builder.py +++ b/module_build_service/builder.py @@ -52,6 +52,7 @@ from OpenSSL.SSL import SysCallError from module_build_service import conf, log, db from module_build_service.models import ModuleBuild +from module_build_service import pdc import module_build_service.scm import module_build_service.utils import module_build_service.scheduler @@ -277,6 +278,24 @@ class GenericBuilder(six.with_metaclass(ABCMeta)): """ raise NotImplementedError() + @classmethod + def default_buildroot_groups(cls, session, module): + try: + pdc_session = pdc.get_pdc_client_session(conf) + pdc_groups = pdc.resolve_profiles(pdc_session, module.mmd(), + ('buildroot', 'srpm-buildroot')) + groups = { + 'build': pdc_groups['buildroot'], + 'srpm-build': pdc_groups['srpm-buildroot'], + } + except ValueError: + reason = "Failed to gather buildroot groups from SCM." + log.exception(reason) + module.transition(conf, state="failed", state_reason=reason) + session.commit() + raise + return groups + class KojiModuleBuilder(GenericBuilder): """ Koji specific builder class """ diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index a95fb088..6b4f0d5d 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -183,6 +183,73 @@ def get_module_tag(session, module_info, strict=False): """ return get_module(session, module_info, strict=strict)['koji_tag'] +def get_module_modulemd(session, module_info, strict=False): + """ + :param session : PDCClient instance + :param module_info: list of module_info dicts + :param strict: Normally this function returns None if no module can be + found. If strict=True, then a ValueError is raised. + :return: ModuleMetadata instance + """ + yaml = get_module(session, module_info, strict=strict)['modulemd'] + if not yaml: + if strict: + raise ValueError("Failed to find modulemd entry in PDC for " + "%r" % module_info) + else: + return None + + mmd = modulemd.ModuleMetadata() + mmd.loads(yaml) + return mmd + +def resolve_profiles(session, mmd, keys, seen=None): + """ + :param session : PDCClient instance + :param mmd: ModuleMetadata instance of module + :param keys: list of modulemd installation profiles to include in + the result. + :return: Dictionary with keys set according to `keys` param and values + set to union of all components defined in all installation + profiles matching the key recursively using the buildrequires. + + https://pagure.io/fm-orchestrator/issue/181 + """ + + seen = seen or [] # Initialize to an empty list. + results = {} + for key in keys: + results[key] = set() + for name, stream in mmd.buildrequires.items(): + # First, guard against infinite recursion + if name in seen: + continue + + # Find the latest of the dep in our db of built modules. + module_info = {} + module_info['variant_id'] = name + module_info['variant_stream'] = stream + dep_mmd = get_module_modulemd(session, module_info, False) + # XXX - We may want to make this fatal one day, but warn for now. + if not dep_mmd: + log.warn("Could not find built dep " + "%s/%s for %r" % (name, stream, mmd.name)) + continue + + # Take note of what rpms are in this dep's profile. + profiles = dep_mmd.profiles + for key in keys: + if key in profiles: + results[key] |= profiles[key].rpms + + # And recurse to all modules that are deps of our dep. + rec_results = resolve_profiles(session, dep_mmd, keys, seen + [name]) + for rec_key, rec_result in rec_results.items(): + results[rec_key] |= rec_result + + # Return the union of all rpms in all profiles of the given keys. + return results + def module_depsolving_wrapper(session, module_list, strict=True): """ :param session : PDCClient instance diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index 7ebab34b..c039b47b 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -77,18 +77,8 @@ def _finalize(config, session, msg, state): builder = module_build_service.builder.GenericBuilder.create( parent.owner, module_name, config.system, config, tag_name=tag) - try: - groups = { - 'build': parent.resolve_profiles(session, 'buildroot'), - 'srpm-build': parent.resolve_profiles(session, 'srpm-buildroot'), - } - except ValueError: - reason = "Failed to gather buildroot groups from SCM." - log.exception(reason) - parent.transition(config, state=models.BUILD_STATES["failed"], - state_reason=reason) - session.commit() - raise + groups = module_build_service.builder.GenericBuilder.default_buildroot_groups( + session, parent) builder.buildroot_connect(groups) diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 240d16c3..d393c8c8 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -68,17 +68,8 @@ def failed(config, session, msg): and c.state != koji.BUILD_STATES["FAILED"]) ] - try: - groups = { - 'build': build.resolve_profiles(session, 'buildroot'), - 'srpm-build': build.resolve_profiles(session, 'srpm-buildroot'), - } - except ValueError: - reason = "Failed to gather buildroot groups from SCM." - log.exception(reason) - build.transition(config, state="failed", state_reason=reason) - session.commit() - raise + groups = module_build_service.builder.GenericBuilder.default_buildroot_groups( + session, build) if build.koji_tag: builder = module_build_service.builder.GenericBuilder.create( @@ -201,17 +192,8 @@ def wait(config, session, msg): session.commit() raise - try: - groups = { - 'build': build.resolve_profiles(session, 'buildroot'), - 'srpm-build': build.resolve_profiles(session, 'srpm-buildroot'), - } - except ValueError: - reason = "Failed to gather buildroot groups from SCM." - log.exception(reason) - build.transition(config, state="failed", state_reason=reason) - session.commit() - raise + groups = module_build_service.builder.GenericBuilder.default_buildroot_groups( + session, build) log.debug("Found tag=%s for module %r" % (tag, build)) # Hang on to this information for later. We need to know which build is diff --git a/module_build_service/scheduler/handlers/repos.py b/module_build_service/scheduler/handlers/repos.py index 5010d086..cf4aa54e 100644 --- a/module_build_service/scheduler/handlers/repos.py +++ b/module_build_service/scheduler/handlers/repos.py @@ -79,17 +79,8 @@ def done(config, session, msg): log.warn("Odd! All components in batch failed for %r." % module_build) return - try: - groups = { - 'build': module_build.resolve_profiles(session, 'buildroot'), - 'srpm-build': module_build.resolve_profiles(session, 'srpm-buildroot'), - } - except ValueError: - reason = "Failed to gather buildroot groups from SCM." - log.exception(reason) - module_build.transition(config, state="failed", state_reason=reason) - session.commit() - raise + groups = module_build_service.builder.GenericBuilder.default_buildroot_groups( + session, module_build) builder = module_build_service.builder.GenericBuilder.create( module_build.owner, module_build.name, config.system, config, diff --git a/tests/__init__.py b/tests/__init__.py index ecc16a69..12c869b2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,10 +28,34 @@ from module_build_service import db from module_build_service.config import init_config from module_build_service.models import ModuleBuild, ComponentBuild from module_build_service.utils import insert_fake_baseruntime +import module_build_service.pdc app = module_build_service.app conf = init_config(app) +def local_resolve_profiles(pdc_session, mmd, keys): + """ + Returns the default srpm-buildroot and "buildroot" groups as they + would have been return from PDC. + + We are not using real PDC for resolve_profiles in tests, + because it makes tests slower and depending on the remote dev + PDC instance which is not ideal. + """ + default_groups = { + 'srpm-buildroot': + set(['shadow-utils', 'fedora-release', 'redhat-rpm-config', + 'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']), + 'buildroot': + set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk', + 'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash', + 'info', 'bzip2', 'grep', 'redhat-rpm-config', + 'diffutils', 'make', 'patch', 'shadow-utils', + 'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])} + + return default_groups + +module_build_service.pdc.resolve_profiles = local_resolve_profiles def init_data(): db.session.remove() diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index da9ec056..ce3488f9 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -96,7 +96,18 @@ class TestModuleBuilder(GenericBuilder): TestModuleBuilder.on_tag_artifacts_cb = None def buildroot_connect(self, groups): - pass + default_groups = { + 'srpm-build': + set(['shadow-utils', 'fedora-release', 'redhat-rpm-config', + 'rpm-build', 'fedpkg-minimal', 'gnupg2', 'bash']), + 'build': + set(['unzip', 'fedora-release', 'tar', 'cpio', 'gawk', + 'gcc', 'xz', 'sed', 'findutils', 'util-linux', 'bash', + 'info', 'bzip2', 'grep', 'redhat-rpm-config', + 'diffutils', 'make', 'patch', 'shadow-utils', + 'coreutils', 'which', 'rpm-build', 'gzip', 'gcc-c++'])} + if groups != default_groups: + raise ValueError("Wrong groups in TestModuleBuilder.buildroot_connect()") def buildroot_prep(self): pass