From 8b807a9fcd33166eb096cc8a96201decf14b18aa Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Mon, 23 Apr 2018 09:41:53 +0800 Subject: [PATCH] Generate koji tag from MBS and use informative name koji now supports tags with max length of 256, we can use more informative tag name instead of the hash one. The new format of koji tag name is: module---- However when the generated tag's length > (256 - len('build')), we fallback to the old way of name in hash format (module-). In this change, koji tag is always generated from MBS itself, even with pdc resolver. FIXES: #918 #925 --- module_build_service/resolver/DBResolver.py | 17 --------------- module_build_service/resolver/PDCResolver.py | 13 ------------ module_build_service/resolver/base.py | 4 ---- .../scheduler/handlers/modules.py | 6 +++--- module_build_service/utils/general.py | 21 +++++++++++++++++++ tests/__init__.py | 8 +++---- tests/test_scheduler/test_module_wait.py | 1 + tests/test_scheduler/test_repo_done.py | 8 +++---- tests/test_utils/test_utils.py | 16 ++++++++++++++ 9 files changed, 49 insertions(+), 45 deletions(-) diff --git a/module_build_service/resolver/DBResolver.py b/module_build_service/resolver/DBResolver.py index 65fd7f67..f6c2b020 100644 --- a/module_build_service/resolver/DBResolver.py +++ b/module_build_service/resolver/DBResolver.py @@ -21,8 +21,6 @@ # # Written by Matt Prahl -import hashlib - from module_build_service import log from module_build_service.resolver.base import GenericResolver from module_build_service import models @@ -67,21 +65,6 @@ class DBResolver(GenericResolver): "Cannot find any module builds for %s:%s" % (name, stream)) return [build.mmd() for build in builds] - def get_module_tag(self, name, stream, version, context, strict=False): - """ - Gets the module tag from the resolver. Since the resolver is the DB, it is just generated - here. - :param name: a string of the module's name - :param stream: a string of the module's stream - :param version: a string or int of the module's version - :param context: a string of the module's context - :kwarg strict: Here solely for compatibility with the base class' function signature - :return: a string of the tag to use - """ - # This algorithm mimicks what pdc-updater does - tag_str = '.'.join([name, stream, str(version), context]) - return 'module-{0}'.format(hashlib.sha1(tag_str).hexdigest()[:16]) - def resolve_profiles(self, mmd, keys): """ Returns a dictionary with keys set according the `keys` parameters and values diff --git a/module_build_service/resolver/PDCResolver.py b/module_build_service/resolver/PDCResolver.py index df52fc1a..e7fbe327 100644 --- a/module_build_service/resolver/PDCResolver.py +++ b/module_build_service/resolver/PDCResolver.py @@ -142,19 +142,6 @@ class PDCResolver(GenericResolver): def _get_module(self, name, stream, version=None, context=None, active=None, strict=False): return self._get_modules(name, stream, version, context, active, strict)[0] - def get_module_tag(self, name, stream, version=None, context=None, strict=False): - """ - :param name: a module's name - :param stream: a module's stream - :param version: a module's version - :param context: a module's context - :param strict: Normally this function returns None if no module can be - found. If strict=True, then an UnprocessableEntity is raised. - :return: koji tag string - """ - return self._get_module( - name, stream, version, context, active=True, strict=strict)['koji_tag'] - def get_module_modulemds(self, name, stream, version=None, context=None, strict=False): """ Gets the module modulemds from the resolver. diff --git a/module_build_service/resolver/base.py b/module_build_service/resolver/base.py index c39ca4e4..c73368e6 100644 --- a/module_build_service/resolver/base.py +++ b/module_build_service/resolver/base.py @@ -98,10 +98,6 @@ class GenericResolver(six.with_metaclass(ABCMeta)): def get_module_modulemds(self, name, stream, version=None, context=None, strict=False): raise NotImplementedError() - @abstractmethod - def get_module_tag(self, name, stream, version, context, strict=False): - raise NotImplementedError() - @abstractmethod def resolve_profiles(self, mmd, keys): raise NotImplementedError() diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 5fb5f229..896b302d 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -31,7 +31,8 @@ import module_build_service.messaging from module_build_service.utils import ( attempt_to_reuse_all_components, record_component_builds, - get_rpm_release) + get_rpm_release, + generate_koji_tag) from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError from module_build_service.builder.KojiContentGenerator import KojiContentGenerator @@ -244,8 +245,7 @@ def wait(config, session, msg): break log.info('Getting tag for {0}'.format(nsvc)) - tag = resolver.get_module_tag( - build.name, build.stream, build.version, build.context, strict=True) + tag = generate_koji_tag(build.name, build.stream, build.version, build.context) return dependencies, tag, cg_build_koji_tag diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 5e06c514..4cf60339 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -93,6 +93,27 @@ def module_build_state_from_msg(msg): return state +def generate_koji_tag(name, stream, version, context): + """ + Generate a koji tag from name, stream, version and context. + + :param name: a module's name + :param stream: a module's stream + :param version: a module's version + :param context: a module's context + :return: a Koji tag + """ + nsvc_list = [name, stream, str(version), context] + nsvc_tag = 'module-' + '-'.join(nsvc_list) + if len(nsvc_tag) + len('-build') > 256: + # Koji supports tag names with a max length of 256, fallback + # to the old way of 'module-' if the generated koji tag + # name is too long + nsvc_hash = hashlib.sha1('.'.join(nsvc_list)).hexdigest()[:16] + return 'module-' + nsvc_hash + return nsvc_tag + + def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'): """ Used as a decorator validates koji tag arg(s)' value(s) diff --git a/tests/__init__.py b/tests/__init__.py index 7e8dc2f2..b781b7d7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -298,7 +298,7 @@ def scheduler_init_data(tangerine_state=None): build_one.state = BUILD_STATES['build'] build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0' build_one.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0' - build_one.koji_tag = 'module-95b214a704c984be' + build_one.koji_tag = 'module-testmodule-master-20170109091357-7c29193d' build_one.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79' if tangerine_state: build_one.batch = 3 @@ -403,7 +403,7 @@ def reuse_component_init_data(): build_one.ref_build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0' build_one.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0' build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb1' - build_one.koji_tag = 'module-de3adf79caf3e1b8' + build_one.koji_tag = 'module-testmodule-master-20170109091357-78e4a6fd' build_one.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79' build_one.batch = 3 build_one.owner = 'Tom Brady' @@ -578,7 +578,7 @@ def reuse_shared_userspace_init_data(): build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a' build_one.state = BUILD_STATES['ready'] build_one.modulemd = mmd.dumps() - build_one.koji_tag = 'module-testmodule-master-20170109091357' + build_one.koji_tag = 'module-shared-userspace-f26-20170601141014-75f92abb' build_one.scmurl = ('git://pkgs.stg.fedoraproject.org/modules/testmodule.' 'git?#7fea453') build_one.batch = 16 @@ -629,7 +629,7 @@ def reuse_shared_userspace_init_data(): build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a' build_one.state = BUILD_STATES['done'] build_one.modulemd = mmd2.dumps() - build_one.koji_tag = 'module-testmodule-master-20170109091357' + build_one.koji_tag = 'module-shared-userspace-f26-20170605091544-75f92abb' build_one.scmurl = ('git://pkgs.stg.fedoraproject.org/modules/testmodule.' 'git?#7fea453') build_one.batch = 0 diff --git a/tests/test_scheduler/test_module_wait.py b/tests/test_scheduler/test_module_wait.py index b5ceb0e3..e8ba1055 100644 --- a/tests/test_scheduler/test_module_wait.py +++ b/tests/test_scheduler/test_module_wait.py @@ -59,6 +59,7 @@ class TestModuleWait: mocked_module_build.name = 'foo' mocked_module_build.stream = 'stream' mocked_module_build.version = '1' + mocked_module_build.context = '1234567' mocked_module_build.state = 1 mocked_module_build.id = 1 mocked_module_build.json.return_value = { diff --git a/tests/test_scheduler/test_repo_done.py b/tests/test_scheduler/test_repo_done.py index 1f5239b9..83eb8878 100644 --- a/tests/test_scheduler/test_repo_done.py +++ b/tests/test_scheduler/test_repo_done.py @@ -67,7 +67,7 @@ class TestRepoDone: build_fn.return_value = 1234, 1, '', None msg = module_build_service.messaging.KojiRepoChange( - 'some_msg_id', 'module-95b214a704c984be-build') + 'some_msg_id', 'module-testmodule-master-20170109091357-7c29193d-build') module_build_service.scheduler.handlers.repos.done( config=conf, session=db.session, msg=msg) build_fn.assert_called_once_with( @@ -101,7 +101,7 @@ class TestRepoDone: build_fn.return_value = None, 4, 'Failed to submit artifact tangerine to Koji', None msg = module_build_service.messaging.KojiRepoChange( - 'some_msg_id', 'module-95b214a704c984be-build') + 'some_msg_id', 'module-testmodule-master-20170109091357-7c29193d-build') module_build_service.scheduler.handlers.repos.done( config=conf, session=db.session, msg=msg) build_fn.assert_called_once_with( @@ -119,7 +119,7 @@ class TestRepoDone: """ scheduler_init_data(1) msg = module_build_service.messaging.KojiRepoChange( - 'some_msg_id', 'module-95b214a704c984be-build') + 'some_msg_id', 'module-testmodule-master-20170109091357-7c29193d-build') component_build = module_build_service.models.ComponentBuild.query\ .filter_by(package='tangerine').one() component_build.tagged = False @@ -156,7 +156,7 @@ class TestRepoDone: build_fn.return_value = None, 4, 'Failed to submit artifact x to Koji', None msg = module_build_service.messaging.KojiRepoChange( - 'some_msg_id', 'module-95b214a704c984be-build') + 'some_msg_id', 'module-testmodule-master-20170109091357-7c29193d-build') module_build_service.scheduler.handlers.repos.done( config=conf, session=db.session, msg=msg) module_build = module_build_service.models.ModuleBuild.query.get(2) diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 60a458c4..6f2efee1 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -535,6 +535,22 @@ class TestUtils: for c in module_build.component_builds: assert c.weight == 1.5 + def test_generate_koji_tag_in_nsvc_format(self): + name, stream, version, context = ('testmodule', 'master', '20170816080815', '37c6c57') + + tag = module_build_service.utils.generate_koji_tag(name, stream, version, context) + + assert tag == 'module-testmodule-master-20170816080815-37c6c57' + + def test_generate_koji_tag_in_hash_format(self): + name, version, context = ('testmodule', '20170816080815', '37c6c57') + stream = 'this-is-a-stream-with-very-looooong-name' + '-blah' * 50 + nsvc_list = [name, stream, version, context] + + tag = module_build_service.utils.generate_koji_tag(*nsvc_list) + expected_tag = 'module-1cf457d452e54dda' + assert tag == expected_tag + class DummyModuleBuilder(GenericBuilder): """