mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-01 18:01:40 +08:00
Add get_modules_build_required_by_module_recursively to get the input for libsolv solver.
This commit is contained in:
Jan Kaluza
@@ -1404,6 +1404,109 @@ def get_reusable_component(session, module, component_name,
|
||||
return reusable_component
|
||||
|
||||
|
||||
def _get_mmds_from_requires(session, requires, mmds, recursive=False):
|
||||
"""
|
||||
Helper method for get_modules_build_required_by_module_recursively returning
|
||||
the list of module metadata objects defined by `requires` dict.
|
||||
|
||||
:param session: SQLAlchemy DB session.
|
||||
:param requires: Modulemetadata requires or buildrequires.
|
||||
:param mmds: Dictionary with already handled name:streams as a keys and lists
|
||||
of resulting mmds as values.
|
||||
:param recursive: If True, the requires are checked recursively.
|
||||
:return: Dict with name:stream as a key and list with mmds as value.
|
||||
"""
|
||||
# To be able to call itself recursively, we need to store list of mmds
|
||||
# we have added to global mmds list in this particular call.
|
||||
added_mmds = {}
|
||||
for name, streams in requires:
|
||||
# Stream can be prefixed with '-' sign to define that this stream should
|
||||
# not appear in a resulting list of streams. There can be two situations:
|
||||
# a) all streams have '-' prefix. In this case, we treat list of streams
|
||||
# as blacklist and we find all the valid streams and just remove those with
|
||||
# '-' prefix.
|
||||
# b) there is at least one stream without '-' prefix. In this case, we can
|
||||
# ignore all the streams with '-' prefix and just add those without
|
||||
# '-' prefix to the list of valid streams.
|
||||
streams_is_blacklist = all([stream[0] == "-" for stream in streams.get()])
|
||||
if streams_is_blacklist or len(streams.get()) == 0:
|
||||
builds = models.ModuleBuild.get_last_build_in_all_streams(
|
||||
session, name)
|
||||
valid_streams = [build.stream for build in builds]
|
||||
else:
|
||||
valid_streams = []
|
||||
for stream in streams.get():
|
||||
if stream.startswith("-"):
|
||||
if streams_is_blacklist and stream[1:] in valid_streams:
|
||||
valid_streams.remove(stream[1:])
|
||||
else:
|
||||
valid_streams.append(stream)
|
||||
|
||||
# For each valid stream, find the last build in a stream and also all
|
||||
# its contexts and add mmds of these builds to `mmds` and `added_mmds`.
|
||||
# Of course only do that if we have not done that already in some
|
||||
# previous call of this method.
|
||||
for stream in valid_streams:
|
||||
ns = "%s:%s" % (name, stream)
|
||||
if ns in mmds:
|
||||
continue
|
||||
|
||||
last_build_in_stream = models.ModuleBuild.get_last_build_in_stream(
|
||||
session, name, stream)
|
||||
builds = models.ModuleBuild.get_builds_in_version(
|
||||
session, name, stream, last_build_in_stream.version)
|
||||
mmds[ns] = [build.mmd() for build in builds]
|
||||
added_mmds[ns] = mmds[ns]
|
||||
|
||||
# Get the requires recursively.
|
||||
if recursive:
|
||||
for mmd_list in added_mmds.values():
|
||||
for mmd in mmd_list:
|
||||
for deps in mmd.get_dependencies():
|
||||
mmds = _get_mmds_from_requires(session, deps.get_requires().items(), mmds, True)
|
||||
|
||||
return mmds
|
||||
|
||||
|
||||
def get_modules_build_required_by_module_recursively(session, mmd):
|
||||
"""
|
||||
Returns the list of Module metadata objects of all modules required while
|
||||
building the module defined by `mmd` module metadata.
|
||||
|
||||
This method finds out latest versions of all the build-requires of
|
||||
the `mmd` module and then also all contexts of these latest versions.
|
||||
|
||||
For each build-required name:stream:version:context module, it checks
|
||||
recursively all the "requires" and finds the latest version of each
|
||||
required module and also all contexts of these latest versions.
|
||||
|
||||
:rtype: list of Modulemd metadata
|
||||
:return: List of all modulemd metadata of all modules required to build
|
||||
the module `mmd`.
|
||||
"""
|
||||
# We use dict with name:stream as a key and list with mmds as value.
|
||||
# That way, we can ensure we won't have any duplicate mmds in a resulting
|
||||
# list and we also don't waste resources on getting the modules we already
|
||||
# handled from DB.
|
||||
mmds = {}
|
||||
|
||||
# At first get all the buildrequires of the module of interest.
|
||||
for deps in mmd.get_dependencies():
|
||||
mmds = _get_mmds_from_requires(session, deps.get_buildrequires().items(), mmds)
|
||||
|
||||
# Now get the requires of buildrequires recursively.
|
||||
for mmd_key in list(mmds.keys()):
|
||||
for mmd in mmds[mmd_key]:
|
||||
for deps in mmd.get_dependencies():
|
||||
mmds = _get_mmds_from_requires(session, deps.get_requires().items(), mmds, True)
|
||||
|
||||
# Make single list from dict of lists.
|
||||
res = []
|
||||
for mmds_list in mmds.values():
|
||||
res += mmds_list
|
||||
return res
|
||||
|
||||
|
||||
def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'):
|
||||
"""
|
||||
Used as a decorator validates koji tag arg(s)' value(s)
|
||||
|
||||
209
tests/test_utils/test_utils_mse.py
Normal file
209
tests/test_utils/test_utils_mse.py
Normal file
@@ -0,0 +1,209 @@
|
||||
# Copyright (c) 2017 Red Hat, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import gi
|
||||
gi.require_version('Modulemd', '1.0') # noqa
|
||||
from gi.repository import Modulemd
|
||||
import module_build_service.utils
|
||||
from module_build_service import models, conf
|
||||
from tests import (db, clean_database)
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
from mock import patch
|
||||
import pytest
|
||||
|
||||
|
||||
class TestUtilsModuleStreamExpansion:
|
||||
|
||||
def setup_method(self, test_method):
|
||||
clean_database()
|
||||
|
||||
def mocked_context(modulebuild_instance):
|
||||
"""
|
||||
Changes the ModuleBuild.context behaviour to return
|
||||
ModuleBuild.build_context instead of computing new context hash.
|
||||
"""
|
||||
return modulebuild_instance.build_context
|
||||
|
||||
# For these tests, we need the ModuleBuild.context to return the well-known
|
||||
# context as we define it in test data. Therefore patch the ModuleBuild.context
|
||||
# to return ModuleBuild.build_context, which we can control.
|
||||
self.modulebuild_context_patcher = patch(
|
||||
"module_build_service.models.ModuleBuild.context", autospec=True)
|
||||
modulebuild_context = self.modulebuild_context_patcher.start()
|
||||
modulebuild_context.side_effect = mocked_context
|
||||
|
||||
def teardown_method(self, test_method):
|
||||
clean_database()
|
||||
self.modulebuild_context_patcher.stop()
|
||||
|
||||
def _make_module(self, nsvc, requires_list, build_requires_list):
|
||||
"""
|
||||
Creates new models.ModuleBuild defined by `nsvc` string with requires
|
||||
and buildrequires set according to `requires_list` and `build_requires_list`.
|
||||
|
||||
:param str nsvc: name:stream:version:context of a module.
|
||||
:param list_of_dicts requires_list: List of dictionaries defining the
|
||||
requires in the mmd requires field format.
|
||||
:param list_of_dicts build_requires_list: List of dictionaries defining the
|
||||
build_requires_list in the mmd build_requires_list field format.
|
||||
:rtype: ModuleBuild
|
||||
:return: New Module Build.
|
||||
"""
|
||||
name, stream, version, context = nsvc.split(":")
|
||||
mmd = Modulemd.Module()
|
||||
mmd.set_mdversion(2)
|
||||
mmd.set_name(name)
|
||||
mmd.set_stream(stream)
|
||||
mmd.set_version(int(version))
|
||||
mmd.set_context(context)
|
||||
mmd.set_summary("foo")
|
||||
mmd.set_description("foo")
|
||||
licenses = Modulemd.SimpleSet()
|
||||
licenses.add("GPL")
|
||||
mmd.set_module_licenses(licenses)
|
||||
|
||||
if not isinstance(requires_list, list):
|
||||
requires_list = [requires_list]
|
||||
if not isinstance(build_requires_list, list):
|
||||
build_requires_list = [build_requires_list]
|
||||
|
||||
deps_list = []
|
||||
for requires, build_requires in zip(requires_list, build_requires_list):
|
||||
deps = Modulemd.Dependencies()
|
||||
for req_name, req_streams in requires.items():
|
||||
deps.add_requires(req_name, req_streams)
|
||||
for req_name, req_streams in build_requires.items():
|
||||
deps.add_buildrequires(req_name, req_streams)
|
||||
deps_list.append(deps)
|
||||
mmd.set_dependencies(deps_list)
|
||||
|
||||
module_build = module_build_service.models.ModuleBuild()
|
||||
module_build.name = name
|
||||
module_build.stream = stream
|
||||
module_build.version = version
|
||||
module_build.state = models.BUILD_STATES['ready']
|
||||
module_build.scmurl = 'git://pkgs.stg.fedoraproject.org/modules/unused.git?#ff1ea79'
|
||||
module_build.batch = 1
|
||||
module_build.owner = 'Tom Brady'
|
||||
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
|
||||
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
|
||||
module_build.rebuild_strategy = 'changed-and-after'
|
||||
module_build.build_context = context
|
||||
module_build.runtime_context = context
|
||||
module_build.modulemd = mmd.dumps()
|
||||
db.session.add(module_build)
|
||||
db.session.commit()
|
||||
|
||||
return module_build
|
||||
|
||||
def _get_modules_build_required_by_module_recursively(self, module_build):
|
||||
"""
|
||||
Convenience wrapper around get_modules_build_required_by_module_recursively
|
||||
returning the list with nsvc strings of modules returned by this the wrapped
|
||||
method.
|
||||
"""
|
||||
modules = module_build_service.utils.get_modules_build_required_by_module_recursively(
|
||||
db.session, module_build.mmd())
|
||||
nsvcs = [":".join([m.get_name(), m.get_stream(), str(m.get_version()), m.get_context()])
|
||||
for m in modules]
|
||||
return nsvcs
|
||||
|
||||
def _generate_default_modules(self):
|
||||
"""
|
||||
Generates gtk:1, gtk:2, foo:1 and foo:2 modules requiring the
|
||||
platform:f28 and platform:f29 modules.
|
||||
"""
|
||||
self._make_module("gtk:1:0:c2", {"platform": ["f28"]}, {})
|
||||
self._make_module("gtk:1:0:c3", {"platform": ["f29"]}, {})
|
||||
self._make_module("gtk:2:0:c4", {"platform": ["f28"]}, {})
|
||||
self._make_module("gtk:2:0:c5", {"platform": ["f29"]}, {})
|
||||
self._make_module("foo:1:0:c2", {"platform": ["f28"]}, {})
|
||||
self._make_module("foo:1:0:c3", {"platform": ["f29"]}, {})
|
||||
self._make_module("foo:2:0:c4", {"platform": ["f28"]}, {})
|
||||
self._make_module("foo:2:0:c5", {"platform": ["f29"]}, {})
|
||||
self._make_module("platform:f28:0:c10", {}, {})
|
||||
self._make_module("platform:f29:0:c11", {}, {})
|
||||
|
||||
@pytest.mark.parametrize('requires,build_requires,expected', [
|
||||
({}, {"gtk": ["1", "2"]},
|
||||
['platform:f29:0:c11', 'gtk:2:0:c4', 'gtk:2:0:c5',
|
||||
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
|
||||
|
||||
({}, {"gtk": ["1"], "foo": ["1"]},
|
||||
['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3',
|
||||
'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']),
|
||||
|
||||
({}, {"gtk": ["1"], "foo": ["1"], "platform": ["f28"]},
|
||||
['platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3',
|
||||
'foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11']),
|
||||
|
||||
([{}, {}], [{"gtk": ["1"], "foo": ["1"]}, {"gtk": ["2"], "foo": ["2"]}],
|
||||
['foo:1:0:c2', 'foo:1:0:c3', 'foo:2:0:c4', 'foo:2:0:c5',
|
||||
'platform:f28:0:c10', 'platform:f29:0:c11', 'gtk:1:0:c2',
|
||||
'gtk:1:0:c3', 'gtk:2:0:c4', 'gtk:2:0:c5']),
|
||||
|
||||
({}, {"gtk": ["-2"], "foo": ["-2"]},
|
||||
['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11',
|
||||
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
|
||||
|
||||
({}, {"gtk": ["-1", "1"], "foo": ["-2", "1"]},
|
||||
['foo:1:0:c2', 'foo:1:0:c3', 'platform:f29:0:c11',
|
||||
'platform:f28:0:c10', 'gtk:1:0:c2', 'gtk:1:0:c3']),
|
||||
])
|
||||
def test_get_required_modules_simple(self, requires, build_requires, expected):
|
||||
module_build = self._make_module("app:1:0:c1", requires, build_requires)
|
||||
self._generate_default_modules()
|
||||
nsvcs = self._get_modules_build_required_by_module_recursively(module_build)
|
||||
print nsvcs
|
||||
assert set(nsvcs) == set(expected)
|
||||
|
||||
def _generate_default_modules_recursion(self):
|
||||
"""
|
||||
Generates the gtk:1 module requiring foo:1 module requiring bar:1
|
||||
and lorem:1 modules which require base:f29 module requiring
|
||||
platform:f29 module :).
|
||||
"""
|
||||
self._make_module("gtk:1:0:c2", {"foo": ["unknown"]}, {})
|
||||
self._make_module("gtk:1:1:c2", {"foo": ["1"]}, {})
|
||||
self._make_module("foo:1:0:c2", {"bar": ["unknown"]}, {})
|
||||
self._make_module("foo:1:1:c2", {"bar": ["1"], "lorem": ["1"]}, {})
|
||||
self._make_module("bar:1:0:c2", {"base": ["unknown"]}, {})
|
||||
self._make_module("bar:1:1:c2", {"base": ["f29"]}, {})
|
||||
self._make_module("lorem:1:0:c2", {"base": ["unknown"]}, {})
|
||||
self._make_module("lorem:1:1:c2", {"base": ["f29"]}, {})
|
||||
self._make_module("base:f29:0:c3", {"platform": ["f29"]}, {})
|
||||
self._make_module("platform:f29:0:c11", {}, {})
|
||||
|
||||
@pytest.mark.parametrize('requires,build_requires,expected', [
|
||||
({}, {"gtk": ["1"]},
|
||||
['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11',
|
||||
'bar:1:1:c2', 'gtk:1:1:c2', 'lorem:1:1:c2']),
|
||||
|
||||
({}, {"foo": ["1"]},
|
||||
['foo:1:1:c2', 'base:f29:0:c3', 'platform:f29:0:c11',
|
||||
'bar:1:1:c2', 'lorem:1:1:c2']),
|
||||
])
|
||||
def test_get_required_modules_recursion(self, requires, build_requires, expected):
|
||||
module_build = self._make_module("app:1:0:c1", requires, build_requires)
|
||||
self._generate_default_modules_recursion()
|
||||
nsvcs = self._get_modules_build_required_by_module_recursively(module_build)
|
||||
print nsvcs
|
||||
assert set(nsvcs) == set(expected)
|
||||
Reference in New Issue
Block a user