modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 03:38:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Split utils/mse.py

Browse Source 
This moves the code used by the backend and API to common/resolve.py
and moves the code used just by the API to web/mse.py.
This commit is contained in:
mprahl
2020-01-03 11:14:48 -05:00
parent b0a27081f4
commit d3d1f2d3dd
10 changed files with 317 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
module_build_service/common/resolve.py Normal file
View File

@@ -0,0 +1,183 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from module_build_service import conf
from module_build_service.errors import StreamAmbigous
from module_build_service.resolver import GenericResolver
def expand_single_mse_streams(
db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False):
"""
Helper method for `expand_mse_stream()` expanding single name:[streams].
Returns list of expanded streams.
:param db_session: SQLAlchemy DB session.
:param str name: Name of the module which will be expanded.
:param streams: List of streams to expand.
:type streams: list[str]
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
"""
from module_build_service import models
default_streams = default_streams or {}
# Stream can be prefixed with '-' sign to define that this stream should
# not appear in a resulting list of streams. There can be two situations:
# a) all streams have '-' prefix. In this case, we treat list of streams
# as blacklist and we find all the valid streams and just remove those with
# '-' prefix.
# b) there is at least one stream without '-' prefix. In this case, we can
# ignore all the streams with '-' prefix and just add those without
# '-' prefix to the list of valid streams.
streams_is_blacklist = all(stream.startswith("-") for stream in streams)
if streams_is_blacklist or len(streams) == 0:
if name in default_streams:
expanded_streams = [default_streams[name]]
elif raise_if_stream_ambigous:
raise StreamAmbigous("There are multiple streams to choose from for module %s." % name)
else:
builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name)
expanded_streams = [build.stream for build in builds]
else:
expanded_streams = []
for stream in streams:
if stream.startswith("-"):
if streams_is_blacklist and stream[1:] in expanded_streams:
expanded_streams.remove(stream[1:])
else:
expanded_streams.append(stream)
if len(expanded_streams) > 1:
if name in default_streams:
expanded_streams = [default_streams[name]]
elif raise_if_stream_ambigous:
raise StreamAmbigous(
"There are multiple streams %r to choose from for module %s."
% (expanded_streams, name)
)
return expanded_streams
def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None):
"""
Returns dict containing the base modules compatible with `base_mmd` grouped by their state.
:param GenericResolver resolver: GenericResolver instance.
:param Modulemd base_mmd: Modulemd instant to return compatible modules for.
:param set ignore_ns: When set, defines the name:stream of base modules which will be ignored
by this function and therefore not returned.
:return dict: Dictionary with module's state name as a key and list of Modulemd objects for
each compatible base module in that state. For example:
{
"ready": [base_mmd_1, base_mmd_2]
"garbage": [base_mmd_3]
}
The input `base_mmd` is always included in the result in "ready" state.
"""
from module_build_service import models
# Add the module to `seen` and `ret`.
ret = {"ready": [], "garbage": []}
ret["ready"].append(base_mmd)
ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()])
seen = set() if not ignore_ns else set(ignore_ns)
seen.add(ns)
# Get the list of compatible virtual streams.
xmd = base_mmd.get_xmd()
virtual_streams = xmd.get("mbs", {}).get("virtual_streams")
# In case there are no virtual_streams in the buildrequired name:stream,
# it is clear that there are no compatible streams, so return just this
# `base_mmd`.
if not virtual_streams:
return ret
if conf.allow_only_compatible_base_modules:
stream_version_lte = True
states = ["ready"]
else:
stream_version_lte = False
states = ["ready", "garbage"]
for state in states:
mmds = resolver.get_compatible_base_module_modulemds(
base_mmd, stream_version_lte, virtual_streams,
[models.BUILD_STATES[state]])
ret_chunk = []
# Add the returned mmds to the `seen` set to avoid querying those
# individually if they are part of the buildrequire streams for this
# base module.
for mmd_ in mmds:
mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()])
# An extra precaution to ensure there are no duplicates. This can happen when there
# are two modules with the same name:stream - one in ready state and one in garbage
# state.
if mmd_ns not in seen:
ret_chunk.append(mmd_)
seen.add(mmd_ns)
ret[state] += ret_chunk
return ret
def get_base_module_mmds(db_session, mmd):
"""
Returns list of MMDs of base modules buildrequired by `mmd` including the compatible
old versions of the base module based on the stream version.
:param Modulemd mmd: Input modulemd metadata.
:rtype: dict with lists of Modulemd
:return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules
buildrequired by `mmd` as a value.
"""
from module_build_service import models
seen = set()
ret = {"ready": [], "garbage": []}
resolver = GenericResolver.create(db_session, conf)
for deps in mmd.get_dependencies():
buildrequires = {
module: deps.get_buildtime_streams(module)
for module in deps.get_buildtime_modules()
}
for name in conf.base_module_names:
if name not in buildrequires.keys():
continue
# Since the query below uses stream_version_lte, we can sort the streams by stream
# version in descending order to not perform unnecessary queries. Otherwise, if the
# streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be
# returned twice. Only one query for that scenario is necessary.
sorted_desc_streams = sorted(
buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version)
for stream in sorted_desc_streams:
ns = ":".join([name, stream])
if ns in seen:
continue
# Get the MMD for particular buildrequired name:stream to find out the virtual
# streams according to which we can find the compatible streams later.
# The returned MMDs are all the module builds for name:stream in the highest
# version. Given the base module does not depend on other modules, it can appear
# only in single context and therefore the `mmds` should always contain just
# zero or one module build.
mmds = resolver.get_module_modulemds(name, stream)
if not mmds:
continue
base_mmd = mmds[0]
new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen)
for state in new_ret.keys():
for mmd_ in new_ret[state]:
mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()])
seen.add(mmd_ns)
ret[state] += new_ret[state]
break
return ret

5
module_build_service/scheduler/default_modules.py
View File

@@ -12,12 +12,13 @@ import six.moves.xmlrpc_client as xmlrpclib
from module_build_service import conf, log, models, Modulemd, scm
from module_build_service.common.koji import get_session, koji_retrying_multicall_map
from module_build_service.common.resolve import (
expand_single_mse_streams, get_compatible_base_module_mmds
)
from module_build_service.common.retry import retry
from module_build_service.db_session import db_session
from module_build_service.errors import UnprocessableEntity
from module_build_service.resolver.base import GenericResolver
from module_build_service.utils.mse import (
get_compatible_base_module_mmds, expand_single_mse_streams)
def add_default_modules(mmd):

1
module_build_service/utils/__init__.py
View File

@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from module_build_service.utils.mse import * # noqa
from module_build_service.utils.views import * # noqa
from module_build_service.utils.reuse import * # noqa
from module_build_service.utils.submit import * # noqa

2
module_build_service/utils/reuse.py
View File

@@ -6,7 +6,7 @@ from module_build_service import log, models, conf
from module_build_service.db_session import db_session
from module_build_service.resolver import GenericResolver
from module_build_service.scheduler import events
from module_build_service.utils.mse import get_base_module_mmds
from module_build_service.common.resolve import get_base_module_mmds
def reuse_component(component, previous_component_build, change_state_now=False,

3
module_build_service/utils/submit.py
View File

@@ -20,6 +20,7 @@ from module_build_service import conf, log, models, Modulemd
from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str, to_text_type
from module_build_service.db_session import db_session
from module_build_service.errors import ValidationError, UnprocessableEntity, Forbidden, Conflict
from module_build_service.web.mse import generate_expanded_mmds
from module_build_service.web.utils import deps_to_dict
@@ -955,8 +956,6 @@ def submit_module_build(db_session, username, mmd, params):
:rtype: list with ModuleBuild
:return: List with submitted module builds.
"""
from .mse import generate_expanded_mmds
log.debug(
"Submitted %s module build for %s:%s:%s",
("scratch" if params.get("scratch", False) else "normal"),

173
module_build_service/utils/mse.py → module_build_service/web/mse.py
View File

@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from module_build_service import log, models, Modulemd, conf
from module_build_service.common.resolve import expand_single_mse_streams, get_base_module_mmds
from module_build_service.common.utils import mmd_to_str
from module_build_service.errors import StreamAmbigous
from module_build_service.errors import UnprocessableEntity
@@ -9,62 +10,6 @@ from module_build_service.web.utils import deps_to_dict
from module_build_service.resolver import GenericResolver
def expand_single_mse_streams(
db_session, name, streams, default_streams=None, raise_if_stream_ambigous=False):
"""
Helper method for `expand_mse_stream()` expanding single name:[streams].
Returns list of expanded streams.
:param db_session: SQLAlchemy DB session.
:param str name: Name of the module which will be expanded.
:param streams: List of streams to expand.
:type streams: list[str]
:param dict default_streams: Dict in {module_name: module_stream, ...} format defining
the default stream to choose for module in case when there are multiple streams to
choose from.
:param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case
there are multiple streams for some dependency of module and the module name is not
defined in `default_streams`, so it is not clear which stream should be used.
"""
default_streams = default_streams or {}
# Stream can be prefixed with '-' sign to define that this stream should
# not appear in a resulting list of streams. There can be two situations:
# a) all streams have '-' prefix. In this case, we treat list of streams
# as blacklist and we find all the valid streams and just remove those with
# '-' prefix.
# b) there is at least one stream without '-' prefix. In this case, we can
# ignore all the streams with '-' prefix and just add those without
# '-' prefix to the list of valid streams.
streams_is_blacklist = all(stream.startswith("-") for stream in streams)
if streams_is_blacklist or len(streams) == 0:
if name in default_streams:
expanded_streams = [default_streams[name]]
elif raise_if_stream_ambigous:
raise StreamAmbigous("There are multiple streams to choose from for module %s." % name)
else:
builds = models.ModuleBuild.get_last_build_in_all_streams(db_session, name)
expanded_streams = [build.stream for build in builds]
else:
expanded_streams = []
for stream in streams:
if stream.startswith("-"):
if streams_is_blacklist and stream[1:] in expanded_streams:
expanded_streams.remove(stream[1:])
else:
expanded_streams.append(stream)
if len(expanded_streams) > 1:
if name in default_streams:
expanded_streams = [default_streams[name]]
elif raise_if_stream_ambigous:
raise StreamAmbigous(
"There are multiple streams %r to choose from for module %s."
% (expanded_streams, name)
)
return expanded_streams
def expand_mse_streams(db_session, mmd, default_streams=None, raise_if_stream_ambigous=False):
"""
Expands streams in both buildrequires/requires sections of MMD.
@@ -186,122 +131,6 @@ def _get_mmds_from_requires(
return mmds
def get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=None):
"""
Returns dict containing the base modules compatible with `base_mmd` grouped by their state.
:param GenericResolver resolver: GenericResolver instance.
:param Modulemd base_mmd: Modulemd instant to return compatible modules for.
:param set ignore_ns: When set, defines the name:stream of base modules which will be ignored
by this function and therefore not returned.
:return dict: Dictionary with module's state name as a key and list of Modulemd objects for
each compatible base module in that state. For example:
{
"ready": [base_mmd_1, base_mmd_2]
"garbage": [base_mmd_3]
}
The input `base_mmd` is always included in the result in "ready" state.
"""
# Add the module to `seen` and `ret`.
ret = {"ready": [], "garbage": []}
ret["ready"].append(base_mmd)
ns = ":".join([base_mmd.get_module_name(), base_mmd.get_stream_name()])
seen = set() if not ignore_ns else set(ignore_ns)
seen.add(ns)
# Get the list of compatible virtual streams.
xmd = base_mmd.get_xmd()
virtual_streams = xmd.get("mbs", {}).get("virtual_streams")
# In case there are no virtual_streams in the buildrequired name:stream,
# it is clear that there are no compatible streams, so return just this
# `base_mmd`.
if not virtual_streams:
return ret
if conf.allow_only_compatible_base_modules:
stream_version_lte = True
states = ["ready"]
else:
stream_version_lte = False
states = ["ready", "garbage"]
for state in states:
mmds = resolver.get_compatible_base_module_modulemds(
base_mmd, stream_version_lte, virtual_streams,
[models.BUILD_STATES[state]])
ret_chunk = []
# Add the returned mmds to the `seen` set to avoid querying those
# individually if they are part of the buildrequire streams for this
# base module.
for mmd_ in mmds:
mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()])
# An extra precaution to ensure there are no duplicates. This can happen when there
# are two modules with the same name:stream - one in ready state and one in garbage
# state.
if mmd_ns not in seen:
ret_chunk.append(mmd_)
seen.add(mmd_ns)
ret[state] += ret_chunk
return ret
def get_base_module_mmds(db_session, mmd):
"""
Returns list of MMDs of base modules buildrequired by `mmd` including the compatible
old versions of the base module based on the stream version.
:param Modulemd mmd: Input modulemd metadata.
:rtype: dict with lists of Modulemd
:return: Dict with "ready" or "garbage" state name as a key and list of MMDs of base modules
buildrequired by `mmd` as a value.
"""
seen = set()
ret = {"ready": [], "garbage": []}
resolver = GenericResolver.create(db_session, conf)
for deps in mmd.get_dependencies():
buildrequires = {
module: deps.get_buildtime_streams(module)
for module in deps.get_buildtime_modules()
}
for name in conf.base_module_names:
if name not in buildrequires.keys():
continue
# Since the query below uses stream_version_lte, we can sort the streams by stream
# version in descending order to not perform unnecessary queries. Otherwise, if the
# streams are f29.1.0 and f29.2.0, then two queries will occur, causing f29.1.0 to be
# returned twice. Only one query for that scenario is necessary.
sorted_desc_streams = sorted(
buildrequires[name], reverse=True, key=models.ModuleBuild.get_stream_version)
for stream in sorted_desc_streams:
ns = ":".join([name, stream])
if ns in seen:
continue
# Get the MMD for particular buildrequired name:stream to find out the virtual
# streams according to which we can find the compatible streams later.
# The returned MMDs are all the module builds for name:stream in the highest
# version. Given the base module does not depend on other modules, it can appear
# only in single context and therefore the `mmds` should always contain just
# zero or one module build.
mmds = resolver.get_module_modulemds(name, stream)
if not mmds:
continue
base_mmd = mmds[0]
new_ret = get_compatible_base_module_mmds(resolver, base_mmd, ignore_ns=seen)
for state in new_ret.keys():
for mmd_ in new_ret[state]:
mmd_ns = ":".join([mmd_.get_module_name(), mmd_.get_stream_name()])
seen.add(mmd_ns)
ret[state] += new_ret[state]
break
return ret
def get_mmds_required_by_module_recursively(
db_session, mmd, default_streams=None, raise_if_stream_ambigous=False
):

113
tests/test_common/test_resolve.py Normal file
View File

@@ -0,0 +1,113 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from mock import patch, PropertyMock
import pytest
from module_build_service.common.utils import load_mmd
from module_build_service.common.resolve import get_base_module_mmds
from module_build_service import Modulemd, models
from module_build_service.db_session import db_session
from tests import clean_database, make_module_in_db, init_data, read_staged_data
class TestResolve:
def setup_method(self, test_method):
clean_database(False)
def teardown_method(self, test_method):
clean_database()
def test__get_base_module_mmds(self):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=True)
mmd = load_mmd(read_staged_data("testmodule_v2.yaml"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "f29.1.0")
new_deps.add_buildtime_stream("platform", "f29.2.0")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = get_base_module_mmds(db_session, mmd)
expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"}
# Verify no duplicates were returned before doing set operations
assert len(mmds["ready"]) == len(expected)
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds["ready"]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected
@pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"]))
def test__get_base_module_mmds_virtual_streams(self, virtual_streams):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=True)
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "f29.2.0")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams)
mmds = get_base_module_mmds(db_session, mmd)
if virtual_streams == ["f29"]:
expected = {
"platform:f29.0.0",
"platform:f29.1.0",
"platform:f29.2.0",
"platform:lp29.1.1"
}
else:
expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"}
# Verify no duplicates were returned before doing set operations
assert len(mmds["ready"]) == len(expected)
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds["ready"]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected
@patch(
"module_build_service.config.Config.allow_only_compatible_base_modules",
new_callable=PropertyMock, return_value=False
)
def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"])
# Mark platform:foo28 as garbage to test that it is still considered as compatible.
platform = db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="foo28").first()
platform.state = "garbage"
db_session.add(platform)
db_session.commit()
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "foo29")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = get_base_module_mmds(db_session, mmd)
expected = {}
expected["ready"] = {"platform:foo29", "platform:foo30"}
expected["garbage"] = {"platform:foo28"}
# Verify no duplicates were returned before doing set operations
assert len(mmds) == len(expected)
for k in expected.keys():
assert len(mmds[k]) == len(expected[k])
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds[k]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected[k]

4
tests/test_utils/test_utils.py
View File

@@ -703,7 +703,7 @@ class TestUtils:
v = module_build_service.utils.submit.get_prefixed_version(mmd)
assert v == 7000120180205135154
@patch("module_build_service.utils.mse.generate_expanded_mmds")
@patch("module_build_service.utils.submit.generate_expanded_mmds")
def test_submit_build_new_mse_build(self, generate_expanded_mmds):
"""
Tests that finished build can be resubmitted in case the resubmitted
@@ -729,7 +729,7 @@ class TestUtils:
assert builds[0].siblings(db_session) == [builds[1].id]
assert builds[1].siblings(db_session) == [builds[0].id]
@patch("module_build_service.utils.mse.generate_expanded_mmds")
@patch("module_build_service.utils.submit.generate_expanded_mmds")
@patch(
"module_build_service.config.Config.scratch_build_only_branches",
new_callable=mock.PropertyMock,

0
tests/test_web/__init__.py Normal file
View File

125
tests/test_utils/test_utils_mse.py → tests/test_web/test_mse.py
View File

@@ -1,17 +1,16 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from mock import patch, PropertyMock
import pytest
import module_build_service.utils
from module_build_service.common.utils import load_mmd
from module_build_service import Modulemd, models
from module_build_service.db_session import db_session
from module_build_service.errors import StreamAmbigous
from tests import clean_database, make_module_in_db, init_data, read_staged_data
from module_build_service.web.mse import (
expand_mse_streams, generate_expanded_mmds, get_mmds_required_by_module_recursively
)
from tests import clean_database, make_module_in_db
class TestUtilsModuleStreamExpansion:
class TestModuleStreamExpansion:
def setup_method(self, test_method):
clean_database(False)
@@ -25,9 +24,8 @@ class TestUtilsModuleStreamExpansion:
method.
"""
mmd = module_build.mmd()
module_build_service.utils.expand_mse_streams(db_session, mmd)
modules = module_build_service.utils.get_mmds_required_by_module_recursively(
db_session, mmd)
expand_mse_streams(db_session, mmd)
modules = get_mmds_required_by_module_recursively(db_session, mmd)
nsvcs = [m.get_nsvc() for m in modules]
return nsvcs
@@ -67,8 +65,7 @@ class TestUtilsModuleStreamExpansion:
},
}],
)
mmds = module_build_service.utils.generate_expanded_mmds(
db_session, module_build.mmd())
mmds = generate_expanded_mmds(db_session, module_build.mmd())
contexts = {mmd.get_context() for mmd in mmds}
assert {"e1e005fb", "ce132a1e"} == contexts
@@ -168,11 +165,10 @@ class TestUtilsModuleStreamExpansion:
# and also that it does not raise an exception otherwise.
if stream_ambigous:
with pytest.raises(StreamAmbigous):
module_build_service.utils.generate_expanded_mmds(
generate_expanded_mmds(
db_session, module_build.mmd(), raise_if_stream_ambigous=True)
else:
module_build_service.utils.generate_expanded_mmds(
db_session, module_build.mmd(), raise_if_stream_ambigous=True)
generate_expanded_mmds(db_session, module_build.mmd(), raise_if_stream_ambigous=True)
# Check that if stream is ambigous and we define the stream, it does not raise
# an exception.
@@ -181,14 +177,14 @@ class TestUtilsModuleStreamExpansion:
for ns in list(expected_buildrequires)[0]:
name, stream = ns.split(":")
default_streams[name] = stream
module_build_service.utils.generate_expanded_mmds(
generate_expanded_mmds(
db_session,
module_build.mmd(),
raise_if_stream_ambigous=True,
default_streams=default_streams,
)
mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd())
mmds = generate_expanded_mmds(db_session, module_build.mmd())
buildrequires_per_mmd_xmd = set()
buildrequires_per_mmd_buildrequires = set()
@@ -255,7 +251,7 @@ class TestUtilsModuleStreamExpansion:
def test_generate_expanded_mmds_requires(self, module_deps, expected):
self._generate_default_modules()
module_build = make_module_in_db("app:1:0:c1", module_deps)
mmds = module_build_service.utils.generate_expanded_mmds(db_session, module_build.mmd())
mmds = generate_expanded_mmds(db_session, module_build.mmd())
requires_per_mmd = set()
for mmd in mmds:
@@ -467,98 +463,3 @@ class TestUtilsModuleStreamExpansion:
self._generate_default_modules_modules_multiple_stream_versions()
nsvcs = self._get_mmds_required_by_module_recursively(module_build, db_session)
assert set(nsvcs) == set(expected)
def test__get_base_module_mmds(self):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=True)
mmd = load_mmd(read_staged_data("testmodule_v2.yaml"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "f29.1.0")
new_deps.add_buildtime_stream("platform", "f29.2.0")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd)
expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"}
# Verify no duplicates were returned before doing set operations
assert len(mmds["ready"]) == len(expected)
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds["ready"]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected
@pytest.mark.parametrize("virtual_streams", (None, ["f29"], ["lp29"]))
def test__get_base_module_mmds_virtual_streams(self, virtual_streams):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=True)
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "f29.2.0")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
make_module_in_db("platform:lp29.1.1:12:c11", virtual_streams=virtual_streams)
mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd)
if virtual_streams == ["f29"]:
expected = {
"platform:f29.0.0",
"platform:f29.1.0",
"platform:f29.2.0",
"platform:lp29.1.1"
}
else:
expected = {"platform:f29.0.0", "platform:f29.1.0", "platform:f29.2.0"}
# Verify no duplicates were returned before doing set operations
assert len(mmds["ready"]) == len(expected)
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds["ready"]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected
@patch(
"module_build_service.config.Config.allow_only_compatible_base_modules",
new_callable=PropertyMock, return_value=False
)
def test__get_base_module_mmds_virtual_streams_only_major_versions(self, cfg):
"""Ensure the correct results are returned without duplicates."""
init_data(data_size=1, multiple_stream_versions=["foo28", "foo29", "foo30"])
# Mark platform:foo28 as garbage to test that it is still considered as compatible.
platform = db_session.query(models.ModuleBuild).filter_by(
name="platform", stream="foo28").first()
platform.state = "garbage"
db_session.add(platform)
db_session.commit()
mmd = load_mmd(read_staged_data("testmodule_v2"))
deps = mmd.get_dependencies()[0]
new_deps = Modulemd.Dependencies()
for stream in deps.get_runtime_streams("platform"):
new_deps.add_runtime_stream("platform", stream)
new_deps.add_buildtime_stream("platform", "foo29")
mmd.remove_dependencies(deps)
mmd.add_dependencies(new_deps)
mmds = module_build_service.utils.mse.get_base_module_mmds(db_session, mmd)
expected = {}
expected["ready"] = {"platform:foo29", "platform:foo30"}
expected["garbage"] = {"platform:foo28"}
# Verify no duplicates were returned before doing set operations
assert len(mmds) == len(expected)
for k in expected.keys():
assert len(mmds[k]) == len(expected[k])
# Verify the expected ones were returned
actual = set()
for mmd_ in mmds[k]:
actual.add("{}:{}".format(mmd_.get_module_name(), mmd_.get_stream_name()))
assert actual == expected[k]