modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-05 11:48:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Allow buildrequring a virtual stream of a base module

Browse Source
This commit is contained in:
mprahl
2019-04-23 13:55:17 -04:00
parent 88078a555b
commit 7b8947f660
10 changed files with 331 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
module_build_service/models.py
View File

@@ -467,6 +467,17 @@ class ModuleBuild(MBSBase):
ret.append(build)
return ret
@staticmethod
def get_module_count(session, **kwargs):
"""
Determine the number of modules that match the provided filter.
:param session: SQLAlchemy session
:return: the number of modules that match the provided filter
:rtype: int
"""
return session.query(func.count(ModuleBuild.id)).filter_by(**kwargs).scalar()
@staticmethod
def get_build_by_koji_tag(session, tag):
"""Get build by its koji_tag"""

33
module_build_service/resolver/DBResolver.py
View File

@@ -28,6 +28,7 @@ from module_build_service import log, db
from module_build_service.resolver.base import GenericResolver
from module_build_service import models
from module_build_service.errors import UnprocessableEntity
from module_build_service.utils.submit import load_mmd
import sqlalchemy
@@ -53,6 +54,37 @@ class DBResolver(GenericResolver):
raise UnprocessableEntity(
'Cannot find any module builds for %s:%s' % (name, stream))
def get_module_count(self, **kwargs):
"""
Determine the number of modules that match the provided filter.
:return: the number of modules that match the provided filter
:rtype: int
"""
with models.make_session(self.config) as session:
return models.ModuleBuild.get_module_count(session, **kwargs)
def get_latest_with_virtual_stream(self, name, virtual_stream):
"""
Get the latest module with the input virtual stream based on the stream version and version.
:param str name: the module name to search for
:param str virtual_stream: the module virtual stream to search for
:return: the module's modulemd or None
:rtype: Modulemd.Module or None
"""
with models.make_session(self.config) as session:
query = session.query(models.ModuleBuild).filter_by(name=name)
query = models.ModuleBuild._add_virtual_streams_filter(session, query, [virtual_stream])
# Cast the version as an integer so that we get proper ordering
module = query.order_by(
models.ModuleBuild.stream_version.desc(),
sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc()
).first()
if module:
return load_mmd(module.modulemd)
def get_module_modulemds(self, name, stream, version=None, context=None, strict=False,
stream_version_lte=False, virtual_streams=None):
"""
@@ -72,7 +104,6 @@ class DBResolver(GenericResolver):
logic. When falsy, no filtering occurs.
:return: List of Modulemd metadata instances matching the query
"""
from module_build_service.utils import load_mmd
if version and context:
mmd = self._get_module(name, stream, version, context, strict=strict)
if mmd is None:

45
module_build_service/resolver/MBSResolver.py
View File

@@ -126,6 +126,51 @@ class MBSResolver(GenericResolver):
if rv:
return rv[0]
def get_module_count(self, **kwargs):
"""
Determine the number of modules that match the provided filter.
:return: the number of modules that match the provided filter
:rtype: int
"""
query = {
"page": 1,
"per_page": 1,
"short": True,
}
query.update(kwargs)
res = self.session.get(self.mbs_prod_url, params=query)
if not res.ok:
raise RuntimeError(self._generic_error % (query, res.status_code))
data = res.json()
return data["meta"]["total"]
def get_latest_with_virtual_stream(self, name, virtual_stream):
"""
Get the latest module with the input virtual stream based on the stream version and version.
:param str name: the module name to search for
:param str virtual_stream: the module virtual stream to search for
:return: the module's modulemd or None
:rtype: Modulemd.Module or None
"""
query = {
"name": name,
"order_desc_by": ["stream_version", "version"],
"page": 1,
"per_page": 1,
"verbose": True,
"virtual_stream": virtual_stream,
}
res = self.session.get(self.mbs_prod_url, params=query)
if not res.ok:
raise RuntimeError(self._generic_error % (query, res.status_code))
data = res.json()
if data["items"]:
return load_mmd(data["items"][0]["modulemd"])
def get_module_modulemds(self, name, stream, version=None, context=None, strict=False,
stream_version_lte=False, virtual_streams=None):
"""

8
module_build_service/resolver/base.py
View File

@@ -105,6 +105,14 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
from module_build_service.utils import load_mmd
return load_mmd(yaml)
@abstractmethod
def get_module_count(self, **kwargs):
raise NotImplementedError()
@abstractmethod
def get_latest_with_virtual_stream(self, name, virtual_stream):
raise NotImplementedError()
@abstractmethod
def get_module_modulemds(self, name, stream, version=None, context=None, strict=False,
stream_version_lte=None, virtual_streams=None):

76
module_build_service/utils/submit.py
View File

@@ -33,18 +33,17 @@ from functools import partial
from multiprocessing.dummy import Pool as ThreadPool
from datetime import datetime
import copy
from module_build_service.utils import to_text_type
import kobo.rpmlib
import requests
from gi.repository import GLib
import module_build_service.scm
from module_build_service import conf, db, log, models, Modulemd
from module_build_service.errors import (
ValidationError, UnprocessableEntity, Forbidden, Conflict)
from module_build_service import glib
from module_build_service.utils import to_text_type
def record_filtered_rpms(mmd):
@@ -603,6 +602,78 @@ def _apply_dep_overrides(mmd, params):
mmd.set_dependencies(deps)
def _handle_base_module_virtual_stream_br(mmd):
"""
Translate a base module virtual stream buildrequire to an actual stream on the input modulemd.
:param Modulemd.Module mmd: the modulemd to apply the overrides on
"""
from module_build_service.resolver import system_resolver
overridden = False
deps = mmd.get_dependencies()
for dep in deps:
brs = dep.get_buildrequires()
for base_module in conf.base_module_names:
if base_module not in brs:
continue
streams = list(brs[base_module].get())
new_streams = copy.copy(streams)
for i, stream in enumerate(streams):
# Ignore streams that start with a minus sign, since those are handled in the
# MSE code
if stream.startswith('-'):
continue
# Check if the base module stream is available
log.debug(
'Checking to see if the base module "%s:%s" is available', base_module, stream)
if system_resolver.get_module_count(name=base_module, stream=stream) > 0:
continue
# If the base module stream is not available, check if there's a virtual stream
log.debug(
'Checking to see if there is a base module "%s" with the virtual stream "%s"',
base_module,
stream
)
base_module_mmd = system_resolver.get_latest_with_virtual_stream(
name=base_module, virtual_stream=stream)
if not base_module_mmd:
# If there isn't this base module stream or virtual stream available, skip it,
# and let the dep solving code deal with it like it normally would
log.warning(
'There is no base module "%s" with stream/virtual stream "%s"',
base_module,
stream
)
continue
latest_stream = base_module_mmd.get_stream()
log.info(
('Replacing the buildrequire "%s:%s" with "%s:%s", since "%s" is a virtual '
'stream'),
base_module,
stream,
base_module,
latest_stream,
stream
)
new_streams[i] = latest_stream
overridden = True
if streams != new_streams:
brs[base_module].set(new_streams)
if overridden:
dep.set_buildrequires(brs)
if overridden:
mmd.set_dependencies(deps)
def submit_module_build(username, mmd, params):
"""
Submits new module build.
@@ -631,6 +702,7 @@ def submit_module_build(username, mmd, params):
if "default_streams" in params:
default_streams = params["default_streams"]
_apply_dep_overrides(mmd, params)
_handle_base_module_virtual_stream_br(mmd)
mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams)
if not mmds:

37
tests/staged_data/testmodule_el8.yaml Normal file
View File

@@ -0,0 +1,37 @@
document: modulemd
version: 1
data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:
buildrequires:
platform: el8
requires:
platform: el8
references:
community: https://docs.pagure.org/modularity/
documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
profiles:
default:
rpms:
- tangerine
api:
rpms:
- perl-Tangerine
- tangerine
components:
rpms:
perl-List-Compare:
rationale: A dependency of tangerine.
ref: master
perl-Tangerine:
rationale: Provides API for this module and is a dependency of tangerine.
ref: master
tangerine:
rationale: Provides API for this module.
buildorder: 10
ref: master

8
tests/test_models/test_models.py
View File

@@ -173,6 +173,14 @@ class TestModelsGetStreamsContexts:
assert builds == set(['platform:f29.1.0:15:c11', 'platform:f29.1.0:15:c11.another',
'platform:f29.2.0:1:c11'])
def test_get_module_count(self):
clean_database(False)
make_module("platform:f29.1.0:10:c11", {}, {})
make_module("platform:f29.1.0:10:c12", {}, {})
with make_session(conf) as session:
count = ModuleBuild.get_module_count(session, name="platform")
assert count == 2
def test_add_virtual_streams_filter(self):
clean_database(False)
make_module("platform:f29.1.0:10:c1", {}, {}, virtual_streams=["f29"])

17
tests/test_resolver/test_db.py
View File

@@ -227,3 +227,20 @@ class TestDBModule:
set(['bar'])
}
assert result == expected
def test_get_latest_with_virtual_stream(self):
tests.init_data(1, multiple_stream_versions=True)
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
mmd = resolver.get_latest_with_virtual_stream('platform', 'f29')
assert mmd
assert mmd.get_stream() == 'f29.2.0'
def test_get_latest_with_virtual_stream_none(self):
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
mmd = resolver.get_latest_with_virtual_stream('platform', 'doesnotexist')
assert not mmd
def test_get_module_count(self):
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
count = resolver.get_module_count(name='platform', stream='f28')
assert count == 1

70
tests/test_resolver/test_mbs.py
View File

@@ -367,3 +367,73 @@ class TestMBSModule:
assert '10' == mmd.get_stream()
assert 1 == mmd.get_version()
assert 'c1' == mmd.get_context()
@patch("requests.Session")
def test_get_module_count(self, mock_session):
mock_res = Mock()
mock_res.ok.return_value = True
mock_res.json.return_value = {
"items": [
{
"name": "platform",
"stream": "f28",
"version": "3",
"context": "00000000",
}
],
"meta": {
"total": 5
}
}
mock_session.return_value.get.return_value = mock_res
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")
count = resolver.get_module_count(name="platform", stream="f28")
assert count == 5
mock_session.return_value.get.assert_called_once_with(
"https://mbs.fedoraproject.org/module-build-service/1/module-builds/",
params={
"name": "platform",
"page": 1,
"per_page": 1,
"short": True,
"stream": "f28",
}
)
@patch("requests.Session")
def test_get_latest_with_virtual_stream(self, mock_session, platform_mmd):
mock_res = Mock()
mock_res.ok.return_value = True
mock_res.json.return_value = {
"items": [
{
"context": "00000000",
"modulemd": platform_mmd,
"name": "platform",
"stream": "f28",
"version": "3",
}
],
"meta": {
"total": 5
}
}
mock_session.return_value.get.return_value = mock_res
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend="mbs")
mmd = resolver.get_latest_with_virtual_stream("platform", "virtualf28")
assert mmd.get_name() == "platform"
mock_session.return_value.get.assert_called_once_with(
"https://mbs.fedoraproject.org/module-build-service/1/module-builds/",
params={
"name": "platform",
"order_desc_by": ["stream_version", "version"],
"page": 1,
"per_page": 1,
"verbose": True,
"virtual_stream": "virtualf28",
}
)

30
tests/test_views/test_views.py
View File

@@ -32,7 +32,7 @@ from os.path import basename, dirname, splitext
from requests.utils import quote
import hashlib
import pytest
from module_build_service.utils import to_text_type, load_mmd_file
from module_build_service.utils import to_text_type, load_mmd_file, load_mmd
import re
from tests import app, init_data, clean_database, reuse_component_init_data
@@ -2111,3 +2111,31 @@ class TestViews:
data = json.loads(rv.data)[0]
mmd = module_build_service.utils.load_mmd(data['modulemd'])
assert mmd.get_xmd()['mbs']['disttag_marking'] == 'product12'
@patch('module_build_service.auth.get_user', return_value=user)
@patch('module_build_service.scm.SCM')
def test_submit_build_request_platform_virtual_stream(self, mocked_scm, mocked_get_user):
# Create a platform with el8.25.0 but with the virtual stream el8
mmd = load_mmd_file(path.join(base_dir, 'staged_data', 'platform.yaml'))
mmd.set_stream('el8.25.0')
xmd = from_variant_dict(mmd.get_xmd())
xmd['mbs']['virtual_streams'] = ['el8']
mmd.set_xmd(dict_values(xmd))
import_mmd(db.session, mmd)
# Use a testmodule that buildrequires platform:el8
FakeSCM(mocked_scm, 'testmodule', 'testmodule_el8.yaml',
'620ec77321b2ea7b0d67d82992dda3e1d67055b4')
post_url = '/module-build-service/2/module-builds/'
scm_url = ('https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe'
'efbd35246a81b6cb8d49')
rv = self.client.post(post_url, data=json.dumps({'branch': 'master', 'scmurl': scm_url}))
data = json.loads(rv.data)
print(data)
mmd = load_mmd(data[0]['modulemd'])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
assert set(dep.get_buildrequires()['platform'].get()) == set(['el8.25.0'])
assert set(dep.get_requires()['platform'].get()) == set(['el8'])