modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-08 07:43:20 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Query the Red Hat Product Pages to see if this is a Z stream build when configured

Browse Source 
In certain use-cases, a module's buildrequires may remain the same
in the modulemd, but a different support stream of the buildrequired
base module should be used. For example, since RHEL 8.0.0 is GA, any
modules that buildrequire platform:el8.0.0 should buildrequire
platform:el8.0.0z instead.
This commit is contained in:
mprahl
2019-05-16 14:18:16 -04:00
parent 00ea4e3bca
commit dd5667665d
4 changed files with 365 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
module_build_service/config.py
View File

@@ -30,6 +30,8 @@ import pkg_resources
import re
import sys
from six import string_types
from module_build_service import logger
@@ -623,6 +625,25 @@ class Config(object):
"type": bool,
"default": False,
"desc": "Allow module scratch builds",
},
"product_pages_url": {
"type": str,
"default": "",
"desc": "The URL to the Product Pages. This is queried to determine if a base module "
"stream has been released. If it has, the stream may be modified automatically "
"to use a different support stream.",
},
"product_pages_module_streams": {
"type": dict,
"default": {},
"desc": "The keys are regexes of base module streams that should be checked in the Red "
"Hat Product Pages. The values are tuples. The first value is a string that "
"should be appended to the stream if there is a match and the release the "
"stream represents has been released. The second value is a template string "
"that represents the release in Product Pages and can accept format kwargs of "
"x, y, and z (represents the version). The third value is an optional template "
"string that represent the Product Pages release for major releases "
"(e.g. 8.0.0). After the first match, the rest will be ignored."
}
}
@@ -838,3 +859,42 @@ class Config(object):
s, ", ".join(SUPPORTED_RESOLVERS.keys()))
)
self._resolver = s
def _setifok_product_pages_module_streams(self, d):
if not isinstance(d, dict):
raise ValueError("PRODUCT_PAGES_MODULE_STREAMS must be a dict")
for regex, values in d.items():
try:
re.compile(regex)
except (TypeError, re.error):
raise ValueError(
'The regex `%r` in the configuration "PRODUCT_PAGES_MODULE_STREAMS" is invalid'
% regex
)
if not isinstance(values, list) and not isinstance(values, tuple):
raise ValueError(
'The values in the configured dictionary for "PRODUCT_PAGES_MODULE_STREAMS" '
"must be a list or tuple"
)
if len(values) != 3:
raise ValueError(
"There must be three entries in each value in the dictionary configured for "
'"PRODUCT_PAGES_MODULE_STREAMS"'
)
for i, value in enumerate(values):
if not isinstance(value, string_types):
# The last value is optional
if value is None and i == 2:
continue
raise ValueError(
'The value in the %i index of the values in "PRODUCT_PAGES_MODULE_STREAMS" '
"must be a string"
% i
)
self._product_pages_module_streams = d

120
module_build_service/utils/submit.py
View File

@@ -753,6 +753,125 @@ def resolve_base_module_virtual_streams(name, streams):
return new_streams
def _process_support_streams(mmd, params):
"""
Check if any buildrequired base modules require a support stream suffix.
This checks the Red Hat Product Pages to see if the buildrequired base module stream has been
released, if yes, then add the appropriate stream suffix.
:param Modulemd.ModuleStream mmd: the modulemd to apply the overrides on
:param dict params: the API parameters passed in by the user
"""
config_msg = (
'Skipping the release date checks for adding a stream suffix since "%s" '
"is not configured"
)
if not conf.product_pages_url:
log.debug(config_msg, "product_pages_url")
return
elif not conf.product_pages_module_streams:
log.debug(config_msg, "product_pages_module_streams")
return
buildrequire_overrides = params.get("buildrequire_overrides", {})
def new_streams_func(name, streams):
if name not in conf.base_module_names:
log.debug("The module %s is not a base module. Skipping the release date check.", name)
return streams
elif name in buildrequire_overrides:
log.debug(
"The module %s is a buildrequire override. Skipping the release date check.", name)
return streams
new_streams = copy.deepcopy(streams)
for i, stream in enumerate(streams):
for regex, values in conf.product_pages_module_streams.items():
if re.match(regex, stream):
log.debug(
'The regex `%s` from the configuration "product_pages_module_streams" '
"matched the stream %s",
regex, stream,
)
stream_suffix, pp_release_template, pp_major_release_template = values
break
else:
log.debug(
'No regexes in the configuration "product_pages_module_streams" matched the '
"stream %s. Skipping the release date check for this stream.",
stream,
)
continue
if stream.endswith(stream_suffix):
log.debug(
'The stream %s already contains the stream suffix of "%s". Skipping the '
"release date check.",
stream, stream_suffix
)
continue
stream_version = models.ModuleBuild.get_stream_version(stream)
if not stream_version:
log.debug("A stream version couldn't be parsed from %s", stream)
continue
# Convert the stream_version float to an int to make the math below deal with only
# integers
stream_version_int = int(stream_version)
# For example 80000 => 8
x = stream_version_int // 10000
# For example 80100 => 1
y = (stream_version_int - x * 10000) // 100
# For example 80104 => 4
z = stream_version_int - x * 10000 - y * 100
# Check if the stream version is x.0.0
if stream_version_int % 10000 == 0 and pp_major_release_template:
# For example, el8.0.0 => rhel-8-0
pp_release = pp_major_release_template.format(x=x, y=y, z=z)
else:
# For example el8.0.1 => rhel-8-0.1
pp_release = pp_release_template.format(x=x, y=y, z=z)
url = "{}/api/v7/releases/{}/?fields=ga_date".format(
conf.product_pages_url.rstrip("/"), pp_release)
try:
pp_rv = requests.get(url, timeout=15)
pp_json = pp_rv.json()
# Catch requests failures and JSON parsing errors
except (requests.exceptions.RequestException, ValueError):
log.exception(
"The query to the Product Pages at %s failed. Assuming it is not yet released.",
url,
)
continue
ga_date = pp_json.get("ga_date")
if not ga_date:
log.debug("A release date for the release %s could not be determined", pp_release)
continue
if datetime.strptime(ga_date, '%Y-%m-%d') > datetime.utcnow():
log.debug(
"The release %s hasn't been released yet. Not adding a stream suffix.",
ga_date
)
continue
new_stream = stream + stream_suffix
log.info(
'Replacing the buildrequire "%s:%s" with "%s:%s", since the stream is released',
name, stream, name, new_stream
)
new_streams[i] = new_stream
return new_streams
_modify_buildtime_streams(mmd, new_streams_func)
def submit_module_build(username, mmd, params):
"""
Submits new module build.
@@ -786,6 +905,7 @@ def submit_module_build(username, mmd, params):
default_streams = params["default_streams"]
_apply_dep_overrides(mmd, params)
_modify_buildtime_streams(mmd, resolve_base_module_virtual_streams)
_process_support_streams(mmd, params)
mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams)
if not mmds:

37
tests/staged_data/testmodule_el821.yaml Normal file
View File

@@ -0,0 +1,37 @@
document: modulemd
version: 1
data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:
buildrequires:
platform: el8.2.1
requires:
platform: el8.0.0
references:
community: https://docs.pagure.org/modularity/
documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
profiles:
default:
rpms:
- tangerine
api:
rpms:
- perl-Tangerine
- tangerine
components:
rpms:
perl-List-Compare:
rationale: A dependency of tangerine.
ref: master
perl-Tangerine:
rationale: Provides API for this module and is a dependency of tangerine.
ref: master
tangerine:
rationale: Provides API for this module.
buildorder: 10
ref: master

148
tests/test_views/test_views.py
View File

@@ -2436,3 +2436,151 @@ class TestViews:
dep = mmd.get_dependencies()[0]
assert dep.get_buildtime_streams("platform") == ["el8.25.0"]
assert dep.get_runtime_streams("platform") == ["el8"]
@pytest.mark.parametrize(
"pp_url, pp_streams, get_rv, br_stream, br_override, expected_stream",
(
# Test a stream of a major release
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"ga_date": "2019-05-07"},
"el8.0.0",
{},
"el8.0.0z",
),
# Test when the releases GA date is far in the future
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"ga_date": "2099-10-30"},
"el8.0.0",
{},
"el8.0.0",
),
# Test when product_pages_url isn't set
(
"",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"ga_date": "2019-05-07"},
"el8.0.0",
{},
"el8.0.0",
),
# Test when the release isn't found in Product Pages
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"detail": "Not found."},
"el8.0.0",
{},
"el8.0.0",
),
# Test when a non-major release stream
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"ga_date": "2019-05-07"},
"el8.2.1",
{},
"el8.2.1z",
),
# Test that when buildrequire overrides is set for platform, nothing changes
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}.{z}", "rhel-{x}-{y}")},
{"ga_date": "2019-05-07"},
"el8.0.0",
{"platform": ["el8.0.0"]},
"el8.0.0",
),
# Test when product_pages_module_streams is not set
(
"https://pp.domain.local/pp/",
{},
{"ga_date": "2019-05-07"},
"el8.0.0",
{},
"el8.0.0",
),
# Test when there is no stream that matches the configured regexes
(
"https://pp.domain.local/pp/",
{r"js.+": ("z", "js-{x}-{y}", "js-{x}-{y}")},
{"ga_date": "2019-05-07"},
"el8.0.0",
{},
"el8.0.0",
),
# Test when there is no configured special Product Pages template for major releases
(
"https://pp.domain.local/pp/",
{r"el.+": ("z", "rhel-{x}-{y}", None)},
{"ga_date": "2019-05-07"},
"el8.0.0",
{},
"el8.0.0z",
),
),
)
@patch(
"module_build_service.config.Config.product_pages_url",
new_callable=PropertyMock,
)
@patch(
"module_build_service.config.Config.product_pages_module_streams",
new_callable=PropertyMock,
)
@patch("requests.get")
@patch("module_build_service.auth.get_user", return_value=user)
@patch("module_build_service.scm.SCM")
def test_submit_build_automatic_z_stream_detection(
self, mocked_scm, mocked_get_user, mock_get, mock_pp_streams, mock_pp_url, pp_url,
pp_streams, get_rv, br_stream, br_override, expected_stream,
):
# Configure the Product Pages URL
mock_pp_url.return_value = pp_url
mock_pp_streams.return_value = pp_streams
# Mock the Product Pages query
mock_get.return_value.json.return_value = get_rv
mmd = load_mmd_file(path.join(base_dir, "staged_data", "platform.yaml"))
# Create the required platforms
for stream in ("el8.0.0", "el8.0.0z", "el8.2.1", "el8.2.1z"):
mmd = mmd.copy(mmd.get_module_name(), stream)
import_mmd(db.session, mmd)
# Use a testmodule that buildrequires platform:el8.0.0 or platform:el8.2.1
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_{}.yaml".format(br_stream.replace(".", "")),
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
payload = {"branch": "master", "scmurl": scm_url}
if br_override:
payload["buildrequire_overrides"] = br_override
rv = self.client.post(post_url, json=payload)
data = json.loads(rv.data)
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
assert dep.get_buildtime_streams("platform") == [expected_stream]
# The runtime stream suffix should remain unchanged
assert dep.get_runtime_streams("platform") == ["el8.0.0"]
if pp_url and not br_override and pp_streams.get(r"el.+"):
if br_stream == "el8.0.0":
pp_release = "rhel-8-0"
else:
pp_release = "rhel-8-2.1"
expected_url = "{}api/v7/releases/{}/?fields=ga_date".format(pp_url, pp_release)
mock_get.assert_called_once_with(expected_url, timeout=15)
else:
mock_get.assert_not_called()