modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-13 10:09:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Split utils/general.py

Browse Source 
This puts backend specific code in either the builder or scheduler
subpackage. This puts API specific code in the new web subpackage.
Lastly, any code shared between the API and backend is placed in the
common subpackage.
This commit is contained in:
mprahl
2020-01-03 09:11:44 -05:00
parent 19e3abe84c
commit 5e401bd598
41 changed files with 1373 additions and 1353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

449
tests/test_builder/test_builder_utils.py
View File

@@ -3,122 +3,351 @@
import tempfile
import shutil
from mock import patch, Mock, call
from mock import call, MagicMock, Mock, patch, PropertyMock
import pytest
from module_build_service import conf
from module_build_service import conf, models
from module_build_service.builder import utils
from module_build_service.db_session import db_session
from module_build_service.errors import ProgrammingError, ValidationError
from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str
from tests import init_data, read_staged_data, scheduler_init_data
class TestBuilderUtils:
@patch("requests.get")
@patch("koji.ClientSession")
@patch("module_build_service.builder.utils.execute_cmd")
def test_create_local_repo_from_koji_tag(self, mock_exec_cmd, mock_koji_session, mock_get):
session = Mock()
rpms = [
{
"arch": "src",
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"size": 6890,
"version": "0.1",
},
{
"arch": "noarch",
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"size": 6890,
"version": "0.1",
},
{
"arch": "x86_64",
"build_id": 875636,
"name": "ed-debuginfo",
"release": "2.module_bd6e0eb1",
"size": 81438,
"version": "1.14.1",
},
{
"arch": "x86_64",
"build_id": 875636,
"name": "ed",
"release": "2.module_bd6e0eb1",
"size": 80438,
"version": "1.14.1",
},
{
"arch": "x86_64",
"build_id": 875640,
"name": "mksh-debuginfo",
"release": "2.module_bd6e0eb1",
"size": 578774,
"version": "54",
},
{
"arch": "x86_64",
"build_id": 875640,
"name": "mksh",
"release": "2.module_bd6e0eb1",
"size": 267042,
"version": "54",
},
]
@patch("requests.get")
@patch("koji.ClientSession")
@patch("module_build_service.builder.utils.execute_cmd")
def test_create_local_repo_from_koji_tag(mock_exec_cmd, mock_koji_session, mock_get):
session = Mock()
rpms = [
{
"arch": "src",
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"size": 6890,
"version": "0.1",
},
{
"arch": "noarch",
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"size": 6890,
"version": "0.1",
},
{
"arch": "x86_64",
"build_id": 875636,
"name": "ed-debuginfo",
"release": "2.module_bd6e0eb1",
"size": 81438,
"version": "1.14.1",
},
{
"arch": "x86_64",
"build_id": 875636,
"name": "ed",
"release": "2.module_bd6e0eb1",
"size": 80438,
"version": "1.14.1",
},
{
"arch": "x86_64",
"build_id": 875640,
"name": "mksh-debuginfo",
"release": "2.module_bd6e0eb1",
"size": 578774,
"version": "54",
},
{
"arch": "x86_64",
"build_id": 875640,
"name": "mksh",
"release": "2.module_bd6e0eb1",
"size": 267042,
"version": "54",
},
]
builds = [
{
"build_id": 875640,
"name": "mksh",
"release": "2.module_bd6e0eb1",
"version": "54",
"volume_name": "prod",
},
{
"build_id": 875636,
"name": "ed",
"release": "2.module_bd6e0eb1",
"version": "1.14.1",
"volume_name": "prod",
},
{
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"version": "0.1",
"volume_name": "prod",
},
]
builds = [
{
"build_id": 875640,
"name": "mksh",
"release": "2.module_bd6e0eb1",
"version": "54",
"volume_name": "prod",
},
{
"build_id": 875636,
"name": "ed",
"release": "2.module_bd6e0eb1",
"version": "1.14.1",
"volume_name": "prod",
},
{
"build_id": 875991,
"name": "module-build-macros",
"release": "1.module_92011fe6",
"version": "0.1",
"volume_name": "prod",
},
]
session.listTaggedRPMS.return_value = (rpms, builds)
session.opts = {"topurl": "https://kojipkgs.stg.fedoraproject.org/"}
mock_koji_session.return_value = session
session.listTaggedRPMS.return_value = (rpms, builds)
session.opts = {"topurl": "https://kojipkgs.stg.fedoraproject.org/"}
mock_koji_session.return_value = session
tag = "module-testmodule-master-20170405123740-build"
temp_dir = tempfile.mkdtemp()
try:
utils.create_local_repo_from_koji_tag(conf, tag, temp_dir)
finally:
shutil.rmtree(temp_dir)
tag = "module-testmodule-master-20170405123740-build"
temp_dir = tempfile.mkdtemp()
try:
utils.create_local_repo_from_koji_tag(conf, tag, temp_dir)
finally:
shutil.rmtree(temp_dir)
url_one = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/module-build-macros/"
"0.1/1.module_92011fe6/noarch/module-build-macros-0.1-1.module_92011fe6.noarch.rpm"
)
url_two = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/ed/1.14.1/"
"2.module_bd6e0eb1/x86_64/ed-1.14.1-2.module_bd6e0eb1.x86_64.rpm"
)
url_three = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/mksh/54/"
"2.module_bd6e0eb1/x86_64/mksh-54-2.module_bd6e0eb1.x86_64.rpm"
)
url_one = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/module-build-macros/"
"0.1/1.module_92011fe6/noarch/module-build-macros-0.1-1.module_92011fe6.noarch.rpm"
)
url_two = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/ed/1.14.1/"
"2.module_bd6e0eb1/x86_64/ed-1.14.1-2.module_bd6e0eb1.x86_64.rpm"
)
url_three = (
"https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/mksh/54/"
"2.module_bd6e0eb1/x86_64/mksh-54-2.module_bd6e0eb1.x86_64.rpm"
)
expected_calls = [
call(url_one, stream=True, timeout=60),
call(url_two, stream=True, timeout=60),
call(url_three, stream=True, timeout=60),
]
for expected_call in expected_calls:
assert expected_call in mock_get.call_args_list
assert len(mock_get.call_args_list) == len(expected_calls)
expected_calls = [
call(url_one, stream=True, timeout=60),
call(url_two, stream=True, timeout=60),
call(url_three, stream=True, timeout=60),
]
for expected_call in expected_calls:
assert expected_call in mock_get.call_args_list
assert len(mock_get.call_args_list) == len(expected_calls)
def test_validate_koji_tag_wrong_tag_arg_during_programming():
""" Test that we fail on a wrong param name (non-existing one) due to
programming error. """
@utils.validate_koji_tag("wrong_tag_arg")
def validate_koji_tag_programming_error(good_tag_arg, other_arg):
pass
with pytest.raises(ProgrammingError):
validate_koji_tag_programming_error("dummy", "other_val")
def test_validate_koji_tag_bad_tag_value():
""" Test that we fail on a bad tag value. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_bad_tag_value(tag_arg):
pass
with pytest.raises(ValidationError):
validate_koji_tag_bad_tag_value("forbiddentagprefix-foo")
def test_validate_koji_tag_bad_tag_value_in_list():
""" Test that we fail on a list containing bad tag value. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_bad_tag_value_in_list(tag_arg):
pass
with pytest.raises(ValidationError):
validate_koji_tag_bad_tag_value_in_list(["module-foo", "forbiddentagprefix-bar"])
def test_validate_koji_tag_good_tag_value():
""" Test that we pass on a good tag value. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_good_tag_value(tag_arg):
return True
assert validate_koji_tag_good_tag_value("module-foo") is True
def test_validate_koji_tag_good_tag_values_in_list():
""" Test that we pass on a list of good tag values. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_good_tag_values_in_list(tag_arg):
return True
assert validate_koji_tag_good_tag_values_in_list(["module-foo", "module-bar"]) is True
def test_validate_koji_tag_good_tag_value_in_dict():
""" Test that we pass on a dict arg with default key
and a good value. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_good_tag_value_in_dict(tag_arg):
return True
assert validate_koji_tag_good_tag_value_in_dict({"name": "module-foo"}) is True
def test_validate_koji_tag_good_tag_value_in_dict_nondefault_key():
""" Test that we pass on a dict arg with non-default key
and a good value. """
@utils.validate_koji_tag("tag_arg", dict_key="nondefault")
def validate_koji_tag_good_tag_value_in_dict_nondefault_key(tag_arg):
return True
assert (
validate_koji_tag_good_tag_value_in_dict_nondefault_key({"nondefault": "module-foo"})
is True
)
def test_validate_koji_tag_double_trouble_good():
""" Test that we pass on a list of tags that are good. """
expected = "foo"
@utils.validate_koji_tag(["tag_arg1", "tag_arg2"])
def validate_koji_tag_double_trouble(tag_arg1, tag_arg2):
return expected
actual = validate_koji_tag_double_trouble("module-1", "module-2")
assert actual == expected
def test_validate_koji_tag_double_trouble_bad():
""" Test that we fail on a list of tags that are bad. """
@utils.validate_koji_tag(["tag_arg1", "tag_arg2"])
def validate_koji_tag_double_trouble(tag_arg1, tag_arg2):
pass
with pytest.raises(ValidationError):
validate_koji_tag_double_trouble("module-1", "BADNEWS-2")
def test_validate_koji_tag_is_None():
""" Test that we fail on a tag which is None. """
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_is_None(tag_arg):
pass
with pytest.raises(ValidationError) as cm:
validate_koji_tag_is_None(None)
assert str(cm.value).endswith(" No value provided.") is True
@patch(
"module_build_service.config.Config.allowed_privileged_module_names",
new_callable=PropertyMock,
return_value=["testmodule"],
)
def test_validate_koji_tag_previleged_module_name(conf_apmn):
@utils.validate_koji_tag("tag_arg")
def validate_koji_tag_priv_mod_name(self, tag_arg):
pass
builder = MagicMock()
builder.module_str = 'testmodule'
validate_koji_tag_priv_mod_name(builder, "abc")
def test_get_rpm_release_mse():
init_data(contexts=True)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release_one = utils.get_rpm_release(db_session, build_one)
assert release_one == "module+2+b8645bbb"
build_two = models.ModuleBuild.get_by_id(db_session, 3)
release_two = utils.get_rpm_release(db_session, build_two)
assert release_two == "module+2+17e35784"
def test_get_rpm_release_platform_stream():
scheduler_init_data(1)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+f28+2+814cfa39"
def test_get_rpm_release_platform_stream_override():
scheduler_init_data(1)
# Set the disttag_marking override on the platform
platform = (
db_session.query(models.ModuleBuild)
.filter_by(name="platform", stream="f28")
.first()
)
platform_mmd = platform.mmd()
platform_xmd = platform_mmd.get_xmd()
platform_xmd["mbs"]["disttag_marking"] = "fedora28"
platform_mmd.set_xmd(platform_xmd)
platform.modulemd = mmd_to_str(platform_mmd)
db_session.add(platform)
db_session.commit()
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+fedora28+2+814cfa39"
@patch(
"module_build_service.config.Config.allowed_privileged_module_names",
new_callable=PropertyMock,
return_value=["build"],
)
def test_get_rpm_release_metadata_br_stream_override(mock_admmn):
"""
Test that when a module buildrequires a module in conf.allowed_privileged_module_names,
and that module has the xmd.mbs.disttag_marking field set, it should influence the disttag.
"""
scheduler_init_data(1)
metadata_mmd = load_mmd(read_staged_data("build_metadata_module"))
import_mmd(db_session, metadata_mmd)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
mmd = build_one.mmd()
deps = mmd.get_dependencies()[0]
deps.add_buildtime_stream("build", "product1.2")
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"]["build"] = {
"filtered_rpms": [],
"ref": "virtual",
"stream": "product1.2",
"version": "1",
"context": "00000000",
}
mmd.set_xmd(xmd)
build_one.modulemd = mmd_to_str(mmd)
db_session.add(build_one)
db_session.commit()
release = utils.get_rpm_release(db_session, build_one)
assert release == "module+product12+2+814cfa39"
def test_get_rpm_release_mse_scratch():
init_data(contexts=True, scratch=True)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release_one = utils.get_rpm_release(db_session, build_one)
assert release_one == "scrmod+2+b8645bbb"
build_two = models.ModuleBuild.get_by_id(db_session, 3)
release_two = utils.get_rpm_release(db_session, build_two)
assert release_two == "scrmod+2+17e35784"
def test_get_rpm_release_platform_stream_scratch():
scheduler_init_data(1, scratch=True)
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release = utils.get_rpm_release(db_session, build_one)
assert release == "scrmod+f28+2+814cfa39"

20
tests/test_builder/test_koji.py
View File

@@ -14,11 +14,11 @@ import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
import module_build_service.models
from module_build_service import conf, Modulemd
from module_build_service.common.utils import mmd_to_str
from module_build_service.db_session import db_session
from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
from module_build_service.builder import GenericBuilder
from module_build_service.scheduler import events
from module_build_service.utils.general import mmd_to_str
from tests import init_data, clean_database, make_module_in_db
@@ -1030,3 +1030,21 @@ class TestGetDistTagSRPM:
# Conflicting ursine RPMs
for nevr in ["pizza-0:4.0-1.fc32", "spaghetti-0:3.0-1.fc32"]:
assert KojiModuleBuilder.format_conflicts_line(nevr) + "\n" in content
def test_generate_koji_tag_in_nsvc_format():
name, stream, version, context = ("testmodule", "master", "20170816080815", "37c6c57")
tag = KojiModuleBuilder.generate_koji_tag(name, stream, version, context)
assert tag == "module-testmodule-master-20170816080815-37c6c57"
def test_generate_koji_tag_in_hash_format():
name, version, context = ("testmodule", "20170816080815", "37c6c57")
stream = "this-is-a-stream-with-very-looooong-name" + "-blah" * 50
nsvc_list = [name, stream, version, context]
tag = KojiModuleBuilder.generate_koji_tag(*nsvc_list)
expected_tag = "module-1cf457d452e54dda"
assert tag == expected_tag

78
tests/test_builder/test_mock.py
View File

@@ -1,19 +1,21 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
import os
import mock
import koji
import tempfile
import shutil
from textwrap import dedent
import mock
import kobo.rpmlib
import koji
from module_build_service import conf
from module_build_service import conf, models
from module_build_service.common.utils import load_mmd, mmd_to_str
from module_build_service.db_session import db_session
from module_build_service.models import ModuleBuild, ComponentBuild
from module_build_service.builder.MockModuleBuilder import MockModuleBuilder
from module_build_service.utils import import_fake_base_module, mmd_to_str, load_mmd
from module_build_service.builder.MockModuleBuilder import (
import_fake_base_module, import_builds_from_local_dnf_repos, MockModuleBuilder,
)
from tests import clean_database, make_module_in_db, read_staged_data
@@ -232,3 +234,69 @@ class TestMockModuleBuilderAddRepos:
assert "repofile 3" in builder.yum_conf
assert set(builder.enabled_modules) == {"foo:1", "app:1"}
class TestOfflineLocalBuilds:
def setup_method(self):
clean_database()
def teardown_method(self):
clean_database()
def test_import_fake_base_module(self):
import_fake_base_module("platform:foo:1:000000")
module_build = models.ModuleBuild.get_build_from_nsvc(
db_session, "platform", "foo", 1, "000000")
assert module_build
mmd = module_build.mmd()
xmd = mmd.get_xmd()
assert xmd == {
"mbs": {
"buildrequires": {},
"commit": "ref_000000",
"koji_tag": "repofile://",
"mse": "true",
"requires": {},
}
}
assert set(mmd.get_profile_names()) == {"buildroot", "srpm-buildroot"}
@mock.patch(
"module_build_service.builder.MockModuleBuilder.open",
create=True,
new_callable=mock.mock_open,
)
def test_import_builds_from_local_dnf_repos(self, patched_open):
with mock.patch("dnf.Base") as dnf_base:
repo = mock.MagicMock()
repo.repofile = "/etc/yum.repos.d/foo.repo"
mmd = load_mmd(read_staged_data("formatted_testmodule"))
repo.get_metadata_content.return_value = mmd_to_str(mmd)
base = dnf_base.return_value
base.repos = {"reponame": repo}
patched_open.return_value.readlines.return_value = ("FOO=bar", "PLATFORM_ID=platform:x")
import_builds_from_local_dnf_repos()
base.read_all_repos.assert_called_once()
repo.load.assert_called_once()
repo.get_metadata_content.assert_called_once_with("modules")
module_build = models.ModuleBuild.get_build_from_nsvc(
db_session, "testmodule", "master", 20180205135154, "9c690d0e")
assert module_build
assert module_build.koji_tag == "repofile:///etc/yum.repos.d/foo.repo"
module_build = models.ModuleBuild.get_build_from_nsvc(
db_session, "platform", "x", 1, "000000")
assert module_build
def test_import_builds_from_local_dnf_repos_platform_id(self):
with mock.patch("dnf.Base"):
import_builds_from_local_dnf_repos("platform:y")
module_build = models.ModuleBuild.get_build_from_nsvc(
db_session, "platform", "y", 1, "000000")
assert module_build