JIRA: RHELBLD-257,RHELBLD-310 - refactor clean_database

Replace clean_database calls with cheaper truncate operation.
Remove duplicate calls of clean_database.
Extract clean_database/init_data into fixtures.
This commit is contained in:
jobrauer
2020-05-15 15:38:05 +02:00
parent 53351670e3
commit c584d84b76
27 changed files with 730 additions and 693 deletions

View File

@@ -98,6 +98,32 @@ def patch_zeromq_time_sleep():
patch_zeromq_time_sleep()
def truncate_tables():
"""Much cheaper operation (up to 2/3 faster) than clean_database (DROP/CREATE)"""
db_session.remove()
db_session.configure(bind=db.session.get_bind())
meta = db.metadata
for table in reversed(meta.sorted_tables):
db_session.execute(table.delete())
if db_session.bind.dialect.name == "postgresql":
# POSTGRES ONLY (!)
# Tests reference test data by IDs, assuming they always start from 1.
# In psql, sequences are created for models' IDs - they need to be reset.
sequences = ["component_builds_id_seq",
"component_builds_trace_id_seq",
"log_messages_id_seq",
"module_arches_id_seq",
"module_builds_id_seq",
"module_builds_trace_id_seq",
"virtual_streams_id_seq"]
sql_cmds = ["alter sequence {} restart with 1;".format(s) for s in sequences]
db_session.execute("".join(sql_cmds))
db_session.commit()
def clean_database(add_platform_module=True, add_default_arches=True):
"""Initialize the test database
@@ -139,8 +165,9 @@ def init_data(data_size=10, contexts=False, multiple_stream_versions=None, scrat
:param list/bool multiple_stream_versions: If true, multiple base modules with
difference stream versions are generated. If set to list, the list defines
the generated base module streams.
(!) This method is not responsible for cleaning the database, use appropriate fixture.
"""
clean_database()
if multiple_stream_versions:
if multiple_stream_versions is True:

View File

@@ -11,9 +11,15 @@ import pytest
import module_build_service
from module_build_service.builder.utils import get_rpm_release
from module_build_service.common.models import BUILD_STATES
from module_build_service.common.utils import load_mmd, mmd_to_str
from module_build_service.common.utils import load_mmd, mmd_to_str, import_mmd
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, read_staged_data, module_build_from_modulemd
from tests import (
clean_database,
init_data,
truncate_tables,
read_staged_data,
module_build_from_modulemd
)
BASE_DIR = os.path.dirname(__file__)
STAGED_DATA_DIR = os.path.join(BASE_DIR, "staged_data")
@@ -50,7 +56,57 @@ def platform_mmd():
@pytest.fixture()
def model_tests_init_data():
def require_empty_database():
"""Provides cleared database"""
truncate_tables()
@pytest.fixture()
def require_platform_and_default_arch(require_empty_database):
"""Provides clean database with platform module and a default arch"""
arch_obj = module_build_service.common.models.ModuleArch(name="x86_64")
db_session.add(arch_obj)
db_session.commit()
mmd = load_mmd(read_staged_data("platform"))
import_mmd(db_session, mmd)
@pytest.fixture()
def provide_test_data(request, require_platform_and_default_arch):
"""Provides clean database with fresh test data based on supplied params:
e.g.: @pytest.mark.parametrize("provide_test_data",
[{"data_size": 1, "contexts": True, "multiple_stream_versions": True}], indirect=True)
This is a fixture version of tests.init_data function.
"""
size = 1
contexts = False
scratch = False
multiple_stream_versions = False
if hasattr(request, "param") and type(request.param) is dict:
if "data_size" in request.param:
size = request.param.get("data_size")
if "contexts" in request.param:
contexts = True
if "multiple_stream_versions" in request.param:
multiple_stream_versions = True
if "scratch" in request.param:
scratch = True
init_data(data_size=size, contexts=contexts,
multiple_stream_versions=multiple_stream_versions,
scratch=scratch)
@pytest.fixture(scope="class")
def provide_test_client(request):
"""Inject REST client into the test class -> self.client"""
request.cls.client = module_build_service.app.test_client()
@pytest.fixture()
def model_tests_init_data(require_platform_and_default_arch):
"""Initialize data for model tests
This is refactored from tests/test_models/__init__.py, which was able to be
@@ -60,7 +116,6 @@ def model_tests_init_data():
rather than create a new one. That would also benefit the whole test suite
to reduce the number of SQLAlchemy session objects.
"""
clean_database()
model_test_data_dir = os.path.join(
os.path.dirname(__file__), "test_common", "test_models", "data"
@@ -76,9 +131,7 @@ def model_tests_init_data():
@pytest.fixture()
def reuse_component_init_data():
clean_database()
def reuse_component_init_data(require_platform_and_default_arch):
mmd = load_mmd(read_staged_data("formatted_testmodule"))
build_one = module_build_service.common.models.ModuleBuild(
@@ -397,3 +450,9 @@ def cleanup_build_logs(request):
module_build_service.common.build_logs.stop(mock_build)
request.addfinalizer(_cleanup_build_logs)
@pytest.fixture(autouse=True, scope="session")
def create_database(request):
"""Drop and recreate all tables"""
clean_database(add_platform_module=False, add_default_arches=False)

View File

@@ -38,7 +38,7 @@ from module_build_service.scheduler.handlers.components import (
import module_build_service.scheduler.handlers.repos
from module_build_service.scheduler.handlers.repos import done as repos_done_handler
from module_build_service.scheduler.handlers.tags import tagged as tagged_handler
from tests import clean_database, read_staged_data, staged_data_filename
from tests import read_staged_data, staged_data_filename
base_dir = dirname(dirname(__file__))
@@ -440,6 +440,7 @@ class BaseTestBuild:
},
},
)
@pytest.mark.usefixtures("require_platform_and_default_arch")
class TestBuild(BaseTestBuild):
# Global variable used for tests if needed
_global_var = None
@@ -447,7 +448,6 @@ class TestBuild(BaseTestBuild):
def setup_method(self, test_method):
GenericBuilder.register_backend_class(FakeModuleBuilder)
self.client = app.test_client()
clean_database()
def on_get_task_info_cb(cls, task_id):
return {"state": koji.TASK_STATES["CLOSED"]}
@@ -1880,13 +1880,13 @@ class TestBuild(BaseTestBuild):
new_callable=PropertyMock,
return_value="testlocal",
)
@pytest.mark.usefixtures("require_empty_database")
class TestLocalBuild(BaseTestBuild):
def setup_method(self, test_method):
FakeModuleBuilder.on_build_cb = None
FakeModuleBuilder.backend = "testlocal"
GenericBuilder.register_backend_class(FakeModuleBuilder)
self.client = app.test_client()
clean_database()
def teardown_method(self, test_method):
FakeModuleBuilder.reset()

View File

@@ -3,6 +3,7 @@
from __future__ import absolute_import
import mock
import pytest
from mock import patch
import module_build_service.builder
@@ -10,12 +11,10 @@ from module_build_service.builder import GenericBuilder
import module_build_service.common.models
import module_build_service.resolver
from module_build_service.scheduler.db_session import db_session
from tests import init_data
@pytest.mark.usefixtures("provide_test_data")
class TestGenericBuilder:
def setup_method(self, test_method):
init_data(1)
@patch("module_build_service.resolver.DBResolver")
@patch("module_build_service.builder.base.GenericResolver")

View File

@@ -13,7 +13,7 @@ from module_build_service.common.config import conf
from module_build_service.common.errors import ProgrammingError, ValidationError
from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str
from module_build_service.scheduler.db_session import db_session
from tests import init_data, read_staged_data, scheduler_init_data
from tests import read_staged_data, scheduler_init_data
@patch("requests.get")
@@ -260,9 +260,8 @@ def test_validate_koji_tag_previleged_module_name(conf_apmn):
validate_koji_tag_priv_mod_name(builder, "abc")
def test_get_rpm_release_mse():
init_data(contexts=True)
@pytest.mark.parametrize("provide_test_data", [{"contexts": True}], indirect=True)
def test_get_rpm_release_mse(provide_test_data):
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release_one = utils.get_rpm_release(db_session, build_one)
assert release_one == "module+2+b8645bbb"
@@ -336,9 +335,9 @@ def test_get_rpm_release_metadata_br_stream_override(mock_admmn):
assert release == "module+product12+2+814cfa39"
def test_get_rpm_release_mse_scratch():
init_data(contexts=True, scratch=True)
@pytest.mark.parametrize("provide_test_data",
[{"contexts": True, "scratch": True}], indirect=True)
def test_get_rpm_release_mse_scratch(provide_test_data):
build_one = models.ModuleBuild.get_by_id(db_session, 2)
release_one = utils.get_rpm_release(db_session, build_one)
assert release_one == "scrmod+2+b8645bbb"

View File

@@ -29,47 +29,53 @@ GET_USER_RV = {
}
# setup/teardown converted to a fixture -> reuse existing fixture hierarchy
@pytest.fixture()
def test_content_generator_fixture(request, require_platform_and_default_arch):
init_data(1, contexts=True)
module = models.ModuleBuild.get_by_id(db_session, 2)
module.cg_build_koji_tag = "f27-module-candidate"
cg = KojiContentGenerator(module, conf)
p_read_config = patch(
"koji.read_config",
return_value={
"authtype": "kerberos",
"timeout": 60,
"server": "http://koji.example.com/",
},
)
mock_read_config = p_read_config.start()
# Ensure that there is no build log from other tests
try:
file_path = build_logs.path(db_session, cg.module)
os.remove(file_path)
except OSError:
pass
request.cls.cg = cg
request.cls.mock_read_config = mock_read_config
yield
p_read_config.stop()
# Necessary to restart the twisted reactor for the next test.
import sys
for mod in ("twisted.internet.reactor", "moksha.hub.reactor", "moksha.hub"):
if mod in sys.modules:
del sys.modules[mod]
import moksha.hub.reactor # noqa
try:
file_path = build_logs.path(db_session, cg.module)
os.remove(file_path)
except OSError:
pass
@pytest.mark.usefixtures("test_content_generator_fixture")
class TestBuild:
def setup_method(self, test_method):
init_data(1, contexts=True)
module = models.ModuleBuild.get_by_id(db_session, 2)
module.cg_build_koji_tag = "f27-module-candidate"
self.cg = KojiContentGenerator(module, conf)
self.p_read_config = patch(
"koji.read_config",
return_value={
"authtype": "kerberos",
"timeout": 60,
"server": "http://koji.example.com/",
},
)
self.mock_read_config = self.p_read_config.start()
# Ensure that there is no build log from other tests
try:
file_path = build_logs.path(db_session, self.cg.module)
os.remove(file_path)
except OSError:
pass
def teardown_method(self, test_method):
self.p_read_config.stop()
# Necessary to restart the twisted reactor for the next test.
import sys
for mod in ("twisted.internet.reactor", "moksha.hub.reactor", "moksha.hub"):
if mod in sys.modules:
del sys.modules[mod]
import moksha.hub.reactor # noqa
try:
file_path = build_logs.path(db_session, self.cg.module)
os.remove(file_path)
except OSError:
pass
@patch("koji.ClientSession")
@patch("subprocess.Popen")

View File

@@ -20,7 +20,7 @@ from module_build_service.common.utils import mmd_to_str
from module_build_service.scheduler import events
from module_build_service.scheduler.db_session import db_session
import module_build_service.scheduler.handlers.repos
from tests import init_data, clean_database, make_module_in_db
from tests import init_data, make_module_in_db
@pytest.fixture(scope="function")
@@ -85,27 +85,32 @@ class FakeKojiModuleBuilder(KojiModuleBuilder):
return ["x86_64"]
# setup/teardown converted to a fixture -> reuse existing fixture hierarchy
@pytest.fixture()
def koji_builder_fixture(request, require_platform_and_default_arch):
init_data(data_size=1)
events.scheduler.reset()
config = mock.Mock()
config.koji_profile = conf.koji_profile
config.koji_repository_url = conf.koji_repository_url
p_read_config = patch(
"koji.read_config",
return_value={
"authtype": "kerberos",
"timeout": 60,
"server": "http://koji.example.com/",
},
)
p_read_config.start()
request.cls.config = config
yield
p_read_config.stop()
events.scheduler.reset()
@pytest.mark.usefixtures("koji_builder_fixture")
class TestKojiBuilder:
def setup_method(self, test_method):
init_data(1)
events.scheduler.reset()
self.config = mock.Mock()
self.config.koji_profile = conf.koji_profile
self.config.koji_repository_url = conf.koji_repository_url
self.p_read_config = patch(
"koji.read_config",
return_value={
"authtype": "kerberos",
"timeout": 60,
"server": "http://koji.example.com/",
},
)
self.p_read_config.start()
def teardown_method(self, test_method):
self.p_read_config.stop()
events.scheduler.reset()
@patch("koji.ClientSession")
def test_tag_to_repo(self, ClientSession):
@@ -777,95 +782,6 @@ class TestKojiBuilder:
assert set(ret) == {"bar-2:1.30-4.el8+1308+551bfa71", "tar-2:1.30-4.el8+1308+551bfa71"}
session.assert_not_called()
@pytest.mark.usefixtures("reuse_component_init_data")
@pytest.mark.parametrize(
"br_filtered_rpms,expected",
(
(
["perl-Tangerine-0.23-1.module+0+d027b723", "not-in-tag-5.0-1.module+0+d027b723"],
["not-in-tag-5.0-1.module+0+d027b723"],
),
(
[
"perl-Tangerine-0.23-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
],
[],
),
(
[
"perl-Tangerine-0.23-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
"perl-Tangerine-0.23-1.module+0+d027b723",
],
[],
),
(
[
"perl-Tangerine-0.23-1.module+0+diff_module",
"not-in-tag-5.0-1.module+0+d027b723",
],
[
"perl-Tangerine-0.23-1.module+0+diff_module",
"not-in-tag-5.0-1.module+0+d027b723",
],
),
([], []),
),
)
@patch("koji.ClientSession")
def test_get_filtered_rpms_on_self_dep(
self, ClientSession, br_filtered_rpms, expected
):
session = ClientSession.return_value
session.listTaggedRPMS.return_value = (
[
{
"build_id": 12345,
"epoch": None,
"name": "perl-Tangerine",
"release": "1.module+0+d027b723",
"version": "0.23",
},
{
"build_id": 23456,
"epoch": None,
"name": "perl-List-Compare",
"release": "5.module+0+d027b723",
"version": "0.53",
},
{
"build_id": 34567,
"epoch": None,
"name": "tangerine",
"release": "3.module+0+d027b723",
"version": "0.22",
},
],
[
{
"build_id": 12345,
"name": "perl-Tangerine",
"nvr": "perl-Tangerine-0.23-1.module+0+d027b723",
},
{
"build_id": 23456,
"name": "perl-List-Compare",
"nvr": "perl-List-Compare-0.53-5.module+0+d027b723",
},
{
"build_id": 34567,
"name": "tangerine",
"nvr": "tangerine-0.22-3.module+0+d027b723",
},
],
)
current_module = module_build_service.common.models.ModuleBuild.get_by_id(db_session, 3)
with patch.object(module_build_service.common.models.ModuleBuild, 'log_message'):
rv = KojiModuleBuilder._get_filtered_rpms_on_self_dep(current_module, br_filtered_rpms)
assert set(rv) == set(expected)
session.assert_not_called()
@pytest.mark.parametrize(
"cg_enabled,cg_devel_enabled", [(False, False), (True, False), (True, True)]
)
@@ -956,12 +872,99 @@ class TestKojiBuilder:
KojiModuleBuilder.get_module_build_arches(module_build)
@pytest.mark.parametrize(
"br_filtered_rpms,expected",
(
(
["perl-Tangerine-0.23-1.module+0+d027b723", "not-in-tag-5.0-1.module+0+d027b723"],
["not-in-tag-5.0-1.module+0+d027b723"],
),
(
[
"perl-Tangerine-0.23-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
],
[],
),
(
[
"perl-Tangerine-0.23-1.module+0+d027b723",
"perl-List-Compare-0.53-5.module+0+d027b723",
"perl-Tangerine-0.23-1.module+0+d027b723",
],
[],
),
(
[
"perl-Tangerine-0.23-1.module+0+diff_module",
"not-in-tag-5.0-1.module+0+d027b723",
],
[
"perl-Tangerine-0.23-1.module+0+diff_module",
"not-in-tag-5.0-1.module+0+d027b723",
],
),
([], []),
),
)
@patch("koji.ClientSession")
@pytest.mark.usefixtures("reuse_component_init_data")
def test_get_filtered_rpms_on_self_dep(ClientSession, br_filtered_rpms, expected):
session = ClientSession.return_value
session.listTaggedRPMS.return_value = (
[
{
"build_id": 12345,
"epoch": None,
"name": "perl-Tangerine",
"release": "1.module+0+d027b723",
"version": "0.23",
},
{
"build_id": 23456,
"epoch": None,
"name": "perl-List-Compare",
"release": "5.module+0+d027b723",
"version": "0.53",
},
{
"build_id": 34567,
"epoch": None,
"name": "tangerine",
"release": "3.module+0+d027b723",
"version": "0.22",
},
],
[
{
"build_id": 12345,
"name": "perl-Tangerine",
"nvr": "perl-Tangerine-0.23-1.module+0+d027b723",
},
{
"build_id": 23456,
"name": "perl-List-Compare",
"nvr": "perl-List-Compare-0.53-5.module+0+d027b723",
},
{
"build_id": 34567,
"name": "tangerine",
"nvr": "tangerine-0.22-3.module+0+d027b723",
},
],
)
current_module = module_build_service.common.models.ModuleBuild.get_by_id(db_session, 3)
with patch.object(module_build_service.common.models.ModuleBuild, 'log_message'):
rv = KojiModuleBuilder._get_filtered_rpms_on_self_dep(current_module, br_filtered_rpms)
assert set(rv) == set(expected)
session.assert_not_called()
@pytest.mark.usefixtures("require_empty_database")
class TestGetDistTagSRPM:
"""Test KojiModuleBuilder.get_disttag_srpm"""
def setup_method(self):
clean_database()
self.tmp_srpm_build_dir = tempfile.mkdtemp(prefix="test-koji-builder-")
self.spec_file = os.path.join(self.tmp_srpm_build_dir, "module-build-macros.spec")
self.srpms_dir = os.path.join(self.tmp_srpm_build_dir, "SRPMS")
@@ -998,7 +1001,6 @@ class TestGetDistTagSRPM:
def teardown_method(self):
shutil.rmtree(self.tmp_srpm_build_dir)
clean_database()
@patch("tempfile.mkdtemp")
@patch("module_build_service.builder.KojiModuleBuilder.execute_cmd")

View File

@@ -23,16 +23,15 @@ from module_build_service.common.models import ModuleBuild, ComponentBuild
from module_build_service.common.utils import load_mmd, mmd_to_str
from module_build_service.scheduler import events
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, make_module_in_db, read_staged_data, staged_data_filename
from tests import make_module_in_db, read_staged_data, staged_data_filename
@pytest.mark.usefixtures("require_empty_database")
class TestMockModuleBuilder:
def setup_method(self, test_method):
clean_database()
self.resultdir = tempfile.mkdtemp()
def teardown_method(self, test_method):
clean_database()
shutil.rmtree(self.resultdir)
def _create_module_with_filters(self, db_session, batch, state):
@@ -190,9 +189,8 @@ class TestMockModuleBuilder:
assert not pkglist
@pytest.mark.usefixtures("require_empty_database")
class TestMockModuleBuilderAddRepos:
def setup_method(self, test_method):
clean_database(add_platform_module=False)
@mock.patch("module_build_service.common.conf.system", new="mock")
@mock.patch(
@@ -243,12 +241,8 @@ class TestMockModuleBuilderAddRepos:
assert set(builder.enabled_modules) == {"foo:1", "app:1"}
@pytest.mark.usefixtures("require_empty_database")
class TestOfflineLocalBuilds:
def setup_method(self):
clean_database()
def teardown_method(self):
clean_database()
def test_import_fake_base_module(self):
import_fake_base_module("platform:foo:1:000000")
@@ -319,13 +313,13 @@ class TestOfflineLocalBuilds:
new_callable=mock.PropertyMock,
return_value="mock",
)
@pytest.mark.usefixtures("require_empty_database")
class TestLocalBuilds:
def setup_method(self):
clean_database()
events.scheduler.reset()
def teardown_method(self):
clean_database()
events.scheduler.reset()
def test_load_local_builds_name(self, conf_system, conf_resultsdir):

View File

@@ -3,6 +3,7 @@
from __future__ import absolute_import
import os
from os import path
import pytest
import shutil
import tempfile
@@ -10,36 +11,38 @@ from module_build_service.common import log, models
from module_build_service.common.logger import ModuleBuildLogs
from module_build_service.scheduler.consumer import MBSConsumer
from module_build_service.scheduler.db_session import db_session
from tests import init_data
@pytest.fixture()
def test_logger_fixture(request, provide_test_data):
log.debug(request.function.__module__)
try:
# py2
test_id = ".".join([
path.splitext(path.basename(__file__))[0],
request.function.im_class.__name__,
request.function.im_func.__name__,
])
except AttributeError:
# py3
test_id = ".".join([
path.splitext(path.basename(__file__))[0],
request.function.__self__.__class__.__name__,
request.function.__self__.__class__.__name__,
])
base = tempfile.mkdtemp(prefix="mbs-", suffix="-%s" % test_id)
name_format = "build-{id}.log"
print("Storing build logs in %r" % base)
request.cls.build_log = ModuleBuildLogs(base, name_format)
request.cls.base = base
yield
MBSConsumer.current_module_build_id = None
shutil.rmtree(base)
@pytest.mark.usefixtures("test_logger_fixture")
class TestLogger:
def setup_method(self, test_method):
init_data(1)
log.debug(test_method.__module__)
try:
# py2
test_id = ".".join([
path.splitext(path.basename(__file__))[0],
test_method.im_class.__name__,
test_method.im_func.__name__,
])
except AttributeError:
# py3
test_id = ".".join([
path.splitext(path.basename(__file__))[0],
test_method.__self__.__class__.__name__,
test_method.__self__.__class__.__name__,
])
self.base = tempfile.mkdtemp(prefix="mbs-", suffix="-%s" % test_id)
self.name_format = "build-{id}.log"
print("Storing build logs in %r" % self.base)
self.build_log = ModuleBuildLogs(self.base, self.name_format)
def teardown_method(self, test_method):
MBSConsumer.current_module_build_id = None
shutil.rmtree(self.base)
def test_module_build_logs(self):
"""

View File

@@ -10,7 +10,6 @@ from module_build_service.common.models import ComponentBuild, ComponentBuildTra
from module_build_service.common.utils import load_mmd, mmd_to_str
from module_build_service.scheduler.db_session import db_session
from tests import (
clean_database,
init_data as init_data_contexts,
make_module_in_db,
module_build_from_modulemd,
@@ -18,7 +17,6 @@ from tests import (
)
@pytest.mark.usefixtures("model_tests_init_data")
class TestModels:
def test_app_sqlalchemy_events(self):
@@ -63,11 +61,10 @@ class TestModels:
assert build.context == "3ee22b28"
assert build.build_context_no_bms == "089df24993c037e10174f3fa7342ab4dc191a4d4"
def test_siblings_property(self):
def test_siblings_property(self, require_empty_database):
""" Tests that the siblings property returns the ID of all modules with
the same name:stream:version
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule"))
for i in range(3):
build = module_build_from_modulemd(mmd_to_str(mmd))
@@ -77,11 +74,11 @@ class TestModels:
db_session.add(build)
db_session.commit()
build_one = ModuleBuild.get_by_id(db_session, 2)
build_one = ModuleBuild.get_by_id(db_session, 1)
sibling_ids = build_one.siblings(db_session)
db_session.commit()
assert sorted(sibling_ids) == [3, 4]
assert sorted(sibling_ids) == [2, 3]
@pytest.mark.parametrize(
"stream,right_pad,expected",
@@ -101,6 +98,7 @@ class TestModels:
assert expected == ModuleBuild.get_stream_version(stream, right_pad)
@pytest.mark.usefixtures("require_platform_and_default_arch")
class TestModelsGetStreamsContexts:
def test_get_last_build_in_all_streams(self):
init_data_contexts(contexts=True)
@@ -145,7 +143,6 @@ class TestModelsGetStreamsContexts:
Tests that get_last_builds_in_stream_version_lte works in case the
name:stream_ver modules have different versions.
"""
clean_database(False)
make_module_in_db(
"platform:f29.1.0:10:old_version", virtual_streams=["f29"])
@@ -175,18 +172,7 @@ class TestModelsGetStreamsContexts:
"platform:f29.2.0:1:c11",
}
def test_get_module_count(self):
clean_database(False)
make_module_in_db("platform:f29.1.0:10:c11")
make_module_in_db("platform:f29.1.0:10:c12")
count = ModuleBuild.get_module_count(db_session, name="platform")
db_session.commit()
assert count == 2
def test_add_virtual_streams_filter(self):
clean_database(False)
make_module_in_db(
"platform:f29.1.0:10:c1", virtual_streams=["f29"])
make_module_in_db(
@@ -201,3 +187,12 @@ class TestModelsGetStreamsContexts:
count = query.count()
db_session.commit()
assert count == 3
def test_get_module_count(require_empty_database):
make_module_in_db("platform:f29.1.0:10:c11")
make_module_in_db("platform:f29.1.0:10:c12")
count = ModuleBuild.get_module_count(db_session, name="platform")
db_session.commit()
assert count == 2

View File

@@ -8,26 +8,23 @@ import pytest
import requests
from six.moves import reload_module
from module_build_service import app
from module_build_service.common import conf, models
import module_build_service.common.monitor
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, init_data, make_module_in_db
from tests import make_module_in_db
num_of_metrics = 18
@pytest.mark.usefixtures("provide_test_client")
class TestViews:
def setup_method(self, test_method):
self.client = app.test_client()
init_data(2)
def test_metrics(self):
def test_metrics(self, provide_test_data):
rv = self.client.get("/module-build-service/1/monitor/metrics")
count = len([
l for l in rv.get_data(as_text=True).splitlines()
if (l.startswith("# TYPE") and "_created " not in l)
line for line in rv.get_data(as_text=True).splitlines()
if (line.startswith("# TYPE") and "_created " not in line)
])
assert count == num_of_metrics
@@ -43,16 +40,15 @@ def test_standalone_metrics_server():
r = requests.get("http://127.0.0.1:10040/metrics")
count = len([
l for l in r.text.splitlines()
if (l.startswith("# TYPE") and "_created " not in l)
line for line in r.text.splitlines()
if (line.startswith("# TYPE") and "_created " not in line)
])
assert count == num_of_metrics
@mock.patch("module_build_service.common.monitor.builder_failed_counter.labels")
@mock.patch("module_build_service.common.monitor.builder_success_counter.inc")
def test_monitor_state_changing_success(succ_cnt, failed_cnt):
clean_database(add_platform_module=False, add_default_arches=False)
def test_monitor_state_changing_success(succ_cnt, failed_cnt, require_empty_database):
b = make_module_in_db(
"pkg:0.1:1:c1",
[
@@ -72,8 +68,7 @@ def test_monitor_state_changing_success(succ_cnt, failed_cnt):
@mock.patch("module_build_service.common.monitor.builder_failed_counter.labels")
@mock.patch("module_build_service.common.monitor.builder_success_counter.inc")
def test_monitor_state_changing_failure(succ_cnt, failed_cnt):
clean_database(add_platform_module=False, add_default_arches=False)
def test_monitor_state_changing_failure(succ_cnt, failed_cnt, require_empty_database):
failure_type = "user"
b = make_module_in_db(
"pkg:0.1:1:c1",

View File

@@ -10,15 +10,11 @@ from module_build_service.common.modulemd import Modulemd
from module_build_service.common.utils import load_mmd
from module_build_service.common.resolve import get_base_module_mmds
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, make_module_in_db, init_data, read_staged_data
from tests import make_module_in_db, init_data, read_staged_data
@pytest.mark.usefixtures("require_platform_and_default_arch")
class TestResolve:
def setup_method(self, test_method):
clean_database(False)
def teardown_method(self, test_method):
clean_database()
def test__get_base_module_mmds(self):
"""Ensure the correct results are returned without duplicates."""

View File

@@ -8,7 +8,7 @@ from module_build_service.common import models
from module_build_service.common.errors import UnprocessableEntity
from module_build_service.common.utils import import_mmd, load_mmd
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, read_staged_data
from tests import read_staged_data
@pytest.mark.parametrize("context", ["c1", None])
@@ -80,8 +80,7 @@ def test_import_mmd_minimal_xmd_from_local_repository():
("f-28", "fedora-28", "The disttag_marking cannot contain a dash"),
),
)
def test_import_mmd_base_module(stream, disttag_marking, error_msg):
clean_database(add_platform_module=False)
def test_import_mmd_base_module(stream, disttag_marking, error_msg, require_empty_database):
mmd = load_mmd(read_staged_data("platform"))
mmd = mmd.copy(mmd.get_module_name(), stream)

View File

@@ -19,7 +19,6 @@ from module_build_service.scheduler.db_session import db_session
import tests
@pytest.mark.usefixtures("reuse_component_init_data")
class TestDBModule:
def test_get_buildrequired_modulemds(self):
@@ -57,8 +56,10 @@ class TestDBModule:
assert nsvcs == {"testmodule:master:20170109091357:123"}
@pytest.mark.parametrize("stream_versions", [False, True])
def test_get_compatible_base_module_modulemds_stream_versions(self, stream_versions):
tests.init_data(1, multiple_stream_versions=True)
@pytest.mark.parametrize("provide_test_data",
[{"multiple_stream_versions": True}], indirect=True)
def test_get_compatible_base_module_modulemds_stream_versions(self, stream_versions,
provide_test_data):
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
platform = db_session.query(ModuleBuild).filter_by(name="platform", stream="f29.1.0").one()
platform_mmd = platform.mmd()
@@ -76,7 +77,7 @@ class TestDBModule:
}
@pytest.mark.parametrize("empty_buildrequires", [False, True])
def test_get_module_build_dependencies(self, empty_buildrequires):
def test_get_module_build_dependencies(self, empty_buildrequires, reuse_component_init_data):
"""
Tests that the buildrequires of testmodule are returned
"""
@@ -98,7 +99,7 @@ class TestDBModule:
"testmodule", "master", "20170109091357", "78e4a6fd").keys()
assert set(result) == expected
def test_get_module_build_dependencies_recursive(self):
def test_get_module_build_dependencies_recursive(self, reuse_component_init_data):
"""
Tests that the buildrequires are returned when it is two layers deep
"""
@@ -141,7 +142,8 @@ class TestDBModule:
new_callable=PropertyMock,
return_value=tests.staged_data_filename("local_builds"),
)
def test_get_module_build_dependencies_recursive_requires(self, resultdir, conf_system):
def test_get_module_build_dependencies_recursive_requires(self, resultdir, conf_system,
reuse_component_init_data):
"""
Tests that it returns the requires of the buildrequires recursively
"""
@@ -173,7 +175,7 @@ class TestDBModule:
}
}
def test_resolve_requires_exception(self):
def test_resolve_requires_exception(self, reuse_component_init_data):
build = models.ModuleBuild.get_by_id(db_session, 2)
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
with pytest.raises(UnprocessableEntity):
@@ -181,8 +183,7 @@ class TestDBModule:
[":".join(["abcdefghi", build.stream, build.version, build.context])]
)
def test_resolve_requires_siblings(self):
tests.clean_database()
def test_resolve_requires_siblings(self, require_platform_and_default_arch):
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
mmd = load_mmd(tests.read_staged_data("formatted_testmodule"))
for i in range(3):
@@ -284,8 +285,9 @@ class TestDBModule:
expected = {"buildroot": {"foo"}, "srpm-buildroot": {"bar"}}
assert result == expected
def test_get_latest_with_virtual_stream(self):
tests.init_data(1, multiple_stream_versions=True)
@pytest.mark.parametrize("provide_test_data",
[{"multiple_stream_versions": True}], indirect=True)
def test_get_latest_with_virtual_stream(self, provide_test_data):
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="db")
mmd = resolver.get_latest_with_virtual_stream("platform", "f29")
assert mmd

View File

@@ -330,9 +330,8 @@ class TestMBSModule:
return_value=tests.staged_data_filename("local_builds")
)
def test_resolve_profiles_local_module(
self, local_builds, conf_system, formatted_testmodule_mmd
self, local_builds, conf_system, formatted_testmodule_mmd, require_empty_database
):
tests.clean_database(add_platform_module=False)
load_local_builds(["platform:f28"])
resolver = mbs_resolver.GenericResolver.create(db_session, conf, backend="mbs")
@@ -455,9 +454,8 @@ class TestMBSModule:
return_value=tests.staged_data_filename("local_builds")
)
def test_get_buildrequired_modulemds_local_builds(
self, local_builds, conf_system
self, local_builds, conf_system, require_empty_database
):
tests.clean_database()
with app.app_context():
load_local_builds(["testmodule"])

View File

@@ -100,7 +100,6 @@ class TestBatches:
events.scheduler.reset()
def teardown_method(self, test_method):
# clean_database()
DummyModuleBuilder.TAGGED_COMPONENTS = []
GenericBuilder.register_backend_class(KojiModuleBuilder)
events.scheduler.reset()

View File

@@ -6,17 +6,15 @@ from mock import patch
from module_build_service.common import models
from module_build_service.common.config import conf
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, make_module_in_db
from tests import make_module_in_db
@patch('module_build_service.common.messaging.publish')
def test_send_messages_after_several_state_transitions(mock_publish):
def test_send_messages_after_several_state_transitions(mock_publish, require_empty_database):
"""
Ensure all module build state change messages are sent after multiple
ModuleBuild.transitions are committed at once
"""
clean_database()
build = make_module_in_db("testmodule:1:2:c3")
build.transition(db_session, conf, models.BUILD_STATES["wait"])

View File

@@ -14,16 +14,15 @@ from module_build_service.common.models import ModuleBuild
from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str
from module_build_service.scheduler import default_modules
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, make_module_in_db, read_staged_data
from tests import make_module_in_db, read_staged_data
@patch("module_build_service.scheduler.default_modules.handle_collisions_with_base_module_rpms")
@patch("module_build_service.scheduler.default_modules._get_default_modules")
def test_add_default_modules(mock_get_dm, mock_hc):
def test_add_default_modules(mock_get_dm, mock_hc, require_platform_and_default_arch):
"""
Test that default modules present in the database are added, and the others are ignored.
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
xmd_brs = mmd.get_xmd()["mbs"]["buildrequires"]
assert set(xmd_brs.keys()) == {"platform"}
@@ -67,11 +66,10 @@ def test_add_default_modules(mock_get_dm, mock_hc):
@patch("module_build_service.scheduler.default_modules._get_default_modules")
def test_add_default_modules_not_linked(mock_get_dm):
def test_add_default_modules_not_linked(mock_get_dm, require_platform_and_default_arch):
"""
Test that no default modules are added when they aren't linked from the base module.
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
assert set(mmd.get_xmd()["mbs"]["buildrequires"].keys()) == {"platform"}
default_modules.add_default_modules(mmd)
@@ -79,13 +77,12 @@ def test_add_default_modules_not_linked(mock_get_dm):
mock_get_dm.assert_not_called()
def test_add_default_modules_platform_not_available():
def test_add_default_modules_platform_not_available(require_empty_database):
"""
Test that an exception is raised when the platform module that is buildrequired is missing.
This error should never occur in practice.
"""
clean_database(False, False)
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))
expected_error = "Failed to retrieve the module platform:f28:3:00000000 from the database"
@@ -94,12 +91,10 @@ def test_add_default_modules_platform_not_available():
@patch("module_build_service.scheduler.default_modules._get_default_modules")
def test_add_default_modules_compatible_platforms(mock_get_dm):
def test_add_default_modules_compatible_platforms(mock_get_dm, require_empty_database):
"""
Test that default modules built against compatible base module streams are added.
"""
clean_database(add_platform_module=False)
# Create compatible base modules.
mmd = load_mmd(read_staged_data("platform"))
for stream in ["f27", "f28"]:
@@ -150,11 +145,10 @@ def test_add_default_modules_compatible_platforms(mock_get_dm):
@patch("module_build_service.scheduler.default_modules._get_default_modules")
def test_add_default_modules_request_failed(mock_get_dm):
def test_add_default_modules_request_failed(mock_get_dm, require_platform_and_default_arch):
"""
Test that an exception is raised when the call to _get_default_modules failed.
"""
clean_database()
make_module_in_db("python:3:12345:1")
make_module_in_db("nodejs:11:2345:2")
mmd = load_mmd(read_staged_data("formatted_testmodule.yaml"))

View File

@@ -7,13 +7,11 @@ from mock import patch, Mock
import pytest
from module_build_service.scheduler.greenwave import greenwave
from tests import clean_database, make_module_in_db
from tests import make_module_in_db
class TestGreenwaveQuery():
def setup_method(self, method):
clean_database()
@pytest.mark.usefixtures("require_empty_database")
class TestGreenwaveQuery:
@patch("module_build_service.scheduler.greenwave.requests")
def test_greenwave_query_decision(self, mock_requests):

View File

@@ -15,15 +15,13 @@ from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.handlers.greenwave import (
decision_update, get_corresponding_module_build
)
from tests import clean_database, make_module_in_db
from tests import make_module_in_db
@pytest.mark.usefixtures("require_empty_database")
class TestGetCorrespondingModuleBuild:
"""Test get_corresponding_module_build"""
def setup_method(self, method):
clean_database()
@patch("koji.ClientSession")
def test_module_build_nvr_does_not_exist_in_koji(self, ClientSession):
ClientSession.return_value.getBuild.return_value = None
@@ -48,7 +46,8 @@ class TestGetCorrespondingModuleBuild:
assert get_corresponding_module_build("n-v-r") is None
@patch("koji.ClientSession")
def test_corresponding_module_build_id_does_not_exist_in_db(self, ClientSession):
def test_corresponding_module_build_id_does_not_exist_in_db(self, ClientSession,
require_platform_and_default_arch):
fake_module_build_id, = db_session.query(func.max(ModuleBuild.id)).first()
ClientSession.return_value.getBuild.return_value = {
@@ -58,7 +57,7 @@ class TestGetCorrespondingModuleBuild:
assert get_corresponding_module_build("n-v-r") is None
@patch("koji.ClientSession")
def test_find_the_module_build(self, ClientSession):
def test_find_the_module_build(self, ClientSession, require_platform_and_default_arch):
expected_module_build = (
db_session.query(ModuleBuild).filter(ModuleBuild.name == "platform").first()
)
@@ -118,9 +117,7 @@ class TestDecisionUpdateHandler:
@patch("module_build_service.common.messaging.publish")
@patch("koji.ClientSession")
def test_transform_from_done_to_ready(self, ClientSession, publish):
clean_database()
def test_transform_from_done_to_ready(self, ClientSession, publish, require_empty_database):
# This build should be queried and transformed to ready state
module_build = make_module_in_db(
"pkg:0.1:1:c1",

View File

@@ -13,7 +13,7 @@ from module_build_service.common.config import conf
from module_build_service.common import models
from module_build_service.scheduler import producer
from module_build_service.scheduler.db_session import db_session
from tests import clean_database, make_module_in_db
from tests import make_module_in_db
@pytest.mark.usefixtures("reuse_component_init_data")
@@ -36,7 +36,6 @@ class TestPoller:
def teardown_method(self, test_method):
self.p_read_config.stop()
clean_database()
@pytest.mark.parametrize("fresh", [True, False])
@patch("module_build_service.scheduler.batches.start_build_component")

View File

@@ -11,7 +11,7 @@ from module_build_service.common.modulemd import Modulemd
from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module
from tests import clean_database, read_staged_data
from tests import read_staged_data
@pytest.mark.usefixtures("reuse_component_init_data")
@@ -239,14 +239,10 @@ class TestUtilsComponentReuse:
class TestReuseSharedUserSpace:
def setup_method(self, test_method):
clean_database()
def teardown_method(self, test_method):
clean_database()
@pytest.mark.usefixtures("reuse_shared_userspace_init_data")
def test_get_reusable_component_shared_userspace_ordering(self):
def test_get_reusable_component_shared_userspace_ordering(self,
require_platform_and_default_arch,
reuse_shared_userspace_init_data):
"""
For modules with lot of components per batch, there is big chance that
the database will return them in different order than what we have for

View File

@@ -17,23 +17,17 @@ from module_build_service.scheduler.submit import (
get_build_arches, format_mmd, record_component_builds, record_module_build_arches
)
from tests import (
clean_database,
init_data,
read_staged_data,
staged_data_filename,
scheduler_init_data,
)
@pytest.mark.usefixtures("require_empty_database")
class TestSubmit:
def setup_method(self, test_method):
clean_database()
def teardown_method(self, test_method):
clean_database()
@mock.patch("koji.ClientSession")
def test_get_build_arches(self, ClientSession):
def test_get_build_arches(self, ClientSession, require_platform_and_default_arch):
session = ClientSession.return_value
session.getTag.return_value = {"arches": "ppc64le"}
mmd = load_mmd(read_staged_data("formatted_testmodule"))
@@ -321,7 +315,6 @@ class TestSubmit:
@mock.patch("module_build_service.common.scm.SCM")
def test_format_mmd_arches(self, mocked_scm):
with app.app_context():
clean_database()
mocked_scm.return_value.commit = "620ec77321b2ea7b0d67d82992dda3e1d67055b4"
mocked_scm.return_value.get_latest.side_effect = [
"4ceea43add2366d8b8c5a622a2fb563b625b9abf",
@@ -358,8 +351,7 @@ class TestSubmit:
@mock.patch("module_build_service.common.scm.SCM")
@mock.patch("module_build_service.scheduler.submit.ThreadPool")
def test_format_mmd_update_time_modified(self, tp, mocked_scm):
init_data()
def test_format_mmd_update_time_modified(self, tp, mocked_scm, provide_test_data):
build = models.ModuleBuild.get_by_id(db_session, 2)
async_result = mock.MagicMock()

View File

@@ -6,7 +6,7 @@ from mock import patch, Mock
from module_build_service.common.config import conf
from module_build_service.scheduler import ursine
from tests import make_module, make_module_in_db, clean_database
from tests import make_module, make_module_in_db
class TestFindModuleKojiTags:
@@ -107,14 +107,9 @@ class TestFindUrsineRootTags:
class TestGetModulemdsFromUrsineContent:
"""Test ursine.get_modulemds_from_ursine_content"""
def setup_method(self):
clean_database(False)
def teardown_method(self, test_method):
clean_database()
@patch("koji.ClientSession")
def test_return_empty_if_no_ursine_build_tag_is_found(self, ClientSession):
def test_return_empty_if_no_ursine_build_tag_is_found(self, ClientSession,
require_empty_database):
koji_session = ClientSession.return_value
# No module koji_tag in ursine content yet. This will result in empty

View File

@@ -9,15 +9,11 @@ from module_build_service.scheduler.db_session import db_session
from module_build_service.web.mse import (
expand_mse_streams, generate_expanded_mmds, get_mmds_required_by_module_recursively
)
from tests import clean_database, make_module_in_db
from tests import make_module_in_db
@pytest.mark.usefixtures("require_empty_database")
class TestModuleStreamExpansion:
def setup_method(self, test_method):
clean_database(False)
def teardown_method(self, test_method):
clean_database()
def _get_mmds_required_by_module_recursively(self, module_build, db_session):
"""

View File

@@ -18,7 +18,6 @@ from module_build_service.web.submit import (
get_prefixed_version, submit_module_build, submit_module_build_from_yaml
)
from tests import (
clean_database,
scheduler_init_data,
make_module_in_db,
make_module,
@@ -26,8 +25,6 @@ from tests import (
class TestSubmit:
def setup_method(self, test_method):
clean_database()
def test_get_prefixed_version_f28(self):
scheduler_init_data(1)

View File

@@ -17,7 +17,7 @@ import pytest
import sqlalchemy
from sqlalchemy.orm import load_only
from module_build_service import app, version
from module_build_service import version
from module_build_service.builder.utils import get_rpm_release
import module_build_service.common.config as mbs_config
from module_build_service.common.errors import UnprocessableEntity
@@ -122,10 +122,10 @@ class FakeSCM(object):
return commit_hash + sha1_hash[len(commit_hash):]
@pytest.mark.usefixtures("provide_test_client")
@pytest.mark.usefixtures("provide_test_data")
@pytest.mark.parametrize('provide_test_data', [{"data_size": 2}], indirect=True)
class TestViews:
def setup_method(self, test_method):
self.client = app.test_client()
init_data(2)
def test_query_build(self):
rv = self.client.get("/module-build-service/1/module-builds/2")
@@ -363,66 +363,6 @@ class TestViews:
module_build["component_builds"].sort()
assert items == expected
def test_query_builds_with_context(self):
clean_database()
init_data(2, contexts=True)
rv = self.client.get("/module-build-service/1/module-builds/?context=3a4057d2")
items = json.loads(rv.data)["items"]
checking_build_id = 3
# Get component build ids dynamically rather than hardcode inside expected output.
component_build_ids = db_session.query(ComponentBuild).filter(
ComponentBuild.module_id == checking_build_id
).order_by(ComponentBuild.id).options(load_only("id")).all()
expected = [
{
"component_builds": [cb.id for cb in component_build_ids],
"context": "3a4057d2",
"id": checking_build_id,
"koji_tag": "module-nginx-1.2",
"name": "nginx",
"owner": "Moe Szyslak",
"rebuild_strategy": "changed-and-after",
"scmurl": (
"git://pkgs.domain.local/modules/nginx"
"?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9"
),
"scratch": False,
"siblings": [2],
"srpms": [],
"state": 5,
"state_name": "ready",
"state_reason": None,
"stream": "0",
"tasks": {
"rpms": {
"module-build-macros": {
"nvr": "module-build-macros-01-1.module+4+0557c87d",
"state": 1,
"state_reason": None,
"task_id": 47383993,
},
"postgresql": {
"nvr": "postgresql-9.5.3-4.module+4+0557c87d",
"state": 1,
"state_reason": None,
"task_id": 2433433,
},
}
},
"time_completed": "2016-09-03T11:25:32Z",
"time_modified": "2016-09-03T11:25:32Z",
"time_submitted": "2016-09-03T11:23:20Z",
"version": "2",
"buildrequires": {},
}
]
# To avoid different order of component builds impact the subsequent assertion.
items[0]['component_builds'] = sorted(items[0]['component_builds'])
assert items == expected
def test_query_builds_with_id_error(self):
rv = self.client.get("/module-build-service/1/module-builds/?id=1")
actual = json.loads(rv.data)
@@ -726,60 +666,6 @@ class TestViews:
"An invalid Zulu ISO 8601 timestamp was " 'provided for the "modified_after" parameter'
assert data["status"] == 400
@pytest.mark.parametrize(
"stream_version_lte",
("280000", "280000.0", "290000", "293000", "invalid"),
)
def test_query_builds_filter_stream_version_lte(self, stream_version_lte):
init_data(data_size=1, multiple_stream_versions=True)
url = (
"/module-build-service/1/module-builds/?name=platform&verbose=true"
"&stream_version_lte={}".format(stream_version_lte)
)
rv = self.client.get(url)
data = json.loads(rv.data)
total = data.get("meta", {}).get("total")
if stream_version_lte == "invalid":
assert data == {
"error": "Bad Request",
"message": (
"An invalid value of stream_version_lte was provided. It must be an "
"integer or float greater than or equal to 10000."
),
"status": 400,
}
elif stream_version_lte in ("280000", "280000.0"):
assert total == 2
elif stream_version_lte == "290000":
assert total == 1
elif stream_version_lte == "293000":
assert total == 3
@pytest.mark.parametrize("virtual_streams", ([], ("f28",), ("f29",), ("f28", "f29")))
def test_query_builds_filter_virtual_streams(self, virtual_streams):
# Populate some platform modules with virtual streams
init_data(data_size=1, multiple_stream_versions=True)
url = "/module-build-service/1/module-builds/?name=platform&verbose=true"
for virtual_stream in virtual_streams:
url += "&virtual_stream={}".format(virtual_stream)
rv = self.client.get(url)
data = json.loads(rv.data)
total = data["meta"]["total"]
if virtual_streams == ("f28",):
assert total == 1
for module in data["items"]:
assert module["virtual_streams"] == ["f28"]
elif virtual_streams == ("f29",):
assert total == 3
for module in data["items"]:
assert module["virtual_streams"] == ["f29"]
elif virtual_streams == ("f28", "f29"):
assert total == 4
for module in data["items"]:
assert len(set(module["virtual_streams"]) - {"f28", "f29"}) == 0
elif len(virtual_streams) == 0:
assert total == 5
def test_query_builds_order_by(self):
build = ModuleBuild.get_by_id(db_session, 2)
build.name = "candy"
@@ -789,29 +675,6 @@ class TestViews:
assert items[0]["name"] == "candy"
assert items[1]["name"] == "nginx"
def test_query_builds_order_by_multiple(self):
init_data(data_size=1, multiple_stream_versions=True)
platform_f28 = ModuleBuild.get_by_id(db_session, 1)
platform_f28.version = "150"
db_session.commit()
# Simply assert the order of all module builds
page_size = db_session.query(ModuleBuild).count()
rv = self.client.get(
"/module-build-service/1/module-builds/?order_desc_by=stream_version"
"&order_desc_by=version&per_page={}".format(page_size)
)
items = json.loads(rv.data)["items"]
actual_ids = [item["id"] for item in items]
expected_ids = [
build.id for build in db_session.query(ModuleBuild).order_by(
ModuleBuild.stream_version.desc(),
sqlalchemy.cast(ModuleBuild.version, sqlalchemy.BigInteger).desc()
).all()
]
assert actual_ids == expected_ids
def test_query_builds_order_desc_by(self):
rv = self.client.get(
"/module-build-service/1/module-builds/?per_page=10&order_desc_by=id")
@@ -820,18 +683,6 @@ class TestViews:
for idx, item in enumerate(items):
assert item["id"] == items[0]["id"] - idx
def test_query_builds_order_desc_by_context(self):
clean_database()
init_data(2, contexts=True)
rv = self.client.get(
"/module-build-service/1/module-builds/?per_page=10&name=nginx&order_desc_by=context")
sorted_items = json.loads(rv.data)["items"]
sorted_contexts = [m["context"] for m in sorted_items]
expected_contexts = ["d5a6c0fa", "795e97c1", "3a4057d2", "10e50d06"]
assert sorted_contexts == expected_contexts
def test_query_builds_order_by_order_desc_by(self):
"""
Test that when both order_by and order_desc_by are set, an error is returned.
@@ -1549,78 +1400,6 @@ class TestViews:
res2 = submit("git://some.custom.url.org/modules/testmodule.git?#68931c9")
assert res2.status_code == 201
@pytest.mark.parametrize(
"br_override_streams, req_override_streams", ((["f28"], None), (["f28"], ["f28"]))
)
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_dep_override(
self, mocked_scm, mocked_get_user, br_override_streams, req_override_streams
):
init_data(data_size=1, multiple_stream_versions=True)
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_platform_f290000.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe"
"efbd35246a81b6cb8d49"
)
json_input = {"branch": "master", "scmurl": scm_url}
if br_override_streams:
json_input["buildrequire_overrides"] = {"platform": br_override_streams}
expected_br = set(br_override_streams)
else:
expected_br = {"f29.0.0"}
if req_override_streams:
json_input["require_overrides"] = {"platform": req_override_streams}
expected_req = set(req_override_streams)
else:
expected_req = {"f29.0.0"}
rv = self.client.post(post_url, data=json.dumps(json_input))
data = json.loads(rv.data)
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
assert set(dep.get_buildtime_streams("platform")) == expected_br
assert set(dep.get_runtime_streams("platform")) == expected_req
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_invalid_basemodule_stream(self, mocked_scm, mocked_get_user):
# By default tests do not provide platform:f28.0.0, but just platform:f28.
# Therefore we want to enable multiple_stream_versions.
init_data(2, multiple_stream_versions=True)
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")
data = {
"branch": "master",
"scmurl": "https://src.stg.fedoraproject.org/modules/"
"testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49",
"buildrequire_overrides": {"platform": ["28.0.0"]},
"require_overrides": {"platform": ["f28.0.0"]},
}
rv = self.client.post("/module-build-service/1/module-builds/", data=json.dumps(data))
result = json.loads(rv.data)
assert result == {
"error": "Unprocessable Entity",
"status": 422,
"message": (
"None of the base module (platform) streams in the buildrequires "
"section could be found"
),
}
assert rv.status_code == 422
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_with_base_module_name(self, mocked_scm, mocked_get_user):
@@ -2398,135 +2177,6 @@ class TestViews:
# but it should still succeed since yaml is always allowed for scratch builds
assert rv.status_code == 201
@pytest.mark.parametrize(
"branch, platform_override",
(("10", None), ("10-rhel-8.0.0", "el8.0.0"), ("10-LP-product1.2", "product1.2")),
)
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@patch.object(
module_build_service.common.config.Config,
"br_stream_override_regexes",
new_callable=PropertyMock,
)
def test_submit_build_dep_override_from_branch(
self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override
):
"""
Test that MBS will parse the SCM branch to determine the platform stream to buildrequire.
"""
mocked_regexes.return_value = [r"(?:rh)(el)(?:\-)(\d+\.\d+\.\d+)$", r"(?:\-LP\-)(.+)$"]
init_data(data_size=1, multiple_stream_versions=True)
# Create a platform for whatever the override is so the build submission succeeds
if platform_override:
platform_mmd = load_mmd(read_staged_data("platform"))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), platform_override)
if platform_override == "el8.0.0":
xmd = platform_mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["el8"]
platform_mmd.set_xmd(xmd)
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
rv = self.client.post(post_url, data=json.dumps({"branch": branch, "scmurl": scm_url}))
data = json.loads(rv.data)
assert rv.status_code == 201
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
if platform_override:
expected_br = {platform_override}
else:
expected_br = {"f28"}
assert set(dep.get_buildtime_streams("platform")) == expected_br
# The requires should not change
assert dep.get_runtime_streams("platform") == ["f28"]
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@patch.object(
module_build_service.common.config.Config,
"br_stream_override_regexes",
new_callable=PropertyMock,
)
def test_submit_build_dep_override_from_branch_br_override(
self, mocked_regexes, mocked_scm, mocked_get_user
):
"""
Test that when the branch includes a stream override for the platform module, that the
provided "buildrequire_override" for the platform module takes precedence.
"""
mocked_regexes.return_value = [r"(?:\-LP\-)(.+)$"]
init_data(data_size=1, multiple_stream_versions=True)
# Create a platform for the override so the build submission succeeds
platform_mmd = load_mmd(read_staged_data('platform'))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "product1.3")
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml", "620ec77321b2ea7b0d67d82992dda3e1d67055b4")
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
json_input = {
"branch": "10-LP-product1.2",
"scmurl": scm_url,
"buildrequire_overrides": {"platform": ["product1.3"]},
}
rv = self.client.post(post_url, data=json.dumps(json_input))
data = json.loads(rv.data)
assert rv.status_code == 201
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
# The buildrequire_override value should take precedence over the stream override from
# parsing the branch
assert dep.get_buildtime_streams("platform") == ["product1.3"]
# The requires should not change
assert dep.get_runtime_streams("platform") == ["f28"]
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_br_xyz_version_no_virtual_streams(self, mocked_scm, mocked_get_user):
"""
Test that when a build is submitted with a buildrequire on a base module with x.y.z
versioning and no virtual streams, that the dependency resolution succeeds.
"""
init_data(data_size=1, multiple_stream_versions=True)
platform_mmd = load_mmd(read_staged_data("platform"))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "el8.0.0")
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_el800.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
rv = self.client.post(post_url, data=json.dumps({"branch": "master", "scmurl": scm_url}))
assert rv.status_code == 201
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@patch(
@@ -2910,10 +2560,362 @@ class TestViews:
)
@pytest.mark.usefixtures("provide_test_client")
@pytest.mark.usefixtures("provide_test_data")
@pytest.mark.parametrize("provide_test_data", [{"data_size": 2, "contexts": True}], indirect=True)
class TestViewsWithContexts:
def test_query_builds_with_context(self):
rv = self.client.get("/module-build-service/1/module-builds/?context=3a4057d2")
items = json.loads(rv.data)["items"]
checking_build_id = 3
# Get component build ids dynamically rather than hardcode inside expected output.
component_build_ids = db_session.query(ComponentBuild).filter(
ComponentBuild.module_id == checking_build_id
).order_by(ComponentBuild.id).options(load_only("id")).all()
expected = [
{
"component_builds": [cb.id for cb in component_build_ids],
"context": "3a4057d2",
"id": checking_build_id,
"koji_tag": "module-nginx-1.2",
"name": "nginx",
"owner": "Moe Szyslak",
"rebuild_strategy": "changed-and-after",
"scmurl": (
"git://pkgs.domain.local/modules/nginx"
"?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9"
),
"scratch": False,
"siblings": [2],
"srpms": [],
"state": 5,
"state_name": "ready",
"state_reason": None,
"stream": "0",
"tasks": {
"rpms": {
"module-build-macros": {
"nvr": "module-build-macros-01-1.module+4+0557c87d",
"state": 1,
"state_reason": None,
"task_id": 47383993,
},
"postgresql": {
"nvr": "postgresql-9.5.3-4.module+4+0557c87d",
"state": 1,
"state_reason": None,
"task_id": 2433433,
},
}
},
"time_completed": "2016-09-03T11:25:32Z",
"time_modified": "2016-09-03T11:25:32Z",
"time_submitted": "2016-09-03T11:23:20Z",
"version": "2",
"buildrequires": {},
}
]
# To avoid different order of component builds impact the subsequent assertion.
items[0]['component_builds'] = sorted(items[0]['component_builds'])
assert items == expected
def test_query_builds_order_desc_by_context(self):
rv = self.client.get(
"/module-build-service/1/module-builds/?per_page=10&name=nginx&order_desc_by=context")
sorted_items = json.loads(rv.data)["items"]
sorted_contexts = [m["context"] for m in sorted_items]
expected_contexts = ["d5a6c0fa", "795e97c1", "3a4057d2", "10e50d06"]
assert sorted_contexts == expected_contexts
@pytest.mark.usefixtures("provide_test_client")
@pytest.mark.usefixtures("provide_test_data")
@pytest.mark.parametrize('provide_test_data',
[{"data_size": 1, "multiple_stream_versions": True}], indirect=True)
class TestViewsWithMultipleStreamVersions:
@pytest.mark.parametrize(
"stream_version_lte",
("280000", "280000.0", "290000", "293000", "invalid"),
)
def test_query_builds_filter_stream_version_lte(self, stream_version_lte,):
url = (
"/module-build-service/1/module-builds/?name=platform&verbose=true"
"&stream_version_lte={}".format(stream_version_lte)
)
rv = self.client.get(url)
data = json.loads(rv.data)
total = data.get("meta", {}).get("total")
if stream_version_lte == "invalid":
assert data == {
"error": "Bad Request",
"message": (
"An invalid value of stream_version_lte was provided. It must be an "
"integer or float greater than or equal to 10000."
),
"status": 400,
}
elif stream_version_lte in ("280000", "280000.0"):
assert total == 2
elif stream_version_lte == "290000":
assert total == 1
elif stream_version_lte == "293000":
assert total == 3
@pytest.mark.parametrize("virtual_streams", ([], ("f28",), ("f29",), ("f28", "f29")))
def test_query_builds_filter_virtual_streams(self, virtual_streams):
url = "/module-build-service/1/module-builds/?name=platform&verbose=true"
for virtual_stream in virtual_streams:
url += "&virtual_stream={}".format(virtual_stream)
rv = self.client.get(url)
data = json.loads(rv.data)
total = data["meta"]["total"]
if virtual_streams == ("f28",):
assert total == 1
for module in data["items"]:
assert module["virtual_streams"] == ["f28"]
elif virtual_streams == ("f29",):
assert total == 3
for module in data["items"]:
assert module["virtual_streams"] == ["f29"]
elif virtual_streams == ("f28", "f29"):
assert total == 4
for module in data["items"]:
assert len(set(module["virtual_streams"]) - {"f28", "f29"}) == 0
elif len(virtual_streams) == 0:
assert total == 5
def test_query_builds_order_by_multiple(self):
platform_f28 = ModuleBuild.get_by_id(db_session, 1)
platform_f28.version = "150"
db_session.commit()
# Simply assert the order of all module builds
page_size = db_session.query(ModuleBuild).count()
rv = self.client.get(
"/module-build-service/1/module-builds/?order_desc_by=stream_version"
"&order_desc_by=version&per_page={}".format(page_size)
)
items = json.loads(rv.data)["items"]
actual_ids = [item["id"] for item in items]
expected_ids = [
build.id for build in db_session.query(ModuleBuild).order_by(
ModuleBuild.stream_version.desc(),
sqlalchemy.cast(ModuleBuild.version, sqlalchemy.BigInteger).desc()
).all()
]
assert actual_ids == expected_ids
@pytest.mark.parametrize(
"br_override_streams, req_override_streams", ((["f28"], None), (["f28"], ["f28"]))
)
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_dep_override(
self, mocked_scm, mocked_get_user, br_override_streams, req_override_streams
):
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_platform_f290000.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#68931c90de214d9d13fe"
"efbd35246a81b6cb8d49"
)
json_input = {"branch": "master", "scmurl": scm_url}
if br_override_streams:
json_input["buildrequire_overrides"] = {"platform": br_override_streams}
expected_br = set(br_override_streams)
else:
expected_br = {"f29.0.0"}
if req_override_streams:
json_input["require_overrides"] = {"platform": req_override_streams}
expected_req = set(req_override_streams)
else:
expected_req = {"f29.0.0"}
rv = self.client.post(post_url, data=json.dumps(json_input))
data = json.loads(rv.data)
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
assert set(dep.get_buildtime_streams("platform")) == expected_br
assert set(dep.get_runtime_streams("platform")) == expected_req
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_invalid_basemodule_stream(self, mocked_scm, mocked_get_user):
# By default tests do not provide platform:f28.0.0, but just platform:f28.
# Therefore we want to enable multiple_stream_versions.
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4")
data = {
"branch": "master",
"scmurl": "https://src.stg.fedoraproject.org/modules/"
"testmodule.git?#68931c90de214d9d13feefbd35246a81b6cb8d49",
"buildrequire_overrides": {"platform": ["28.0.0"]},
"require_overrides": {"platform": ["f28.0.0"]},
}
rv = self.client.post("/module-build-service/1/module-builds/", data=json.dumps(data))
result = json.loads(rv.data)
assert result == {
"error": "Unprocessable Entity",
"status": 422,
"message": (
"None of the base module (platform) streams in the buildrequires "
"section could be found"
),
}
assert rv.status_code == 422
@pytest.mark.parametrize(
"branch, platform_override",
(("10", None), ("10-rhel-8.0.0", "el8.0.0"), ("10-LP-product1.2", "product1.2")),
)
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@patch.object(
module_build_service.common.config.Config,
"br_stream_override_regexes",
new_callable=PropertyMock,
)
def test_submit_build_dep_override_from_branch(
self, mocked_regexes, mocked_scm, mocked_get_user, branch, platform_override
):
"""
Test that MBS will parse the SCM branch to determine the platform stream to buildrequire.
"""
mocked_regexes.return_value = [r"(?:rh)(el)(?:\-)(\d+\.\d+\.\d+)$", r"(?:\-LP\-)(.+)$"]
# Create a platform for whatever the override is so the build submission succeeds
if platform_override:
platform_mmd = load_mmd(read_staged_data("platform"))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), platform_override)
if platform_override == "el8.0.0":
xmd = platform_mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = ["el8"]
platform_mmd.set_xmd(xmd)
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4")
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
rv = self.client.post(post_url, data=json.dumps({"branch": branch, "scmurl": scm_url}))
data = json.loads(rv.data)
assert rv.status_code == 201
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
if platform_override:
expected_br = {platform_override}
else:
expected_br = {"f28"}
assert set(dep.get_buildtime_streams("platform")) == expected_br
# The requires should not change
assert dep.get_runtime_streams("platform") == ["f28"]
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@patch.object(
module_build_service.common.config.Config,
"br_stream_override_regexes",
new_callable=PropertyMock,
)
def test_submit_build_dep_override_from_branch_br_override(
self, mocked_regexes, mocked_scm, mocked_get_user
):
"""
Test that when the branch includes a stream override for the platform module, that the
provided "buildrequire_override" for the platform module takes precedence.
"""
mocked_regexes.return_value = [r"(?:\-LP\-)(.+)$"]
# Create a platform for the override so the build submission succeeds
platform_mmd = load_mmd(read_staged_data('platform'))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "product1.3")
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm, "testmodule", "testmodule.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4")
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
json_input = {
"branch": "10-LP-product1.2",
"scmurl": scm_url,
"buildrequire_overrides": {"platform": ["product1.3"]},
}
rv = self.client.post(post_url, data=json.dumps(json_input))
data = json.loads(rv.data)
assert rv.status_code == 201
mmd = load_mmd(data[0]["modulemd"])
assert len(mmd.get_dependencies()) == 1
dep = mmd.get_dependencies()[0]
# The buildrequire_override value should take precedence over the stream override from
# parsing the branch
assert dep.get_buildtime_streams("platform") == ["product1.3"]
# The requires should not change
assert dep.get_runtime_streams("platform") == ["f28"]
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_br_xyz_version_no_virtual_streams(self, mocked_scm, mocked_get_user):
"""
Test that when a build is submitted with a buildrequire on a base module with x.y.z
versioning and no virtual streams, that the dependency resolution succeeds.
"""
platform_mmd = load_mmd(read_staged_data("platform"))
platform_mmd = platform_mmd.copy(platform_mmd.get_module_name(), "el8.0.0")
import_mmd(db_session, platform_mmd)
FakeSCM(
mocked_scm,
"testmodule",
"testmodule_el800.yaml",
"620ec77321b2ea7b0d67d82992dda3e1d67055b4",
)
post_url = "/module-build-service/2/module-builds/"
scm_url = (
"https://src.stg.fedoraproject.org/modules/testmodule.git?#"
"68931c90de214d9d13feefbd35246a81b6cb8d49"
)
rv = self.client.post(post_url, data=json.dumps({"branch": "master", "scmurl": scm_url}))
assert rv.status_code == 201
@pytest.mark.usefixtures("provide_test_client")
class TestLogMessageViews:
def setup_method(self, test_method):
self.client = app.test_client()
clean_database()
init_data(2)
self.module_id = 2
self.module_build = ModuleBuild.get_by_id(db_session, self.module_id)