Merge #1277 Take the list of arches for -build Koji tag from buildrequired modules.

tests/__init__.py

@@ -103,10 +103,16 @@ def patch_zeromq_time_sleep():
 patch_zeromq_time_sleep()
 
 
-def clean_database(add_platform_module=True):
+def clean_database(add_platform_module=True, add_default_arches=True):
     db.session.commit()
     db.drop_all()
     db.create_all()
+
+    if add_default_arches:
+        arch_obj = module_build_service.models.ModuleArch(name="x86_64")
+        db.session.add(arch_obj)
+        db.session.commit()
+
     if add_platform_module:
         mmd = load_mmd_file(os.path.join(base_dir, "staged_data", "platform.yaml"))
         import_mmd(db.session, mmd)
@@ -143,6 +149,7 @@ def init_data(data_size=10, contexts=False, multiple_stream_versions=False, scra
 
 
 def _populate_data(session, data_size=10, contexts=False, scratch=False):
+    arch = module_build_service.models.ModuleArch.query.get(1)
     num_contexts = 2 if contexts else 1
     for index in range(data_size):
         for context in range(num_contexts):
@@ -164,6 +171,7 @@ def _populate_data(session, data_size=10, contexts=False, scratch=False):
                 time_completed=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)),
                 rebuild_strategy="changed-and-after",
             )
+            build_one.arches.append(arch)
 
             if contexts:
                 build_one.stream = str(index)
@@ -229,6 +237,7 @@ def _populate_data(session, data_size=10, contexts=False, scratch=False):
             time_completed=datetime(2016, 9, 3, 11, 27, 19) + timedelta(minutes=(index * 10)),
             rebuild_strategy="changed-and-after",
         )
+        build_two.arches.append(arch)
 
         session.add(build_two)
         session.commit()
@@ -328,6 +337,7 @@ def scheduler_init_data(tangerine_state=None, scratch=False):
     mmd.get_rpm_component("tangerine").set_buildorder(0)
 
     platform_br = module_build_service.models.ModuleBuild.query.get(1)
+    arch = module_build_service.models.ModuleArch.query.get(1)
 
     module_build = module_build_service.models.ModuleBuild(
         name="testmodule",
@@ -351,6 +361,7 @@ def scheduler_init_data(tangerine_state=None, scratch=False):
         modulemd=mmd_to_str(mmd),
     )
 
+    module_build.arches.append(arch)
     module_build.buildrequires.append(platform_br)
     build_one_component_release = get_rpm_release(module_build)
 
@@ -425,6 +436,7 @@ def reuse_component_init_data():
     mmd = load_mmd_file(formatted_testmodule_yml_path)
 
     platform_br = module_build_service.models.ModuleBuild.query.get(1)
+    arch = module_build_service.models.ModuleArch.query.get(1)
 
     build_one = module_build_service.models.ModuleBuild(
         name="testmodule",
@@ -453,6 +465,7 @@ def reuse_component_init_data():
     xmd["mbs"]["commit"] = "ff1ea79fc952143efeed1851aa0aa006559239ba"
     mmd.set_xmd(xmd)
     build_one.modulemd = mmd_to_str(mmd)
+    build_one.arches.append(arch)
     build_one.buildrequires.append(platform_br)
 
     build_one.component_builds.extend([
@@ -535,6 +548,7 @@ def reuse_component_init_data():
     xmd["mbs"]["commit"] = "55f4a0a2e6cc255c88712a905157ab39315b8fd8"
     mmd.set_xmd(xmd)
     build_two.modulemd = mmd_to_str(mmd)
+    build_two.arches.append(arch)
     build_two.buildrequires.append(platform_br)
 
     build_two.component_builds.extend([
@@ -705,6 +719,7 @@ def make_module(
     xmd=None,
     store_to_db=True,
     virtual_streams=None,
+    arches=None,
 ):
     """
     Creates new models.ModuleBuild defined by `nsvc` string with requires
@@ -724,6 +739,8 @@ def make_module(
     :param bool store_to_db: whether to store created module metadata to the
         database.
     :param list virtual_streams: List of virtual streams provided by this module.
+    :param list arches: List of architectures this module is built against.
+        If set to None, ["x86_64"] is used as a default.
     :return: New Module Build if set to store module metadata to database,
         otherwise the module metadata is returned.
     :rtype: ModuleBuild or Modulemd.Module
@@ -821,4 +838,18 @@ def make_module(
                 module_build.virtual_streams.append(vs_obj)
                 db.session.commit()
 
+    if arches is None:
+        arches = ["x86_64"]
+    for arch in arches:
+        arch_obj = db.session.query(module_build_service.models.ModuleArch).filter_by(
+            name=arch).first()
+        if not arch_obj:
+            arch_obj = module_build_service.models.ModuleArch(name=arch)
+            db.session.add(arch_obj)
+            db.session.commit()
+
+        if arch_obj not in module_build.arches:
+            module_build.arches.append(arch_obj)
+            db.session.commit()
+
     return module_build

tests/test_build/test_build.py

@@ -136,6 +136,10 @@ class FakeModuleBuilder(GenericBuilder):
         FakeModuleBuilder.DEFAULT_GROUPS = None
         FakeModuleBuilder.backend = "test"
 
+    @classmethod
+    def get_module_build_arches(cls, module):
+        return ["x86_64"]
+
     def buildroot_connect(self, groups):
         default_groups = FakeModuleBuilder.DEFAULT_GROUPS or {
             "srpm-build": set([

tests/test_builder/test_koji.py

@@ -98,6 +98,10 @@ class FakeKojiModuleBuilder(KojiModuleBuilder):
 
         return koji_session
 
+    @classmethod
+    def get_module_build_arches(cls, module):
+        return ["x86_64"]
+
 
 class TestKojiBuilder:
     def setup_method(self, test_method):
@@ -463,13 +467,11 @@ class TestKojiBuilder:
         assert weights == {"httpd": 1.5, "apr": 1.5}
         session.krb_login.assert_called_once()
 
-    @patch.object(conf, "base_module_arches", new={"platform:xx": ["x86_64", "i686"]})
     @pytest.mark.parametrize("blocklist", [False, True])
     @pytest.mark.parametrize("custom_whitelist", [False, True])
     @pytest.mark.parametrize("repo_include_all", [False, True])
-    @pytest.mark.parametrize("override_arches", [False, True])
     def test_buildroot_connect(
-        self, custom_whitelist, blocklist, repo_include_all, override_arches
+        self, custom_whitelist, blocklist, repo_include_all
     ):
         if blocklist:
             mmd = self.module.mmd()
@@ -495,14 +497,7 @@ class TestKojiBuilder:
             mmd.set_xmd(xmd)
             self.module.modulemd = mmd_to_str(mmd)
 
-        if override_arches:
-            mmd = self.module.mmd()
-            xmd = mmd.get_xmd()
-            mbs_options = xmd["mbs"] if "mbs" in xmd.keys() else {}
-            mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}
-            xmd["mbs"] = mbs_options
-            mmd.set_xmd(xmd)
-            self.module.modulemd = mmd_to_str(mmd)
+        self.module.arches.append(module_build_service.models.ModuleArch(name="i686"))
 
         builder = FakeKojiModuleBuilder(
             owner=self.module.owner,
@@ -549,10 +544,7 @@ class TestKojiBuilder:
         expected_calls = []
         assert session.packageListBlock.mock_calls == expected_calls
 
-        if override_arches:
-            expected_arches = "x86_64 i686"
-        else:
-            expected_arches = "i686 armv7hl x86_64"
+        expected_arches = "x86_64 i686"
 
         expected_calls = [
             mock.call(
@@ -824,9 +816,18 @@ class TestKojiBuilder:
     @patch.dict("sys.modules", krbV=MagicMock())
     @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
     def test_ensure_builder_use_a_logged_in_koji_session(self, ClientSession):
-        builder = KojiModuleBuilder("owner", MagicMock(), conf, "module-tag", [])
+        builder = KojiModuleBuilder("owner", self.module, conf, "module-tag", [])
         builder.koji_session.krb_login.assert_called_once()
 
+    @patch.dict("sys.modules", krbV=MagicMock())
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_module_build_arches(self, ClientSession):
+        arches = "x86_64 i686 ppc64le aarch64 s390x"
+        session = ClientSession.return_value
+        session.getTag.return_value = {"arches": arches}
+        ret = KojiModuleBuilder.get_module_build_arches(self.module)
+        assert " ".join(ret) == arches
+
 
 class TestGetDistTagSRPM:
     """Test KojiModuleBuilder.get_disttag_srpm"""

tests/test_scheduler/test_module_init.py

@@ -59,7 +59,8 @@ class TestModuleInit:
     )
     @patch("module_build_service.scm.SCM")
     @patch("module_build_service.scheduler.handlers.modules.handle_stream_collision_modules")
-    def test_init_basic(self, rscm, mocked_scm, built_rpms):
+    @patch("module_build_service.utils.submit.get_build_arches", return_value=["x86_64"])
+    def test_init_basic(self, get_build_arches, rscm, mocked_scm, built_rpms):
         FakeSCM(
             mocked_scm,
             "testmodule",
@@ -119,7 +120,8 @@ class TestModuleInit:
         assert mmd_to_str(old_mmd) == mmd_to_str(new_mmd)
 
     @patch("module_build_service.scm.SCM")
-    def test_init_scm_not_available(self, mocked_scm):
+    @patch("module_build_service.utils.submit.get_build_arches", return_value=["x86_64"])
+    def test_init_scm_not_available(self, get_build_arches, mocked_scm):
         def mocked_scm_get_latest():
             raise RuntimeError("Failed in mocked_scm_get_latest")
 
@@ -141,7 +143,8 @@ class TestModuleInit:
         return_value=True,
     )
     @patch("module_build_service.scm.SCM")
-    def test_init_includedmodule(self, mocked_scm, mocked_mod_allow_repo):
+    @patch("module_build_service.utils.submit.get_build_arches", return_value=["x86_64"])
+    def test_init_includedmodule(self, get_build_arches, mocked_scm, mocked_mod_allow_repo):
         FakeSCM(mocked_scm, "includedmodules", ["testmodule_init.yaml"])
         includedmodules_yml_path = os.path.join(self.staged_data_dir, "includedmodules.yaml")
         mmd = load_mmd_file(includedmodules_yml_path)
@@ -177,7 +180,9 @@ class TestModuleInit:
 
     @patch("module_build_service.models.ModuleBuild.from_module_event")
     @patch("module_build_service.scm.SCM")
-    def test_init_when_get_latest_raises(self, mocked_scm, mocked_from_module_event):
+    @patch("module_build_service.utils.submit.get_build_arches", return_value=["x86_64"])
+    def test_init_when_get_latest_raises(
+            self, get_build_arches, mocked_scm, mocked_from_module_event):
         FakeSCM(
             mocked_scm,
             "testmodule",

tests/test_utils/test_utils.py

@@ -311,6 +311,40 @@ class TestUtils:
     def teardown_method(self, test_method):
         clean_database()
 
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
+    def test_get_build_arches(self, ClientSession):
+        session = ClientSession.return_value
+        session.getTag.return_value = {"arches": "ppc64le"}
+        mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))
+        r = module_build_service.utils.get_build_arches(mmd, conf)
+        assert r == ["ppc64le"]
+
+    @patch(
+        "module_build_service.config.Config.allowed_privileged_module_names",
+        new_callable=mock.PropertyMock,
+        return_value=["testmodule"],
+    )
+    def test_get_build_arches_koji_tag_arches(self, cfg):
+        mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))
+        xmd = mmd.get_xmd()
+        xmd["mbs"]["koji_tag_arches"] = ["ppc64", "ppc64le"]
+        mmd.set_xmd(xmd)
+
+        r = module_build_service.utils.get_build_arches(mmd, conf)
+        assert r == ["ppc64", "ppc64le"]
+
+    @patch.object(conf, "base_module_arches", new={"platform:xx": ["x86_64", "i686"]})
+    def test_get_build_arches_base_module_override(self):
+        mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))
+        xmd = mmd.get_xmd()
+        mbs_options = xmd["mbs"] if "mbs" in xmd.keys() else {}
+        mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}
+        xmd["mbs"] = mbs_options
+        mmd.set_xmd(xmd)
+
+        r = module_build_service.utils.get_build_arches(mmd, conf)
+        assert r == ["x86_64", "i686"]
+
     @pytest.mark.parametrize("context", ["c1", None])
     def test_import_mmd_contexts(self, context):
         mmd = load_mmd_file(path.join(BASE_DIR, "..", "staged_data", "formatted_testmodule.yaml"))
@@ -425,13 +459,13 @@ class TestUtils:
         assert release == "module+fedora28+2+814cfa39"
 
     @patch(
-        "module_build_service.config.Config.allowed_disttag_marking_module_names",
+        "module_build_service.config.Config.allowed_privileged_module_names",
         new_callable=mock.PropertyMock,
         return_value=["build"],
     )
     def test_get_rpm_release_metadata_br_stream_override(self, mock_admmn):
         """
-        Test that when a module buildrequires a module in conf.allowed_disttag_marking_module_names,
+        Test that when a module buildrequires a module in conf.allowed_privileged_module_names,
         and that module has the xmd.mbs.disttag_marking field set, it should influence the disttag.
         """
         scheduler_init_data(1)
@@ -477,6 +511,17 @@ class TestUtils:
         release = module_build_service.utils.get_rpm_release(build_one)
         assert release == "scrmod+f28+2+814cfa39"
 
+    @patch("module_build_service.utils.submit.get_build_arches")
+    def test_record_module_build_arches(self, get_build_arches):
+        get_build_arches.return_value = ["x86_64", "i686"]
+        scheduler_init_data(1)
+        build = models.ModuleBuild.query.get(2)
+        build.arches = []
+        module_build_service.utils.record_module_build_arches(build.mmd(), build, db.session)
+
+        arches = set([arch.name for arch in build.arches])
+        assert arches == set(get_build_arches.return_value)
+
     @pytest.mark.parametrize(
         "scmurl",
         [

tests/test_views/test_views.py

@@ -2373,7 +2373,7 @@ class TestViews:
     @patch("module_build_service.auth.get_user", return_value=user)
     @patch("module_build_service.scm.SCM")
     @patch(
-        "module_build_service.config.Config.allowed_disttag_marking_module_names",
+        "module_build_service.config.Config.allowed_privileged_module_names",
         new_callable=PropertyMock,
         return_value=["build"],
     )