buildonly support

This fixes #1307

Signed-off-by: Valerij Maljulin <vmaljuli@redhat.com>

module_build_service/migrations/versions/4d1e2e13e514_buildonly.py

@@ -0,0 +1,23 @@
+"""Add buildonly column
+
+Revision ID: 4d1e2e13e514
+Revises: 1817e62719f9
+Create Date: 2019-10-01 13:49:20.446641
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4d1e2e13e514'
+down_revision = '1817e62719f9'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column("component_builds", sa.Column("buildonly", sa.Boolean(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("component_builds", schema=None) as batch_op:
+        batch_op.drop_column("buildonly")

module_build_service/models.py

@@ -1219,6 +1219,8 @@ class ComponentBuild(MBSBase):
     # True when this component build is build-time only (should be tagged only
     # to -build tag)
     build_time_only = db.Column(db.Boolean, default=False)
+    # True if buildonly is True in modulemd
+    buildonly = db.Column(db.Boolean, default=False)
 
     # A monotonically increasing integer that represents which batch or
     # iteration this *component* is currently in.  This relates to the owning

module_build_service/scheduler/handlers/components.py

@@ -3,11 +3,11 @@
 """ Handlers for koji component build events on the message bus. """
 
 import logging
-
+import koji
 import module_build_service.builder
 
-import koji
-
+from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder
+from module_build_service.utils.general import mmd_to_str
 from module_build_service import models, log, messaging
 
 logging.basicConfig(level=logging.DEBUG)
@@ -56,6 +56,19 @@ def _finalize(config, db_session, msg, state):
         db_session.commit()
         return
 
+    if (
+        component_build.buildonly
+        and config.system in ["koji", "test"]
+        and state == koji.BUILD_STATES["COMPLETE"]
+    ):
+        koji_session = KojiModuleBuilder.get_session(config)
+        rpms = koji_session.listBuildRPMs(component_build.nvr)
+        mmd = parent.mmd()
+        for artifact in rpms:
+            mmd.add_rpm_filter(artifact["name"])
+        parent.modulemd = mmd_to_str(mmd)
+        db_session.commit()
+
     further_work = []
 
     parent_current_batch = parent.current_batch()

module_build_service/utils/submit.py

@@ -175,8 +175,6 @@ def format_mmd(mmd, scmurl, module=None, db_session=None):
                     "Custom component caches aren't allowed.  "
                     "%r bears cache %r" % (pkgname, pkg.get_cache())
                 )
-            if pkg.get_buildonly() is True:
-                raise ValidationError('The usage of "buildonly" is not yet supported')
             if pkg.get_buildafter():
                 raise ValidationError('The usage of "buildafter" is not yet supported')
             if not pkg.get_repository():
@@ -508,6 +506,7 @@ def record_component_builds(
             batch=batch,
             ref=component_ref,
             weight=rpm_weights[package],
+            buildonly=component.get_buildonly()
         )
         db_session.add(build)
 

tests/staged_data/testmodule_v2_buildonly.yaml

@@ -0,0 +1,39 @@
+document: modulemd
+version: 2
+data:
+  summary: A test module in all its beautiful beauty
+  description: >-
+    This module demonstrates how to write simple modulemd files And can be used for
+    testing the build and release pipeline. ’
+  license:
+    module:
+    - MIT
+  dependencies:
+  - buildrequires:
+      platform: [f28]
+    requires:
+      platform: [f28]
+  references:
+    community: https://docs.pagure.org/modularity/
+    documentation: https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules
+  profiles:
+    default:
+      rpms:
+      - tangerine
+  api:
+    rpms:
+    - perl-Tangerine
+    - tangerine
+  components:
+    rpms:
+      perl-List-Compare:
+        rationale: A dependency of tangerine.
+        ref: master
+        buildonly: true
+      perl-Tangerine:
+        rationale: Provides API for this module and is a dependency of tangerine.
+        ref: master
+      tangerine:
+        rationale: Provides API for this module.
+        ref: master
+        buildorder: 10

tests/test_build/test_build.py

@@ -18,7 +18,7 @@ from module_build_service.errors import Forbidden
 from module_build_service import models, conf, build_logs
 from module_build_service.scheduler import make_simple_stop_condition
 
-from mock import patch, PropertyMock, Mock
+from mock import patch, PropertyMock, Mock, MagicMock
 from werkzeug.datastructures import FileStorage
 import kobo
 import pytest
@@ -497,6 +497,74 @@ class TestBuild(BaseTestBuild):
         assert module_build.module_builds_trace[4].state == models.BUILD_STATES["ready"]
         assert len(module_build.module_builds_trace) == 5
 
+    @patch("module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder.get_session")
+    @patch("module_build_service.auth.get_user", return_value=user)
+    @patch("module_build_service.scm.SCM")
+    def test_submit_build_buildonly(
+        self, mocked_scm, mocked_get_user, mocked_get_session, conf_system, dbg, hmsc, db_session
+    ):
+        def list_build_rpms(nvr):
+            assert nvr == "perl-List-Compare-1-1"
+            return [{"name": "perl-List-Compare"}]
+        koji_session = MagicMock()
+        koji_session.listBuildRPMs = list_build_rpms
+        mocked_get_session.return_value = koji_session
+
+        yaml_file = "testmodule_v2_buildonly.yaml"
+        FakeSCM(mocked_scm, "testmodule", yaml_file, "620ec77321b2ea7b0d67d82992dda3e1d67055b4")
+
+        rv = self.client.post(
+            "/module-build-service/1/module-builds/",
+            data=json.dumps({
+                "branch": "master",
+                "scmurl": "https://src.stg.fedoraproject.org/modules/"
+                "testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4",
+            }),
+        )
+
+        data = json.loads(rv.data)
+        module_build_id = data["id"]
+
+        # Check that components are tagged after the batch is built.
+        tag_groups = [
+            {"perl-Tangerine-1-1", "perl-List-Compare-1-1"},
+            {"tangerine-1-1"},
+        ]
+
+        def on_finalize_cb(cls, succeeded):
+            assert succeeded is True
+
+        def on_tag_artifacts_cb(cls, artifacts, dest_tag=True):
+            assert tag_groups.pop(0) == set(artifacts)
+
+        FakeModuleBuilder.on_finalize_cb = on_finalize_cb
+        FakeModuleBuilder.on_tag_artifacts_cb = on_tag_artifacts_cb
+
+        # Check that the components are added to buildroot after the batch
+        # is built.
+        buildroot_groups = [
+            {"module-build-macros-1-1"},
+            {"perl-Tangerine-1-1", "perl-List-Compare-1-1"},
+            {"tangerine-1-1"},
+        ]
+
+        def on_buildroot_add_artifacts_cb(cls, artifacts, install):
+            assert buildroot_groups.pop(0) == set(artifacts)
+
+        FakeModuleBuilder.on_buildroot_add_artifacts_cb = on_buildroot_add_artifacts_cb
+
+        self.run_scheduler(db_session)
+
+        # All components should be built and module itself should be in "done"
+        # or "ready" state.
+        for build in db_session.query(models.ComponentBuild).filter_by(
+                module_id=module_build_id).all():
+            assert build.state == koji.BUILD_STATES["COMPLETE"]
+            assert build.module_build.state in [
+                models.BUILD_STATES["done"],
+                models.BUILD_STATES["ready"],
+            ]
+
     @pytest.mark.parametrize("gating_result", (True, False))
     @patch("module_build_service.auth.get_user", return_value=user)
     @patch("module_build_service.scm.SCM")

tests/test_models/test_models.py

@@ -24,6 +24,7 @@ class TestModels:
             nvr="before_models_committed-0.0.0-0.module_before_models_committed_0_0",
             batch=1,
             module_id=1,
+            buildonly=True,
         )
 
         db_session.add(component_build)