Merge #1498 Add integration test to check scratch module builds

docker/test-py3.sh

@@ -22,4 +22,4 @@ sed -i \
 
 # Since tox seems to ignore `usedevelop` when we have `sitepackages` on, we have to run it manually
 python3 setup.py develop --no-deps
-/usr/bin/tox -e flake8,py3 "$@"
+/usr/bin/tox -e flake8,py3,intflake "$@"

tests/integration/conftest.py

@@ -2,14 +2,17 @@
 # SPDX-License-Identifier: MIT
 
 import os
+import tempfile
 
-import yaml
 import pytest
+from sh import git, pushd
+import yaml
 
-from utils import MBS, Git, Koji
+import utils
 
 
-def load_test_env():
+@pytest.fixture(scope="session")
+def test_env():
     """Load test environment configuration
 
     :return: Test environment configuration.
@@ -21,19 +24,43 @@ def load_test_env():
     return env
 
 
-test_env = load_test_env()
+@pytest.fixture(scope="function")
+def repo(request, test_env):
+    """Clone the git repo to be used by the test
+
+    Find out the name of the test (anything that follow "test_"),
+    and get the corresponding git repo configuration from the test
+    environment configuration.
+
+    Do a shallow clone of the git repo in a temporary location and
+    switch the current working directory into it.
+
+    :param pytest.FixtureRequest request: request object giving access
+        to the requesting test context
+    :param pytest.fixture test_env: test environment fixture
+    :return: repository object the tests can work with
+    :rtype: utils.Repo
+    """
+    with tempfile.TemporaryDirectory() as tempdir:
+        testname = request.function.__name__.split("test_", 1)[1]
+        repo_conf = test_env["testdata"][testname]
+        url = test_env["git_url"] + repo_conf["module"]
+        args = [
+            "--branch",
+            repo_conf["branch"],
+            "--single-branch",
+            "--depth",
+            "1",
+            url,
+            tempdir,
+        ]
+        git("clone", *args)
+        with pushd(tempdir):
+            yield utils.Repo(repo_conf["module"])
 
 
 @pytest.fixture(scope="session")
-def mbs():
-    return MBS(test_env["mbs_api"])
-
-
-@pytest.fixture(scope="session")
-def git():
-    return Git(test_env["git_url"])
-
-
-@pytest.fixture(scope="session")
-def koji():
-    return Koji(**test_env["koji"])
+def koji(test_env):
+    """Koji session for the instance MBS is configured to work with
+    """
+    return utils.Koji(**test_env["koji"])

tests/integration/example.test.env.yaml

@@ -1,4 +1,5 @@
 ---
+packaging_utility: fedpkg
 # API endpoint of the MBS instance under test.
 mbs_api: https://mbs.fedoraproject.org/module-build-service/2/module-builds/
 # Git instance used by the build system.
@@ -7,3 +8,12 @@ git_url: https://src.fedoraproject.org/
 koji:
   server: https://koji.fedoraproject.org/kojihub
   topurl: https://kojipkgs.fedoraproject.org/
+# Test data to be used by the tests.
+# Items in here are mapped by their name to the tests that use them.
+# For example test_scratch_build will use scratch_build.
+testdata:
+  scratch_build:
+    # Name of the module.
+    module: testmodule
+    # Branch which is going to be built for this test.
+    branch: scratch-build-branch

tests/integration/test_normal_build.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# SPDX-License-Identifier: MIT
-
-
-def test_normal_build():
-    """ TODO(csomh): implement this test
-    """
-    assert True

tests/integration/test_scratch_build.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+
+import utils
+
+
+def test_scratch_build(test_env, repo, koji):
+    """
+    Run a scratch build with "rebuild_strategy=all".
+
+    Check that:
+    * the module build is done with the correct components
+    * the module build completes in the "done" state
+      (as opposed to the "ready" state)
+    * no content generator builds are created in Koji
+    """
+    build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])
+    build.run("--watch", "--scratch", "--optional", "rebuild_strategy=all")
+
+    assert build.state_name == "done"
+    assert sorted(build.components()) == sorted(
+        repo.components + ["module-build-macros"]
+    )
+
+    cg_build = koji.get_build(build.nvr())
+    cg_devel_build = koji.get_build(build.nvr(name_suffix="-devel"))
+    assert not (cg_build or cg_devel_build)

tests/integration/utils.py

@@ -1,21 +1,136 @@
 # -*- coding: utf-8 -*-
 # SPDX-License-Identifier: MIT
 
+import re
 
-class MBS:
-
-    def __init__(self, api):
-        self._api = api
-
-
-class Git:
-
-    def __init__(self, url):
-        self._url = url
+from kobo import rpmlib
+import koji
+import yaml
+import requests
+from sh import Command
 
 
 class Koji:
+    """Wrapper class to work with Koji
+
+    :attribute string _server: URL of the Koji hub
+    :attribute string _topurl: URL of the top-level Koji download location
+    :attribute koji.ClientSession _session: Koji session
+    """
 
     def __init__(self, server, topurl):
         self._server = server
         self._topurl = topurl
+        self._session = koji.ClientSession(self._server)
+
+    def get_build(self, nvr_dict):
+        """Koji build data for NVR
+
+        :param dict nvr_dict: NVR dictionary as expected by kobo.rpmlib.make_nvr()
+        :return: Dictionary with Koji build data or None, if build is not found
+        :rtype: dict or None
+        """
+        nvr_string = rpmlib.make_nvr(nvr_dict)
+        return self._session.getBuild(nvr_string)
+
+
+class Repo:
+    """Wrapper class to work with module git repositories
+
+    :attribute string module_name: name of the module stored in this repo
+    :attribute dict _modulemd: Modulemd file as read from the repo
+    """
+
+    def __init__(self, module_name):
+        self.module_name = module_name
+        self._modulemd = None
+
+    @property
+    def modulemd(self):
+        """Modulemd file as read from the repo
+
+        :return: Modulemd file as read from the repo
+        :rtype: dict
+        """
+        if self._modulemd is None:
+            modulemd_file = self.module_name + ".yaml"
+            with open(modulemd_file, "r") as f:
+                self._modulemd = yaml.safe_load(f)
+        return self._modulemd
+
+    @property
+    def components(self):
+        """List of components as defined in the modulemd file
+
+        :return: List of components as defined in the modulemd file
+        :rtype: list of strings
+        """
+        return list(self.modulemd["data"]["components"]["rpms"])
+
+
+class Build:
+    """Wrapper class to work with module builds
+
+    :attribute sh.Command _packaging_utility: packaging utility command used to
+        kick off this build
+    :attribute string _mbs_api: URL of the MBS API (including trailing '/')
+    :attribute string _url: URL of this MBS module build
+    :attribute string _data: Module build data cache for this build fetched from MBS
+    """
+
+    def __init__(self, packaging_utility, mbs_api):
+        self._packaging_utility = Command(packaging_utility)
+        self._mbs_api = mbs_api
+        self._url = None
+        self._data = None
+
+    def run(self, *args):
+        """Run a module build
+
+        :param args: Options and arguments for the build command
+        :return: MBS API URL for the build created
+        :rtype: string
+        """
+        stdout = self._packaging_utility("module-build", *args).stdout.decode("utf-8")
+        self._url = re.search(self._mbs_api + r"module-builds/\d+", stdout).group(0)
+        return self._url
+
+    @property
+    def data(self):
+        """Module build data cache for this build fetched from MBS"""
+        if self._data is None:
+            r = requests.get(self._url)
+            r.raise_for_status()
+            self._data = r.json()
+        return self._data
+
+    @property
+    def state_name(self):
+        """Name of the state of this module build"""
+        return self.data["state_name"]
+
+    def components(self, state="COMPLETE"):
+        """Components of this module build which are in some state
+
+        :param string state: Koji build state the components should be in
+        :return: List of components
+        :rtype: list of strings
+        """
+        comps = []
+        for rpm, info in self.data["tasks"]["rpms"].items():
+            if info["state"] == koji.BUILD_STATES[state]:
+                comps.append(rpm)
+        return comps
+
+    def nvr(self, name_suffix=""):
+        """NVR dictionary of this module build
+
+        :param string name_suffix: an optional suffix for the name component of the NVR
+        :return: dictionary with NVR components
+        :rtype: dict
+        """
+        return {
+            "name": f'{self.data["name"]}{name_suffix}',
+            "version": self.data["stream"].replace("-", "_"),
+            "release": f'{self.data["version"]}.{self.data["context"]}',
+        }

tox.ini

@@ -4,17 +4,12 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = flake8, py27, py3
+envlist = flake8, intflake, py27, py3
 
 [flake8]
 ignore = E731,W503
 max-line-length = 100
-exclude =
-    ./.tox
-    ./.git
-    ./module_build_service/migrations
-    ./build
-    ./.env
+exclude = .tox,.git,module_build_service/migrations,build,.env
 
 [testenv]
 usedevelop = true
@@ -49,7 +44,16 @@ commands =
 basepython = python2
 skip_install = true
 deps = flake8
-commands = flake8
+# doing this until --extend-exclude support becomes available
+# https://flake8.readthedocs.io/en/latest/user/options.html#cmdoption-flake8-extend-exclude
+commands = flake8 --exclude={[flake8]exclude},tests/integration
+
+[testenv:intflake]
+basepython = python3
+skip_install = true
+sitepackages = false
+deps = flake8
+commands = flake8 tests/integration
 
 [testenv:bandit]
 basepython = python2
@@ -66,9 +70,12 @@ skip_install = true
 sitepackages = false
 # let's handle integration test deps separately
 deps =
+    kobo
+    koji
     pytest
-    requests
     PyYAML
+    requests
+    sh
 # Set this to /etc/pki/tls/certs/ca-bundle.crt, for example,
 # if the instance tested has a self-signed certificate.
 passenv = REQUESTS_CA_BUNDLE MBS_TEST_CONFIG