Merge #1538 Various test improvements

.gitignore

@@ -24,3 +24,4 @@ requirements.txt.orig
 mbstest.db
 htmlcov/
 test.env.yaml
+report.html

tests/integration/conftest.py

@@ -2,14 +2,18 @@
 # SPDX-License-Identifier: MIT
 
 import os
+import sys
 import tempfile
 
 import pytest
-from sh import git, pushd
+import sh
 import yaml
 
 import utils
 
+our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True)
+from our_sh import pushd, Command  # noqa
+
 
 @pytest.fixture(scope="session")
 def test_env():
@@ -26,14 +30,13 @@ def test_env():
 
 @pytest.fixture(scope="function")
 def repo(request, test_env):
-    """Clone the git repo to be used by the test
+    """Clone the module repo to be used by the test
 
-    Find out the name of the test (anything that follow "test_"),
-    and get the corresponding git repo configuration from the test
-    environment configuration.
+    Find out the name of the test (anything that follow "test_"), and get
+    the corresponding module repo from the test environment configuration.
 
-    Do a shallow clone of the git repo in a temporary location and
-    switch the current working directory into it.
+    Clone the repo in a temporary location and switch the current working
+    directory into it.
 
     :param pytest.FixtureRequest request: request object giving access
         to the requesting test context
@@ -44,14 +47,16 @@ def repo(request, test_env):
     with tempfile.TemporaryDirectory() as tempdir:
         testname = request.function.__name__.split("test_", 1)[1]
         repo_conf = test_env["testdata"][testname]
-        url = test_env["git_url"] + repo_conf["module"]
+        packaging_util = Command(test_env["packaging_utility"]).bake(
+            _out=sys.stdout, _err=sys.stderr, _tee=True
+        )
         args = [
             "--branch",
             repo_conf["branch"],
-            url,
+            f"modules/{repo_conf['module']}",
             tempdir,
         ]
-        git("clone", *args)
+        packaging_util("clone", *args)
         with pushd(tempdir):
             yield utils.Repo(repo_conf["module"])
 

tests/integration/example.test.env.yaml

@@ -1,14 +1,14 @@
 ---
+# Utility to be used to clone and build the modules.
+# It's configuration points to the dist-git where
+# test modules are to be found.
 packaging_utility: fedpkg
 # API endpoint of the MBS instance under test.
 mbs_api: https://mbs.fedoraproject.org/module-build-service/2/module-builds/
-# Git instance used by the build system.
-git_url: https://src.fedoraproject.org/
 # Koji instance the MBS instance under test communicates with.
 koji:
   server: https://koji.fedoraproject.org/kojihub
   topurl: https://kojipkgs.fedoraproject.org/
-  weburl: https://brewweb.stage.engineering.redhat.com/brewroot
 # Test data to be used by the tests.
 # Items in here are mapped by their name to the tests that use them.
 # For example test_scratch_build will use scratch_build.

tests/integration/test_failed_build.py

@@ -14,13 +14,13 @@ def test_failed_build(test_env, repo, koji):
         cancelled, if not completed.
     """
     build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])
+    repo.bump()
     build.run(
-        "--watch",
-        "--scratch",
         "--optional",
         "rebuild_strategy=all",
         reuse=test_env["testdata"]["failed_build"].get("build_id"),
     )
+    build.watch()
 
     assert build.state_name == "failed"
     batch = test_env["testdata"]["failed_build"]["batch"]

tests/integration/test_normal_build.py

@@ -18,13 +18,14 @@ def test_normal_build(test_env, repo, koji):
         if a Platform stream is representing a GA RHEL release.
     """
     build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])
+    repo.bump()
     build_id = build.run(
-        "--watch",
-        "--scratch",
         "--optional",
         "rebuild_strategy=all",
         reuse=test_env["testdata"]["normal_build"].get("build_id"),
     )
+    build.watch()
+
     assert sorted(build.component_names()) == sorted(repo.components + ["module-build-macros"])
 
     expected_buildorder = test_env["testdata"]["normal_build"]["buildorder"]

tests/integration/test_scratch_build.py

@@ -16,12 +16,12 @@ def test_scratch_build(test_env, repo, koji):
     """
     build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"])
     build.run(
-        "--watch",
         "--scratch",
         "--optional",
         "rebuild_strategy=all",
         reuse=test_env["testdata"]["scratch_build"].get("build_id"),
     )
+    build.watch()
 
     assert build.state_name == "done"
     assert sorted(build.component_names(state="COMPLETE")) == sorted(

tests/integration/utils.py

@@ -2,13 +2,17 @@
 # SPDX-License-Identifier: MIT
 
 import re
+import sys
 import time
 
 from kobo import rpmlib
 import koji
 import yaml
 import requests
-from sh import Command, git
+import sh
+
+our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True)
+from our_sh import Command, git  # noqa
 
 
 class Koji:
@@ -16,17 +20,15 @@ class Koji:
 
     :attribute string _server: URL of the Koji hub
     :attribute string _topurl: URL of the top-level Koji download location
-    :attribute string _weburl: URL of the web interface
     :attribute koji.ClientSession _session: Koji session
     :attribute koji.PathInfo _pathinfo: Koji path
     """
 
-    def __init__(self, server, topurl, weburl):
+    def __init__(self, server, topurl):
         self._server = server
         self._topurl = topurl
-        self._weburl = weburl
         self._session = koji.ClientSession(self._server)
-        self._pathinfo = koji.PathInfo(self._weburl)
+        self._pathinfo = koji.PathInfo(self._topurl)
 
     def get_build(self, nvr_dict):
         """Koji build data for NVR
@@ -123,7 +125,9 @@ class Build:
     """
 
     def __init__(self, packaging_utility, mbs_api):
-        self._packaging_utility = Command(packaging_utility)
+        self._packaging_utility = Command(packaging_utility).bake(
+            _out=sys.stdout, _err=sys.stderr, _tee=True
+        )
         self._mbs_api = mbs_api
         self._data = None
         self._component_data = None
@@ -148,6 +152,17 @@ class Build:
             self._build_id = int(re.search(self._mbs_api + r"module-builds/(\d+)", stdout).group(1))
         return self._build_id
 
+    def watch(self):
+        """Watch the build till the finish"""
+        if self._build_id is None:
+            raise RuntimeError("Build was not started. Cannot watch.")
+
+        stdout = self._packaging_utility(
+            "module-build-watch", str(self._build_id)
+        ).stdout.decode("utf-8")
+
+        return stdout
+
     def cancel(self):
         """Cancel the module build
 
@@ -249,7 +264,7 @@ class Build:
 
         return batches
 
-    def wait_for_koji_task_id(self, package, batch, timeout=60, sleep=10):
+    def wait_for_koji_task_id(self, package, batch, timeout=300, sleep=10):
         """Wait until the component is submitted to Koji (has a task_id)
 
         :param string: name of component (package)

tox.ini

@@ -73,6 +73,7 @@ deps =
     kobo
     koji
     pytest
+    pytest-html
     pytest-xdist
     PyYAML
     requests
@@ -81,4 +82,9 @@ deps =
 # if the instance tested has a self-signed certificate.
 passenv = REQUESTS_CA_BUNDLE MBS_TEST_CONFIG MBS_TEST_WORKERS HOME
 commands =
-    pytest -vv --confcutdir=tests/integration -n {env:MBS_TEST_WORKERS:0} {posargs:tests/integration}
+    pytest -rA -vv \
+        --confcutdir=tests/integration \
+        -n {env:MBS_TEST_WORKERS:0} \
+        --html=report.html \
+        --self-contained-html \
+        {posargs:tests/integration}