From 30215a1ac5b7a15aa84384f012a663e92ec05199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:11:15 +0100 Subject: [PATCH 1/6] Tests: improve reporting when running with tox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate a short report at the end of each test run and a self-contained HTML report, too. Signed-off-by: Hunor Csomortáni --- .gitignore | 1 + tox.ini | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index de8bdcac..87431d68 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ requirements.txt.orig mbstest.db htmlcov/ test.env.yaml +report.html diff --git a/tox.ini b/tox.ini index deab2549..f36548e5 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,7 @@ deps = kobo koji pytest + pytest-html pytest-xdist PyYAML requests @@ -81,4 +82,9 @@ deps = # if the instance tested has a self-signed certificate. passenv = REQUESTS_CA_BUNDLE MBS_TEST_CONFIG MBS_TEST_WORKERS HOME commands = - pytest -vv --confcutdir=tests/integration -n {env:MBS_TEST_WORKERS:0} {posargs:tests/integration} + pytest -rA -vv \ + --confcutdir=tests/integration \ + -n {env:MBS_TEST_WORKERS:0} \ + --html=report.html \ + --self-contained-html \ + {posargs:tests/integration} From cfb3549e79d6408afbaa38af73c85a38dd9c8e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:18:11 +0100 Subject: [PATCH 2/6] Tests: clone modules with the packaging utility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is how packagers do. Git URL configuration can also be removed. Also: redirect command outputs, so that pytest can capture them. Signed-off-by: Hunor Csomortáni --- tests/integration/conftest.py | 25 +++++++++++++++---------- tests/integration/example.test.env.yaml | 5 +++-- tests/integration/utils.py | 10 ++++++++-- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 813b916a..1a710a30 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,14 +2,18 @@ # SPDX-License-Identifier: MIT import os +import sys import tempfile import pytest -from sh import git, pushd +import sh import yaml import utils +our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True) +from our_sh import pushd, Command # noqa + @pytest.fixture(scope="session") def test_env(): @@ -26,14 +30,13 @@ def test_env(): @pytest.fixture(scope="function") def repo(request, test_env): - """Clone the git repo to be used by the test + """Clone the module repo to be used by the test - Find out the name of the test (anything that follow "test_"), - and get the corresponding git repo configuration from the test - environment configuration. + Find out the name of the test (anything that follow "test_"), and get + the corresponding module repo from the test environment configuration. - Do a shallow clone of the git repo in a temporary location and - switch the current working directory into it. + Clone the repo in a temporary location and switch the current working + directory into it. :param pytest.FixtureRequest request: request object giving access to the requesting test context @@ -44,14 +47,16 @@ def repo(request, test_env): with tempfile.TemporaryDirectory() as tempdir: testname = request.function.__name__.split("test_", 1)[1] repo_conf = test_env["testdata"][testname] - url = test_env["git_url"] + repo_conf["module"] + packaging_util = Command(test_env["packaging_utility"]).bake( + _out=sys.stdout, _err=sys.stderr, _tee=True + ) args = [ "--branch", repo_conf["branch"], - url, + f"modules/{repo_conf['module']}", tempdir, ] - git("clone", *args) + packaging_util("clone", *args) with pushd(tempdir): yield utils.Repo(repo_conf["module"]) diff --git a/tests/integration/example.test.env.yaml b/tests/integration/example.test.env.yaml index 69eb5cb0..595494b2 100644 --- a/tests/integration/example.test.env.yaml +++ b/tests/integration/example.test.env.yaml @@ -1,9 +1,10 @@ --- +# Utility to be used to clone and build the modules. +# It's configuration points to the dist-git where +# test modules are to be found. packaging_utility: fedpkg # API endpoint of the MBS instance under test. mbs_api: https://mbs.fedoraproject.org/module-build-service/2/module-builds/ -# Git instance used by the build system. -git_url: https://src.fedoraproject.org/ # Koji instance the MBS instance under test communicates with. koji: server: https://koji.fedoraproject.org/kojihub diff --git a/tests/integration/utils.py b/tests/integration/utils.py index cd8c239b..de0840e2 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,13 +2,17 @@ # SPDX-License-Identifier: MIT import re +import sys import time from kobo import rpmlib import koji import yaml import requests -from sh import Command, git +import sh + +our_sh = sh(_out=sys.stdout, _err=sys.stderr, _tee=True) +from our_sh import Command, git # noqa class Koji: @@ -123,7 +127,9 @@ class Build: """ def __init__(self, packaging_utility, mbs_api): - self._packaging_utility = Command(packaging_utility) + self._packaging_utility = Command(packaging_utility).bake( + _out=sys.stdout, _err=sys.stderr, _tee=True + ) self._mbs_api = mbs_api self._data = None self._component_data = None From 72bb5a29435d7863737c4787fe0a7b92c0f2dde4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:23:35 +0100 Subject: [PATCH 3/6] Tests: bump the commit before builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whenever the tests start a "normal" (non-scratch) build, the commit needs to be bumped. Signed-off-by: Hunor Csomortáni --- tests/integration/test_failed_build.py | 1 + tests/integration/test_normal_build.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration/test_failed_build.py b/tests/integration/test_failed_build.py index 422c44bc..eb63c4b5 100644 --- a/tests/integration/test_failed_build.py +++ b/tests/integration/test_failed_build.py @@ -14,6 +14,7 @@ def test_failed_build(test_env, repo, koji): cancelled, if not completed. """ build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"]) + repo.bump() build.run( "--watch", "--scratch", diff --git a/tests/integration/test_normal_build.py b/tests/integration/test_normal_build.py index a1ca5d66..a414b015 100644 --- a/tests/integration/test_normal_build.py +++ b/tests/integration/test_normal_build.py @@ -18,6 +18,7 @@ def test_normal_build(test_env, repo, koji): if a Platform stream is representing a GA RHEL release. """ build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"]) + repo.bump() build_id = build.run( "--watch", "--scratch", From 343f1b98186c77158eca49d9d2a90b7af34f8488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:25:07 +0100 Subject: [PATCH 4/6] Tests: separate starting and watching the builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the tests more explicit, but also prepares the ground to enable canceling test builds during session teardown in the future. Signed-off-by: Hunor Csomortáni --- tests/integration/test_failed_build.py | 3 +-- tests/integration/test_normal_build.py | 4 ++-- tests/integration/test_scratch_build.py | 2 +- tests/integration/utils.py | 11 +++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_failed_build.py b/tests/integration/test_failed_build.py index eb63c4b5..9ef644c8 100644 --- a/tests/integration/test_failed_build.py +++ b/tests/integration/test_failed_build.py @@ -16,12 +16,11 @@ def test_failed_build(test_env, repo, koji): build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"]) repo.bump() build.run( - "--watch", - "--scratch", "--optional", "rebuild_strategy=all", reuse=test_env["testdata"]["failed_build"].get("build_id"), ) + build.watch() assert build.state_name == "failed" batch = test_env["testdata"]["failed_build"]["batch"] diff --git a/tests/integration/test_normal_build.py b/tests/integration/test_normal_build.py index a414b015..0dece3e2 100644 --- a/tests/integration/test_normal_build.py +++ b/tests/integration/test_normal_build.py @@ -20,12 +20,12 @@ def test_normal_build(test_env, repo, koji): build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"]) repo.bump() build_id = build.run( - "--watch", - "--scratch", "--optional", "rebuild_strategy=all", reuse=test_env["testdata"]["normal_build"].get("build_id"), ) + build.watch() + assert sorted(build.component_names()) == sorted(repo.components + ["module-build-macros"]) expected_buildorder = test_env["testdata"]["normal_build"]["buildorder"] diff --git a/tests/integration/test_scratch_build.py b/tests/integration/test_scratch_build.py index 0636a85e..f7182f61 100644 --- a/tests/integration/test_scratch_build.py +++ b/tests/integration/test_scratch_build.py @@ -16,12 +16,12 @@ def test_scratch_build(test_env, repo, koji): """ build = utils.Build(test_env["packaging_utility"], test_env["mbs_api"]) build.run( - "--watch", "--scratch", "--optional", "rebuild_strategy=all", reuse=test_env["testdata"]["scratch_build"].get("build_id"), ) + build.watch() assert build.state_name == "done" assert sorted(build.component_names(state="COMPLETE")) == sorted( diff --git a/tests/integration/utils.py b/tests/integration/utils.py index de0840e2..5f76572d 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -154,6 +154,17 @@ class Build: self._build_id = int(re.search(self._mbs_api + r"module-builds/(\d+)", stdout).group(1)) return self._build_id + def watch(self): + """Watch the build till the finish""" + if self._build_id is None: + raise RuntimeError("Build was not started. Cannot watch.") + + stdout = self._packaging_utility( + "module-build-watch", str(self._build_id) + ).stdout.decode("utf-8") + + return stdout + def cancel(self): """Cancel the module build From b2485a615029a1d179803d78576d63c558bfcaa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:30:10 +0100 Subject: [PATCH 5/6] Tests: fix koji.PathInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should use "topurl". Remove the "weburl" config as it's not needed anymore. Signed-off-by: Hunor Csomortáni --- tests/integration/example.test.env.yaml | 1 - tests/integration/utils.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/integration/example.test.env.yaml b/tests/integration/example.test.env.yaml index 595494b2..b364754e 100644 --- a/tests/integration/example.test.env.yaml +++ b/tests/integration/example.test.env.yaml @@ -9,7 +9,6 @@ mbs_api: https://mbs.fedoraproject.org/module-build-service/2/module-builds/ koji: server: https://koji.fedoraproject.org/kojihub topurl: https://kojipkgs.fedoraproject.org/ - weburl: https://brewweb.stage.engineering.redhat.com/brewroot # Test data to be used by the tests. # Items in here are mapped by their name to the tests that use them. # For example test_scratch_build will use scratch_build. diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 5f76572d..d5a71a67 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -20,17 +20,15 @@ class Koji: :attribute string _server: URL of the Koji hub :attribute string _topurl: URL of the top-level Koji download location - :attribute string _weburl: URL of the web interface :attribute koji.ClientSession _session: Koji session :attribute koji.PathInfo _pathinfo: Koji path """ - def __init__(self, server, topurl, weburl): + def __init__(self, server, topurl): self._server = server self._topurl = topurl - self._weburl = weburl self._session = koji.ClientSession(self._server) - self._pathinfo = koji.PathInfo(self._weburl) + self._pathinfo = koji.PathInfo(self._topurl) def get_build(self, nvr_dict): """Koji build data for NVR From 35c1097920a0082a10edc12db47bd0cf3ec1698d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hunor=20Csomort=C3=A1ni?= Date: Fri, 6 Dec 2019 12:31:58 +0100 Subject: [PATCH 6/6] Tests: increase koji task timeout to 5 minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When running multiple test builds, the 1 minute timeout was often not enough. Signed-off-by: Hunor Csomortáni --- tests/integration/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d5a71a67..1ea3e183 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -264,7 +264,7 @@ class Build: return batches - def wait_for_koji_task_id(self, package, batch, timeout=60, sleep=10): + def wait_for_koji_task_id(self, package, batch, timeout=300, sleep=10): """Wait until the component is submitted to Koji (has a task_id) :param string: name of component (package)