From e94dee5c508ebdbaf443f35dfecb2011df73700b Mon Sep 17 00:00:00 2001 From: jobrauer Date: Mon, 17 Feb 2020 10:53:29 +0100 Subject: [PATCH 1/7] Add test_buildrequire_module_not_in_tag --- .../test_buildrequire_module_not_in_tag.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/integration/test_buildrequire_module_not_in_tag.py diff --git a/tests/integration/test_buildrequire_module_not_in_tag.py b/tests/integration/test_buildrequire_module_not_in_tag.py new file mode 100644 index 00000000..6a5b9a33 --- /dev/null +++ b/tests/integration/test_buildrequire_module_not_in_tag.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MIT +import pytest +from sh import ErrorReturnCode + + +def test_buildrequire_module_not_in_tag(pkg_util, scenario, repo, koji): + """ + Run a build with with an invalid 'build_require' module. + I.e.: required module is picked in such a way, + that it is not tagged according to the the base module (platform) requirements, + see platform's modulemd file and its 'koji_tag_with_modules' attribute + (e.g.: platform: el-8.1.0 --> rhel-8.1.0-modules-build). + + Koji resolver is expected to not be able to satisfy this build requirement and hence fail the build. + + Assert that: + * the module build hasn't been accepted by MBS - rhpkg utility returns something else than 0 + + If assert fails: + * cancel all triggered module builds + + """ + + repo.bump() + + try: + builds = pkg_util.run( + "--optional", + "rebuild_strategy=all" + ) + + for build in builds: + print("Canceling module-build {}...".format(build.id)) + pkg_util.cancel(build) + + pytest.fail("build_require satisfied and module build accepted by MBS!") + + except ErrorReturnCode as e: + # expected outcome is that build submission fails + pass From cdd0dcbb376cdc2af9e73e1a983139484709182f Mon Sep 17 00:00:00 2001 From: jobrauer Date: Mon, 17 Feb 2020 13:29:03 +0100 Subject: [PATCH 2/7] Add better assert, remove stdout/err redirect --- ...py => test_buildrequire_invalid_module.py} | 26 +++++++++++-------- tests/integration/utils.py | 6 ++++- 2 files changed, 20 insertions(+), 12 deletions(-) rename tests/integration/{test_buildrequire_module_not_in_tag.py => test_buildrequire_invalid_module.py} (51%) diff --git a/tests/integration/test_buildrequire_module_not_in_tag.py b/tests/integration/test_buildrequire_invalid_module.py similarity index 51% rename from tests/integration/test_buildrequire_module_not_in_tag.py rename to tests/integration/test_buildrequire_invalid_module.py index 6a5b9a33..02d2bc7b 100644 --- a/tests/integration/test_buildrequire_module_not_in_tag.py +++ b/tests/integration/test_buildrequire_invalid_module.py @@ -4,7 +4,11 @@ import pytest from sh import ErrorReturnCode -def test_buildrequire_module_not_in_tag(pkg_util, scenario, repo, koji): +# expected error message +CANNOT_FIND_ANY_MODULE_BUILDS = "Cannot find any module builds" # ... + + +def test_buildrequire_invalid_module(pkg_util, scenario, repo, koji): """ Run a build with with an invalid 'build_require' module. I.e.: required module is picked in such a way, @@ -12,30 +16,30 @@ def test_buildrequire_module_not_in_tag(pkg_util, scenario, repo, koji): see platform's modulemd file and its 'koji_tag_with_modules' attribute (e.g.: platform: el-8.1.0 --> rhel-8.1.0-modules-build). - Koji resolver is expected to not be able to satisfy this build requirement and hence fail the build. + Koji resolver is expected to not be able to satisfy this build requirement + and hence fail the build. Assert that: - * the module build hasn't been accepted by MBS - rhpkg utility returns something else than 0 + * the module build hasn't been accepted by MBS: + rhpkg utility returns something else than 0 + * "Cannot find any module build..." found on STDERR If assert fails: - * cancel all triggered module builds + * cancel all triggered module builds. """ repo.bump() try: - builds = pkg_util.run( - "--optional", - "rebuild_strategy=all" - ) + builds = pkg_util.run("--optional", "rebuild_strategy=all") for build in builds: print("Canceling module-build {}...".format(build.id)) pkg_util.cancel(build) - pytest.fail("build_require satisfied and module build accepted by MBS!") + pytest.fail("Invalid 'build_require' was satisfied and module build was accepted by MBS!") except ErrorReturnCode as e: - # expected outcome is that build submission fails - pass + # expected outcome: build submission fails + assert CANNOT_FIND_ANY_MODULE_BUILDS in e.stderr.decode("utf-8") diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 480db8a9..7eb12b1c 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -167,7 +167,11 @@ class PackagingUtility: def __init__(self, packaging_utility, mbs_api): self._packaging_utility = Command(packaging_utility).bake( - _out=sys.stdout, _err=sys.stderr, _tee=True + # review: is redirect necessary? + # In case of failure, I can't find the stderr in the resulting exception object.. + #_out=sys.stdout, + #_err=sys.stderr, + _tee=True ) self._mbs_api = mbs_api From 938c243450d2345f18eff79ff0ecda212b463258 Mon Sep 17 00:00:00 2001 From: jobrauer Date: Wed, 19 Feb 2020 16:03:30 +0100 Subject: [PATCH 3/7] Satisfy flake --- tests/integration/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7eb12b1c..5642c58f 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -169,8 +169,8 @@ class PackagingUtility: self._packaging_utility = Command(packaging_utility).bake( # review: is redirect necessary? # In case of failure, I can't find the stderr in the resulting exception object.. - #_out=sys.stdout, - #_err=sys.stderr, + # _out=sys.stdout, + # _err=sys.stderr, _tee=True ) self._mbs_api = mbs_api From a40f87e7d1252fe8a1bf94195dda67662c9ca7bf Mon Sep 17 00:00:00 2001 From: jobrauer Date: Mon, 24 Feb 2020 11:09:36 +0100 Subject: [PATCH 4/7] Address CRW - add pytest.raises - add **kwargs in PackagingUtility.run(...) --- .../test_buildrequire_invalid_module.py | 57 +++++++++---------- tests/integration/utils.py | 10 ++-- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/tests/integration/test_buildrequire_invalid_module.py b/tests/integration/test_buildrequire_invalid_module.py index 02d2bc7b..cc14cd20 100644 --- a/tests/integration/test_buildrequire_invalid_module.py +++ b/tests/integration/test_buildrequire_invalid_module.py @@ -4,42 +4,39 @@ import pytest from sh import ErrorReturnCode -# expected error message -CANNOT_FIND_ANY_MODULE_BUILDS = "Cannot find any module builds" # ... - - def test_buildrequire_invalid_module(pkg_util, scenario, repo, koji): """ - Run a build with with an invalid 'build_require' module. - I.e.: required module is picked in such a way, - that it is not tagged according to the the base module (platform) requirements, - see platform's modulemd file and its 'koji_tag_with_modules' attribute - (e.g.: platform: el-8.1.0 --> rhel-8.1.0-modules-build). + Run a build with with an invalid 'build_require' module. + I.e.: required module is picked in such a way, + that it is not tagged according to the the base module (platform) requirements, + see platform's modulemd file and its 'koji_tag_with_modules' attribute + (e.g.: platform: el-8.1.0 --> rhel-8.1.0-modules-build). - Koji resolver is expected to not be able to satisfy this build requirement - and hence fail the build. + Koji resolver is expected to not be able to satisfy this build requirement + and hence fail the build. - Assert that: - * the module build hasn't been accepted by MBS: - rhpkg utility returns something else than 0 - * "Cannot find any module build..." found on STDERR + Assert that: + * the module build hasn't been accepted by MBS: + rhpkg utility returns something else than 0 + * "Cannot find any module build..." found on STDERR - If assert fails: - * cancel all triggered module builds. + If assert fails: + * cancel all triggered module builds. - """ + """ repo.bump() - try: - builds = pkg_util.run("--optional", "rebuild_strategy=all") - - for build in builds: - print("Canceling module-build {}...".format(build.id)) - pkg_util.cancel(build) - - pytest.fail("Invalid 'build_require' was satisfied and module build was accepted by MBS!") - - except ErrorReturnCode as e: - # expected outcome: build submission fails - assert CANNOT_FIND_ANY_MODULE_BUILDS in e.stderr.decode("utf-8") + expected_error = "Cannot find any module builds" + with pytest.raises(ErrorReturnCode) as excinfo: + # Override 'baked' (_err=sys.stderr) stderr redirect: + # Here we are fine with what plain sh.Command gives us + # (otherwise ErrorReturnCode.stderr is incomplete). + builds = pkg_util.run("--optional", "rebuild_strategy=all", _err=None) + try: + for build in builds: + print("Canceling module-build {}...".format(build.id)) + pkg_util.cancel(build) + except: + pass + assert expected_error in excinfo.value.stderr.decode("utf-8") diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 5642c58f..62dcbc00 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -167,15 +167,13 @@ class PackagingUtility: def __init__(self, packaging_utility, mbs_api): self._packaging_utility = Command(packaging_utility).bake( - # review: is redirect necessary? - # In case of failure, I can't find the stderr in the resulting exception object.. - # _out=sys.stdout, - # _err=sys.stderr, + _out=sys.stdout, + _err=sys.stderr, _tee=True ) self._mbs_api = mbs_api - def run(self, *args, reuse=None): + def run(self, *args, reuse=None, **kwargs): """Run one or more module builds :param args: Options and arguments for the build command @@ -195,7 +193,7 @@ class PackagingUtility: else: build_ids = [reuse] else: - stdout = self._packaging_utility("module-build", *args).stdout.decode("utf-8") + stdout = self._packaging_utility("module-build", *args, **kwargs).stdout.decode("utf-8") build_ids = re.findall(self._mbs_api + r"module-builds/(\d+)", stdout) return [Build(self._mbs_api, int(build_id)) for build_id in build_ids] From d6aefc8d5501d35edd92e23f67f646db66265579 Mon Sep 17 00:00:00 2001 From: jobrauer Date: Mon, 24 Feb 2020 11:39:44 +0100 Subject: [PATCH 5/7] Add except ErrorReturnCode to cancel build --- tests/integration/test_buildrequire_invalid_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_buildrequire_invalid_module.py b/tests/integration/test_buildrequire_invalid_module.py index cc14cd20..487cdc3a 100644 --- a/tests/integration/test_buildrequire_invalid_module.py +++ b/tests/integration/test_buildrequire_invalid_module.py @@ -37,6 +37,6 @@ def test_buildrequire_invalid_module(pkg_util, scenario, repo, koji): for build in builds: print("Canceling module-build {}...".format(build.id)) pkg_util.cancel(build) - except: + except ErrorReturnCode: pass assert expected_error in excinfo.value.stderr.decode("utf-8") From 3fa8e5a61067875b3c8a1fe7db3a66901503ce00 Mon Sep 17 00:00:00 2001 From: jobrauer Date: Tue, 25 Feb 2020 07:50:50 +0100 Subject: [PATCH 6/7] Add comment --- tests/integration/test_buildrequire_invalid_module.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_buildrequire_invalid_module.py b/tests/integration/test_buildrequire_invalid_module.py index 487cdc3a..cf06af35 100644 --- a/tests/integration/test_buildrequire_invalid_module.py +++ b/tests/integration/test_buildrequire_invalid_module.py @@ -38,5 +38,7 @@ def test_buildrequire_invalid_module(pkg_util, scenario, repo, koji): print("Canceling module-build {}...".format(build.id)) pkg_util.cancel(build) except ErrorReturnCode: + # Do nothing, this is just a clean-up of accidentally started builds + # in case that the test-case fails pass assert expected_error in excinfo.value.stderr.decode("utf-8") From 5d8a54db2381f94509f25d55c00b223842ca5a1b Mon Sep 17 00:00:00 2001 From: jobrauer Date: Mon, 9 Mar 2020 08:57:38 +0100 Subject: [PATCH 7/7] Add test.env example --- tests/integration/example.test.env.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/example.test.env.yaml b/tests/integration/example.test.env.yaml index 6666a123..16636cb3 100644 --- a/tests/integration/example.test.env.yaml +++ b/tests/integration/example.test.env.yaml @@ -88,3 +88,6 @@ testdata: # For this scenario reusing former builds doesn't make sense. module: testmodule2 branch: test-stream-expans-branch + buildrequire_invalid_module: + module: testmodule + branch: test-buildrequire-invalid-module