diff --git a/docs/GATING.rst b/docs/GATING.rst index 0a613175..925062b2 100644 --- a/docs/GATING.rst +++ b/docs/GATING.rst @@ -1,20 +1,8 @@ -Modules gating using Greenwave -============================== +Modules gating +============== Every successfully built module is moved to the ``done`` state. Modules in this state cannot be used as a build dependency for other modules. They need to be moved to the ``ready`` state. -By default, MBS moves the module from the ``done`` state to the ``ready`` state automatically, -but MBS can also be configured to gate the ``done`` to ``ready`` state transition using -`Greenwave `_. - -When Greenwave integration is configured, the following additional MBS features are enabled: - -- When the module is moved to the ``done`` state, Greenwave is queried to find out whether - the module can be moved to the ``ready`` state instantly. -- If the module cannot be moved to the ``ready`` state yet, MBS keeps the module build in the - ``done`` state and waits for a message from Greenwave. If this message says that all the - tests defined by Greenwave policy have passed, then the module build is moved to the ``ready`` - state. -- MBS also queries Greenwave periodically to find out the current gating status for modules - in the ``done`` state. This is useful in case a message from Greenwave was missed. +By default, MBS moves the module from the ``done`` state to the ``ready`` state automatically. +The Koji resolver, if used, will handle gating naturally. diff --git a/module_build_service/common/config.py b/module_build_service/common/config.py index e76bba80..ea6ad0f3 100644 --- a/module_build_service/common/config.py +++ b/module_build_service/common/config.py @@ -62,11 +62,6 @@ class TestConfiguration(BaseConfiguration): ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - # Greenwave configuration - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" - STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} # Ensures task.delay executes locally instead of scheduling a task to a queue. @@ -605,11 +600,6 @@ class Config(object): ], "desc": ("The list packages for offline module build RPM buildroot."), }, - "greenwave_decision_context": { - "type": str, - "default": "", - "desc": "The Greenwave decision context that determines a module's gating status.", - }, "allowed_privileged_module_names": { "type": list, "default": [], @@ -628,22 +618,6 @@ class Config(object): "corresponding suffix added to formatted stream version. " 'For example, {r"regexp": 0.1, ...}', }, - "greenwave_url": { - "type": str, - "default": "", - "desc": "The URL of the server where Greenwave is running (should include " - "the root of the API)" - }, - "greenwave_subject_type": { - "type": str, - "default": "", - "desc": "Subject type for Greenwave requests" - }, - "greenwave_timeout": { - "type": int, - "default": 60, - "desc": "Greenwave response timeout" - }, "modules_allow_scratch": { "type": bool, "default": False, @@ -755,7 +729,6 @@ class Config(object): "module_build_service.scheduler.handlers.modules", "module_build_service.scheduler.handlers.repos", "module_build_service.scheduler.handlers.tags", - "module_build_service.scheduler.handlers.greenwave", "module_build_service.scheduler.producer", ], "desc": "The list Python paths for the Celery application to import.", diff --git a/module_build_service/common/errors.py b/module_build_service/common/errors.py index 7c370a88..44946e06 100644 --- a/module_build_service/common/errors.py +++ b/module_build_service/common/errors.py @@ -37,9 +37,5 @@ class StreamAmbigous(ValueError): pass -class GreenwaveError(RuntimeError): - pass - - class IgnoreMessage(Exception): """Raise if message received from message bus should be ignored""" diff --git a/module_build_service/common/messaging.py b/module_build_service/common/messaging.py index 60c08d58..90fec373 100644 --- a/module_build_service/common/messaging.py +++ b/module_build_service/common/messaging.py @@ -95,7 +95,7 @@ def _in_memory_publish(topic, msg, conf, service): _initial_messages.append(wrapped_msg) -known_fedmsg_services = ["buildsys", "mbs", "greenwave"] +known_fedmsg_services = ["buildsys", "mbs"] _fedmsg_backend = { diff --git a/module_build_service/scheduler/consumer.py b/module_build_service/scheduler/consumer.py index d2fa2c4e..10350d5d 100644 --- a/module_build_service/scheduler/consumer.py +++ b/module_build_service/scheduler/consumer.py @@ -28,7 +28,7 @@ import module_build_service.common.monitor as monitor from module_build_service.scheduler import events from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler.handlers import components, repos, modules, greenwave, tags +from module_build_service.scheduler.handlers import components, repos, modules, tags def no_op_handler(*args, **kwargs): @@ -57,7 +57,6 @@ ON_MODULE_CHANGE_HANDLERS = { # Only one kind of repo change event, though... ON_REPO_CHANGE_HANDLER = repos.done ON_TAG_CHANGE_HANDLER = tags.tagged -ON_DECISION_UPDATE_HANDLER = greenwave.decision_update class MBSConsumer(fedmsg.consumers.FedmsgConsumer): @@ -234,12 +233,6 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer): models.ModuleBuild.get_by_id(db_session, event_info["module_build_id"]) ) - if event == events.GREENWAVE_DECISION_UPDATE: - return ( - ON_DECISION_UPDATE_HANDLER, - greenwave.get_corresponding_module_build(event_info["subject_identifier"]) - ) - return None, None def process_message(self, event_info): diff --git a/module_build_service/scheduler/events.py b/module_build_service/scheduler/events.py index f0060b2c..b02dc7bf 100644 --- a/module_build_service/scheduler/events.py +++ b/module_build_service/scheduler/events.py @@ -24,7 +24,6 @@ KOJI_BUILD_CHANGE = "koji_build_change" KOJI_TAG_CHANGE = "koji_tag_change" KOJI_REPO_CHANGE = "koji_repo_change" MBS_MODULE_STATE_CHANGE = "mbs_module_state_change" -GREENWAVE_DECISION_UPDATE = "greenwave_decision_update" class Scheduler(sched.scheduler): diff --git a/module_build_service/scheduler/greenwave.py b/module_build_service/scheduler/greenwave.py deleted file mode 100644 index 9f1b770c..00000000 --- a/module_build_service/scheduler/greenwave.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from __future__ import absolute_import -from functools import reduce -import json - -import requests - -from module_build_service.common import log, conf -from module_build_service.common.errors import GreenwaveError - - -class Greenwave(object): - def __init__(self): - """ - Initialize greenwave instance with config - """ - self.url = conf.greenwave_url - self._decision_context = conf.greenwave_decision_context - if not self.decision_context: - raise RuntimeError("No Greenwave decision context set") - self._subj_type = conf.greenwave_subject_type - self._gw_timeout = conf.greenwave_timeout - self.error_occurred = False - - def _greenwave_query(self, query_type, payload=None): - """ - Make a query to greenwave - :param query_type: will be part of url - :type query_type: str - :param payload: request payload used in 'decision' query - :type payload: str - :return: response - :rtype: dict - """ - query_func = requests.post if payload else requests.get - kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout} - - if payload: - kwargs["headers"] = {"Content-Type": "application/json"} - kwargs["data"] = payload - - try: - response = query_func(**kwargs) - except requests.exceptions.Timeout: - raise GreenwaveError("Greenwave request timed out") - except Exception as exc: - error_message = "Unspecified greenwave request error " \ - '(original exception was: "{0}")'.format(str(exc)) - log.exception(error_message) - raise GreenwaveError(error_message) - - try: - resp_json = response.json() - except ValueError: - log.debug("Greenwave response content (status {0}): {1}".format( - response.status_code, response.text - )) - raise GreenwaveError("Greenwave returned invalid JSON.") - - log.debug( - 'Query to Greenwave (%s) result: status=%d, content="%s"', - kwargs["url"], response.status_code, resp_json - ) - - if response.status_code == 200: - return resp_json - - try: - err_msg = resp_json["message"] - except KeyError: - err_msg = response.text - raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format( - response.status_code, err_msg - )) - - def query_decision(self, build, prod_version): - """ - Query decision to greenwave - :param build: build object - :type build: module_build_service.common.models.ModuleBuild - :param prod_version: The product version string used for querying WaiverDB - :type prod_version: str - :return: response - :rtype: dict - """ - payload = { - "decision_context": self.decision_context, - "product_version": prod_version, - "subject_type": self.subject_type, - "subject_identifier": build.nvr_string - } - return self._greenwave_query('decision', json.dumps(payload)) - - def query_policies(self, return_all=False): - """ - Query policies to greenwave - :param return_all: Return all policies, if False select by subject_type and decision_context - :type return_all: bool - :return: response - :rtype: dict - """ - response = self._greenwave_query('policies') - - if return_all: - return response - - try: - selective_resp = { - "policies": [ - pol for pol in response["policies"] - if pol["decision_context"] == self.decision_context - and pol["subject_type"] == self.subject_type - ] - } - except KeyError: - log.exception("Incorrect greenwave response (Mandatory key is missing)") - raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)") - return selective_resp - - def get_product_versions(self): - """ - Return a set of product versions according to decision_context and subject_type - :return: product versions - :rtype: set - """ - return reduce( - lambda old, new: old.union(new), - [pol["product_versions"] for pol in self.query_policies()["policies"]], - set() - ) - - def check_gating(self, build): - """ - Query decision to greenwave - :param build: build object - :type build: module_build_service.common.models.ModuleBuild - :return: True if at least one GW response contains policies_satisfied set to true - :rtype: bool - """ - self.error_occurred = False - try: - versions = self.get_product_versions() - except GreenwaveError: - log.warning('An error occured while getting a product versions') - self.error_occurred = True - return False - - for ver in versions: - try: - if self.query_decision(build, ver)["policies_satisfied"]: - # at least one positive result is enough - return True - except (KeyError, GreenwaveError) as exc: - self.error_occurred = True - log.warning('Incorrect greenwave result "%s", ignoring', str(exc)) - - return False - - @property - def url(self): - return self._url - - @url.setter - def url(self, value): - value = value.rstrip("/") - if not value: - raise RuntimeError("No Greenwave URL set") - self._url = value - - @property - def decision_context(self): - return self._decision_context - - @property - def subject_type(self): - return self._subj_type - - @property - def timeout(self): - return self._gw_timeout - - @timeout.setter - def timeout(self, value): - self._gw_timeout = value - - -try: - greenwave = Greenwave() -except RuntimeError: - log.warning('Greenwave is not configured or configured improperly') - greenwave = None diff --git a/module_build_service/scheduler/handlers/greenwave.py b/module_build_service/scheduler/handlers/greenwave.py deleted file mode 100644 index e8898c9b..00000000 --- a/module_build_service/scheduler/handlers/greenwave.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from __future__ import absolute_import - -from module_build_service.common import conf, log -from module_build_service.common.koji import get_session -from module_build_service.common.models import ModuleBuild, BUILD_STATES -from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler import celery_app, events - - -def get_corresponding_module_build(nvr): - """Find corresponding module build from database and return - - :param str nvr: module build NVR. This is the subject_identifier included - inside ``greenwave.decision.update`` message. - :return: the corresponding module build object. For whatever the reason, - if the original module build id cannot be found from the Koji build of - ``nvr``, None will be returned. - :rtype: :class:`ModuleBuild` or None - """ - koji_session = get_session(conf, login=False) - build_info = koji_session.getBuild(nvr) - if build_info is None: - return None - - try: - module_build_id = build_info["extra"]["typeinfo"]["module"]["module_build_service_id"] - except KeyError: - # If any of the keys is not present, the NVR is not the one for - # handling Greenwave event. - return None - - return ModuleBuild.get_by_id(db_session, module_build_id) - - -@celery_app.task -@events.mbs_event_handler -def decision_update(msg_id, decision_context, subject_identifier, policies_satisfied): - """Move module build to ready or failed according to Greenwave result - - :param str msg_id: the original id of the message being handled which is - received from the message bus. - :param str decision_context: the context of the greewave decision. Refer to - the messaging document for detailed information. - :param str subject_identifier: usually a build NVR. Refer to - https://docs.pagure.org/greenwave/messaging.html for detailed information. - :param bool policies_satisfied: whether the build satisfies Greenwave rules. - Refer to the messaging document for detailed information. - """ - if not conf.greenwave_decision_context: - log.debug( - "Skip Greenwave message %s as MBS does not have GREENWAVE_DECISION_CONTEXT " - "configured", - msg_id, - ) - return - - if decision_context != conf.greenwave_decision_context: - log.debug( - "Skip Greenwave message %s as MBS only handles messages with the " - 'decision context "%s"', - msg_id, - conf.greenwave_decision_context, - ) - return - - module_build_nvr = subject_identifier - - if not policies_satisfied: - log.debug( - "Skip to handle module build %s because it has not satisfied Greenwave policies.", - module_build_nvr, - ) - return - - build = get_corresponding_module_build(module_build_nvr) - - if build is None: - log.debug( - "No corresponding module build of subject_identifier %s is found.", module_build_nvr) - return - - if build.state == BUILD_STATES["done"]: - build.transition( - db_session, - conf, - BUILD_STATES["ready"], - state_reason="Module build {} has satisfied Greenwave policies.".format( - module_build_nvr - ), - ) - else: - log.warning( - "Module build %s is not in done state but Greenwave tells " - "it passes tests in decision context %s", - module_build_nvr, decision_context, - ) - - db_session.commit() diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 3b858dd1..f955495a 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -3,7 +3,6 @@ """ Handlers for module change events on the message bus. """ from __future__ import absolute_import -from datetime import datetime import logging import os @@ -29,7 +28,6 @@ from module_build_service.scheduler import celery_app, events from module_build_service.scheduler.db_session import db_session from module_build_service.scheduler.default_modules import ( add_default_modules, handle_collisions_with_base_module_rpms) -from module_build_service.scheduler.greenwave import greenwave from module_build_service.scheduler.reuse import attempt_to_reuse_all_components from module_build_service.scheduler.submit import format_mmd, get_module_srpm_overrides from module_build_service.scheduler.ursine import handle_stream_collision_modules @@ -144,13 +142,7 @@ def done(msg_id, module_build_id, module_build_state): # Scratch builds stay in 'done' state if not build.scratch: - if greenwave is None or greenwave.check_gating(build): - build.transition(db_session, conf, state=models.BUILD_STATES["ready"]) - else: - build.state_reason = "Gating failed" - if greenwave.error_occurred: - build.state_reason += " (Error occured while querying Greenwave)" - build.time_modified = datetime.utcnow() + build.transition(db_session, conf, state=models.BUILD_STATES["ready"]) db_session.commit() build_logs.stop(build) diff --git a/module_build_service/scheduler/parser.py b/module_build_service/scheduler/parser.py index e960c762..1f6a4590 100644 --- a/module_build_service/scheduler/parser.py +++ b/module_build_service/scheduler/parser.py @@ -12,7 +12,7 @@ class MessageParser(object): :param topic_categories: list of known services, that MBS can handle the messages sent from them. For example, a value could be - ``["buildsys", "mbs", "greenwave"]``. + ``["buildsys", "mbs"]``. :type topic_categories: list[str] """ @@ -114,13 +114,3 @@ class FedmsgMessageParser(MessageParser): "module_build_id": msg_inner_msg.get("id"), "module_build_state": msg_inner_msg.get("state"), } - - if (category == "greenwave" - and object == "decision" and subobject is None and event == "update"): - return { - "msg_id": msg_id, - "event": events.GREENWAVE_DECISION_UPDATE, - "decision_context": msg_inner_msg.get("decision_context"), - "policies_satisfied": msg_inner_msg.get("policies_satisfied"), - "subject_identifier": msg_inner_msg.get("subject_identifier"), - } diff --git a/module_build_service/scheduler/producer.py b/module_build_service/scheduler/producer.py index b6f08f46..496da5fd 100644 --- a/module_build_service/scheduler/producer.py +++ b/module_build_service/scheduler/producer.py @@ -20,7 +20,6 @@ from module_build_service.scheduler.batches import ( start_next_batch_build, ) from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler.greenwave import greenwave from module_build_service.scheduler.handlers.components import build_task_finalize from module_build_service.scheduler.handlers.tags import tagged @@ -36,7 +35,6 @@ def setup_periodic_tasks(sender, **kwargs): (cleanup_stale_failed_builds, "Cleanup stale failed builds"), (cancel_stuck_module_builds, "Cancel stuck module builds"), (sync_koji_build_tags, "Sync Koji build tags"), - (poll_greenwave, "Gating module build to ready state"), ) for task, name in tasks: @@ -442,32 +440,6 @@ def sync_koji_build_tags(): tagged.delay("internal:sync_koji_build_tags", build_tag, c.nvr) -@celery_app.task -def poll_greenwave(): - """Polls Greenwave for all builds in done state""" - if greenwave is None: - return - - module_builds = db_session.query(models.ModuleBuild).filter_by( - state=models.BUILD_STATES["done"], - scratch=False - ).all() - - log.info("Checking Greenwave for %d builds", len(module_builds)) - - for build in module_builds: - if greenwave.check_gating(build): - build.transition(db_session, conf, state=models.BUILD_STATES["ready"]) - else: - build.state_reason = "Gating failed (MBS will retry in {0} seconds)".format( - conf.polling_interval - ) - if greenwave.error_occurred: - build.state_reason += " (Error occured while querying Greenwave)" - build.time_modified = datetime.utcnow() - db_session.commit() - - def has_missed_new_repo_message(module_build, koji_session): """ Returns whether or not a new repo message has probably been missed. diff --git a/module_build_service/scheduler/route.py b/module_build_service/scheduler/route.py index c74c78c6..b92bc187 100644 --- a/module_build_service/scheduler/route.py +++ b/module_build_service/scheduler/route.py @@ -7,7 +7,6 @@ import inspect from module_build_service.common import conf, log, models from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler.handlers.greenwave import get_corresponding_module_build def route_task(name, args, kwargs, options, task=None, **kw): @@ -57,11 +56,6 @@ def route_task(name, args, kwargs, options, task=None, **kw): module_build = models.ModuleBuild.get_by_tag(db_session, tag_name) if module_build: module_build_id = module_build.id - elif "subject_identifier" in handler_args: - module_build_nvr = _get_handler_arg("subject_identifier") - module_build = get_corresponding_module_build(module_build_nvr) - if module_build is not None: - module_build_id = module_build.id if module_build_id is not None: queue_name = "mbs-{}".format(module_build_id % num_workers) diff --git a/openshift/integration/koji/README.md b/openshift/integration/koji/README.md index e158ab88..0f9b595a 100644 --- a/openshift/integration/koji/README.md +++ b/openshift/integration/koji/README.md @@ -83,9 +83,5 @@ you need to run the trigger jobs once manually so Jenkins can setup required mes following jobs should be triggered manually: - mbs-trigger-on-latest-tag - mbs-trigger-on-stage-tag -- mbs-backend-greenwave-promote-to-stage -- mbs-backend-greenwave-promote-to-prod -- mbs-frontend-greenwave-promote-to-stage -- mbs-frontend-greenwave-promote-to-prod [OpenShift secret for a private registry]: https://docs.openshift.com/container-platform/3.11/dev_guide/builds/build_inputs.html#using-docker-credentials-for-private-registries diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 104c49df..1f8c5152 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -454,11 +454,6 @@ class TestBuild(BaseTestBuild): FakeModuleBuilder.on_get_task_info_cb = on_get_task_info_cb - self.p_check_gating = patch( - "module_build_service.scheduler.greenwave.Greenwave.check_gating", - return_value=True) - self.mock_check_gating = self.p_check_gating.start() - self.patch_config_broker = patch.object( module_build_service.common.config.Config, "celery_broker_url", @@ -469,7 +464,6 @@ class TestBuild(BaseTestBuild): self.patch_config_broker.start() def teardown_method(self, test_method): - self.p_check_gating.stop() self.patch_config_broker.stop() FakeModuleBuilder.reset() cleanup_moksha() @@ -628,16 +622,14 @@ class TestBuild(BaseTestBuild): models.BUILD_STATES["ready"], ] - @pytest.mark.parametrize("gating_result", (True, False)) @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.common.scm.SCM") def test_submit_build_no_components( - self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc, gating_result + self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc ): """ Tests the build of a module with no components """ - self.mock_check_gating.return_value = gating_result FakeSCM( mocked_scm, "python3", @@ -663,12 +655,6 @@ class TestBuild(BaseTestBuild): # Make sure no component builds were registered assert len(module_build.component_builds) == 0 - # Make sure the build is done - if gating_result: - assert module_build.state == models.BUILD_STATES["ready"] - else: - assert module_build.state == models.BUILD_STATES["done"] - assert module_build.state_reason == "Gating failed" @patch( "module_build_service.common.config.Config.check_for_eol", @@ -1913,7 +1899,6 @@ class TestLocalBuild(BaseTestBuild): except Exception: pass - @patch("module_build_service.scheduler.greenwave.Greenwave.query_policies") @patch("module_build_service.scheduler.handlers.modules.handle_stream_collision_modules") @patch("module_build_service.web.auth.get_user", return_value=user) @patch("module_build_service.common.scm.SCM") @@ -1923,7 +1908,7 @@ class TestLocalBuild(BaseTestBuild): return_value=staged_data_filename('local_builds'), ) def test_submit_build_local_dependency( - self, resultsdir, mocked_scm, mocked_get_user, conf_system, hmsc, mocked_greenwave + self, resultsdir, mocked_scm, mocked_get_user, conf_system, hmsc ): """ Tests local module build dependency. diff --git a/tests/test_memory/mbs_configuration.py b/tests/test_memory/mbs_configuration.py index 8f0e0430..b911af0d 100644 --- a/tests/test_memory/mbs_configuration.py +++ b/tests/test_memory/mbs_configuration.py @@ -22,9 +22,6 @@ class TestConfiguration: KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos" SCMURLS = ["https://src.stg.fedoraproject.org/modules/"] ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"} - GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/" - GREENWAVE_DECISION_CONTEXT = "test_dec_context" - GREENWAVE_SUBJECT_TYPE = "some-module" STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1} CELERY_TASK_ALWAYS_EAGER = True diff --git a/tests/test_memory/mbs_debug.py b/tests/test_memory/mbs_debug.py index a02f3d8e..6746ab43 100644 --- a/tests/test_memory/mbs_debug.py +++ b/tests/test_memory/mbs_debug.py @@ -42,9 +42,7 @@ class SimpleMock: new_callable=SimpleMock) @mock.patch("module_build_service.scheduler.handlers.modules.record_module_build_arches", new_callable=SimpleMock) -@mock.patch("module_build_service.scheduler.greenwave.Greenwave.check_gating", - new_callable=SimpleMock) -def run_debug_instance(mock_1, mock_2, mock_3, host=None, port=None): +def run_debug_instance(mock_1, mock_2, host=None, port=None): def handle_pdb(sig, frame): import pdb diff --git a/tests/test_scheduler/test_celery_route_task.py b/tests/test_scheduler/test_celery_route_task.py index 25c8d779..95d865c8 100644 --- a/tests/test_scheduler/test_celery_route_task.py +++ b/tests/test_scheduler/test_celery_route_task.py @@ -6,7 +6,7 @@ import mock from module_build_service.common.config import conf from module_build_service.scheduler import celery_app -from module_build_service.scheduler.handlers import components, greenwave, modules, repos, tags +from module_build_service.scheduler.handlers import components, modules, repos, tags from module_build_service.scheduler.producer import fail_lost_builds from tests import scheduler_init_data @@ -95,22 +95,6 @@ class TestCeleryRouteTask: qname = queue.__dict__.get("name") assert qname == "mbs-2" - @mock.patch("koji.ClientSession") - def test_route_greenwave_decision_update_task(self, kojisession, send_task_message): - kojisession.return_value.getBuild.return_value = { - "extra": {"typeinfo": {"module": {"module_build_service_id": 1}}} - } - scheduler_init_data() - greenwave.decision_update.delay( - "fakemsg", - decision_context="test_dec_context", - subject_identifier="module-testmodule-master-20170109091357-7c29193d-build", - policies_satisfied=False - ) - queue = send_task_message.call_args[1].get("queue") - qname = queue.__dict__.get("name") - assert qname == "mbs-1" - def test_route_fail_lost_builds_task(self, send_task_message): fail_lost_builds.delay() queue = send_task_message.call_args[1].get("queue") diff --git a/tests/test_scheduler/test_greenwave.py b/tests/test_scheduler/test_greenwave.py deleted file mode 100644 index c551de32..00000000 --- a/tests/test_scheduler/test_greenwave.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from __future__ import absolute_import -import json - -from mock import patch, Mock -import pytest - -from module_build_service.scheduler.greenwave import greenwave -from tests import make_module_in_db - - -@pytest.mark.usefixtures("require_empty_database") -class TestGreenwaveQuery: - - @patch("module_build_service.scheduler.greenwave.requests") - def test_greenwave_query_decision(self, mock_requests): - resp_status = 200 - resp_content = { - "applicable_policies": ["osci_compose_modules"], - "policies_satisfied": True, - "satisfied_requirements": [ - { - "result_id": 7336633, - "testcase": "test-ci.test-module.tier1", - "type": "test-result-passed" - }, - { - "result_id": 7336650, - "testcase": "test-ci.test-module.tier2", - "type": "test-result-passed" - } - ], - "summary": "All required tests passed", - "unsatisfied_requirements": [] - } - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.post.return_value = response - - fake_build = make_module_in_db( - "pkg:0.1:1:c1", [{ - "requires": {"platform": ["el8"]}, - "buildrequires": {"platform": ["el8"]}, - }], - ) - got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8") - - assert got_response == resp_content - assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == { - "decision_context": "test_dec_context", - "product_version": "xxxx-8", "subject_type": "some-module", - "subject_identifier": "pkg-0.1-1.c1"} - assert mock_requests.post.call_args_list[0][1]["headers"] == { - "Content-Type": "application/json"} - assert mock_requests.post.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/decision" - - @pytest.mark.parametrize("return_all", (False, True)) - @patch("module_build_service.scheduler.greenwave.requests") - def test_greenwave_query_policies(self, mock_requests, return_all): - resp_status = 200 - resp_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver2"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "decision_context_2", - "product_versions": ["ver4"], - "rules": [], - "subject_type": "subject_type_2" - } - ] - } - selected_policies = {"policies": resp_content["policies"][:-1]} - - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.get.return_value = response - - got_response = greenwave.query_policies(return_all) - - if return_all: - assert got_response == resp_content - else: - assert got_response == selected_policies - assert mock_requests.get.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/policies" - - @patch("module_build_service.scheduler.greenwave.requests") - def test_greenwave_get_product_versions(self, mock_requests): - resp_status = 200 - resp_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver2"], - "rules": [], - "subject_type": "some-module" - }, - { - "decision_context": "decision_context_2", - "product_versions": ["ver4"], - "rules": [], - "subject_type": "subject_type_2" - } - ] - } - expected_versions = {"ver1", "ver2", "ver3"} - - response = Mock() - response.json.return_value = resp_content - response.status_code = resp_status - mock_requests.get.return_value = response - - versions_set = greenwave.get_product_versions() - - assert versions_set == expected_versions - assert mock_requests.get.call_args_list[0][1]["url"] == \ - "https://greenwave.example.local/api/v1.0/policies" - - @pytest.mark.parametrize("policies_satisfied", (True, False)) - @patch("module_build_service.scheduler.greenwave.requests") - def test_greenwave_check_gating(self, mock_requests, policies_satisfied): - resp_status = 200 - policies_content = { - "policies": [ - { - "decision_context": "test_dec_context", - "product_versions": ["ver1", "ver3"], - "rules": [], - "subject_type": "some-module" - } - ] - } - - responses = [Mock() for i in range(3)] - for r in responses: - r.status_code = resp_status - responses[0].json.return_value = policies_content - responses[1].json.return_value = {"policies_satisfied": False} - responses[2].json.return_value = {"policies_satisfied": policies_satisfied} - mock_requests.get.return_value = responses[0] - mock_requests.post.side_effect = responses[1:] - - fake_build = make_module_in_db( - "pkg:0.1:1:c1", [{ - "requires": {"platform": ["el8"]}, - "buildrequires": {"platform": ["el8"]}, - }], - ) - result = greenwave.check_gating(fake_build) - - assert result == policies_satisfied diff --git a/tests/test_scheduler/test_greenwave_handler.py b/tests/test_scheduler/test_greenwave_handler.py deleted file mode 100644 index 384c37a2..00000000 --- a/tests/test_scheduler/test_greenwave_handler.py +++ /dev/null @@ -1,174 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT -from __future__ import absolute_import - -from mock import call, patch, PropertyMock, Mock -import pytest -from sqlalchemy import func - - -from module_build_service.common.config import conf -import module_build_service.common.config -from module_build_service.common.models import BUILD_STATES, ModuleBuild -from module_build_service.scheduler.consumer import MBSConsumer -from module_build_service.scheduler.db_session import db_session -from module_build_service.scheduler.handlers.greenwave import ( - decision_update, get_corresponding_module_build -) -from tests import make_module_in_db - - -@pytest.mark.usefixtures("require_empty_database") -class TestGetCorrespondingModuleBuild: - """Test get_corresponding_module_build""" - - @patch("koji.ClientSession") - def test_module_build_nvr_does_not_exist_in_koji(self, ClientSession): - ClientSession.return_value.getBuild.return_value = None - - assert get_corresponding_module_build("n-v-r") is None - - @pytest.mark.parametrize( - "build_info", - [ - # Build info does not have key extra - {"id": 1000, "name": "ed"}, - # Build info contains key extra, but it is not for the module build - {"extra": {"submitter": "osbs", "image": {}}}, - # Key module_build_service_id is missing - {"extra": {"typeinfo": {"module": {}}}}, - ], - ) - @patch("koji.ClientSession") - def test_cannot_find_module_build_id_from_build_info(self, ClientSession, build_info): - ClientSession.return_value.getBuild.return_value = build_info - - assert get_corresponding_module_build("n-v-r") is None - - @patch("koji.ClientSession") - def test_corresponding_module_build_id_does_not_exist_in_db(self, ClientSession, - require_platform_and_default_arch): - fake_module_build_id, = db_session.query(func.max(ModuleBuild.id)).first() - - ClientSession.return_value.getBuild.return_value = { - "extra": {"typeinfo": {"module": {"module_build_service_id": fake_module_build_id + 1}}} - } - - assert get_corresponding_module_build("n-v-r") is None - - @patch("koji.ClientSession") - def test_find_the_module_build(self, ClientSession, require_platform_and_default_arch): - expected_module_build = ( - db_session.query(ModuleBuild).filter(ModuleBuild.name == "platform").first() - ) - - ClientSession.return_value.getBuild.return_value = { - "extra": {"typeinfo": {"module": {"module_build_service_id": expected_module_build.id}}} - } - - build = get_corresponding_module_build("n-v-r") - - assert expected_module_build.id == build.id - assert expected_module_build.name == build.name - - -class TestDecisionUpdateHandler: - """Test handler decision_update""" - - def setup_method(self, test_method): - self.patch_config_broker = patch.object( - module_build_service.common.config.Config, - "celery_broker_url", - create=True, - new_callable=PropertyMock, - return_value=False, - ) - self.patch_config_broker.start() - - def teardown_method(self, test_method): - self.patch_config_broker.stop() - - @patch("module_build_service.scheduler.handlers.greenwave.log") - def test_decision_context_is_not_match(self, log): - decision_update( - msg_id="msg-id-1", - decision_context="bodhi_update_push_testing", - policies_satisfied=True, - subject_identifier="xxx", - ) - log.debug.assert_called_once_with( - 'Skip Greenwave message %s as MBS only handles messages with the decision context "%s"', - "msg-id-1", - "test_dec_context" - ) - - @patch("module_build_service.scheduler.handlers.greenwave.log") - def test_not_satisfy_policies(self, log): - subject_identifier = "pkg-0.1-1.c1" - decision_update( - msg_id="msg-id-1", - decision_context="test_dec_context", - policies_satisfied=False, - subject_identifier=subject_identifier) - log.debug.assert_called_once_with( - "Skip to handle module build %s because it has not satisfied Greenwave policies.", - subject_identifier, - ) - - @patch("module_build_service.common.messaging.publish") - @patch("koji.ClientSession") - def test_transform_from_done_to_ready(self, ClientSession, publish, require_empty_database): - # This build should be queried and transformed to ready state - module_build = make_module_in_db( - "pkg:0.1:1:c1", - [ - { - "requires": {"platform": ["el8"]}, - "buildrequires": {"platform": ["el8"]}, - } - ], - ) - module_build.transition( - db_session, conf, BUILD_STATES["done"], "Move to done directly for running test." - ) - db_session.commit() - - # Assert this call below - first_publish_call = call( - "module.state.change", - module_build.json(db_session, show_tasks=False), - conf, - "mbs", - ) - - ClientSession.return_value.getBuild.return_value = { - "extra": {"typeinfo": {"module": {"module_build_service_id": module_build.id}}} - } - - msg = { - "msg_id": "msg-id-1", - "topic": "org.fedoraproject.prod.greenwave.decision.update", - "msg": { - "decision_context": "test_dec_context", - "policies_satisfied": True, - "subject_identifier": "pkg-0.1-1.c1", - }, - } - hub = Mock(config={"validate_signatures": False}) - consumer = MBSConsumer(hub) - consumer.consume(msg) - - db_session.add(module_build) - # Load module build again to check its state is moved correctly - db_session.refresh(module_build) - assert BUILD_STATES["ready"] == module_build.state - - publish.assert_has_calls([ - first_publish_call, - call( - "module.state.change", - module_build.json(db_session, show_tasks=False), - conf, - "mbs" - ), - ]) diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index 891c58ff..2ff41dd0 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: MIT from __future__ import absolute_import from datetime import datetime, timedelta -import re import koji import mock @@ -13,7 +12,6 @@ from module_build_service.common.config import conf from module_build_service.common import models from module_build_service.scheduler import producer from module_build_service.scheduler.db_session import db_session -from tests import make_module_in_db @pytest.mark.usefixtures("reuse_component_init_data") @@ -554,44 +552,3 @@ class TestPoller: tagged_handler.delay.assert_has_calls( expected_tagged_calls, any_order=True) - - @pytest.mark.parametrize("greenwave_result", [True, False]) - @patch("module_build_service.scheduler.greenwave.Greenwave.check_gating") - def test_poll_greenwave(self, mock_gw, create_builder, dbg, greenwave_result): - - module_build1 = models.ModuleBuild.get_by_id(db_session, 1) - module_build1.state = models.BUILD_STATES["ready"] - - module_build2 = models.ModuleBuild.get_by_id(db_session, 2) - module_build2.state = models.BUILD_STATES["done"] - - module_build3 = models.ModuleBuild.get_by_id(db_session, 3) - module_build3.state = models.BUILD_STATES["init"] - - module_build4 = make_module_in_db("foo:1:1:1", {}) - module_build4.state = models.BUILD_STATES["done"] - module_build4.scratch = True - - db_session.commit() - - mock_gw.return_value = greenwave_result - - producer.poll_greenwave() - - mock_gw.assert_called_once() - modules = models.ModuleBuild.by_state(db_session, "ready") - - if greenwave_result: - assert len(modules) == 2 - assert {m.id for m in modules} == {1, 2} - else: - assert len(modules) == 1 - assert modules[0].id == 1 - modules = models.ModuleBuild.by_state(db_session, "done") - assert len(modules) == 2 - for module in modules: - assert module.id in [2, 4] - if module.id == 2: - assert re.match("Gating failed.*", module.state_reason) - else: - assert module.state_reason is None