modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-02 20:59:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove Greenwave Code

Browse Source 
No implementations of MBS are using Greenwave, and there are no current plans
to do so. Koji Resolver will be sufficient for any usecase dependent on gating.
This commit is contained in:
Brendan Reilly
2022-01-06 13:46:25 -05:00
parent 24e1efd394
commit 4a30847bea
20 changed files with 12 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
docs/GATING.rst
View File

@@ -1,20 +1,8 @@
Modules gating using Greenwave
==============================
Modules gating
==============
Every successfully built module is moved to the ``done`` state. Modules in this state cannot
be used as a build dependency for other modules. They need to be moved to the ``ready`` state.
By default, MBS moves the module from the ``done`` state to the ``ready`` state automatically,
but MBS can also be configured to gate the ``done`` to ``ready`` state transition using
`Greenwave <https://pagure.io/docs/greenwave/>`_.
When Greenwave integration is configured, the following additional MBS features are enabled:
- When the module is moved to the ``done`` state, Greenwave is queried to find out whether
the module can be moved to the ``ready`` state instantly.
- If the module cannot be moved to the ``ready`` state yet, MBS keeps the module build in the
``done`` state and waits for a message from Greenwave. If this message says that all the
tests defined by Greenwave policy have passed, then the module build is moved to the ``ready``
state.
- MBS also queries Greenwave periodically to find out the current gating status for modules
in the ``done`` state. This is useful in case a message from Greenwave was missed.
By default, MBS moves the module from the ``done`` state to the ``ready`` state automatically.
The Koji resolver, if used, will handle gating naturally.

27
module_build_service/common/config.py
View File

@@ -62,11 +62,6 @@ class TestConfiguration(BaseConfiguration):
ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
# Greenwave configuration
GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
GREENWAVE_DECISION_CONTEXT = "test_dec_context"
GREENWAVE_SUBJECT_TYPE = "some-module"
STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
# Ensures task.delay executes locally instead of scheduling a task to a queue.
@@ -605,11 +600,6 @@ class Config(object):
],
"desc": ("The list packages for offline module build RPM buildroot."),
},
"greenwave_decision_context": {
"type": str,
"default": "",
"desc": "The Greenwave decision context that determines a module's gating status.",
},
"allowed_privileged_module_names": {
"type": list,
"default": [],
@@ -628,22 +618,6 @@ class Config(object):
"corresponding suffix added to formatted stream version. "
'For example, {r"regexp": 0.1, ...}',
},
"greenwave_url": {
"type": str,
"default": "",
"desc": "The URL of the server where Greenwave is running (should include "
"the root of the API)"
},
"greenwave_subject_type": {
"type": str,
"default": "",
"desc": "Subject type for Greenwave requests"
},
"greenwave_timeout": {
"type": int,
"default": 60,
"desc": "Greenwave response timeout"
},
"modules_allow_scratch": {
"type": bool,
"default": False,
@@ -755,7 +729,6 @@ class Config(object):
"module_build_service.scheduler.handlers.modules",
"module_build_service.scheduler.handlers.repos",
"module_build_service.scheduler.handlers.tags",
"module_build_service.scheduler.handlers.greenwave",
"module_build_service.scheduler.producer",
],
"desc": "The list Python paths for the Celery application to import.",

4
module_build_service/common/errors.py
View File

@@ -37,9 +37,5 @@ class StreamAmbigous(ValueError):
pass
class GreenwaveError(RuntimeError):
pass
class IgnoreMessage(Exception):
"""Raise if message received from message bus should be ignored"""

2
module_build_service/common/messaging.py
View File

@@ -95,7 +95,7 @@ def _in_memory_publish(topic, msg, conf, service):
_initial_messages.append(wrapped_msg)
known_fedmsg_services = ["buildsys", "mbs", "greenwave"]
known_fedmsg_services = ["buildsys", "mbs"]
_fedmsg_backend = {

9
module_build_service/scheduler/consumer.py
View File

@@ -28,7 +28,7 @@ import module_build_service.common.monitor as monitor
from module_build_service.scheduler import events
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.handlers import components, repos, modules, greenwave, tags
from module_build_service.scheduler.handlers import components, repos, modules, tags
def no_op_handler(*args, **kwargs):
@@ -57,7 +57,6 @@ ON_MODULE_CHANGE_HANDLERS = {
# Only one kind of repo change event, though...
ON_REPO_CHANGE_HANDLER = repos.done
ON_TAG_CHANGE_HANDLER = tags.tagged
ON_DECISION_UPDATE_HANDLER = greenwave.decision_update
class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
@@ -234,12 +233,6 @@ class MBSConsumer(fedmsg.consumers.FedmsgConsumer):
models.ModuleBuild.get_by_id(db_session, event_info["module_build_id"])
)
if event == events.GREENWAVE_DECISION_UPDATE:
return (
ON_DECISION_UPDATE_HANDLER,
greenwave.get_corresponding_module_build(event_info["subject_identifier"])
)
return None, None
def process_message(self, event_info):

1
module_build_service/scheduler/events.py
View File

@@ -24,7 +24,6 @@ KOJI_BUILD_CHANGE = "koji_build_change"
KOJI_TAG_CHANGE = "koji_tag_change"
KOJI_REPO_CHANGE = "koji_repo_change"
MBS_MODULE_STATE_CHANGE = "mbs_module_state_change"
GREENWAVE_DECISION_UPDATE = "greenwave_decision_update"
class Scheduler(sched.scheduler):

192
module_build_service/scheduler/greenwave.py
View File

@@ -1,192 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from functools import reduce
import json
import requests
from module_build_service.common import log, conf
from module_build_service.common.errors import GreenwaveError
class Greenwave(object):
def __init__(self):
"""
Initialize greenwave instance with config
"""
self.url = conf.greenwave_url
self._decision_context = conf.greenwave_decision_context
if not self.decision_context:
raise RuntimeError("No Greenwave decision context set")
self._subj_type = conf.greenwave_subject_type
self._gw_timeout = conf.greenwave_timeout
self.error_occurred = False
def _greenwave_query(self, query_type, payload=None):
"""
Make a query to greenwave
:param query_type: will be part of url
:type query_type: str
:param payload: request payload used in 'decision' query
:type payload: str
:return: response
:rtype: dict
"""
query_func = requests.post if payload else requests.get
kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout}
if payload:
kwargs["headers"] = {"Content-Type": "application/json"}
kwargs["data"] = payload
try:
response = query_func(**kwargs)
except requests.exceptions.Timeout:
raise GreenwaveError("Greenwave request timed out")
except Exception as exc:
error_message = "Unspecified greenwave request error " \
'(original exception was: "{0}")'.format(str(exc))
log.exception(error_message)
raise GreenwaveError(error_message)
try:
resp_json = response.json()
except ValueError:
log.debug("Greenwave response content (status {0}): {1}".format(
response.status_code, response.text
))
raise GreenwaveError("Greenwave returned invalid JSON.")
log.debug(
'Query to Greenwave (%s) result: status=%d, content="%s"',
kwargs["url"], response.status_code, resp_json
)
if response.status_code == 200:
return resp_json
try:
err_msg = resp_json["message"]
except KeyError:
err_msg = response.text
raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format(
response.status_code, err_msg
))
def query_decision(self, build, prod_version):
"""
Query decision to greenwave
:param build: build object
:type build: module_build_service.common.models.ModuleBuild
:param prod_version: The product version string used for querying WaiverDB
:type prod_version: str
:return: response
:rtype: dict
"""
payload = {
"decision_context": self.decision_context,
"product_version": prod_version,
"subject_type": self.subject_type,
"subject_identifier": build.nvr_string
}
return self._greenwave_query('decision', json.dumps(payload))
def query_policies(self, return_all=False):
"""
Query policies to greenwave
:param return_all: Return all policies, if False select by subject_type and decision_context
:type return_all: bool
:return: response
:rtype: dict
"""
response = self._greenwave_query('policies')
if return_all:
return response
try:
selective_resp = {
"policies": [
pol for pol in response["policies"]
if pol["decision_context"] == self.decision_context
and pol["subject_type"] == self.subject_type
]
}
except KeyError:
log.exception("Incorrect greenwave response (Mandatory key is missing)")
raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)")
return selective_resp
def get_product_versions(self):
"""
Return a set of product versions according to decision_context and subject_type
:return: product versions
:rtype: set
"""
return reduce(
lambda old, new: old.union(new),
[pol["product_versions"] for pol in self.query_policies()["policies"]],
set()
)
def check_gating(self, build):
"""
Query decision to greenwave
:param build: build object
:type build: module_build_service.common.models.ModuleBuild
:return: True if at least one GW response contains policies_satisfied set to true
:rtype: bool
"""
self.error_occurred = False
try:
versions = self.get_product_versions()
except GreenwaveError:
log.warning('An error occured while getting a product versions')
self.error_occurred = True
return False
for ver in versions:
try:
if self.query_decision(build, ver)["policies_satisfied"]:
# at least one positive result is enough
return True
except (KeyError, GreenwaveError) as exc:
self.error_occurred = True
log.warning('Incorrect greenwave result "%s", ignoring', str(exc))
return False
@property
def url(self):
return self._url
@url.setter
def url(self, value):
value = value.rstrip("/")
if not value:
raise RuntimeError("No Greenwave URL set")
self._url = value
@property
def decision_context(self):
return self._decision_context
@property
def subject_type(self):
return self._subj_type
@property
def timeout(self):
return self._gw_timeout
@timeout.setter
def timeout(self, value):
self._gw_timeout = value
try:
greenwave = Greenwave()
except RuntimeError:
log.warning('Greenwave is not configured or configured improperly')
greenwave = None

100
module_build_service/scheduler/handlers/greenwave.py
View File

@@ -1,100 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from module_build_service.common import conf, log
from module_build_service.common.koji import get_session
from module_build_service.common.models import ModuleBuild, BUILD_STATES
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler import celery_app, events
def get_corresponding_module_build(nvr):
"""Find corresponding module build from database and return
:param str nvr: module build NVR. This is the subject_identifier included
inside ``greenwave.decision.update`` message.
:return: the corresponding module build object. For whatever the reason,
if the original module build id cannot be found from the Koji build of
``nvr``, None will be returned.
:rtype: :class:`ModuleBuild` or None
"""
koji_session = get_session(conf, login=False)
build_info = koji_session.getBuild(nvr)
if build_info is None:
return None
try:
module_build_id = build_info["extra"]["typeinfo"]["module"]["module_build_service_id"]
except KeyError:
# If any of the keys is not present, the NVR is not the one for
# handling Greenwave event.
return None
return ModuleBuild.get_by_id(db_session, module_build_id)
@celery_app.task
@events.mbs_event_handler
def decision_update(msg_id, decision_context, subject_identifier, policies_satisfied):
"""Move module build to ready or failed according to Greenwave result
:param str msg_id: the original id of the message being handled which is
received from the message bus.
:param str decision_context: the context of the greewave decision. Refer to
the messaging document for detailed information.
:param str subject_identifier: usually a build NVR. Refer to
https://docs.pagure.org/greenwave/messaging.html for detailed information.
:param bool policies_satisfied: whether the build satisfies Greenwave rules.
Refer to the messaging document for detailed information.
"""
if not conf.greenwave_decision_context:
log.debug(
"Skip Greenwave message %s as MBS does not have GREENWAVE_DECISION_CONTEXT "
"configured",
msg_id,
)
return
if decision_context != conf.greenwave_decision_context:
log.debug(
"Skip Greenwave message %s as MBS only handles messages with the "
'decision context "%s"',
msg_id,
conf.greenwave_decision_context,
)
return
module_build_nvr = subject_identifier
if not policies_satisfied:
log.debug(
"Skip to handle module build %s because it has not satisfied Greenwave policies.",
module_build_nvr,
)
return
build = get_corresponding_module_build(module_build_nvr)
if build is None:
log.debug(
"No corresponding module build of subject_identifier %s is found.", module_build_nvr)
return
if build.state == BUILD_STATES["done"]:
build.transition(
db_session,
conf,
BUILD_STATES["ready"],
state_reason="Module build {} has satisfied Greenwave policies.".format(
module_build_nvr
),
)
else:
log.warning(
"Module build %s is not in done state but Greenwave tells "
"it passes tests in decision context %s",
module_build_nvr, decision_context,
)
db_session.commit()

10
module_build_service/scheduler/handlers/modules.py
View File

@@ -3,7 +3,6 @@
""" Handlers for module change events on the message bus. """
from __future__ import absolute_import
from datetime import datetime
import logging
import os
@@ -29,7 +28,6 @@ from module_build_service.scheduler import celery_app, events
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.default_modules import (
add_default_modules, handle_collisions_with_base_module_rpms)
from module_build_service.scheduler.greenwave import greenwave
from module_build_service.scheduler.reuse import attempt_to_reuse_all_components
from module_build_service.scheduler.submit import format_mmd, get_module_srpm_overrides
from module_build_service.scheduler.ursine import handle_stream_collision_modules
@@ -144,13 +142,7 @@ def done(msg_id, module_build_id, module_build_state):
# Scratch builds stay in 'done' state
if not build.scratch:
if greenwave is None or greenwave.check_gating(build):
build.transition(db_session, conf, state=models.BUILD_STATES["ready"])
else:
build.state_reason = "Gating failed"
if greenwave.error_occurred:
build.state_reason += " (Error occured while querying Greenwave)"
build.time_modified = datetime.utcnow()
build.transition(db_session, conf, state=models.BUILD_STATES["ready"])
db_session.commit()
build_logs.stop(build)

12
module_build_service/scheduler/parser.py
View File

@@ -12,7 +12,7 @@ class MessageParser(object):
:param topic_categories: list of known services, that MBS can handle the
messages sent from them. For example, a value could be
``["buildsys", "mbs", "greenwave"]``.
``["buildsys", "mbs"]``.
:type topic_categories: list[str]
"""
@@ -114,13 +114,3 @@ class FedmsgMessageParser(MessageParser):
"module_build_id": msg_inner_msg.get("id"),
"module_build_state": msg_inner_msg.get("state"),
}
if (category == "greenwave"
and object == "decision" and subobject is None and event == "update"):
return {
"msg_id": msg_id,
"event": events.GREENWAVE_DECISION_UPDATE,
"decision_context": msg_inner_msg.get("decision_context"),
"policies_satisfied": msg_inner_msg.get("policies_satisfied"),
"subject_identifier": msg_inner_msg.get("subject_identifier"),
}

28
module_build_service/scheduler/producer.py
View File

@@ -20,7 +20,6 @@ from module_build_service.scheduler.batches import (
start_next_batch_build,
)
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.greenwave import greenwave
from module_build_service.scheduler.handlers.components import build_task_finalize
from module_build_service.scheduler.handlers.tags import tagged
@@ -36,7 +35,6 @@ def setup_periodic_tasks(sender, **kwargs):
(cleanup_stale_failed_builds, "Cleanup stale failed builds"),
(cancel_stuck_module_builds, "Cancel stuck module builds"),
(sync_koji_build_tags, "Sync Koji build tags"),
(poll_greenwave, "Gating module build to ready state"),
)
for task, name in tasks:
@@ -442,32 +440,6 @@ def sync_koji_build_tags():
tagged.delay("internal:sync_koji_build_tags", build_tag, c.nvr)
@celery_app.task
def poll_greenwave():
"""Polls Greenwave for all builds in done state"""
if greenwave is None:
return
module_builds = db_session.query(models.ModuleBuild).filter_by(
state=models.BUILD_STATES["done"],
scratch=False
).all()
log.info("Checking Greenwave for %d builds", len(module_builds))
for build in module_builds:
if greenwave.check_gating(build):
build.transition(db_session, conf, state=models.BUILD_STATES["ready"])
else:
build.state_reason = "Gating failed (MBS will retry in {0} seconds)".format(
conf.polling_interval
)
if greenwave.error_occurred:
build.state_reason += " (Error occured while querying Greenwave)"
build.time_modified = datetime.utcnow()
db_session.commit()
def has_missed_new_repo_message(module_build, koji_session):
"""
Returns whether or not a new repo message has probably been missed.

6
module_build_service/scheduler/route.py
View File

@@ -7,7 +7,6 @@ import inspect
from module_build_service.common import conf, log, models
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.handlers.greenwave import get_corresponding_module_build
def route_task(name, args, kwargs, options, task=None, **kw):
@@ -57,11 +56,6 @@ def route_task(name, args, kwargs, options, task=None, **kw):
module_build = models.ModuleBuild.get_by_tag(db_session, tag_name)
if module_build:
module_build_id = module_build.id
elif "subject_identifier" in handler_args:
module_build_nvr = _get_handler_arg("subject_identifier")
module_build = get_corresponding_module_build(module_build_nvr)
if module_build is not None:
module_build_id = module_build.id
if module_build_id is not None:
queue_name = "mbs-{}".format(module_build_id % num_workers)

4
openshift/integration/koji/README.md
View File

@@ -83,9 +83,5 @@ you need to run the trigger jobs once manually so Jenkins can setup required mes
following jobs should be triggered manually:
- mbs-trigger-on-latest-tag
- mbs-trigger-on-stage-tag
- mbs-backend-greenwave-promote-to-stage
- mbs-backend-greenwave-promote-to-prod
- mbs-frontend-greenwave-promote-to-stage
- mbs-frontend-greenwave-promote-to-prod
[OpenShift secret for a private registry]: https://docs.openshift.com/container-platform/3.11/dev_guide/builds/build_inputs.html#using-docker-credentials-for-private-registries

19
tests/test_build/test_build.py
View File

@@ -454,11 +454,6 @@ class TestBuild(BaseTestBuild):
FakeModuleBuilder.on_get_task_info_cb = on_get_task_info_cb
self.p_check_gating = patch(
"module_build_service.scheduler.greenwave.Greenwave.check_gating",
return_value=True)
self.mock_check_gating = self.p_check_gating.start()
self.patch_config_broker = patch.object(
module_build_service.common.config.Config,
"celery_broker_url",
@@ -469,7 +464,6 @@ class TestBuild(BaseTestBuild):
self.patch_config_broker.start()
def teardown_method(self, test_method):
self.p_check_gating.stop()
self.patch_config_broker.stop()
FakeModuleBuilder.reset()
cleanup_moksha()
@@ -628,16 +622,14 @@ class TestBuild(BaseTestBuild):
models.BUILD_STATES["ready"],
]
@pytest.mark.parametrize("gating_result", (True, False))
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
def test_submit_build_no_components(
self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc, gating_result
self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc
):
"""
Tests the build of a module with no components
"""
self.mock_check_gating.return_value = gating_result
FakeSCM(
mocked_scm,
"python3",
@@ -663,12 +655,6 @@ class TestBuild(BaseTestBuild):
# Make sure no component builds were registered
assert len(module_build.component_builds) == 0
# Make sure the build is done
if gating_result:
assert module_build.state == models.BUILD_STATES["ready"]
else:
assert module_build.state == models.BUILD_STATES["done"]
assert module_build.state_reason == "Gating failed"
@patch(
"module_build_service.common.config.Config.check_for_eol",
@@ -1913,7 +1899,6 @@ class TestLocalBuild(BaseTestBuild):
except Exception:
pass
@patch("module_build_service.scheduler.greenwave.Greenwave.query_policies")
@patch("module_build_service.scheduler.handlers.modules.handle_stream_collision_modules")
@patch("module_build_service.web.auth.get_user", return_value=user)
@patch("module_build_service.common.scm.SCM")
@@ -1923,7 +1908,7 @@ class TestLocalBuild(BaseTestBuild):
return_value=staged_data_filename('local_builds'),
)
def test_submit_build_local_dependency(
self, resultsdir, mocked_scm, mocked_get_user, conf_system, hmsc, mocked_greenwave
self, resultsdir, mocked_scm, mocked_get_user, conf_system, hmsc
):
"""
Tests local module build dependency.

3
tests/test_memory/mbs_configuration.py
View File

@@ -22,9 +22,6 @@ class TestConfiguration:
KOJI_REPOSITORY_URL = "https://kojipkgs.stg.fedoraproject.org/repos"
SCMURLS = ["https://src.stg.fedoraproject.org/modules/"]
ALLOWED_GROUPS_TO_IMPORT_MODULE = {"mbs-import-module"}
GREENWAVE_URL = "https://greenwave.example.local/api/v1.0/"
GREENWAVE_DECISION_CONTEXT = "test_dec_context"
GREENWAVE_SUBJECT_TYPE = "some-module"
STREAM_SUFFIXES = {r"^el\d+\.\d+\.\d+\.z$": 0.1}
CELERY_TASK_ALWAYS_EAGER = True

4
tests/test_memory/mbs_debug.py
View File

@@ -42,9 +42,7 @@ class SimpleMock:
new_callable=SimpleMock)
@mock.patch("module_build_service.scheduler.handlers.modules.record_module_build_arches",
new_callable=SimpleMock)
@mock.patch("module_build_service.scheduler.greenwave.Greenwave.check_gating",
new_callable=SimpleMock)
def run_debug_instance(mock_1, mock_2, mock_3, host=None, port=None):
def run_debug_instance(mock_1, mock_2, host=None, port=None):
def handle_pdb(sig, frame):
import pdb

18
tests/test_scheduler/test_celery_route_task.py
View File

@@ -6,7 +6,7 @@ import mock
from module_build_service.common.config import conf
from module_build_service.scheduler import celery_app
from module_build_service.scheduler.handlers import components, greenwave, modules, repos, tags
from module_build_service.scheduler.handlers import components, modules, repos, tags
from module_build_service.scheduler.producer import fail_lost_builds
from tests import scheduler_init_data
@@ -95,22 +95,6 @@ class TestCeleryRouteTask:
qname = queue.__dict__.get("name")
assert qname == "mbs-2"
@mock.patch("koji.ClientSession")
def test_route_greenwave_decision_update_task(self, kojisession, send_task_message):
kojisession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": 1}}}
}
scheduler_init_data()
greenwave.decision_update.delay(
"fakemsg",
decision_context="test_dec_context",
subject_identifier="module-testmodule-master-20170109091357-7c29193d-build",
policies_satisfied=False
)
queue = send_task_message.call_args[1].get("queue")
qname = queue.__dict__.get("name")
assert qname == "mbs-1"
def test_route_fail_lost_builds_task(self, send_task_message):
fail_lost_builds.delay()
queue = send_task_message.call_args[1].get("queue")

172
tests/test_scheduler/test_greenwave.py
View File

@@ -1,172 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
import json
from mock import patch, Mock
import pytest
from module_build_service.scheduler.greenwave import greenwave
from tests import make_module_in_db
@pytest.mark.usefixtures("require_empty_database")
class TestGreenwaveQuery:
@patch("module_build_service.scheduler.greenwave.requests")
def test_greenwave_query_decision(self, mock_requests):
resp_status = 200
resp_content = {
"applicable_policies": ["osci_compose_modules"],
"policies_satisfied": True,
"satisfied_requirements": [
{
"result_id": 7336633,
"testcase": "test-ci.test-module.tier1",
"type": "test-result-passed"
},
{
"result_id": 7336650,
"testcase": "test-ci.test-module.tier2",
"type": "test-result-passed"
}
],
"summary": "All required tests passed",
"unsatisfied_requirements": []
}
response = Mock()
response.json.return_value = resp_content
response.status_code = resp_status
mock_requests.post.return_value = response
fake_build = make_module_in_db(
"pkg:0.1:1:c1", [{
"requires": {"platform": ["el8"]},
"buildrequires": {"platform": ["el8"]},
}],
)
got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8")
assert got_response == resp_content
assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == {
"decision_context": "test_dec_context",
"product_version": "xxxx-8", "subject_type": "some-module",
"subject_identifier": "pkg-0.1-1.c1"}
assert mock_requests.post.call_args_list[0][1]["headers"] == {
"Content-Type": "application/json"}
assert mock_requests.post.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/decision"
@pytest.mark.parametrize("return_all", (False, True))
@patch("module_build_service.scheduler.greenwave.requests")
def test_greenwave_query_policies(self, mock_requests, return_all):
resp_status = 200
resp_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver2"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "decision_context_2",
"product_versions": ["ver4"],
"rules": [],
"subject_type": "subject_type_2"
}
]
}
selected_policies = {"policies": resp_content["policies"][:-1]}
response = Mock()
response.json.return_value = resp_content
response.status_code = resp_status
mock_requests.get.return_value = response
got_response = greenwave.query_policies(return_all)
if return_all:
assert got_response == resp_content
else:
assert got_response == selected_policies
assert mock_requests.get.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/policies"
@patch("module_build_service.scheduler.greenwave.requests")
def test_greenwave_get_product_versions(self, mock_requests):
resp_status = 200
resp_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver2"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "decision_context_2",
"product_versions": ["ver4"],
"rules": [],
"subject_type": "subject_type_2"
}
]
}
expected_versions = {"ver1", "ver2", "ver3"}
response = Mock()
response.json.return_value = resp_content
response.status_code = resp_status
mock_requests.get.return_value = response
versions_set = greenwave.get_product_versions()
assert versions_set == expected_versions
assert mock_requests.get.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/policies"
@pytest.mark.parametrize("policies_satisfied", (True, False))
@patch("module_build_service.scheduler.greenwave.requests")
def test_greenwave_check_gating(self, mock_requests, policies_satisfied):
resp_status = 200
policies_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
}
]
}
responses = [Mock() for i in range(3)]
for r in responses:
r.status_code = resp_status
responses[0].json.return_value = policies_content
responses[1].json.return_value = {"policies_satisfied": False}
responses[2].json.return_value = {"policies_satisfied": policies_satisfied}
mock_requests.get.return_value = responses[0]
mock_requests.post.side_effect = responses[1:]
fake_build = make_module_in_db(
"pkg:0.1:1:c1", [{
"requires": {"platform": ["el8"]},
"buildrequires": {"platform": ["el8"]},
}],
)
result = greenwave.check_gating(fake_build)
assert result == policies_satisfied

174
tests/test_scheduler/test_greenwave_handler.py
View File

@@ -1,174 +0,0 @@
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from mock import call, patch, PropertyMock, Mock
import pytest
from sqlalchemy import func
from module_build_service.common.config import conf
import module_build_service.common.config
from module_build_service.common.models import BUILD_STATES, ModuleBuild
from module_build_service.scheduler.consumer import MBSConsumer
from module_build_service.scheduler.db_session import db_session
from module_build_service.scheduler.handlers.greenwave import (
decision_update, get_corresponding_module_build
)
from tests import make_module_in_db
@pytest.mark.usefixtures("require_empty_database")
class TestGetCorrespondingModuleBuild:
"""Test get_corresponding_module_build"""
@patch("koji.ClientSession")
def test_module_build_nvr_does_not_exist_in_koji(self, ClientSession):
ClientSession.return_value.getBuild.return_value = None
assert get_corresponding_module_build("n-v-r") is None
@pytest.mark.parametrize(
"build_info",
[
# Build info does not have key extra
{"id": 1000, "name": "ed"},
# Build info contains key extra, but it is not for the module build
{"extra": {"submitter": "osbs", "image": {}}},
# Key module_build_service_id is missing
{"extra": {"typeinfo": {"module": {}}}},
],
)
@patch("koji.ClientSession")
def test_cannot_find_module_build_id_from_build_info(self, ClientSession, build_info):
ClientSession.return_value.getBuild.return_value = build_info
assert get_corresponding_module_build("n-v-r") is None
@patch("koji.ClientSession")
def test_corresponding_module_build_id_does_not_exist_in_db(self, ClientSession,
require_platform_and_default_arch):
fake_module_build_id, = db_session.query(func.max(ModuleBuild.id)).first()
ClientSession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": fake_module_build_id + 1}}}
}
assert get_corresponding_module_build("n-v-r") is None
@patch("koji.ClientSession")
def test_find_the_module_build(self, ClientSession, require_platform_and_default_arch):
expected_module_build = (
db_session.query(ModuleBuild).filter(ModuleBuild.name == "platform").first()
)
ClientSession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": expected_module_build.id}}}
}
build = get_corresponding_module_build("n-v-r")
assert expected_module_build.id == build.id
assert expected_module_build.name == build.name
class TestDecisionUpdateHandler:
"""Test handler decision_update"""
def setup_method(self, test_method):
self.patch_config_broker = patch.object(
module_build_service.common.config.Config,
"celery_broker_url",
create=True,
new_callable=PropertyMock,
return_value=False,
)
self.patch_config_broker.start()
def teardown_method(self, test_method):
self.patch_config_broker.stop()
@patch("module_build_service.scheduler.handlers.greenwave.log")
def test_decision_context_is_not_match(self, log):
decision_update(
msg_id="msg-id-1",
decision_context="bodhi_update_push_testing",
policies_satisfied=True,
subject_identifier="xxx",
)
log.debug.assert_called_once_with(
'Skip Greenwave message %s as MBS only handles messages with the decision context "%s"',
"msg-id-1",
"test_dec_context"
)
@patch("module_build_service.scheduler.handlers.greenwave.log")
def test_not_satisfy_policies(self, log):
subject_identifier = "pkg-0.1-1.c1"
decision_update(
msg_id="msg-id-1",
decision_context="test_dec_context",
policies_satisfied=False,
subject_identifier=subject_identifier)
log.debug.assert_called_once_with(
"Skip to handle module build %s because it has not satisfied Greenwave policies.",
subject_identifier,
)
@patch("module_build_service.common.messaging.publish")
@patch("koji.ClientSession")
def test_transform_from_done_to_ready(self, ClientSession, publish, require_empty_database):
# This build should be queried and transformed to ready state
module_build = make_module_in_db(
"pkg:0.1:1:c1",
[
{
"requires": {"platform": ["el8"]},
"buildrequires": {"platform": ["el8"]},
}
],
)
module_build.transition(
db_session, conf, BUILD_STATES["done"], "Move to done directly for running test."
)
db_session.commit()
# Assert this call below
first_publish_call = call(
"module.state.change",
module_build.json(db_session, show_tasks=False),
conf,
"mbs",
)
ClientSession.return_value.getBuild.return_value = {
"extra": {"typeinfo": {"module": {"module_build_service_id": module_build.id}}}
}
msg = {
"msg_id": "msg-id-1",
"topic": "org.fedoraproject.prod.greenwave.decision.update",
"msg": {
"decision_context": "test_dec_context",
"policies_satisfied": True,
"subject_identifier": "pkg-0.1-1.c1",
},
}
hub = Mock(config={"validate_signatures": False})
consumer = MBSConsumer(hub)
consumer.consume(msg)
db_session.add(module_build)
# Load module build again to check its state is moved correctly
db_session.refresh(module_build)
assert BUILD_STATES["ready"] == module_build.state
publish.assert_has_calls([
first_publish_call,
call(
"module.state.change",
module_build.json(db_session, show_tasks=False),
conf,
"mbs"
),
])

43
tests/test_scheduler/test_poller.py
View File

@@ -2,7 +2,6 @@
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from datetime import datetime, timedelta
import re
import koji
import mock
@@ -13,7 +12,6 @@ from module_build_service.common.config import conf
from module_build_service.common import models
from module_build_service.scheduler import producer
from module_build_service.scheduler.db_session import db_session
from tests import make_module_in_db
@pytest.mark.usefixtures("reuse_component_init_data")
@@ -554,44 +552,3 @@ class TestPoller:
tagged_handler.delay.assert_has_calls(
expected_tagged_calls, any_order=True)
@pytest.mark.parametrize("greenwave_result", [True, False])
@patch("module_build_service.scheduler.greenwave.Greenwave.check_gating")
def test_poll_greenwave(self, mock_gw, create_builder, dbg, greenwave_result):
module_build1 = models.ModuleBuild.get_by_id(db_session, 1)
module_build1.state = models.BUILD_STATES["ready"]
module_build2 = models.ModuleBuild.get_by_id(db_session, 2)
module_build2.state = models.BUILD_STATES["done"]
module_build3 = models.ModuleBuild.get_by_id(db_session, 3)
module_build3.state = models.BUILD_STATES["init"]
module_build4 = make_module_in_db("foo:1:1:1", {})
module_build4.state = models.BUILD_STATES["done"]
module_build4.scratch = True
db_session.commit()
mock_gw.return_value = greenwave_result
producer.poll_greenwave()
mock_gw.assert_called_once()
modules = models.ModuleBuild.by_state(db_session, "ready")
if greenwave_result:
assert len(modules) == 2
assert {m.id for m in modules} == {1, 2}
else:
assert len(modules) == 1
assert modules[0].id == 1
modules = models.ModuleBuild.by_state(db_session, "done")
assert len(modules) == 2
for module in modules:
assert module.id in [2, 4]
if module.id == 2:
assert re.match("Gating failed.*", module.state_reason)
else:
assert module.state_reason is None