modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-03 10:48:03 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add greenwave query to done handler

Browse Source 
Signed-off-by: Valerij Maljulin <vmaljuli@redhat.com>
This commit is contained in:
Valerij Maljulin
2019-05-09 18:42:25 +02:00
parent 4b5618f079
commit 95bacc4e15
4 changed files with 258 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
module_build_service/scheduler/handlers/modules.py
View File

@@ -37,6 +37,7 @@ from module_build_service.utils import (
)
from module_build_service.errors import UnprocessableEntity, Forbidden, ValidationError
from module_build_service.utils.ursine import handle_stream_collision_modules
from module_build_service.utils.greenwave import greenwave
from requests.exceptions import ConnectionError
from module_build_service.utils import mmd_to_str
@@ -131,10 +132,11 @@ def done(config, session, msg):
# This is ok.. it's a race condition we can ignore.
pass
# Scratch builds stay in 'done' state, otherwise move to 'ready'
# Scratch builds stay in 'done' state
if not build.scratch:
build.transition(config, state="ready")
session.commit()
if greenwave is None or greenwave.check_gating(build):
build.transition(config, state="ready")
session.commit()
build_logs.stop(build)
module_build_service.builder.GenericBuilder.clear_cache(build)

153
module_build_service/utils/greenwave.py
View File

@@ -24,6 +24,7 @@
import requests
import json
from functools import reduce
from module_build_service import log, conf
from module_build_service.errors import GreenwaveError
@@ -34,14 +35,63 @@ class Greenwave(object):
Initialize greenwave instance with config
"""
self.url = conf.greenwave_url
if not self.url:
raise GreenwaveError("No Greenwave URL set")
self._decision_context = conf.greenwave_decision_context
if not self.decision_context:
raise GreenwaveError("No Greenwave decision context set")
self._subj_type = conf.greenwave_subject_type
self._gw_timeout = conf.greenwave_timeout
def _greenwave_query(self, query_type, payload=None):
"""
Make a query to greenwave
:param query_type: will be part of url
:type query_type: str
:param payload: request payload used in 'decision' query
:type payload: str
:return: response
:rtype: dict
"""
query_func = requests.post if payload else requests.get
kwargs = {"url": "{0}/{1}".format(self.url, query_type), "timeout": self.timeout}
if payload:
kwargs["headers"] = {"Content-Type": "application/json"}
kwargs["data"] = payload
try:
response = query_func(**kwargs)
except requests.exceptions.Timeout:
raise GreenwaveError("Greenwave request timed out")
except Exception as exc:
error_message = "Unspecified greenwave request error" \
'(original exception was: "{0}")'.format(str(exc))
log.exception(error_message)
raise GreenwaveError(error_message)
try:
resp_json = response.json()
except ValueError:
log.debug("Greenwave response content (status {0}): {1}".format(
response.status_code, response.text
))
raise GreenwaveError("Greenwave returned invalid JSON.")
log.debug(
'Query to Greenwave (%s) result: status=%d, content="%s"',
(kwargs["url"], response.status_code, resp_json)
)
if response.status_code == 200:
return resp_json
try:
err_msg = resp_json["message"]
except KeyError:
err_msg = response.text
raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format(
response.status_code, err_msg
))
def query_decision(self, build, prod_version):
"""
Query decision to greenwave
@@ -58,36 +108,69 @@ class Greenwave(object):
"subject_type": self.subject_type,
"subject_identifier": build.nvr_string
}
url = "{0}/decision".format(self.url)
headers = {"Content-Type": "application/json"}
try:
response = requests.post(
url=url, headers=headers, data=json.dumps(payload), timeout=self.timeout)
except requests.exceptions.Timeout:
raise GreenwaveError("Greenwave request timed out")
except Exception as exc:
log.exception(str(exc))
raise GreenwaveError("Greenwave request error")
return self._greenwave_query('decision', json.dumps(payload))
def query_policies(self, return_all=False):
"""
Query policies to greenwave
:param return_all: Return all policies, if False select by subject_type and decision_context
:type return_all: bool
:return: response
:rtype: dict
"""
response = self._greenwave_query('policies')
if return_all:
return response
try:
resp_json = response.json()
except ValueError:
log.debug("Greenwave response content (status {0}): {1}".format(
response.status_code, response.text))
raise GreenwaveError("Greenwave returned invalid JSON.")
log.debug('Query to Greenwave result: status=%d, content="%s"',
(response.status_code, resp_json))
if response.status_code == 200:
return resp_json
try:
err_msg = resp_json["message"]
selective_resp = {
"policies": [
pol for pol in response["policies"]
if pol["decision_context"] == self.decision_context
and pol["subject_type"] == self.subject_type
]
}
except KeyError:
err_msg = response.text
raise GreenwaveError("Greenwave returned {0} status code. Message: {1}".format(
response.status_code, err_msg))
log.exception("Incorrect greenwave response (Mandatory key is missing)")
raise GreenwaveError("Incorrect greenwave response (Mandatory key is missing)")
return selective_resp
def get_product_versions(self):
"""
Return a set of product versions according to decision_context and subject_type
:return: product versions
:rtype: set
"""
return reduce(
lambda old, new: old.union(new),
[pol["product_versions"] for pol in self.query_policies()["policies"]],
set()
)
def check_gating(self, build):
"""
Query decision to greenwave
:param build: build object
:type build: module_build_service.models.ModuleBuild
:return: True if at least one GW response contains policies_satisfied set to true
:rtype: bool
"""
try:
versions = self.get_product_versions()
except GreenwaveError:
log.warning('An error occured while getting a product versions')
return False
for ver in versions:
try:
if self.query_decision(build, ver)["policies_satisfied"]:
# at least one positive result is enough
return True
except (KeyError, GreenwaveError) as exc:
log.warning('Incorrect greenwave result "%s", ignoring', str(exc))
return False
@property
def url(self):
@@ -96,8 +179,9 @@ class Greenwave(object):
@url.setter
def url(self, value):
value = value.rstrip("/")
if value:
self._url = value
if not value:
raise GreenwaveError("No Greenwave URL set")
self._url = value
@property
def decision_context(self):
@@ -114,3 +198,10 @@ class Greenwave(object):
@timeout.setter
def timeout(self, value):
self._gw_timeout = value
try:
greenwave = Greenwave()
except GreenwaveError:
log.warning('Greenwave is not configured or configured improperly')
greenwave = None

23
tests/test_build/test_build.py
View File

@@ -406,9 +406,12 @@ class TestBuild:
pass
@pytest.mark.parametrize("mmd_version", [1, 2])
@patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True)
@patch("module_build_service.auth.get_user", return_value=user)
@patch("module_build_service.scm.SCM")
def test_submit_build(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc, mmd_version):
def test_submit_build(
self, mocked_scm, mocked_get_user, mocked_greenwave, conf_system, dbg, hmsc, mmd_version
):
"""
Tests the build of testmodule.yaml using FakeModuleBuilder which
succeeds everytime.
@@ -481,12 +484,17 @@ class TestBuild:
assert module_build.module_builds_trace[4].state == models.BUILD_STATES["ready"]
assert len(module_build.module_builds_trace) == 5
@pytest.mark.parametrize("gating_result", (True, False))
@patch("module_build_service.utils.greenwave.Greenwave.check_gating")
@patch("module_build_service.auth.get_user", return_value=user)
@patch("module_build_service.scm.SCM")
def test_submit_build_no_components(self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc):
def test_submit_build_no_components(
self, mocked_scm, mocked_get_user, mocked_greenwave, conf_system, dbg, hmsc, gating_result
):
"""
Tests the build of a module with no components
"""
mocked_greenwave.return_value = gating_result
FakeSCM(
mocked_scm,
"python3",
@@ -512,7 +520,10 @@ class TestBuild:
# Make sure no component builds were registered
assert len(module_build.component_builds) == 0
# Make sure the build is done
assert module_build.state == models.BUILD_STATES["ready"]
if gating_result:
assert module_build.state == models.BUILD_STATES["ready"]
else:
assert module_build.state == models.BUILD_STATES["done"]
@patch(
"module_build_service.config.Config.check_for_eol",
@@ -1412,10 +1423,11 @@ class TestBuild:
models.BUILD_STATES["ready"],
]
@patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True)
@patch("module_build_service.auth.get_user", return_value=user)
@patch("module_build_service.scm.SCM")
def test_submit_build_resume_init_fail(
self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc
self, mocked_scm, mocked_get_user, mock_greenwave, conf_system, dbg, hmsc
):
"""
Tests that resuming the build fails when the build is in init state
@@ -1654,10 +1666,11 @@ class TestBuild:
module = db.session.query(models.ModuleBuild).get(module_build_id)
assert module.state == models.BUILD_STATES["build"]
@patch("module_build_service.utils.greenwave.Greenwave.check_gating", return_value=True)
@patch("module_build_service.auth.get_user", return_value=user)
@patch("module_build_service.scm.SCM")
def test_submit_br_metadata_only_module(
self, mocked_scm, mocked_get_user, conf_system, dbg, hmsc
self, mocked_scm, mocked_get_user, mock_greenwave, conf_system, dbg, hmsc
):
"""
Test that when a build is submitted with a buildrequire without a Koji tag,

117
tests/test_utils/test_greenwave.py
View File

@@ -23,13 +23,14 @@
import json
from mock import patch, Mock
import module_build_service.utils.greenwave
import pytest
from module_build_service.utils.greenwave import greenwave
from tests import make_module
class TestGreenwaveQuery():
@patch("module_build_service.utils.greenwave.requests")
def test_greenwave_decision(self, mock_requests):
def test_greenwave_query_decision(self, mock_requests):
resp_status = 200
resp_content = {
"applicable_policies": ["osci_compose_modules"],
@@ -56,8 +57,7 @@ class TestGreenwaveQuery():
fake_build = make_module("pkg:0.1:1:c1", requires_list={"platform": "el8"})
gw = module_build_service.utils.greenwave.Greenwave()
got_response = gw.query_decision(fake_build, prod_version="xxxx-8")
got_response = greenwave.query_decision(fake_build, prod_version="xxxx-8")
assert got_response == resp_content
assert json.loads(mock_requests.post.call_args_list[0][1]["data"]) == {
@@ -68,3 +68,112 @@ class TestGreenwaveQuery():
"Content-Type": "application/json"}
assert mock_requests.post.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/decision"
@pytest.mark.parametrize("return_all", (False, True))
@patch("module_build_service.utils.greenwave.requests")
def test_greenwave_query_policies(self, mock_requests, return_all):
resp_status = 200
resp_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver2"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "decision_context_2",
"product_versions": ["ver4"],
"rules": [],
"subject_type": "subject_type_2"
}
]
}
selected_policies = {"policies": resp_content["policies"][:-1]}
response = Mock()
response.json.return_value = resp_content
response.status_code = resp_status
mock_requests.get.return_value = response
got_response = greenwave.query_policies(return_all)
if return_all:
assert got_response == resp_content
else:
assert got_response == selected_policies
assert mock_requests.get.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/policies"
@patch("module_build_service.utils.greenwave.requests")
def test_greenwave_get_product_versions(self, mock_requests):
resp_status = 200
resp_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver2"],
"rules": [],
"subject_type": "some-module"
},
{
"decision_context": "decision_context_2",
"product_versions": ["ver4"],
"rules": [],
"subject_type": "subject_type_2"
}
]
}
expected_versions = {"ver1", "ver2", "ver3"}
response = Mock()
response.json.return_value = resp_content
response.status_code = resp_status
mock_requests.get.return_value = response
versions_set = greenwave.get_product_versions()
assert versions_set == expected_versions
assert mock_requests.get.call_args_list[0][1]["url"] == \
"https://greenwave.example.local/api/v1.0/policies"
@pytest.mark.parametrize("policies_satisfied", (True, False))
@patch("module_build_service.utils.greenwave.requests")
def test_greenwave_check_gating(self, mock_requests, policies_satisfied):
resp_status = 200
policies_content = {
"policies": [
{
"decision_context": "test_dec_context",
"product_versions": ["ver1", "ver3"],
"rules": [],
"subject_type": "some-module"
}
]
}
responses = [Mock() for i in range(3)]
for r in responses:
r.status_code = resp_status
responses[0].json.return_value = policies_content
responses[1].json.return_value = {"policies_satisfied": False}
responses[2].json.return_value = {"policies_satisfied": policies_satisfied}
mock_requests.get.return_value = responses[0]
mock_requests.post.side_effect = responses[1:]
fake_build = make_module("pkg:0.1:1:c1", requires_list={"platform": "el8"})
result = greenwave.check_gating(fake_build)
assert result == policies_satisfied