modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-06-16 15:06:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
ff32aaef47e32cbd8b5ecde8e011206a60dff3a2
fm-orchestrator/module_build_service/web/views.py
Jan Kaluza bfd9a13205 Allow overriding RPM components refs while submitting the module build. 
There is a need to rebuild the module builds done in CentOS 9 Stream
internally in MBS to include them in RHEL. This is currenly a hard task,
because the RPM components included in a module are usually
taken from HEAD of the branch defined by their `ref` value.

For the rebuild task, it means we would have to ensure that the HEAD
of all RPM components points to right commit hash right before we start
rebuilding CentOS 9 Stream module in internal MBS. This is very hard
and fragile thing to do, especially if there are two different modules
using the RPM component from the same branch. This is prone to race
condition and makes the rebuilds quite complex and in some cases
not possible to do without force pushes to RPM component repositories
which is not acceptable by internal dist-git policy.

This commit fixes it by allowing overriding the commit hash while
submitting the module build. This helps in the mentioned situation,
because we can keep internal RPM components branches in 1:1 sync with
CentOS 9 Stream branches and HEAD can always point to the same commit
in both internal and CentOS 9 Stream repositories.

When the module rebuild is submitted in internal MBS,
we can use this new feature to override the `ref` for each RPM component
so it points to particular commit and the requirement for HEAD to point
to this commit is no longer there.

The `ref` is overriden only internally in MBS (but it is recorded in logs
and in XMD section), so the input modulemd file is not altered. This is
the same logic as used for other overrides (`buildrequire_overrides` or
`side_tag`).

This does not bring any security problem, because it is already possible
to use commit hash in `ref`, so the package maintainer can already change
the commit hash to any particular commit by using this `ref` value.

Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
2021-07-28 08:48:12 +02:00

688 lines
26 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
""" The module build orchestrator for Modularity, API.
This is the implementation of the orchestrator's public RESTful API.
"""
from __future__ import absolute_import
from io import BytesIO
import json
import sqlalchemy.event
from flask import request, url_for, Blueprint, Response
from flask.views import MethodView
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
from six import string_types
from module_build_service import app, db, version, api_version as max_api_version
from module_build_service.common import conf, log, models
from module_build_service.common.errors import (
ValidationError, Forbidden, NotFound, ProgrammingError,
Unauthorized, UnprocessableEntity, Conflict
)
from module_build_service.common.models import send_message_after_module_build_state_change
from module_build_service.common.monitor import registry
from module_build_service.common.submit import fetch_mmd
from module_build_service.common.utils import import_mmd
import module_build_service.web.auth
from module_build_service.web.backports import jsonify
from module_build_service.web.submit import (
submit_module_build_from_scm, submit_module_build_from_yaml
)
from module_build_service.web.utils import (
cors_header,
filter_component_builds,
filter_module_builds,
get_scm_url_re,
pagination_metadata,
str_to_bool,
validate_api_version,
)
api_routes = {
"module_builds": {
"url": "/module-build-service/<int:api_version>/module-builds/",
"options": {"methods": ["POST"]},
},
"module_builds_list": {
"url": "/module-build-service/<int:api_version>/module-builds/",
"options": {"defaults": {"id": None}, "methods": ["GET"]},
},
"module_build": {
"url": "/module-build-service/<int:api_version>/module-builds/<int:id>",
"options": {"methods": ["GET", "PATCH"]},
},
"component_builds_list": {
"url": "/module-build-service/<int:api_version>/component-builds/",
"options": {"defaults": {"id": None}, "methods": ["GET"]},
},
"component_build": {
"url": "/module-build-service/<int:api_version>/component-builds/<int:id>",
"options": {"methods": ["GET"]},
},
"about": {
"url": "/module-build-service/<int:api_version>/about/",
"options": {"methods": ["GET"]},
},
"rebuild_strategies_list": {
"url": "/module-build-service/<int:api_version>/rebuild-strategies/",
"options": {"methods": ["GET"]},
},
"import_module": {
"url": "/module-build-service/<int:api_version>/import-module/",
"options": {"methods": ["POST"]},
},
"log_messages_module_build": {
"url": "/module-build-service/<int:api_version>/module-builds/<int:id>/messages",
"options": {"methods": ["GET"], "defaults": {"model": models.ModuleBuild}},
},
"log_messages_component_build": {
"url": "/module-build-service/<int:api_version>/component-builds/<int:id>/messages",
"options": {"methods": ["GET"], "defaults": {"model": models.ComponentBuild}},
},
"final_modulemd": {
"url": "/module-build-service/<int:api_version>/final-modulemd/<int:id>",
"options": {"methods": ["GET"]},
},
}
class AbstractQueryableBuildAPI(MethodView):
""" An abstract class, housing some common functionality. """
@cors_header()
@validate_api_version()
def get(self, api_version, id):
id_flag = request.args.get("id")
if id_flag:
endpoint = request.endpoint.split("s_list")[0]
raise ValidationError(
'The "id" query option is invalid. Did you mean to go to "{0}"?'.format(
url_for(endpoint, api_version=api_version, id=id_flag)
)
)
verbose_flag = request.args.get("verbose", "false").lower()
short_flag = request.args.get("short", "false").lower()
json_func_kwargs = {}
json_func_name = "json"
if id is None:
# Lists all tracked builds
p_query = self.query_filter(request)
json_data = {"meta": pagination_metadata(p_query, api_version, request.args)}
if verbose_flag == "true" or verbose_flag == "1":
json_func_name = "extended_json"
json_func_kwargs["show_state_url"] = True
json_func_kwargs["api_version"] = api_version
elif short_flag == "true" or short_flag == "1":
if p_query.items and hasattr(p_query.items[0], "short_json"):
json_func_name = "short_json"
if json_func_name == "json" or json_func_name == "extended_json":
# Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
json_func_kwargs["db_session"] = db.session
json_data["items"] = [
getattr(item, json_func_name)(**json_func_kwargs) for item in p_query.items
]
return jsonify(json_data), 200
else:
# Lists details for the specified build
instance = self.model.query.filter_by(id=id).first()
if instance:
if verbose_flag == "true" or verbose_flag == "1":
json_func_name = "extended_json"
json_func_kwargs["show_state_url"] = True
json_func_kwargs["api_version"] = api_version
elif short_flag == "true" or short_flag == "1":
if getattr(instance, "short_json", None):
json_func_name = "short_json"
if json_func_name == "json" or json_func_name == "extended_json":
# Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
json_func_kwargs["db_session"] = db.session
return jsonify(getattr(instance, json_func_name)(**json_func_kwargs)), 200
else:
raise NotFound("No such %s found." % self.kind)
class ComponentBuildAPI(AbstractQueryableBuildAPI):
kind = "component"
query_filter = staticmethod(filter_component_builds)
model = models.ComponentBuild
class ModuleBuildAPI(AbstractQueryableBuildAPI):
kind = "module"
query_filter = staticmethod(filter_module_builds)
model = models.ModuleBuild
@staticmethod
def check_groups(username, groups, allowed_groups=conf.allowed_groups):
# If the user is part of the whitelist, then the group membership check is skipped
if username in conf.allowed_users:
return
if allowed_groups and not (allowed_groups & groups):
raise Forbidden("%s is not in any of %r, only %r" % (username, allowed_groups, groups))
# Additional POST and DELETE handlers for modules follow.
@validate_api_version()
def post(self, api_version):
data = _dict_from_request(request)
if "modulemd" in data or (hasattr(request, "files") and "yaml" in request.files):
handler = YAMLFileHandler(request, data)
else:
handler = SCMHandler(request, data)
if conf.no_auth is True and handler.username == "anonymous" and "owner" in handler.data:
handler.username = handler.data["owner"]
self.check_groups(handler.username, handler.groups)
handler.validate()
modules = handler.post()
if api_version == 1:
# Only show the first module build for backwards-compatibility
rv = modules[0].extended_json(db.session, True, api_version)
else:
rv = [module.extended_json(db.session, True, api_version) for module in modules]
return jsonify(rv), 201
@validate_api_version()
def patch(self, api_version, id):
username, groups = module_build_service.web.auth.get_user(request)
try:
r = json.loads(request.get_data().decode("utf-8"))
except Exception:
log.exception("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
if "owner" in r:
if conf.no_auth is not True:
raise ValidationError(
"The request contains 'owner' parameter, however NO_AUTH is not allowed"
)
elif username == "anonymous":
username = r["owner"]
self.check_groups(username, groups)
module = models.ModuleBuild.query.filter_by(id=id).first()
if not module:
raise NotFound("No such module found.")
if module.owner != username and not (conf.admin_groups & groups):
raise Forbidden("You are not owner of this build and therefore cannot modify it.")
if not r.get("state"):
log.error("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
state = r["state"]
valid_input_states = ("failed", str(models.BUILD_STATES["failed"]))
if state not in valid_input_states:
raise ValidationError(
"An invalid state was submitted. Valid states values are: {}"
.format(", ".join(valid_input_states))
)
valid_states_to_cancel = ("build", "init", "wait")
module_state_name = models.INVERSE_BUILD_STATES[module.state]
if module_state_name not in valid_states_to_cancel:
log.error(
"The user %s attempted to cancel a build in the %s state",
username, module_state_name,
)
raise ValidationError(
"To cancel a module build, it must be in one of the following states: {}"
.format(", ".join(valid_states_to_cancel))
)
module.transition(
db.session, conf, models.BUILD_STATES["failed"], "Canceled by %s." % username)
db.session.add(module)
db.session.commit()
return jsonify(module.extended_json(db.session, True, api_version)), 200
class AboutAPI(MethodView):
@cors_header()
@validate_api_version()
def get(self, api_version):
json = {"version": version, "api_version": max_api_version}
config_items = ["auth_method"]
for item in config_items:
config_item = getattr(conf, item)
# All config items have a default, so if doesn't exist it is a programming error
if not config_item:
raise ProgrammingError('An invalid config item of "{0}" was specified'.format(item))
json[item] = config_item
return jsonify(json), 200
class RebuildStrategies(MethodView):
@cors_header()
@validate_api_version()
def get(self, api_version):
items = []
# Sort the items list by name
for strategy in sorted(models.ModuleBuild.rebuild_strategies.keys()):
default = False
if strategy == conf.rebuild_strategy:
default = True
allowed = True
elif (
conf.rebuild_strategy_allow_override and strategy in conf.rebuild_strategies_allowed
):
allowed = True
else:
allowed = False
items.append({
"name": strategy,
"description": models.ModuleBuild.rebuild_strategies[strategy],
"allowed": allowed,
"default": default,
})
return jsonify({"items": items}), 200
class ImportModuleAPI(MethodView):
@validate_api_version()
def post(self, api_version):
# disable this API endpoint if no groups are defined
if not conf.allowed_groups_to_import_module:
log.error(
"Import module API is disabled. Set 'ALLOWED_GROUPS_TO_IMPORT_MODULE'"
" configuration value first."
)
raise Forbidden("Import module API is disabled.")
# auth checks
username, groups = module_build_service.web.auth.get_user(request)
ModuleBuildAPI.check_groups(
username, groups, allowed_groups=conf.allowed_groups_to_import_module)
# process request using SCM handler
handler = SCMHandler(request)
handler.validate(skip_branch=True, skip_optional_params=True)
mmd, _ = fetch_mmd(handler.data["scmurl"], mandatory_checks=False)
build, messages = import_mmd(db.session, mmd)
json_data = {
"module": build.json(db.session, show_tasks=False),
"messages": messages
}
# return 201 Created if we reach this point
return jsonify(json_data), 201
class LogMessageAPI(MethodView):
@validate_api_version()
def get(self, api_version, id, model):
if not model:
raise ValidationError("Model is not set for this log messages endpoint")
query = model.query.filter_by(id=id).first().log_messages.order_by(
models.LogMessage.time_created.desc())
page = request.args.get("page", 1, type=int)
per_page = request.args.get("per_page", 10, type=int)
p_query = query.paginate(page, per_page, False)
request_args = {"id": id}
json_data = {"meta": pagination_metadata(p_query, api_version, request_args)}
json_data["messages"] = [
getattr(message, "json")() for message in p_query.items
]
return jsonify(json_data), 200
class FinalModulemdAPI(MethodView):
@validate_api_version()
def get(self, api_version, id):
module = models.ModuleBuild.get_by_id(db.session, id)
if not module:
raise ValidationError("The module could not be found")
if conf.system == "koji":
# We are importing KojiContentGenerator here so we can generate the final modulemds.
# If we imported this regularly we would have gotten a circular import error.
from module_build_service.builder.KojiContentGenerator import KojiContentGenerator # noqa
cg = KojiContentGenerator(module, conf)
finalmmds = cg.get_final_mmds()
else:
raise ValidationError("Configured builder not able to generate final modulemds!")
return jsonify(finalmmds), 200
class BaseHandler(object):
valid_params = {
"branch",
"buildrequire_overrides",
"rpm_component_ref_overrides",
"modulemd",
"module_name",
"module_stream",
"owner",
"rebuild_strategy",
"reuse_components_from",
"require_overrides",
"scmurl",
"scratch",
"side_tag",
"srpms",
}
def __init__(self, request, data=None):
self.username, self.groups = module_build_service.web.auth.get_user(request)
self.data = data or _dict_from_request(request)
# canonicalize and validate scratch option
if "scratch" in self.data and str_to_bool(str(self.data["scratch"])):
self.data["scratch"] = True
if conf.modules_allow_scratch is not True:
raise Forbidden("Scratch builds are not enabled")
else:
self.data["scratch"] = False
# canonicalize and validate srpms list
if "srpms" in self.data and self.data["srpms"]:
if not self.data["scratch"]:
raise Forbidden("srpms may only be specified for scratch builds")
if not isinstance(self.data["srpms"], list):
raise ValidationError("srpms must be specified as a list")
else:
self.data["srpms"] = []
def _validate_dep_overrides_format(self, key):
"""
Validate any dependency overrides provided to the API.
:param str key: the override key to validate
:raises ValidationError: when the overrides are an invalid format
"""
if not self.data.get(key):
return
invalid_override_msg = (
'The "{}" parameter must be an object with the keys as module '
"names and the values as arrays of streams".format(key)
)
if not isinstance(self.data[key], dict):
raise ValidationError(invalid_override_msg)
for streams in self.data[key].values():
if not isinstance(streams, list):
raise ValidationError(invalid_override_msg)
for stream in streams:
if not isinstance(stream, string_types):
raise ValidationError(invalid_override_msg)
def _validate_ref_overrides_format(self, key):
"""
Validate any dependency overrides provided to the API.
:param str key: the override key to validate
:raises ValidationError: when the overrides are an invalid format
"""
if not self.data.get(key):
return
invalid_override_msg = (
'The "{}" parameter must be an object with the keys as component '
"names and the values as strings of refs".format(key)
)
if not isinstance(self.data[key], dict):
raise ValidationError(invalid_override_msg)
for ref in self.data[key].values():
if not isinstance(ref, string_types):
raise ValidationError(invalid_override_msg)
def validate_optional_params(self):
forbidden_params = [k for k in self.data if k not in self.valid_params]
if forbidden_params:
raise ValidationError(
"The request contains invalid parameters: {}".format(
", ".join(forbidden_params))
)
if not conf.no_auth and "owner" in self.data:
raise ValidationError(
"The request contains 'owner' parameter, however NO_AUTH is not allowed")
if not conf.rebuild_strategy_allow_override and "rebuild_strategy" in self.data:
raise ValidationError(
'The request contains the "rebuild_strategy" parameter but '
"overriding the default isn't allowed"
)
if "rebuild_strategy" in self.data:
if self.data["rebuild_strategy"] not in conf.rebuild_strategies_allowed:
raise ValidationError(
'The rebuild method of "{0}" is not allowed. Choose from: {1}.'.format(
self.data["rebuild_strategy"], ", ".join(conf.rebuild_strategies_allowed))
)
self._validate_dep_overrides_format("buildrequire_overrides")
self._validate_dep_overrides_format("require_overrides")
self._validate_ref_overrides_format("rpm_component_ref_overrides")
if "reuse_components_from" in self.data:
if "rebuild_strategy" in self.data and self.data["rebuild_strategy"] == "all":
raise ValidationError(
'You cannot specify the parameter "reuse_components_from" when the '
'"rebuild_strategy" parameter is set to "all"'
)
invalid_identifier_msg = (
'The parameter "reuse_components_from" contains an invalid module identifier')
if isinstance(self.data["reuse_components_from"], int):
reuse_module = models.ModuleBuild.get_by_id(
db.session, self.data["reuse_components_from"])
elif isinstance(self.data["reuse_components_from"], string_types):
try:
n, s, v, c = self.data["reuse_components_from"].split(":")
except ValueError:
raise ValidationError(invalid_identifier_msg)
reuse_module = models.ModuleBuild.get_build_from_nsvc(db.session, n, s, v, c)
else:
raise ValidationError(invalid_identifier_msg)
if not reuse_module:
raise ValidationError(
'The module in the parameter "reuse_components_from" could not be found')
if reuse_module.state != models.BUILD_STATES["ready"]:
raise ValidationError(
'The module in the parameter "reuse_components_from" must be in the ready state'
)
# Normalize the value so that it simplifies any code that uses this value
self.data["reuse_components_from"] = reuse_module.id
class SCMHandler(BaseHandler):
def validate(self, skip_branch=False, skip_optional_params=False):
if "scmurl" not in self.data:
log.error("Missing scmurl")
raise ValidationError("Missing scmurl")
url = self.data["scmurl"]
allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls)
if not conf.allow_custom_scmurls and not allowed_prefix:
log.error("The submitted scmurl %r is not allowed" % url)
raise Forbidden("The submitted scmurl %s is not allowed" % url)
if not get_scm_url_re().match(url):
log.error("The submitted scmurl %r is not valid" % url)
raise ValidationError("The submitted scmurl %s is not valid" % url)
if not skip_branch and "branch" not in self.data:
log.error("Missing branch")
raise ValidationError("Missing branch")
if "module_name" in self.data:
log.error("Module name override is only allowed when a YAML file is submitted")
raise ValidationError(
"Module name override is only allowed when a YAML file is submitted"
)
if "module_stream" in self.data:
log.error("Stream name override is only allowed when a YAML file is submitted")
raise ValidationError(
"Stream name override is only allowed when a YAML file is submitted"
)
if not skip_optional_params:
self.validate_optional_params()
def post(self):
return submit_module_build_from_scm(
db.session, self.username, self.data, allow_local_url=False)
class YAMLFileHandler(BaseHandler):
def __init__(self, request, data=None):
super(YAMLFileHandler, self).__init__(request, data)
if not self.data["scratch"] and not conf.yaml_submit_allowed:
raise Forbidden("YAML submission is not enabled")
def validate(self):
if (
"modulemd" not in self.data
and (not hasattr(request, "files") or "yaml" not in request.files)
):
log.error("Invalid file submitted")
raise ValidationError("Invalid file submitted")
self.validate_optional_params()
def post(self):
if "modulemd" in self.data:
handle = BytesIO(self.data["modulemd"].encode("utf-8"))
else:
handle = request.files["yaml"]
if self.data.get("module_name"):
handle.filename = self.data["module_name"]
stream_name = self.data.get("module_stream", None)
return submit_module_build_from_yaml(
db.session, self.username, handle, self.data, stream=stream_name)
def _dict_from_request(request):
if "multipart/form-data" in request.headers.get("Content-Type", ""):
data = request.form.to_dict()
else:
try:
data = json.loads(request.get_data().decode("utf-8"))
except Exception:
log.exception("Invalid JSON submitted")
raise ValidationError("Invalid JSON submitted")
return data
monitor_api = Blueprint(
"monitor", __name__, url_prefix="/module-build-service/<int:api_version>/monitor")
@cors_header()
@validate_api_version()
@monitor_api.route("/metrics")
def metrics(api_version):
return Response(generate_latest(registry), content_type=CONTENT_TYPE_LATEST)
def register_api():
""" Registers the MBS API. """
module_view = ModuleBuildAPI.as_view("module_builds")
component_view = ComponentBuildAPI.as_view("component_builds")
about_view = AboutAPI.as_view("about")
rebuild_strategies_view = RebuildStrategies.as_view("rebuild_strategies")
import_module = ImportModuleAPI.as_view("import_module")
log_message = LogMessageAPI.as_view("log_messages")
final_modulemd = FinalModulemdAPI.as_view("final_modulemd")
for key, val in api_routes.items():
if key.startswith("component_build"):
app.add_url_rule(val["url"], endpoint=key, view_func=component_view, **val["options"])
elif key.startswith("module_build"):
app.add_url_rule(val["url"], endpoint=key, view_func=module_view, **val["options"])
elif key.startswith("about"):
app.add_url_rule(val["url"], endpoint=key, view_func=about_view, **val["options"])
elif key == "rebuild_strategies_list":
app.add_url_rule(
val["url"], endpoint=key, view_func=rebuild_strategies_view, **val["options"]
)
elif key == "import_module":
app.add_url_rule(val["url"], endpoint=key, view_func=import_module, **val["options"])
elif key.startswith("log_message"):
app.add_url_rule(val["url"], endpoint=key, view_func=log_message, **val["options"])
elif key.startswith("final_modulemd"):
app.add_url_rule(val["url"], endpoint=key, view_func=final_modulemd, **val["options"])
else:
raise NotImplementedError("Unhandled api key.")
app.register_blueprint(monitor_api)
register_api()
def json_error(status, error, message):
response = jsonify({"status": status, "error": error, "message": message})
response.status_code = status
return response
@app.errorhandler(ValidationError)
def validationerror_error(e):
"""Flask error handler for ValidationError exceptions"""
return json_error(400, "Bad Request", str(e))
@app.errorhandler(Unauthorized)
def unauthorized_error(e):
"""Flask error handler for NotAuthorized exceptions"""
return json_error(401, "Unauthorized", str(e))
@app.errorhandler(Forbidden)
def forbidden_error(e):
"""Flask error handler for Forbidden exceptions"""
return json_error(403, "Forbidden", str(e))
@app.errorhandler(RuntimeError)
def runtimeerror_error(e):
"""Flask error handler for RuntimeError exceptions"""
log.exception("RuntimeError exception raised")
return json_error(500, "Internal Server Error", str(e))
@app.errorhandler(UnprocessableEntity)
def unprocessableentity_error(e):
"""Flask error handler for UnprocessableEntity exceptions"""
return json_error(422, "Unprocessable Entity", str(e))
@app.errorhandler(Conflict)
def conflict_error(e):
"""Flask error handler for Conflict exceptions"""
return json_error(409, "Conflict", str(e))
@app.errorhandler(NotFound)
def notfound_error(e):
"""Flask error handler for Conflict exceptions"""
return json_error(404, "Not Found", str(e))
# Ensure the event handler is called on db.session
sqlalchemy.event.listen(
db.session, "after_commit", send_message_after_module_build_state_change)
Reference in New Issue View Git Blame Copy Permalink