Move views.py to web/views.py

module_build_service/web/views.py

@@ -0,0 +1,564 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MIT
+""" The module build orchestrator for Modularity, API.
+This is the implementation of the orchestrator's public RESTful API.
+"""
+
+import json
+from flask import request, url_for, Blueprint, Response
+from flask.views import MethodView
+from six import string_types
+from io import BytesIO
+from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
+
+from module_build_service import app, conf, log, models, db, version, api_version as max_api_version
+from module_build_service.common.utils import import_mmd
+from module_build_service.common.errors import (
+    ValidationError, Forbidden, NotFound, ProgrammingError
+)
+from module_build_service.web.backports import jsonify
+from module_build_service.common.monitor import registry
+from module_build_service.common.submit import fetch_mmd
+import module_build_service.web.auth
+from module_build_service.web.submit import (
+    submit_module_build_from_scm, submit_module_build_from_yaml
+)
+from module_build_service.web.utils import (
+    cors_header,
+    filter_component_builds,
+    filter_module_builds,
+    get_scm_url_re,
+    pagination_metadata,
+    str_to_bool,
+    validate_api_version,
+)
+
+
+api_routes = {
+    "module_builds": {
+        "url": "/module-build-service/<int:api_version>/module-builds/",
+        "options": {"methods": ["POST"]},
+    },
+    "module_builds_list": {
+        "url": "/module-build-service/<int:api_version>/module-builds/",
+        "options": {"defaults": {"id": None}, "methods": ["GET"]},
+    },
+    "module_build": {
+        "url": "/module-build-service/<int:api_version>/module-builds/<int:id>",
+        "options": {"methods": ["GET", "PATCH"]},
+    },
+    "component_builds_list": {
+        "url": "/module-build-service/<int:api_version>/component-builds/",
+        "options": {"defaults": {"id": None}, "methods": ["GET"]},
+    },
+    "component_build": {
+        "url": "/module-build-service/<int:api_version>/component-builds/<int:id>",
+        "options": {"methods": ["GET"]},
+    },
+    "about": {
+        "url": "/module-build-service/<int:api_version>/about/",
+        "options": {"methods": ["GET"]},
+    },
+    "rebuild_strategies_list": {
+        "url": "/module-build-service/<int:api_version>/rebuild-strategies/",
+        "options": {"methods": ["GET"]},
+    },
+    "import_module": {
+        "url": "/module-build-service/<int:api_version>/import-module/",
+        "options": {"methods": ["POST"]},
+    },
+    "log_messages_module_build": {
+        "url": "/module-build-service/<int:api_version>/module-builds/<int:id>/messages",
+        "options": {"methods": ["GET"], "defaults": {"model": models.ModuleBuild}},
+    },
+    "log_messages_component_build": {
+        "url": "/module-build-service/<int:api_version>/component-builds/<int:id>/messages",
+        "options": {"methods": ["GET"], "defaults": {"model": models.ComponentBuild}},
+    },
+}
+
+
+class AbstractQueryableBuildAPI(MethodView):
+    """ An abstract class, housing some common functionality. """
+
+    @cors_header()
+    @validate_api_version()
+    def get(self, api_version, id):
+        id_flag = request.args.get("id")
+        if id_flag:
+            endpoint = request.endpoint.split("s_list")[0]
+            raise ValidationError(
+                'The "id" query option is invalid. Did you mean to go to "{0}"?'.format(
+                    url_for(endpoint, api_version=api_version, id=id_flag)
+                )
+            )
+        verbose_flag = request.args.get("verbose", "false").lower()
+        short_flag = request.args.get("short", "false").lower()
+        json_func_kwargs = {}
+        json_func_name = "json"
+
+        if id is None:
+            # Lists all tracked builds
+            p_query = self.query_filter(request)
+            json_data = {"meta": pagination_metadata(p_query, api_version, request.args)}
+
+            if verbose_flag == "true" or verbose_flag == "1":
+                json_func_name = "extended_json"
+                json_func_kwargs["show_state_url"] = True
+                json_func_kwargs["api_version"] = api_version
+            elif short_flag == "true" or short_flag == "1":
+                if p_query.items and hasattr(p_query.items[0], "short_json"):
+                    json_func_name = "short_json"
+            if json_func_name == "json" or json_func_name == "extended_json":
+                # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
+                json_func_kwargs["db_session"] = db.session
+            json_data["items"] = [
+                getattr(item, json_func_name)(**json_func_kwargs) for item in p_query.items
+            ]
+
+            return jsonify(json_data), 200
+        else:
+            # Lists details for the specified build
+            instance = self.model.query.filter_by(id=id).first()
+            if instance:
+                if verbose_flag == "true" or verbose_flag == "1":
+                    json_func_name = "extended_json"
+                    json_func_kwargs["show_state_url"] = True
+                    json_func_kwargs["api_version"] = api_version
+                elif short_flag == "true" or short_flag == "1":
+                    if getattr(instance, "short_json", None):
+                        json_func_name = "short_json"
+                if json_func_name == "json" or json_func_name == "extended_json":
+                    # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
+                    json_func_kwargs["db_session"] = db.session
+                return jsonify(getattr(instance, json_func_name)(**json_func_kwargs)), 200
+            else:
+                raise NotFound("No such %s found." % self.kind)
+
+
+class ComponentBuildAPI(AbstractQueryableBuildAPI):
+    kind = "component"
+    query_filter = staticmethod(filter_component_builds)
+    model = models.ComponentBuild
+
+
+class ModuleBuildAPI(AbstractQueryableBuildAPI):
+    kind = "module"
+    query_filter = staticmethod(filter_module_builds)
+    model = models.ModuleBuild
+
+    @staticmethod
+    def check_groups(username, groups, allowed_groups=conf.allowed_groups):
+        # If the user is part of the whitelist, then the group membership check is skipped
+        if username in conf.allowed_users:
+            return
+        if allowed_groups and not (allowed_groups & groups):
+            raise Forbidden("%s is not in any of %r, only %r" % (username, allowed_groups, groups))
+
+    # Additional POST and DELETE handlers for modules follow.
+    @validate_api_version()
+    def post(self, api_version):
+        data = _dict_from_request(request)
+        if "modulemd" in data or (hasattr(request, "files") and "yaml" in request.files):
+            handler = YAMLFileHandler(request, data)
+        else:
+            handler = SCMHandler(request, data)
+
+        if conf.no_auth is True and handler.username == "anonymous" and "owner" in handler.data:
+            handler.username = handler.data["owner"]
+
+        self.check_groups(handler.username, handler.groups)
+
+        handler.validate()
+        modules = handler.post()
+        if api_version == 1:
+            # Only show the first module build for backwards-compatibility
+            rv = modules[0].extended_json(db.session, True, api_version)
+        else:
+            rv = [module.extended_json(db.session, True, api_version) for module in modules]
+        return jsonify(rv), 201
+
+    @validate_api_version()
+    def patch(self, api_version, id):
+        username, groups = module_build_service.web.auth.get_user(request)
+
+        try:
+            r = json.loads(request.get_data().decode("utf-8"))
+        except Exception:
+            log.exception("Invalid JSON submitted")
+            raise ValidationError("Invalid JSON submitted")
+
+        if "owner" in r:
+            if conf.no_auth is not True:
+                raise ValidationError(
+                    "The request contains 'owner' parameter, however NO_AUTH is not allowed"
+                )
+            elif username == "anonymous":
+                username = r["owner"]
+
+        self.check_groups(username, groups)
+
+        module = models.ModuleBuild.query.filter_by(id=id).first()
+        if not module:
+            raise NotFound("No such module found.")
+
+        if module.owner != username and not (conf.admin_groups & groups):
+            raise Forbidden("You are not owner of this build and therefore cannot modify it.")
+
+        if not r.get("state"):
+            log.error("Invalid JSON submitted")
+            raise ValidationError("Invalid JSON submitted")
+
+        state = r["state"]
+        valid_input_states = ("failed", str(models.BUILD_STATES["failed"]))
+        if state not in valid_input_states:
+            raise ValidationError(
+                "An invalid state was submitted. Valid states values are: {}"
+                .format(", ".join(valid_input_states))
+            )
+
+        valid_states_to_cancel = ("build", "init", "wait")
+        module_state_name = models.INVERSE_BUILD_STATES[module.state]
+        if module_state_name not in valid_states_to_cancel:
+            log.error(
+                "The user %s attempted to cancel a build in the %s state",
+                username, module_state_name,
+            )
+            raise ValidationError(
+                "To cancel a module build, it must be in one of the following states: {}"
+                .format(", ".join(valid_states_to_cancel))
+            )
+
+        module.transition(
+            db.session, conf, models.BUILD_STATES["failed"], "Canceled by %s." % username)
+        db.session.add(module)
+        db.session.commit()
+
+        return jsonify(module.extended_json(db.session, True, api_version)), 200
+
+
+class AboutAPI(MethodView):
+    @cors_header()
+    @validate_api_version()
+    def get(self, api_version):
+        json = {"version": version, "api_version": max_api_version}
+        config_items = ["auth_method"]
+        for item in config_items:
+            config_item = getattr(conf, item)
+            # All config items have a default, so if doesn't exist it is a programming error
+            if not config_item:
+                raise ProgrammingError('An invalid config item of "{0}" was specified'.format(item))
+            json[item] = config_item
+        return jsonify(json), 200
+
+
+class RebuildStrategies(MethodView):
+    @cors_header()
+    @validate_api_version()
+    def get(self, api_version):
+        items = []
+        # Sort the items list by name
+        for strategy in sorted(models.ModuleBuild.rebuild_strategies.keys()):
+            default = False
+            if strategy == conf.rebuild_strategy:
+                default = True
+                allowed = True
+            elif (
+                conf.rebuild_strategy_allow_override and strategy in conf.rebuild_strategies_allowed
+            ):
+                allowed = True
+            else:
+                allowed = False
+            items.append({
+                "name": strategy,
+                "description": models.ModuleBuild.rebuild_strategies[strategy],
+                "allowed": allowed,
+                "default": default,
+            })
+
+        return jsonify({"items": items}), 200
+
+
+class ImportModuleAPI(MethodView):
+    @validate_api_version()
+    def post(self, api_version):
+        # disable this API endpoint if no groups are defined
+        if not conf.allowed_groups_to_import_module:
+            log.error(
+                "Import module API is disabled. Set 'ALLOWED_GROUPS_TO_IMPORT_MODULE'"
+                " configuration value first."
+            )
+            raise Forbidden("Import module API is disabled.")
+
+        # auth checks
+        username, groups = module_build_service.web.auth.get_user(request)
+        ModuleBuildAPI.check_groups(
+            username, groups, allowed_groups=conf.allowed_groups_to_import_module)
+
+        # process request using SCM handler
+        handler = SCMHandler(request)
+        handler.validate(skip_branch=True, skip_optional_params=True)
+
+        mmd, _ = fetch_mmd(handler.data["scmurl"], mandatory_checks=False)
+        build, messages = import_mmd(db.session, mmd)
+        json_data = {
+            "module": build.json(db.session, show_tasks=False),
+            "messages": messages
+        }
+
+        # return 201 Created if we reach this point
+        return jsonify(json_data), 201
+
+
+class LogMessageAPI(MethodView):
+
+    @validate_api_version()
+    def get(self, api_version, id, model):
+
+        if not model:
+            raise ValidationError("Model is not set for this log messages endpoint")
+
+        query = model.query.filter_by(id=id).first().log_messages.order_by(
+            models.LogMessage.time_created.desc())
+
+        page = request.args.get("page", 1, type=int)
+        per_page = request.args.get("per_page", 10, type=int)
+        p_query = query.paginate(page, per_page, False)
+
+        request_args = {"id": id}
+        json_data = {"meta": pagination_metadata(p_query, api_version, request_args)}
+        json_data["messages"] = [
+            getattr(message, "json")() for message in p_query.items
+        ]
+
+        return jsonify(json_data), 200
+
+
+class BaseHandler(object):
+    valid_params = {
+        "branch",
+        "buildrequire_overrides",
+        "modulemd",
+        "module_name",
+        "owner",
+        "rebuild_strategy",
+        "reuse_components_from",
+        "require_overrides",
+        "scmurl",
+        "scratch",
+        "srpms",
+    }
+
+    def __init__(self, request, data=None):
+        self.username, self.groups = module_build_service.web.auth.get_user(request)
+        self.data = data or _dict_from_request(request)
+
+        # canonicalize and validate scratch option
+        if "scratch" in self.data and str_to_bool(str(self.data["scratch"])):
+            self.data["scratch"] = True
+            if conf.modules_allow_scratch is not True:
+                raise Forbidden("Scratch builds are not enabled")
+        else:
+            self.data["scratch"] = False
+
+        # canonicalize and validate srpms list
+        if "srpms" in self.data and self.data["srpms"]:
+            if not self.data["scratch"]:
+                raise Forbidden("srpms may only be specified for scratch builds")
+            if not isinstance(self.data["srpms"], list):
+                raise ValidationError("srpms must be specified as a list")
+        else:
+            self.data["srpms"] = []
+
+    def _validate_dep_overrides_format(self, key):
+        """
+        Validate any dependency overrides provided to the API.
+
+        :param str key: the override key to validate
+        :raises ValidationError: when the overrides are an invalid format
+        """
+        if not self.data.get(key):
+            return
+        invalid_override_msg = (
+            'The "{}" parameter must be an object with the keys as module '
+            "names and the values as arrays of streams".format(key)
+        )
+        if not isinstance(self.data[key], dict):
+            raise ValidationError(invalid_override_msg)
+        for streams in self.data[key].values():
+            if not isinstance(streams, list):
+                raise ValidationError(invalid_override_msg)
+            for stream in streams:
+                if not isinstance(stream, string_types):
+                    raise ValidationError(invalid_override_msg)
+
+    def validate_optional_params(self):
+        forbidden_params = [k for k in self.data if k not in self.valid_params]
+        if forbidden_params:
+            raise ValidationError(
+                "The request contains unspecified parameters: {}".format(
+                    ", ".join(forbidden_params))
+            )
+
+        if not conf.no_auth and "owner" in self.data:
+            raise ValidationError(
+                "The request contains 'owner' parameter, however NO_AUTH is not allowed")
+
+        if not conf.rebuild_strategy_allow_override and "rebuild_strategy" in self.data:
+            raise ValidationError(
+                'The request contains the "rebuild_strategy" parameter but '
+                "overriding the default isn't allowed"
+            )
+
+        if "rebuild_strategy" in self.data:
+            if self.data["rebuild_strategy"] not in conf.rebuild_strategies_allowed:
+                raise ValidationError(
+                    'The rebuild method of "{0}" is not allowed. Choose from: {1}.'.format(
+                        self.data["rebuild_strategy"], ", ".join(conf.rebuild_strategies_allowed))
+                )
+
+        self._validate_dep_overrides_format("buildrequire_overrides")
+        self._validate_dep_overrides_format("require_overrides")
+
+        if "reuse_components_from" in self.data:
+            if "rebuild_strategy" in self.data and self.data["rebuild_strategy"] == "all":
+                raise ValidationError(
+                    'You cannot specify the parameter "reuse_components_from" when the '
+                    '"rebuild_strategy" parameter is set to "all"'
+                )
+
+            invalid_identifier_msg = (
+                'The parameter "reuse_components_from" contains an invalid module identifier')
+
+            if isinstance(self.data["reuse_components_from"], int):
+                reuse_module = models.ModuleBuild.get_by_id(
+                    db.session, self.data["reuse_components_from"])
+            elif isinstance(self.data["reuse_components_from"], string_types):
+                try:
+                    n, s, v, c = self.data["reuse_components_from"].split(":")
+                except ValueError:
+                    raise ValidationError(invalid_identifier_msg)
+                reuse_module = models.ModuleBuild.get_build_from_nsvc(db.session, n, s, v, c)
+            else:
+                raise ValidationError(invalid_identifier_msg)
+
+            if not reuse_module:
+                raise ValidationError(
+                    'The module in the parameter "reuse_components_from" could not be found')
+
+            if reuse_module.state != models.BUILD_STATES["ready"]:
+                raise ValidationError(
+                    'The module in the parameter "reuse_components_from" must be in the ready state'
+                )
+
+            # Normalize the value so that it simplifies any code that uses this value
+            self.data["reuse_components_from"] = reuse_module.id
+
+
+class SCMHandler(BaseHandler):
+    def validate(self, skip_branch=False, skip_optional_params=False):
+        if "scmurl" not in self.data:
+            log.error("Missing scmurl")
+            raise ValidationError("Missing scmurl")
+
+        url = self.data["scmurl"]
+        allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls)
+        if not conf.allow_custom_scmurls and not allowed_prefix:
+            log.error("The submitted scmurl %r is not allowed" % url)
+            raise Forbidden("The submitted scmurl %s is not allowed" % url)
+
+        if not get_scm_url_re().match(url):
+            log.error("The submitted scmurl %r is not valid" % url)
+            raise ValidationError("The submitted scmurl %s is not valid" % url)
+
+        if not skip_branch and "branch" not in self.data:
+            log.error("Missing branch")
+            raise ValidationError("Missing branch")
+
+        if not skip_optional_params:
+            self.validate_optional_params()
+
+    def post(self):
+        return submit_module_build_from_scm(
+            db.session, self.username, self.data, allow_local_url=False)
+
+
+class YAMLFileHandler(BaseHandler):
+    def __init__(self, request, data=None):
+        super(YAMLFileHandler, self).__init__(request, data)
+        if not self.data["scratch"] and not conf.yaml_submit_allowed:
+            raise Forbidden("YAML submission is not enabled")
+
+    def validate(self):
+        if (
+            "modulemd" not in self.data
+            and (not hasattr(request, "files") or "yaml" not in request.files)
+        ):
+            log.error("Invalid file submitted")
+            raise ValidationError("Invalid file submitted")
+        self.validate_optional_params()
+
+    def post(self):
+        if "modulemd" in self.data:
+            handle = BytesIO(self.data["modulemd"].encode("utf-8"))
+            if self.data.get("module_name"):
+                handle.filename = self.data["module_name"]
+        else:
+            handle = request.files["yaml"]
+        return submit_module_build_from_yaml(
+            db.session, self.username, handle, self.data)
+
+
+def _dict_from_request(request):
+    if "multipart/form-data" in request.headers.get("Content-Type", ""):
+        data = request.form.to_dict()
+    else:
+        try:
+            data = json.loads(request.get_data().decode("utf-8"))
+        except Exception:
+            log.exception("Invalid JSON submitted")
+            raise ValidationError("Invalid JSON submitted")
+    return data
+
+
+monitor_api = Blueprint(
+    "monitor", __name__, url_prefix="/module-build-service/<int:api_version>/monitor")
+
+
+@cors_header()
+@validate_api_version()
+@monitor_api.route("/metrics")
+def metrics(api_version):
+    return Response(generate_latest(registry), content_type=CONTENT_TYPE_LATEST)
+
+
+def register_api():
+    """ Registers the MBS API. """
+    module_view = ModuleBuildAPI.as_view("module_builds")
+    component_view = ComponentBuildAPI.as_view("component_builds")
+    about_view = AboutAPI.as_view("about")
+    rebuild_strategies_view = RebuildStrategies.as_view("rebuild_strategies")
+    import_module = ImportModuleAPI.as_view("import_module")
+    log_message = LogMessageAPI.as_view("log_messages")
+    for key, val in api_routes.items():
+        if key.startswith("component_build"):
+            app.add_url_rule(val["url"], endpoint=key, view_func=component_view, **val["options"])
+        elif key.startswith("module_build"):
+            app.add_url_rule(val["url"], endpoint=key, view_func=module_view, **val["options"])
+        elif key.startswith("about"):
+            app.add_url_rule(val["url"], endpoint=key, view_func=about_view, **val["options"])
+        elif key == "rebuild_strategies_list":
+            app.add_url_rule(
+                val["url"], endpoint=key, view_func=rebuild_strategies_view, **val["options"]
+            )
+        elif key == "import_module":
+            app.add_url_rule(val["url"], endpoint=key, view_func=import_module, **val["options"])
+        elif key.startswith("log_message"):
+            app.add_url_rule(val["url"], endpoint=key, view_func=log_message, **val["options"])
+        else:
+            raise NotImplementedError("Unhandled api key.")
+
+    app.register_blueprint(monitor_api)
+
+
+register_api()