fm-orchestrator/module_build_service/views.py

# -*- coding: utf-8 -*-
# Copyright (c) 2016  Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Petr Šabata <contyk@redhat.com>
#            Matt Prahl <mprahl@redhat.com>

""" The module build orchestrator for Modularity, API.
This is the implementation of the orchestrator's public RESTful API.
"""

import json
import module_build_service.auth
from flask import request, url_for
from flask.views import MethodView
from six import string_types
from io import BytesIO

from module_build_service import app, conf, log, models, db, version, api_version as max_api_version
from module_build_service.utils import (
    pagination_metadata,
    filter_module_builds,
    filter_component_builds,
    submit_module_build_from_scm,
    submit_module_build_from_yaml,
    get_scm_url_re,
    cors_header,
    validate_api_version,
    import_mmd,
    get_mmd_from_scm,
    str_to_bool,
)
from module_build_service.errors import ValidationError, Forbidden, NotFound, ProgrammingError
from module_build_service.backports import jsonify
from module_build_service.monitor import monitor_api


api_routes = {
    "module_builds": {
        "url": "/module-build-service/<int:api_version>/module-builds/",
        "options": {"methods": ["POST"]},
    },
    "module_builds_list": {
        "url": "/module-build-service/<int:api_version>/module-builds/",
        "options": {"defaults": {"id": None}, "methods": ["GET"]},
    },
    "module_build": {
        "url": "/module-build-service/<int:api_version>/module-builds/<int:id>",
        "options": {"methods": ["GET", "PATCH"]},
    },
    "component_builds_list": {
        "url": "/module-build-service/<int:api_version>/component-builds/",
        "options": {"defaults": {"id": None}, "methods": ["GET"]},
    },
    "component_build": {
        "url": "/module-build-service/<int:api_version>/component-builds/<int:id>",
        "options": {"methods": ["GET"]},
    },
    "about": {
        "url": "/module-build-service/<int:api_version>/about/",
        "options": {"methods": ["GET"]},
    },
    "rebuild_strategies_list": {
        "url": "/module-build-service/<int:api_version>/rebuild-strategies/",
        "options": {"methods": ["GET"]},
    },
    "import_module": {
        "url": "/module-build-service/<int:api_version>/import-module/",
        "options": {"methods": ["POST"]},
    },
}


class AbstractQueryableBuildAPI(MethodView):
    """ An abstract class, housing some common functionality. """

    @cors_header()
    @validate_api_version()
    def get(self, api_version, id):
        id_flag = request.args.get("id")
        if id_flag:
            endpoint = request.endpoint.split("s_list")[0]
            raise ValidationError(
                'The "id" query option is invalid. Did you mean to go to "{0}"?'.format(
                    url_for(endpoint, api_version=api_version, id=id_flag)
                )
            )
        verbose_flag = request.args.get("verbose", "false").lower()
        short_flag = request.args.get("short", "false").lower()
        json_func_kwargs = {}
        json_func_name = "json"

        if id is None:
            # Lists all tracked builds
            p_query = self.query_filter(request)
            json_data = {"meta": pagination_metadata(p_query, api_version, request.args)}

            if verbose_flag == "true" or verbose_flag == "1":
                json_func_name = "extended_json"
                json_func_kwargs["show_state_url"] = True
                json_func_kwargs["api_version"] = api_version
            elif short_flag == "true" or short_flag == "1":
                if hasattr(p_query.items[0], "short_json"):
                    json_func_name = "short_json"
            if json_func_name == "json" or json_func_name == "extended_json":
                # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
                json_func_kwargs["db_session"] = db.session
            json_data["items"] = [
                getattr(item, json_func_name)(**json_func_kwargs) for item in p_query.items
            ]

            return jsonify(json_data), 200
        else:
            # Lists details for the specified build
            instance = self.model.query.filter_by(id=id).first()
            if instance:
                if verbose_flag == "true" or verbose_flag == "1":
                    json_func_name = "extended_json"
                    json_func_kwargs["show_state_url"] = True
                    json_func_kwargs["api_version"] = api_version
                elif short_flag == "true" or short_flag == "1":
                    if getattr(instance, "short_json", None):
                        json_func_name = "short_json"
                if json_func_name == "json" or json_func_name == "extended_json":
                    # Only ModuleBuild.json and ModuleBuild.extended_json has argument db_session
                    json_func_kwargs["db_session"] = db.session
                return jsonify(getattr(instance, json_func_name)(**json_func_kwargs)), 200
            else:
                raise NotFound("No such %s found." % self.kind)


class ComponentBuildAPI(AbstractQueryableBuildAPI):
    kind = "component"
    query_filter = staticmethod(filter_component_builds)
    model = models.ComponentBuild


class ModuleBuildAPI(AbstractQueryableBuildAPI):
    kind = "module"
    query_filter = staticmethod(filter_module_builds)
    model = models.ModuleBuild

    @staticmethod
    def check_groups(username, groups, allowed_groups=conf.allowed_groups):
        # If the user is part of the whitelist, then the group membership check is skipped
        if username in conf.allowed_users:
            return
        if allowed_groups and not (allowed_groups & groups):
            raise Forbidden("%s is not in any of %r, only %r" % (username, allowed_groups, groups))

    # Additional POST and DELETE handlers for modules follow.
    @validate_api_version()
    def post(self, api_version):
        data = _dict_from_request(request)
        if "modulemd" in data or (hasattr(request, "files") and "yaml" in request.files):
            handler = YAMLFileHandler(request, data)
        else:
            handler = SCMHandler(request, data)

        if conf.no_auth is True and handler.username == "anonymous" and "owner" in handler.data:
            handler.username = handler.data["owner"]

        self.check_groups(handler.username, handler.groups)

        handler.validate()
        modules = handler.post()
        if api_version == 1:
            # Only show the first module build for backwards-compatibility
            rv = modules[0].extended_json(db.session, True, api_version)
        else:
            rv = [module.extended_json(db.session, True, api_version) for module in modules]
        return jsonify(rv), 201

    @validate_api_version()
    def patch(self, api_version, id):
        username, groups = module_build_service.auth.get_user(request)

        try:
            r = json.loads(request.get_data().decode("utf-8"))
        except Exception:
            log.exception("Invalid JSON submitted")
            raise ValidationError("Invalid JSON submitted")

        if "owner" in r:
            if conf.no_auth is not True:
                raise ValidationError(
                    "The request contains 'owner' parameter, however NO_AUTH is not allowed"
                )
            elif username == "anonymous":
                username = r["owner"]

        self.check_groups(username, groups)

        module = models.ModuleBuild.query.filter_by(id=id).first()
        if not module:
            raise NotFound("No such module found.")

        if module.owner != username and not (conf.admin_groups & groups):
            raise Forbidden("You are not owner of this build and therefore cannot modify it.")

        if not r.get("state"):
            log.error("Invalid JSON submitted")
            raise ValidationError("Invalid JSON submitted")

        if module.state == models.BUILD_STATES["failed"]:
            raise Forbidden("You can't cancel a failed module")

        if r["state"] == "failed" or r["state"] == str(models.BUILD_STATES["failed"]):
            module.transition(
                db.session, conf, models.BUILD_STATES["failed"], "Canceled by %s." % username)
        else:
            log.error('The provided state change of "{}" is not supported'.format(r["state"]))
            raise ValidationError("The provided state change is not supported")
        db.session.add(module)
        db.session.commit()

        return jsonify(module.extended_json(db.session, True, api_version)), 200


class AboutAPI(MethodView):
    @cors_header()
    @validate_api_version()
    def get(self, api_version):
        json = {"version": version, "api_version": max_api_version}
        config_items = ["auth_method"]
        for item in config_items:
            config_item = getattr(conf, item)
            # All config items have a default, so if doesn't exist it is a programming error
            if not config_item:
                raise ProgrammingError('An invalid config item of "{0}" was specified'.format(item))
            json[item] = config_item
        return jsonify(json), 200


class RebuildStrategies(MethodView):
    @cors_header()
    @validate_api_version()
    def get(self, api_version):
        items = []
        # Sort the items list by name
        for strategy in sorted(models.ModuleBuild.rebuild_strategies.keys()):
            default = False
            if strategy == conf.rebuild_strategy:
                default = True
                allowed = True
            elif (
                conf.rebuild_strategy_allow_override and strategy in conf.rebuild_strategies_allowed
            ):
                allowed = True
            else:
                allowed = False
            items.append({
                "name": strategy,
                "description": models.ModuleBuild.rebuild_strategies[strategy],
                "allowed": allowed,
                "default": default,
            })

        return jsonify({"items": items}), 200


class ImportModuleAPI(MethodView):
    @validate_api_version()
    def post(self, api_version):
        # disable this API endpoint if no groups are defined
        if not conf.allowed_groups_to_import_module:
            log.error(
                "Import module API is disabled. Set 'ALLOWED_GROUPS_TO_IMPORT_MODULE'"
                " configuration value first."
            )
            raise Forbidden("Import module API is disabled.")

        # auth checks
        username, groups = module_build_service.auth.get_user(request)
        ModuleBuildAPI.check_groups(
            username, groups, allowed_groups=conf.allowed_groups_to_import_module)

        # process request using SCM handler
        handler = SCMHandler(request)
        handler.validate(skip_branch=True, skip_optional_params=True)

        mmd = get_mmd_from_scm(handler.data["scmurl"])
        build, messages = import_mmd(db.session, mmd)
        json_data = {
            "module": build.json(db.session, show_tasks=False),
            "messages": messages
        }

        # return 201 Created if we reach this point
        return jsonify(json_data), 201


class BaseHandler(object):
    valid_params = set([
        "branch",
        "buildrequire_overrides",
        "modulemd",
        "module_name",
        "owner",
        "rebuild_strategy",
        "reuse_components_from",
        "require_overrides",
        "scmurl",
        "scratch",
        "srpms",
    ])

    def __init__(self, request, data=None):
        self.username, self.groups = module_build_service.auth.get_user(request)
        self.data = data or _dict_from_request(request)

        # canonicalize and validate scratch option
        if "scratch" in self.data and str_to_bool(str(self.data["scratch"])):
            self.data["scratch"] = True
            if conf.modules_allow_scratch is not True:
                raise Forbidden("Scratch builds are not enabled")
        else:
            self.data["scratch"] = False

        # canonicalize and validate srpms list
        if "srpms" in self.data and self.data["srpms"]:
            if not self.data["scratch"]:
                raise Forbidden("srpms may only be specified for scratch builds")
            if not isinstance(self.data["srpms"], list):
                raise ValidationError("srpms must be specified as a list")
        else:
            self.data["srpms"] = []

    def _validate_dep_overrides_format(self, key):
        """
        Validate any dependency overrides provided to the API.

        :param str key: the override key to validate
        :raises ValidationError: when the overrides are an invalid format
        """
        if not self.data.get(key):
            return
        invalid_override_msg = (
            'The "{}" parameter must be an object with the keys as module '
            "names and the values as arrays of streams".format(key)
        )
        if not isinstance(self.data[key], dict):
            raise ValidationError(invalid_override_msg)
        for streams in self.data[key].values():
            if not isinstance(streams, list):
                raise ValidationError(invalid_override_msg)
            for stream in streams:
                if not isinstance(stream, string_types):
                    raise ValidationError(invalid_override_msg)

    def validate_optional_params(self):
        forbidden_params = [k for k in self.data if k not in self.valid_params]
        if forbidden_params:
            raise ValidationError(
                "The request contains unspecified parameters: {}".format(
                    ", ".join(forbidden_params))
            )

        if not conf.no_auth and "owner" in self.data:
            raise ValidationError(
                "The request contains 'owner' parameter, however NO_AUTH is not allowed")

        if not conf.rebuild_strategy_allow_override and "rebuild_strategy" in self.data:
            raise ValidationError(
                'The request contains the "rebuild_strategy" parameter but '
                "overriding the default isn't allowed"
            )

        if "rebuild_strategy" in self.data:
            if self.data["rebuild_strategy"] not in conf.rebuild_strategies_allowed:
                raise ValidationError(
                    'The rebuild method of "{0}" is not allowed. Choose from: {1}.'.format(
                        self.data["rebuild_strategy"], ", ".join(conf.rebuild_strategies_allowed))
                )

        self._validate_dep_overrides_format("buildrequire_overrides")
        self._validate_dep_overrides_format("require_overrides")

        if "reuse_components_from" in self.data:
            if "rebuild_strategy" in self.data and self.data["rebuild_strategy"] == "all":
                raise ValidationError(
                    'You cannot specify the parameter "reuse_components_from" when the '
                    '"rebuild_strategy" parameter is set to "all"'
                )

            invalid_identifier_msg = (
                'The parameter "reuse_components_from" contains an invalid module identifier')

            if isinstance(self.data["reuse_components_from"], int):
                reuse_module = models.ModuleBuild.get_by_id(
                    db.session, self.data["reuse_components_from"])
            elif isinstance(self.data["reuse_components_from"], string_types):
                try:
                    n, s, v, c = self.data["reuse_components_from"].split(":")
                except ValueError:
                    raise ValidationError(invalid_identifier_msg)
                reuse_module = models.ModuleBuild.get_build_from_nsvc(db.session, n, s, v, c)
            else:
                raise ValidationError(invalid_identifier_msg)

            if not reuse_module:
                raise ValidationError(
                    'The module in the parameter "reuse_components_from" could not be found')

            if reuse_module.state != models.BUILD_STATES["ready"]:
                raise ValidationError(
                    'The module in the parameter "reuse_components_from" must be in the ready state'
                )

            # Normalize the value so that it simplifies any code that uses this value
            self.data["reuse_components_from"] = reuse_module.id


class SCMHandler(BaseHandler):
    def validate(self, skip_branch=False, skip_optional_params=False):
        if "scmurl" not in self.data:
            log.error("Missing scmurl")
            raise ValidationError("Missing scmurl")

        url = self.data["scmurl"]
        allowed_prefix = any(url.startswith(prefix) for prefix in conf.scmurls)
        if not conf.allow_custom_scmurls and not allowed_prefix:
            log.error("The submitted scmurl %r is not allowed" % url)
            raise Forbidden("The submitted scmurl %s is not allowed" % url)

        if not get_scm_url_re().match(url):
            log.error("The submitted scmurl %r is not valid" % url)
            raise ValidationError("The submitted scmurl %s is not valid" % url)

        if not skip_branch and "branch" not in self.data:
            log.error("Missing branch")
            raise ValidationError("Missing branch")

        if not skip_optional_params:
            self.validate_optional_params()

    def post(self):
        return submit_module_build_from_scm(
            db.session, self.username, self.data, allow_local_url=False)


class YAMLFileHandler(BaseHandler):
    def __init__(self, request, data=None):
        super(YAMLFileHandler, self).__init__(request, data)
        if not self.data["scratch"] and not conf.yaml_submit_allowed:
            raise Forbidden("YAML submission is not enabled")

    def validate(self):
        if (
            "modulemd" not in self.data
            and (not hasattr(request, "files") or "yaml" not in request.files)
        ):
            log.error("Invalid file submitted")
            raise ValidationError("Invalid file submitted")
        self.validate_optional_params()

    def post(self):
        if "modulemd" in self.data:
            handle = BytesIO(self.data["modulemd"].encode("utf-8"))
            if self.data.get("module_name"):
                handle.filename = self.data["module_name"]
        else:
            handle = request.files["yaml"]
        return submit_module_build_from_yaml(
            db.session, self.username, handle, self.data)


def _dict_from_request(request):
    if "multipart/form-data" in request.headers.get("Content-Type", ""):
        data = request.form.to_dict()
    else:
        try:
            data = json.loads(request.get_data().decode("utf-8"))
        except Exception:
            log.exception("Invalid JSON submitted")
            raise ValidationError("Invalid JSON submitted")
    return data


def register_api():
    """ Registers the MBS API. """
    module_view = ModuleBuildAPI.as_view("module_builds")
    component_view = ComponentBuildAPI.as_view("component_builds")
    about_view = AboutAPI.as_view("about")
    rebuild_strategies_view = RebuildStrategies.as_view("rebuild_strategies")
    import_module = ImportModuleAPI.as_view("import_module")
    for key, val in api_routes.items():
        if key.startswith("component_build"):
            app.add_url_rule(val["url"], endpoint=key, view_func=component_view, **val["options"])
        elif key.startswith("module_build"):
            app.add_url_rule(val["url"], endpoint=key, view_func=module_view, **val["options"])
        elif key.startswith("about"):
            app.add_url_rule(val["url"], endpoint=key, view_func=about_view, **val["options"])
        elif key == "rebuild_strategies_list":
            app.add_url_rule(
                val["url"], endpoint=key, view_func=rebuild_strategies_view, **val["options"]
            )
        elif key == "import_module":
            app.add_url_rule(val["url"], endpoint=key, view_func=import_module, **val["options"])
        else:
            raise NotImplementedError("Unhandled api key.")

    app.register_blueprint(monitor_api)


register_api()