fm-orchestrator/module_build_service/scheduler/submit.py

# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from datetime import datetime
import json
from multiprocessing.dummy import Pool as ThreadPool
import os

import kobo.rpmlib

import module_build_service.common.scm
from module_build_service.common import conf, log, models
from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden
from module_build_service.common.modulemd import Modulemd
from module_build_service.common.submit import fetch_mmd
from module_build_service.common.utils import to_text_type
from module_build_service.scheduler.db_session import db_session


def get_build_arches(mmd, config):
    """
    Returns the list of architectures for which the module `mmd` should be built.

    :param mmd: Module MetaData
    :param config: config (module_build_service.common.config.Config instance)
    :return list of architectures
    """
    # Imported here to allow import of utils in GenericBuilder.
    from module_build_service.builder import GenericBuilder

    nsvc = mmd.get_nsvc()

    # At first, handle BASE_MODULE_ARCHES - this overrides any other option.
    # Find out the base modules in buildrequires section of XMD and
    # set the Koji tag arches according to it.
    if "mbs" in mmd.get_xmd():
        for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
            ns = ":".join([req_name, req_data["stream"]])
            if ns in config.base_module_arches:
                arches = config.base_module_arches[ns]
                log.info("Setting build arches of %s to %r based on the BASE_MODULE_ARCHES." % (
                    nsvc, arches))
                return arches

    # Check whether the module contains the `koji_tag_arches`. This is used only
    # by special modules defining the layered products.
    try:
        arches = mmd.get_xmd()["mbs"]["koji_tag_arches"]
        log.info("Setting build arches of %s to %r based on the koji_tag_arches." % (
            nsvc, arches))
        return arches
    except KeyError:
        pass

    # Check the base/layered-product module this module buildrequires and try to get the
    # list of arches from there.
    try:
        buildrequires = mmd.get_xmd()["mbs"]["buildrequires"]
    except (ValueError, KeyError):
        log.warning(
            "Module {0} does not have buildrequires in its xmd".format(mmd.get_nsvc()))
        buildrequires = None
    if buildrequires:
        # Looping through all the privileged modules that are allowed to set koji tag arches
        # and the base modules to see what the koji tag arches should be. Doing it this way
        # preserves the order in the configurations.
        for module in conf.allowed_privileged_module_names + conf.base_module_names:
            module_in_xmd = buildrequires.get(module)

            if not module_in_xmd:
                continue

            module_obj = models.ModuleBuild.get_build_from_nsvc(
                db_session,
                module,
                module_in_xmd["stream"],
                module_in_xmd["version"],
                module_in_xmd["context"],
            )
            if not module_obj:
                continue
            arches = GenericBuilder.get_module_build_arches(module_obj)
            if arches:
                log.info("Setting build arches of %s to %r based on the buildrequired "
                         "module %r." % (nsvc, arches, module_obj))
                return arches

    # As a last resort, return just the preconfigured list of arches.
    arches = config.arches
    log.info("Setting build arches of %s to %r based on default ARCHES." % (nsvc, arches))
    return arches


def record_module_build_arches(mmd, build):
    """
    Finds out the list of build arches against which the ModuleBuld `build` should be built
    and records them to `build.arches`.

    :param Modulemd mmd: The MMD file associated with a ModuleBuild.
    :param ModuleBuild build: The ModuleBuild.
    """
    arches = get_build_arches(mmd, conf)
    for arch in arches:
        arch_obj = db_session.query(models.ModuleArch).filter_by(name=arch).first()
        if not arch_obj:
            arch_obj = models.ModuleArch(name=arch)
        build.arches.append(arch_obj)

    db_session.commit()


def record_filtered_rpms(mmd):
    """Record filtered RPMs that should not be installed into buildroot

    These RPMs are filtered:

    * Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms"
      list containing the NVRs of filtered RPMs in a buildrequired module.

    :param Modulemd mmd: Modulemd that will be built next.
    :rtype: Modulemd.Module
    :return: Modulemd extended with the "filtered_rpms" in XMD section.
    """
    # Imported here to allow import of utils in GenericBuilder.
    from module_build_service.builder import GenericBuilder
    from module_build_service.resolver import GenericResolver

    resolver = GenericResolver.create(db_session, conf)
    builder = GenericBuilder.backends[conf.system]

    new_buildrequires = {}
    for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
        # In case this is module resubmit or local build, the filtered_rpms
        # will already be there, so there is no point in generating them again.
        if "filtered_rpms" in req_data:
            new_buildrequires[req_name] = req_data
            continue

        # We can just get the first modulemd data from result right here thanks to
        # strict=True, so in case the module cannot be found, get_module_modulemds
        # raises an exception.
        req_mmd = resolver.get_module_modulemds(
            req_name, req_data["stream"], req_data["version"], req_data["context"], True)[0]

        # Find out the particular NVR of filtered packages
        filtered_rpms = []
        rpm_filter = req_mmd.get_rpm_filters()
        if rpm_filter:
            built_nvrs = builder.get_built_rpms_in_module_build(req_mmd)
            for nvr in built_nvrs:
                parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
                if parsed_nvr["name"] in rpm_filter:
                    filtered_rpms.append(nvr)
        req_data["filtered_rpms"] = filtered_rpms

        new_buildrequires[req_name] = req_data

    # Replace the old buildrequires with new ones.
    xmd = mmd.get_xmd()
    xmd["mbs"]["buildrequires"] = new_buildrequires
    mmd.set_xmd(xmd)
    return mmd


def _scm_get_latest(pkg):
    try:
        # If the modulemd specifies that the 'f25' branch is what
        # we want to pull from, we need to resolve that f25 branch
        # to the specific commit available at the time of
        # submission (now).
        repo = pkg.get_repository()
        ref = pkg.get_ref()
        log.debug("Getting the commit hash for the ref %s on the repo %s", ref, repo)
        pkgref = module_build_service.common.scm.SCM(repo).get_latest(ref)
    except Exception as e:
        log.exception(e)
        return {
            "error": "Failed to get the latest commit for %s#%s"
            % (pkg.get_repository(), pkg.get_ref())
        }

    return {"pkg_name": pkg.get_name(), "pkg_ref": pkgref, "error": None}


def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None):
    """
    Prepares the modulemd for the MBS. This does things such as replacing the
    branches of components with commit hashes and adding metadata in the xmd
    dictionary.
    :param mmd: the Modulemd.ModuleStream object to format
    :param scmurl: the url to the modulemd
    :param module: When specified together with `session`, the time_modified
        of a module is updated regularly in case this method takes lot of time.
    :param db_session: Database session to update the `module`.
    :param dict srpm_overrides: Mapping of package names to SRPM links for all
        component packages which have custom SRPM overrides specified.
    """
    srpm_overrides = srpm_overrides or {}

    xmd = mmd.get_xmd()
    if "mbs" not in xmd:
        xmd["mbs"] = {}
    if "scmurl" not in xmd["mbs"]:
        xmd["mbs"]["scmurl"] = scmurl or ""
    if "commit" not in xmd["mbs"]:
        xmd["mbs"]["commit"] = ""

    # If module build was submitted via yaml file, there is no scmurl
    if scmurl:
        scm = module_build_service.common.scm.SCM(scmurl)
        # We want to make sure we have the full commit hash for consistency
        if module_build_service.common.scm.SCM.is_full_commit_hash(scm.scheme, scm.commit):
            full_scm_hash = scm.commit
        else:
            full_scm_hash = scm.get_full_commit_hash()

        xmd["mbs"]["commit"] = full_scm_hash

    if mmd.get_rpm_component_names() or mmd.get_module_component_names():
        if "rpms" not in xmd["mbs"]:
            xmd["mbs"]["rpms"] = {}
        # Add missing data in RPM components
        for pkgname in mmd.get_rpm_component_names():
            pkg = mmd.get_rpm_component(pkgname)
            # In case of resubmit of existing module which have been
            # cancelled/failed during the init state, the package
            # was maybe already handled by MBS, so skip it in this case.
            if pkgname in xmd["mbs"]["rpms"]:
                continue
            if pkg.get_repository() and not conf.rpms_allow_repository:
                raise Forbidden(
                    "Custom component repositories aren't allowed.  "
                    "%r bears repository %r" % (pkgname, pkg.get_repository())
                )
            if pkg.get_cache() and not conf.rpms_allow_cache:
                raise Forbidden(
                    "Custom component caches aren't allowed.  "
                    "%r bears cache %r" % (pkgname, pkg.get_cache())
                )
            if pkg.get_buildafter():
                raise ValidationError('The usage of "buildafter" is not yet supported')
            if not pkg.get_repository():
                pkg.set_repository(conf.rpms_default_repository + pkgname)
            if not pkg.get_cache():
                pkg.set_cache(conf.rpms_default_cache + pkgname)
            if not pkg.get_ref():
                pkg.set_ref("master")
            if not pkg.get_arches():
                for arch in conf.arches:
                    pkg.add_restricted_arch(arch)

        # Add missing data in included modules components
        for modname in mmd.get_module_component_names():
            mod = mmd.get_module_component(modname)
            if mod.get_repository() and not conf.modules_allow_repository:
                raise Forbidden(
                    "Custom module repositories aren't allowed.  "
                    "%r bears repository %r" % (modname, mod.get_repository())
                )
            if not mod.get_repository():
                mod.set_repository(conf.modules_default_repository + modname)
            if not mod.get_ref():
                mod.set_ref("master")

        # Check that SCM URL is valid and replace potential branches in pkg refs
        # by real SCM hash and store the result to our private xmd place in modulemd.
        pool = ThreadPool(20)
        try:
            # Filter out the packages which we have already resolved in possible
            # previous runs of this method (can be caused by module build resubmition)
            # or which have custom SRPMs and shouldn't be resolved.
            pkgs_to_resolve = []
            for name in mmd.get_rpm_component_names():
                if name not in xmd["mbs"]["rpms"]:
                    if name in srpm_overrides:
                        # If this package has a custom SRPM, store an empty
                        # ref entry so no further verification takes place.
                        xmd["mbs"]["rpms"][name] = {"ref": None}
                    else:
                        pkgs_to_resolve.append(mmd.get_rpm_component(name))

            async_result = pool.map_async(_scm_get_latest, pkgs_to_resolve)

            # For modules with lot of components, the _scm_get_latest can take a lot of time.
            # We need to bump time_modified from time to time, otherwise poller could think
            # that module is stuck in "init" state and it would send fake "init" message.
            while not async_result.ready():
                async_result.wait(60)
                if module and db_session:
                    module.time_modified = datetime.utcnow()
                    db_session.commit()
            pkg_dicts = async_result.get()
        finally:
            pool.close()

        err_msg = ""
        for pkg_dict in pkg_dicts:
            if pkg_dict["error"]:
                err_msg += pkg_dict["error"] + "\n"
            else:
                pkg_name = pkg_dict["pkg_name"]
                pkg_ref = pkg_dict["pkg_ref"]
                xmd["mbs"]["rpms"][pkg_name] = {"ref": pkg_ref}
        if err_msg:
            raise UnprocessableEntity(err_msg)

    # Set the modified xmd back to the modulemd
    mmd.set_xmd(xmd)


def merge_included_mmd(mmd, included_mmd):
    """
    Merges two modulemds. This merges only metadata which are needed in
    the `main` when it includes another module defined by `included_mmd`
    """
    included_xmd = included_mmd.get_xmd()
    if "rpms" in included_xmd["mbs"]:
        xmd = mmd.get_xmd()
        if "rpms" not in xmd["mbs"]:
            xmd["mbs"]["rpms"] = included_xmd["mbs"]["rpms"]
        else:
            xmd["mbs"]["rpms"].update(included_xmd["mbs"]["rpms"])
    # Set the modified xmd back to the modulemd
    mmd.set_xmd(xmd)


def get_module_srpm_overrides(module):
    """
    Make necessary preparations to use any provided custom SRPMs.

    :param module: ModuleBuild object representing the module being submitted.
    :type module: :class:`models.ModuleBuild`
    :return: mapping of package names to SRPM links for all packages which
             have custom SRPM overrides specified
    :rtype: dict[str, str]

    """
    overrides = {}

    if not module.srpms:
        return overrides

    try:
        # Make sure we can decode the custom SRPM list
        srpms = json.loads(module.srpms)
        assert isinstance(srpms, list)
    except Exception:
        raise ValueError("Invalid srpms list encountered: {}".format(module.srpms))

    for source in srpms:
        if source.startswith("cli-build/") and source.endswith(".src.rpm"):
            # This is a custom srpm that has been uploaded to koji by rpkg
            # using the package name as the basename suffixed with .src.rpm
            rpm_name = os.path.basename(source)[: -len(".src.rpm")]
        else:
            # This should be a local custom srpm path
            if not os.path.exists(source):
                raise IOError("Provided srpm is missing: {}".format(source))
            # Get package name from rpm headers
            try:
                rpm_hdr = kobo.rpmlib.get_rpm_header(source)
                rpm_name = to_text_type(kobo.rpmlib.get_header_field(rpm_hdr, "name"))
            except Exception:
                raise ValueError("Provided srpm is invalid: {}".format(source))

        if rpm_name in overrides:
            log.warning(
                'Encountered duplicate custom SRPM "{0}" for package {1}'
                .format(source, rpm_name)
            )
            continue

        log.debug('Using custom SRPM "{0}" for package {1}'.format(source, rpm_name))
        overrides[rpm_name] = source

    return overrides


def record_component_builds(
    mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None
):
    # Imported here to allow import of utils in GenericBuilder.
    from module_build_service.builder import GenericBuilder

    # When main_mmd is set, merge the metadata from this mmd to main_mmd,
    # otherwise our current mmd is main_mmd.
    if main_mmd:
        # Check for components that are in both MMDs before merging since MBS
        # currently can't handle that situation.
        main_mmd_rpms = main_mmd.get_rpm_component_names()
        mmd_rpms = mmd.get_rpm_component_names()
        duplicate_components = [
            rpm for rpm in main_mmd_rpms
            if rpm in mmd_rpms
        ]
        if duplicate_components:
            error_msg = (
                'The included module "{0}" in "{1}" have the following '
                "conflicting components: {2}".format(
                    mmd.get_module_name(), main_mmd.get_module_name(),
                    ", ".join(duplicate_components)
                )
            )
            raise UnprocessableEntity(error_msg)
        merge_included_mmd(main_mmd, mmd)
    else:
        main_mmd = mmd

    # If the modulemd yaml specifies components, then submit them for build
    rpm_components = [
        mmd.get_rpm_component(name)
        for name in mmd.get_rpm_component_names()
    ]
    module_components = [
        mmd.get_module_component(name)
        for name in mmd.get_module_component_names()
    ]
    all_components = list(rpm_components) + list(module_components)
    if not all_components:
        return

    # Get map of packages that have SRPM overrides
    srpm_overrides = get_module_srpm_overrides(module)

    rpm_weights = GenericBuilder.get_build_weights(
        [c.get_name() for c in rpm_components]
    )
    all_components.sort(key=lambda x: x.get_buildorder())
    # We do not start with batch = 0 here, because the first batch is
    # reserved for module-build-macros. First real components must be
    # planned for batch 2 and following.
    batch = initial_batch

    for component in all_components:
        # Increment the batch number when buildorder increases.
        if previous_buildorder != component.get_buildorder():
            previous_buildorder = component.get_buildorder()
            batch += 1

        # If the component is another module, we fetch its modulemd file
        # and record its components recursively with the initial_batch
        # set to our current batch, so the components of this module
        # are built in the right global order.
        if isinstance(component, Modulemd.ComponentModule):
            full_url = component.get_repository() + "?#" + component.get_ref()
            # It is OK to whitelist all URLs here, because the validity
            # of every URL have been already checked in format_mmd(...).
            included_mmd = fetch_mmd(full_url, whitelist_url=True)[0]
            format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides)
            batch = record_component_builds(
                included_mmd, module, batch, previous_buildorder, main_mmd)
            continue

        package = component.get_name()
        if package in srpm_overrides:
            component_ref = None
            full_url = srpm_overrides[package]
            log.info('Building custom SRPM "{0}"' " for package {1}".format(full_url, package))
        else:
            component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"]
            full_url = component.get_repository() + "?#" + component_ref

        # Skip the ComponentBuild if it already exists in database. This can happen
        # in case of module build resubmition.
        existing_build = models.ComponentBuild.from_component_name(db_session, package, module.id)
        if existing_build:
            # Check that the existing build has the same most important attributes.
            # This should never be a problem, but it's good to be defensive here so
            # we do not mess things during resubmition.
            if (
                existing_build.batch != batch
                or existing_build.scmurl != full_url
                or existing_build.ref != component_ref
            ):
                raise ValidationError(
                    "Component build %s of module build %s (id: %d) already "
                    "exists in database, but its attributes are different from"
                    " resubmitted one." % (
                        component.get_name(), module.name, module.id)
                )
            continue

        build = models.ComponentBuild(
            module_id=module.id,
            package=package,
            format="rpms",
            scmurl=full_url,
            batch=batch,
            ref=component_ref,
            weight=rpm_weights[package],
            buildonly=component.get_buildonly()
        )
        db_session.add(build)

    return batch