Files
Owen W. Taylor 35086174ba Pass koji_tag to get_built_rpms_in_module_build()
For local builds, required modules are not necessarily in the local
database, so the method of looking up the build to find the koji tag
doesn't work reliably. However, the caller has the koji_tag - so just
pass it in.
2023-03-15 17:53:38 +00:00

578 lines
24 KiB
Python

# -*- coding: utf-8 -*-
# SPDX-License-Identifier: MIT
from __future__ import absolute_import
from datetime import datetime
import json
from multiprocessing.dummy import Pool as ThreadPool
import os
from collections import namedtuple
import kobo.rpmlib
import module_build_service.common.scm
from module_build_service.common import conf, log, models
from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden
from module_build_service.common.modulemd import Modulemd
from module_build_service.common.submit import fetch_mmd
from module_build_service.common.utils import to_text_type
from module_build_service.scheduler.db_session import db_session
# The namedtuple to store data passed to _scm_get_latest with following keys:
# - rpm_component - The Modulemd.ComponentRPM instance representing
# the component.
# - ref_override - The str instance used to override the component's ref
# or None.
SCMGetLatestData = namedtuple("SCMGetLatestData", "rpm_component ref_override")
def get_build_arches(mmd, config):
"""
Returns the list of architectures for which the module `mmd` should be built.
:param mmd: Module MetaData
:param config: config (module_build_service.common.config.Config instance)
:return: list of architectures
"""
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
nsvc = mmd.get_nsvc()
def _conditional_log(msg, arches, new_arches):
# Checks if the arch list returned by _check_buildopts_arches is the same one passed to it
# If it is, it outputs the message
if arches is new_arches:
log.info(msg)
# At first, handle BASE_MODULE_ARCHES - this overrides any other option.
# Find out the base modules in buildrequires section of XMD and
# set the Koji tag arches according to it.
if "mbs" in mmd.get_xmd():
for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
ns = ":".join([req_name, req_data["stream"]])
if ns in config.base_module_arches:
arches = config.base_module_arches[ns]
new_arches = _check_buildopts_arches(mmd, arches)
msg = "Setting build arches of %s to %r based on the BASE_MODULE_ARCHES." % (
nsvc, new_arches)
_conditional_log(msg, arches, new_arches)
return new_arches
# Check whether the module contains the `koji_tag_arches`. This is used only
# by special modules defining the layered products.
try:
arches = mmd.get_xmd()["mbs"]["koji_tag_arches"]
new_arches = _check_buildopts_arches(mmd, arches)
msg = "Setting build arches of %s to %r based on the koji_tag_arches." % (
nsvc, new_arches)
_conditional_log(msg, arches, new_arches)
return new_arches
except KeyError:
pass
# Check the base/layered-product module this module buildrequires and try to get the
# list of arches from there.
try:
buildrequires = mmd.get_xmd()["mbs"]["buildrequires"]
except (ValueError, KeyError):
log.warning(
"Module {0} does not have buildrequires in its xmd".format(mmd.get_nsvc()))
buildrequires = None
if buildrequires:
# Looping through all the privileged modules that are allowed to set koji tag arches
# and the base modules to see what the koji tag arches should be. Doing it this way
# preserves the order in the configurations.
for module in conf.allowed_privileged_module_names + conf.base_module_names:
module_in_xmd = buildrequires.get(module)
if not module_in_xmd:
continue
module_obj = models.ModuleBuild.get_build_from_nsvc(
db_session,
module,
module_in_xmd["stream"],
module_in_xmd["version"],
module_in_xmd["context"],
)
if not module_obj:
continue
arches = GenericBuilder.get_module_build_arches(module_obj)
if arches:
new_arches = _check_buildopts_arches(mmd, arches)
msg = "Setting build arches of %s to %r based on the buildrequired module %r." % (
nsvc, new_arches, module_obj)
_conditional_log(msg, arches, new_arches)
return new_arches
# As a last resort, return just the preconfigured list of arches.
arches = config.arches
new_arches = _check_buildopts_arches(mmd, arches)
msg = "Setting build arches of %s to %r based on default ARCHES." % (nsvc, new_arches)
_conditional_log(msg, arches, new_arches)
return new_arches
def _check_buildopts_arches(mmd, arches):
"""
Returns buildopts arches if valid, or otherwise the arches provided.
:param mmd: Module MetaData
:param arches: list of architectures
:return: list of architectures
"""
buildopts = mmd.get_buildopts()
if not buildopts:
return arches
try:
buildopts_arches = buildopts.get_arches()
except AttributeError:
# libmodulemd version < 2.8.3
return arches
# Must be a subset of the input module arches
unsupported_arches = set(buildopts_arches) - set(arches)
# If there are unsupported architectures, and the local one is one of them,
# then we should fail, otherwise continue.
if unsupported_arches and set(arches) in unsupported_arches:
raise ValidationError("The following buildopts arches are not supported with these "
"buildrequires: %r" % unsupported_arches)
if buildopts_arches:
log.info("Setting build arches of %s to %r based on the buildopts arches." % (
mmd.get_nsvc(), buildopts_arches))
return buildopts_arches
return arches
def record_module_build_arches(mmd, build):
"""
Finds out the list of build arches against which the ModuleBuld `build` should be built
and records them to `build.arches`.
:param Modulemd mmd: The MMD file associated with a ModuleBuild.
:param ModuleBuild build: The ModuleBuild.
"""
arches = get_build_arches(mmd, conf)
for arch in arches:
arch_obj = db_session.query(models.ModuleArch).filter_by(name=arch).first()
if not arch_obj:
arch_obj = models.ModuleArch(name=arch)
if arch_obj not in build.arches:
build.arches.append(arch_obj)
db_session.commit()
def record_filtered_rpms(mmd):
"""Record filtered RPMs that should not be installed into buildroot
These RPMs are filtered:
* Reads the mmd["xmd"]["buildrequires"] and extends it with "filtered_rpms"
list containing the NVRs of filtered RPMs in a buildrequired module.
:param Modulemd mmd: Modulemd that will be built next.
:rtype: Modulemd.Module
:return: Modulemd extended with the "filtered_rpms" in XMD section.
"""
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
from module_build_service.resolver import GenericResolver
resolver = GenericResolver.create(db_session, conf)
builder = GenericBuilder.backends[conf.system]
new_buildrequires = {}
for req_name, req_data in mmd.get_xmd()["mbs"]["buildrequires"].items():
# In case this is module resubmit or local build, the filtered_rpms
# will already be there, so there is no point in generating them again.
if "filtered_rpms" in req_data:
new_buildrequires[req_name] = req_data
continue
# We can just get the first modulemd data from result right here thanks to
# strict=True, so in case the module cannot be found, get_module_modulemds
# raises an exception.
req_mmd = resolver.get_module_modulemds(
req_name, req_data["stream"], req_data["version"], req_data["context"], True)[0]
# Find out the particular NVR of filtered packages
filtered_rpms = []
rpm_filter = req_mmd.get_rpm_filters()
if rpm_filter:
built_nvrs = builder.get_built_rpms_in_module_build(req_mmd, req_data["koji_tag"])
for nvr in built_nvrs:
parsed_nvr = kobo.rpmlib.parse_nvr(nvr)
if parsed_nvr["name"] in rpm_filter:
filtered_rpms.append(nvr)
req_data["filtered_rpms"] = filtered_rpms
new_buildrequires[req_name] = req_data
# Replace the old buildrequires with new ones.
xmd = mmd.get_xmd()
xmd["mbs"]["buildrequires"] = new_buildrequires
mmd.set_xmd(xmd)
return mmd
def _scm_get_latest(data):
"""
Resolves the git ref for the package defined in the `data`. Applies git
ref override if set in the `data`. Returns a dict with resolved git ref
or possible resolving error.
:param SCMGetLatestData data: Information about the packages to resolve.
:return: Dict with following keys:
- pkg_name - Name of the resolved package.
- pkg_ref - Resolve git ref.
- error - Contains the error if any or it is set to `None`.
"""
try:
# If the modulemd specifies that the 'f25' branch is what
# we want to pull from, we need to resolve that f25 branch
# to the specific commit available at the time of
# submission (now).
repo = data.rpm_component.get_repository()
ref = data.ref_override or data.rpm_component.get_ref()
log.debug("Getting the commit hash for the ref %s on the repo %s", ref, repo)
pkgref = module_build_service.common.scm.SCM(repo).get_latest(ref)
except Exception as e:
log.exception(e)
return {
"error": "Failed to get the latest commit for %s#%s"
% (data.rpm_component.get_repository(), data.rpm_component.get_ref())
}
return {"pkg_name": data.rpm_component.get_name(), "pkg_ref": pkgref, "error": None}
def format_mmd(mmd, scmurl, module=None, db_session=None, srpm_overrides=None, default_ref=None):
"""
Prepares the modulemd for the MBS. This does things such as replacing the
branches of components with commit hashes and adding metadata in the xmd
dictionary.
:param mmd: the Modulemd.ModuleStream object to format
:param scmurl: the url to the modulemd
:param module: When specified together with `session`, the time_modified
of a module is updated regularly in case this method takes lot of time.
:param db_session: Database session to update the `module`.
:param dict srpm_overrides: Mapping of package names to SRPM links for all
component packages which have custom SRPM overrides specified.
"""
srpm_overrides = srpm_overrides or {}
xmd = mmd.get_xmd()
if "mbs" not in xmd:
xmd["mbs"] = {}
if "scmurl" not in xmd["mbs"]:
xmd["mbs"]["scmurl"] = scmurl or ""
if "commit" not in xmd["mbs"]:
xmd["mbs"]["commit"] = ""
# If module build was submitted via yaml file, there is no scmurl
if scmurl:
scm = module_build_service.common.scm.SCM(scmurl)
# We want to make sure we have the full commit hash for consistency
if module_build_service.common.scm.SCM.is_full_commit_hash(scm.scheme, scm.commit):
full_scm_hash = scm.commit
else:
full_scm_hash = scm.get_full_commit_hash()
xmd["mbs"]["commit"] = full_scm_hash
if default_ref is None:
default_ref = xmd["mbs"].get("branch") or conf.default_branch
if mmd.get_rpm_component_names() or mmd.get_module_component_names():
if "rpms" not in xmd["mbs"]:
xmd["mbs"]["rpms"] = {}
# Add missing data in RPM components
for pkgname in mmd.get_rpm_component_names():
pkg = mmd.get_rpm_component(pkgname)
# In case of resubmit of existing module which have been
# cancelled/failed during the init state, the package
# was maybe already handled by MBS, so skip it in this case.
if pkgname in xmd["mbs"]["rpms"]:
continue
if pkg.get_repository() and not conf.rpms_allow_repository:
raise Forbidden(
"Custom component repositories aren't allowed. "
"%r bears repository %r" % (pkgname, pkg.get_repository())
)
if pkg.get_cache() and not conf.rpms_allow_cache:
raise Forbidden(
"Custom component caches aren't allowed. "
"%r bears cache %r" % (pkgname, pkg.get_cache())
)
if pkg.get_buildafter():
raise ValidationError('The usage of "buildafter" is not yet supported')
if not pkg.get_repository():
pkg.set_repository(conf.rpms_default_repository + pkgname)
if not pkg.get_cache():
pkg.set_cache(conf.rpms_default_cache + pkgname)
if not pkg.get_ref():
pkg.set_ref(default_ref)
if not pkg.get_arches():
for arch in conf.arches:
pkg.add_restricted_arch(arch)
# Add missing data in included modules components
for modname in mmd.get_module_component_names():
mod = mmd.get_module_component(modname)
if mod.get_repository() and not conf.modules_allow_repository:
raise Forbidden(
"Custom module repositories aren't allowed. "
"%r bears repository %r" % (modname, mod.get_repository())
)
if not mod.get_repository():
mod.set_repository(conf.modules_default_repository + modname)
if not mod.get_ref():
mod.set_ref(default_ref)
# It is possible to override the ref of RPM component using
# the rpm_component_ref_overrides.
ref_overrides = xmd["mbs"].get("rpm_component_ref_overrides", {})
# Check that SCM URL is valid and replace potential branches in pkg refs
# by real SCM hash and store the result to our private xmd place in modulemd.
pool = ThreadPool(20)
try:
# Filter out the packages which we have already resolved in possible
# previous runs of this method (can be caused by module build resubmition)
# or which have custom SRPMs and shouldn't be resolved.
scm_get_latest_data_list = []
for name in mmd.get_rpm_component_names():
if name not in xmd["mbs"]["rpms"]:
if name in srpm_overrides:
# If this package has a custom SRPM, store an empty
# ref entry so no further verification takes place.
xmd["mbs"]["rpms"][name] = {"ref": None}
else:
# Apply possible ref override.
if name in ref_overrides:
ref_override = ref_overrides[name]
log.info("Applying rpm_component_ref_overrides - "
"%s, new ref is %s." % (name, ref_override))
else:
ref_override = None
scm_get_latest_data_list.append(
SCMGetLatestData(mmd.get_rpm_component(name), ref_override))
async_result = pool.map_async(_scm_get_latest, scm_get_latest_data_list)
# For modules with lot of components, the _scm_get_latest can take a lot of time.
# We need to bump time_modified from time to time, otherwise poller could think
# that module is stuck in "init" state and it would send fake "init" message.
while not async_result.ready():
async_result.wait(60)
if module and db_session:
module.time_modified = datetime.utcnow()
db_session.commit()
pkg_dicts = async_result.get()
finally:
pool.close()
err_msg = ""
for pkg_dict in pkg_dicts:
if pkg_dict["error"]:
err_msg += pkg_dict["error"] + "\n"
else:
pkg_name = pkg_dict["pkg_name"]
pkg_ref = pkg_dict["pkg_ref"]
xmd["mbs"]["rpms"][pkg_name] = {"ref": pkg_ref}
if err_msg:
raise UnprocessableEntity(err_msg)
# Set the modified xmd back to the modulemd
mmd.set_xmd(xmd)
def merge_included_mmd(mmd, included_mmd):
"""
Merges two modulemds. This merges only metadata which are needed in
the `main` when it includes another module defined by `included_mmd`
"""
included_xmd = included_mmd.get_xmd()
if "rpms" in included_xmd["mbs"]:
xmd = mmd.get_xmd()
if "rpms" not in xmd["mbs"]:
xmd["mbs"]["rpms"] = included_xmd["mbs"]["rpms"]
else:
xmd["mbs"]["rpms"].update(included_xmd["mbs"]["rpms"])
# Set the modified xmd back to the modulemd
mmd.set_xmd(xmd)
def get_module_srpm_overrides(module):
"""
Make necessary preparations to use any provided custom SRPMs.
:param module: ModuleBuild object representing the module being submitted.
:type module: :class:`models.ModuleBuild`
:return: mapping of package names to SRPM links for all packages which
have custom SRPM overrides specified
:rtype: dict[str, str]
"""
overrides = {}
if not module.srpms:
return overrides
try:
# Make sure we can decode the custom SRPM list
srpms = json.loads(module.srpms)
assert isinstance(srpms, list)
except Exception:
raise ValueError("Invalid srpms list encountered: {}".format(module.srpms))
for source in srpms:
if source.startswith("cli-build/") and source.endswith(".src.rpm"):
# This is a custom srpm that has been uploaded to koji by rpkg
# using the package name as the basename suffixed with .src.rpm
rpm_name = os.path.basename(source)[: -len(".src.rpm")]
else:
# This should be a local custom srpm path
if not os.path.exists(source):
raise IOError("Provided srpm is missing: {}".format(source))
# Get package name from rpm headers
try:
rpm_hdr = kobo.rpmlib.get_rpm_header(source)
rpm_name = to_text_type(kobo.rpmlib.get_header_field(rpm_hdr, "name"))
except Exception:
raise ValueError("Provided srpm is invalid: {}".format(source))
if rpm_name in overrides:
log.warning(
'Encountered duplicate custom SRPM "{0}" for package {1}'
.format(source, rpm_name)
)
continue
log.debug('Using custom SRPM "{0}" for package {1}'.format(source, rpm_name))
overrides[rpm_name] = source
return overrides
def record_component_builds(
mmd, module, initial_batch=1, previous_buildorder=None, main_mmd=None
):
# Imported here to allow import of utils in GenericBuilder.
from module_build_service.builder import GenericBuilder
# When main_mmd is set, merge the metadata from this mmd to main_mmd,
# otherwise our current mmd is main_mmd.
if main_mmd:
# Check for components that are in both MMDs before merging since MBS
# currently can't handle that situation.
main_mmd_rpms = main_mmd.get_rpm_component_names()
mmd_rpms = mmd.get_rpm_component_names()
duplicate_components = [
rpm for rpm in main_mmd_rpms
if rpm in mmd_rpms
]
if duplicate_components:
error_msg = (
'The included module "{0}" in "{1}" have the following '
"conflicting components: {2}".format(
mmd.get_module_name(), main_mmd.get_module_name(),
", ".join(duplicate_components)
)
)
raise UnprocessableEntity(error_msg)
merge_included_mmd(main_mmd, mmd)
else:
main_mmd = mmd
# If the modulemd yaml specifies components, then submit them for build
rpm_components = [
mmd.get_rpm_component(name)
for name in mmd.get_rpm_component_names()
]
module_components = [
mmd.get_module_component(name)
for name in mmd.get_module_component_names()
]
all_components = list(rpm_components) + list(module_components)
if not all_components:
return
# Get map of packages that have SRPM overrides
srpm_overrides = get_module_srpm_overrides(module)
rpm_weights = GenericBuilder.get_build_weights(
[c.get_name() for c in rpm_components]
)
all_components.sort(key=lambda x: x.get_buildorder())
# We do not start with batch = 0 here, because the first batch is
# reserved for module-build-macros. First real components must be
# planned for batch 2 and following.
batch = initial_batch
default_ref = mmd.get_xmd()["mbs"].get("branch")
for component in all_components:
# Increment the batch number when buildorder increases.
if previous_buildorder != component.get_buildorder():
previous_buildorder = component.get_buildorder()
batch += 1
# If the component is another module, we fetch its modulemd file
# and record its components recursively with the initial_batch
# set to our current batch, so the components of this module
# are built in the right global order.
if isinstance(component, Modulemd.ComponentModule):
full_url = component.get_repository() + "?#" + component.get_ref()
# It is OK to whitelist all URLs here, because the validity
# of every URL have been already checked in format_mmd(...).
included_mmd = fetch_mmd(full_url, whitelist_url=True)[0]
format_mmd(included_mmd, module.scmurl, module, db_session, srpm_overrides, default_ref)
batch = record_component_builds(
included_mmd, module, batch, previous_buildorder, main_mmd)
continue
package = component.get_name()
if package in srpm_overrides:
component_ref = None
full_url = srpm_overrides[package]
log.info('Building custom SRPM "{0}"' " for package {1}".format(full_url, package))
else:
component_ref = mmd.get_xmd()["mbs"]["rpms"][package]["ref"]
full_url = component.get_repository() + "?#" + component_ref
# Skip the ComponentBuild if it already exists in database. This can happen
# in case of module build resubmition.
existing_build = models.ComponentBuild.from_component_name(db_session, package, module.id)
if existing_build:
# Check that the existing build has the same most important attributes.
# This should never be a problem, but it's good to be defensive here so
# we do not mess things during resubmition.
if (
existing_build.batch != batch
or existing_build.scmurl != full_url
or existing_build.ref != component_ref
):
raise ValidationError(
"Component build %s of module build %s (id: %d) already "
"exists in database, but its attributes are different from"
" resubmitted one." % (
component.get_name(), module.name, module.id)
)
continue
build = models.ComponentBuild(
module_id=module.id,
package=package,
format="rpms",
scmurl=full_url,
batch=batch,
ref=component_ref,
weight=rpm_weights[package],
buildonly=component.get_buildonly()
)
db_session.add(build)
return batch