mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-07 12:48:40 +08:00
473f7e5e5b
To support multiple backend, we need to get rid of `further_work` concept which is used in multiple places in the MBS code. Before this commit, if one handler wanted to execute another handler, it planned this work by constructing fake message and returning it. MBSConsumer then planned its execution by adding it into the event loop. In this commit, the new `events.scheduler` instance of new Scheduler class is used to acomplish this. If handler wants to execute another handler, it simply schedules it using `events.scheduler.add` method. In the end of each handler, the `events.scheduler.run` method is executed which calls all the scheduled handlers. The idea is that when Celery is enabled, we can change the `Scheduler.run` method to execute the handlers using the Celery, while during the local builds, we could execute them directly without Celery. Use of Scheduler also fixes the issue with ordering of such calls. If we would call the handlers directly, they could have been executed in the middle of another handler leading to behavior incompatible with the current `further_work` concept. Using the Scheduler, these calls are executed always in the end of the handler no matter when they have been scheduled.
448 lines
20 KiB
Python
448 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
import kobo.rpmlib
|
|
|
|
from module_build_service import log, models, conf
|
|
from module_build_service.db_session import db_session
|
|
from module_build_service.resolver import GenericResolver
|
|
from module_build_service.scheduler import events
|
|
from module_build_service.utils.mse import get_base_module_mmds
|
|
|
|
|
|
def reuse_component(component, previous_component_build, change_state_now=False,
|
|
schedule_fake_events=True):
|
|
"""
|
|
Reuses component build `previous_component_build` instead of building
|
|
component `component`
|
|
|
|
Please remember to commit the changes where the function is called.
|
|
This allows callers to reuse multiple component builds and commit them all
|
|
at once.
|
|
|
|
:param ComponentBuild component: Component whihch will reuse previous module build.
|
|
:param ComponentBuild previous_component_build: Previous component build to reuse.
|
|
:param bool change_state_now: When True, the component.state will be set to
|
|
previous_component_build.state. Otherwise, the component.state will be set to BUILDING.
|
|
:param bool schedule_fake_events: When True, the `events.scheduler.add` will be used to
|
|
schedule handlers.component.build_task_finalize handler call.
|
|
"""
|
|
|
|
import koji
|
|
from module_build_service.scheduler.handlers.components import (
|
|
build_task_finalize as build_task_finalize_handler)
|
|
|
|
log.info(
|
|
'Reusing component "{0}" from a previous module '
|
|
'build with the nvr "{1}"'.format(component.package, previous_component_build.nvr)
|
|
)
|
|
component.reused_component_id = previous_component_build.id
|
|
component.task_id = previous_component_build.task_id
|
|
if change_state_now:
|
|
component.state = previous_component_build.state
|
|
else:
|
|
# Use BUILDING state here, because we want the state to change to
|
|
# COMPLETE by scheduling a internal buildsys.build.state.change message
|
|
# we are generating few lines below.
|
|
# If we would set it to the right state right here, we would miss the
|
|
# code path handling that event which works only when switching from
|
|
# BUILDING to COMPLETE.
|
|
component.state = koji.BUILD_STATES["BUILDING"]
|
|
component.state_reason = "Reused component from previous module build"
|
|
component.nvr = previous_component_build.nvr
|
|
nvr_dict = kobo.rpmlib.parse_nvr(component.nvr)
|
|
# Add this event to scheduler so that the reused component will be tagged properly.
|
|
if schedule_fake_events:
|
|
args = (
|
|
"reuse_component: fake msg", None, component.task_id, previous_component_build.state,
|
|
nvr_dict["name"], nvr_dict["version"], nvr_dict["release"], component.module_id,
|
|
component.state_reason)
|
|
events.scheduler.add(build_task_finalize_handler, args)
|
|
|
|
|
|
def get_reusable_module(module):
|
|
"""
|
|
Returns previous module build of the module `module` in case it can be
|
|
used as a source module to get the components to reuse from.
|
|
|
|
In case there is no such module, returns None.
|
|
|
|
:param module: the ModuleBuild object of module being built.
|
|
:return: ModuleBuild object which can be used for component reuse.
|
|
"""
|
|
|
|
if module.reused_module:
|
|
return module.reused_module
|
|
|
|
mmd = module.mmd()
|
|
previous_module_build = None
|
|
|
|
# The `base_mmds` will contain the list of base modules against which the possible modules
|
|
# to reuse are built. There are three options how these base modules are found:
|
|
#
|
|
# 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should
|
|
# not try to find any compatible base modules in its DB and simply use the buildrequired
|
|
# base module as it is.
|
|
# 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means
|
|
# that MBS should try to find the compatible modules using its database.
|
|
# The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of
|
|
# all of them.
|
|
# 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This
|
|
# means that MBS should *not* try to find any compatible base modules in its DB, but
|
|
# instead just query Koji using KojiResolver later to find out the module to
|
|
# reuse. The list of compatible base modules is defined by Koji tag inheritance directly
|
|
# in Koji.
|
|
# The `get_base_module_mmds` in this case returns just the buildrequired base module.
|
|
if conf.allow_only_compatible_base_modules:
|
|
log.debug("Checking for compatible base modules")
|
|
base_mmds = get_base_module_mmds(db_session, mmd)["ready"]
|
|
# Sort the base_mmds based on the stream version, higher version first.
|
|
base_mmds.sort(
|
|
key=lambda mmd: models.ModuleBuild.get_stream_version(mmd.get_stream_name(), False),
|
|
reverse=True)
|
|
else:
|
|
log.debug("Skipping the check for compatible base modules")
|
|
base_mmds = []
|
|
for br in module.buildrequires:
|
|
if br.name in conf.base_module_names:
|
|
base_mmds.append(br.mmd())
|
|
|
|
for base_mmd in base_mmds:
|
|
previous_module_build = (
|
|
db_session.query(models.ModuleBuild)
|
|
.filter_by(name=mmd.get_module_name())
|
|
.filter_by(stream=mmd.get_stream_name())
|
|
.filter_by(state=models.BUILD_STATES["ready"])
|
|
.filter(models.ModuleBuild.scmurl.isnot(None))
|
|
.order_by(models.ModuleBuild.time_completed.desc()))
|
|
|
|
koji_resolver_enabled = base_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules")
|
|
if koji_resolver_enabled:
|
|
# Find ModuleBuilds tagged in the Koji tag using KojiResolver.
|
|
resolver = GenericResolver.create(db_session, conf, backend="koji")
|
|
possible_modules_to_reuse = resolver.get_buildrequired_modules(
|
|
module.name, module.stream, base_mmd)
|
|
|
|
# Limit the query to these modules.
|
|
possible_module_ids = [m.id for m in possible_modules_to_reuse]
|
|
previous_module_build = previous_module_build.filter(
|
|
models.ModuleBuild.id.in_(possible_module_ids))
|
|
|
|
# Limit the query to modules sharing the same `build_context_no_bms`. That means they
|
|
# have the same buildrequirements.
|
|
previous_module_build = previous_module_build.filter_by(
|
|
build_context_no_bms=module.build_context_no_bms)
|
|
else:
|
|
# Recompute the build_context with compatible base module stream.
|
|
mbs_xmd = mmd.get_xmd()["mbs"]
|
|
if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]:
|
|
previous_module_build = None
|
|
continue
|
|
mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \
|
|
= base_mmd.get_stream_name()
|
|
build_context = module.calculate_build_context(mbs_xmd["buildrequires"])
|
|
|
|
# Limit the query to find only modules sharing the same build_context.
|
|
previous_module_build = previous_module_build.filter_by(build_context=build_context)
|
|
|
|
# If we are rebuilding with the "changed-and-after" option, then we can't reuse
|
|
# components from modules that were built more liberally
|
|
if module.rebuild_strategy == "changed-and-after":
|
|
previous_module_build = previous_module_build.filter(
|
|
models.ModuleBuild.rebuild_strategy.in_(["all", "changed-and-after"])
|
|
)
|
|
|
|
previous_module_build = previous_module_build.first()
|
|
|
|
if previous_module_build:
|
|
break
|
|
|
|
# The component can't be reused if there isn't a previous build in the done
|
|
# or ready state
|
|
if not previous_module_build:
|
|
log.info("Cannot re-use. %r is the first module build." % module)
|
|
return None
|
|
|
|
module.reused_module_id = previous_module_build.id
|
|
db_session.commit()
|
|
|
|
return previous_module_build
|
|
|
|
|
|
def attempt_to_reuse_all_components(builder, module):
|
|
"""
|
|
Tries to reuse all the components in a build. The components are also
|
|
tagged to the tags using the `builder`.
|
|
|
|
Returns True if all components could be reused, otherwise False. When
|
|
False is returned, no component has been reused.
|
|
"""
|
|
|
|
previous_module_build = get_reusable_module(module)
|
|
if not previous_module_build:
|
|
return False
|
|
|
|
mmd = module.mmd()
|
|
old_mmd = previous_module_build.mmd()
|
|
|
|
# [(component, component_to_reuse), ...]
|
|
component_pairs = []
|
|
|
|
# Find out if we can reuse all components and cache component and
|
|
# component to reuse pairs.
|
|
for c in module.component_builds:
|
|
if c.package == "module-build-macros":
|
|
continue
|
|
component_to_reuse = get_reusable_component(
|
|
module,
|
|
c.package,
|
|
previous_module_build=previous_module_build,
|
|
mmd=mmd,
|
|
old_mmd=old_mmd,
|
|
)
|
|
if not component_to_reuse:
|
|
return False
|
|
|
|
component_pairs.append((c, component_to_reuse))
|
|
|
|
# Stores components we will tag to buildroot and final tag.
|
|
components_to_tag = []
|
|
|
|
# Reuse all components.
|
|
for c, component_to_reuse in component_pairs:
|
|
# Set the module.batch to the last batch we have.
|
|
if c.batch > module.batch:
|
|
module.batch = c.batch
|
|
|
|
# Reuse the component
|
|
reuse_component(c, component_to_reuse, True, False)
|
|
components_to_tag.append(c.nvr)
|
|
|
|
# Tag them
|
|
builder.buildroot_add_artifacts(components_to_tag, install=False)
|
|
builder.tag_artifacts(components_to_tag, dest_tag=True)
|
|
|
|
return True
|
|
|
|
|
|
def get_reusable_components(module, component_names, previous_module_build=None):
|
|
"""
|
|
Returns the list of ComponentBuild instances belonging to previous module
|
|
build which can be reused in the build of module `module`.
|
|
|
|
The ComponentBuild instances in returned list are in the same order as
|
|
their names in the component_names input list.
|
|
|
|
In case some component cannot be reused, None is used instead of a
|
|
ComponentBuild instance in the returned list.
|
|
|
|
:param module: the ModuleBuild object of module being built.
|
|
:param component_names: List of component names to be reused.
|
|
:kwarg previous_module_build: the ModuleBuild instance of a module build
|
|
which contains the components to reuse. If not passed, get_reusable_module
|
|
is called to get the ModuleBuild instance.
|
|
:return: List of ComponentBuild instances to reuse in the same
|
|
order as `component_names`
|
|
"""
|
|
# We support components reusing only for koji and test backend.
|
|
if conf.system not in ["koji", "test"]:
|
|
return [None] * len(component_names)
|
|
|
|
if not previous_module_build:
|
|
previous_module_build = get_reusable_module(module)
|
|
if not previous_module_build:
|
|
return [None] * len(component_names)
|
|
|
|
mmd = module.mmd()
|
|
old_mmd = previous_module_build.mmd()
|
|
|
|
ret = []
|
|
for component_name in component_names:
|
|
ret.append(
|
|
get_reusable_component(
|
|
module, component_name, previous_module_build, mmd, old_mmd)
|
|
)
|
|
|
|
return ret
|
|
|
|
|
|
def get_reusable_component(
|
|
module, component_name, previous_module_build=None, mmd=None, old_mmd=None
|
|
):
|
|
"""
|
|
Returns the component (RPM) build of a module that can be reused
|
|
instead of needing to rebuild it
|
|
|
|
:param module: the ModuleBuild object of module being built with a formatted
|
|
mmd
|
|
:param component_name: the name of the component (RPM) that you'd like to
|
|
reuse a previous build of
|
|
:param previous_module_build: the ModuleBuild instances of a module build
|
|
which contains the components to reuse. If not passed, get_reusable_module
|
|
is called to get the ModuleBuild instance. Consider passing the ModuleBuild
|
|
instance in case you plan to call get_reusable_component repeatedly for the
|
|
same module to make this method faster.
|
|
:param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from
|
|
module.mmd(). Consider passing this arg in case you plan to call
|
|
get_reusable_component repeatedly for the same module to make this method faster.
|
|
:param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed,
|
|
it is taken from previous_module_build.mmd(). Consider passing this arg in
|
|
case you plan to call get_reusable_component repeatedly for the same
|
|
module to make this method faster.
|
|
:return: the component (RPM) build SQLAlchemy object, if one is not found,
|
|
None is returned
|
|
"""
|
|
|
|
# We support component reusing only for koji and test backend.
|
|
if conf.system not in ["koji", "test"]:
|
|
return None
|
|
|
|
# If the rebuild strategy is "all", that means that nothing can be reused
|
|
if module.rebuild_strategy == "all":
|
|
message = ("Cannot reuse the component {component_name} because the module "
|
|
"rebuild strategy is \"all\".").format(
|
|
component_name=component_name)
|
|
module.log_message(db_session, message)
|
|
return None
|
|
|
|
if not previous_module_build:
|
|
previous_module_build = get_reusable_module(module)
|
|
if not previous_module_build:
|
|
message = ("Cannot reuse because no previous build of "
|
|
"module {module_name} found!").format(
|
|
module_name=module.name)
|
|
module.log_message(db_session, message)
|
|
return None
|
|
|
|
if not mmd:
|
|
mmd = module.mmd()
|
|
if not old_mmd:
|
|
old_mmd = previous_module_build.mmd()
|
|
|
|
# If the chosen component for some reason was not found in the database,
|
|
# or the ref is missing, something has gone wrong and the component cannot
|
|
# be reused
|
|
new_module_build_component = models.ComponentBuild.from_component_name(
|
|
db_session, component_name, module.id)
|
|
if (
|
|
not new_module_build_component
|
|
or not new_module_build_component.batch
|
|
or not new_module_build_component.ref
|
|
):
|
|
message = ("Cannot reuse the component {} because it can't be found in the "
|
|
"database").format(component_name)
|
|
module.log_message(db_session, message)
|
|
return None
|
|
|
|
prev_module_build_component = models.ComponentBuild.from_component_name(
|
|
db_session, component_name, previous_module_build.id
|
|
)
|
|
# If the component to reuse for some reason was not found in the database,
|
|
# or the ref is missing, something has gone wrong and the component cannot
|
|
# be reused
|
|
if (
|
|
not prev_module_build_component
|
|
or not prev_module_build_component.batch
|
|
or not prev_module_build_component.ref
|
|
):
|
|
message = ("Cannot reuse the component {} because a previous build of "
|
|
"it can't be found in the database").format(component_name)
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
# Make sure the ref for the component that is trying to be reused
|
|
# hasn't changed since the last build
|
|
if prev_module_build_component.ref != new_module_build_component.ref:
|
|
message = ("Cannot reuse the component because the commit hash changed"
|
|
" since the last build")
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
# At this point we've determined that both module builds contain the component
|
|
# and the components share the same commit hash
|
|
if module.rebuild_strategy == "changed-and-after":
|
|
# Make sure the batch number for the component that is trying to be reused
|
|
# hasn't changed since the last build
|
|
if prev_module_build_component.batch != new_module_build_component.batch:
|
|
message = ("Cannot reuse the component because it is being built in "
|
|
"a different batch than in the compatible module build")
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
# If the mmd.buildopts.macros.rpms changed, we cannot reuse
|
|
buildopts = mmd.get_buildopts()
|
|
if buildopts:
|
|
modulemd_macros = buildopts.get_rpm_macros()
|
|
else:
|
|
modulemd_macros = None
|
|
|
|
old_buildopts = old_mmd.get_buildopts()
|
|
if old_buildopts:
|
|
old_modulemd_macros = old_buildopts.get_rpm_macros()
|
|
else:
|
|
old_modulemd_macros = None
|
|
|
|
if modulemd_macros != old_modulemd_macros:
|
|
message = ("Cannot reuse the component because the modulemd's macros are"
|
|
" different than those of the compatible module build")
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
# At this point we've determined that both module builds contain the component
|
|
# with the same commit hash and they are in the same batch. We've also determined
|
|
# that both module builds depend(ed) on the same exact module builds. Now it's time
|
|
# to determine if the components before it have changed.
|
|
#
|
|
# Convert the component_builds to a list and sort them by batch
|
|
new_component_builds = list(module.component_builds)
|
|
new_component_builds.sort(key=lambda x: x.batch)
|
|
prev_component_builds = list(previous_module_build.component_builds)
|
|
prev_component_builds.sort(key=lambda x: x.batch)
|
|
|
|
new_module_build_components = []
|
|
previous_module_build_components = []
|
|
# Create separate lists for the new and previous module build. These lists
|
|
# will have an entry for every build batch *before* the component's
|
|
# batch except for 1, which is reserved for the module-build-macros RPM.
|
|
# Each batch entry will contain a set of "(name, ref, arches)" with the name,
|
|
# ref (commit), and arches of the component.
|
|
for i in range(new_module_build_component.batch - 1):
|
|
# This is the first batch which we want to skip since it will always
|
|
# contain only the module-build-macros RPM and it gets built every time
|
|
if i == 0:
|
|
continue
|
|
|
|
new_module_build_components.append({
|
|
(value.package, value.ref,
|
|
tuple(sorted(mmd.get_rpm_component(value.package).get_arches())))
|
|
for value in new_component_builds
|
|
if value.batch == i + 1
|
|
})
|
|
|
|
previous_module_build_components.append({
|
|
(value.package, value.ref,
|
|
tuple(sorted(old_mmd.get_rpm_component(value.package).get_arches())))
|
|
for value in prev_component_builds
|
|
if value.batch == i + 1
|
|
})
|
|
|
|
# If the previous batches don't have the same ordering, hashes, and arches, then the
|
|
# component can't be reused
|
|
if previous_module_build_components != new_module_build_components:
|
|
message = ("Cannot reuse the component because a component in a previous"
|
|
" batch has been added, removed, or rebuilt")
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
# check that arches have not changed
|
|
pkg = mmd.get_rpm_component(component_name)
|
|
if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(component_name).get_arches()):
|
|
message = ("Cannot reuse the component because its architectures"
|
|
" have changed since the compatible module build").format(component_name)
|
|
new_module_build_component.log_message(db_session, message)
|
|
return None
|
|
|
|
reusable_component = db_session.query(models.ComponentBuild).filter_by(
|
|
package=component_name, module_id=previous_module_build.id).one()
|
|
log.debug("Found reusable component!")
|
|
return reusable_component
|