# -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT import kobo.rpmlib import module_build_service.messaging from module_build_service import log, models, conf from module_build_service.db_session import db_session from module_build_service.utils.mse import get_base_module_mmds from module_build_service.resolver import GenericResolver def reuse_component(component, previous_component_build, change_state_now=False): """ Reuses component build `previous_component_build` instead of building component `component` Please remember to commit the changes where the function is called. This allows callers to reuse multiple component builds and commit them all at once. Returns the list of BaseMessage instances to be handled later by the scheduler. """ import koji log.info( 'Reusing component "{0}" from a previous module ' 'build with the nvr "{1}"'.format(component.package, previous_component_build.nvr) ) component.reused_component_id = previous_component_build.id component.task_id = previous_component_build.task_id if change_state_now: component.state = previous_component_build.state else: # Use BUILDING state here, because we want the state to change to # COMPLETE by the fake KojiBuildChange message we are generating # few lines below. If we would set it to the right state right # here, we would miss the code path handling the KojiBuildChange # which works only when switching from BUILDING to COMPLETE. component.state = koji.BUILD_STATES["BUILDING"] component.state_reason = "Reused component from previous module build" component.nvr = previous_component_build.nvr nvr_dict = kobo.rpmlib.parse_nvr(component.nvr) # Add this message to further_work so that the reused # component will be tagged properly return [ module_build_service.messaging.KojiBuildChange( msg_id="reuse_component: fake msg", build_id=None, task_id=component.task_id, build_new_state=previous_component_build.state, build_name=nvr_dict["name"], build_version=nvr_dict["version"], build_release=nvr_dict["release"], module_build_id=component.module_id, state_reason=component.state_reason, ) ] def get_reusable_module(module): """ Returns previous module build of the module `module` in case it can be used as a source module to get the components to reuse from. In case there is no such module, returns None. :param module: the ModuleBuild object of module being built. :return: ModuleBuild object which can be used for component reuse. """ if module.reused_module: return module.reused_module mmd = module.mmd() previous_module_build = None # The `base_mmds` will contain the list of base modules against which the possible modules # to reuse are built. There are three options how these base modules are found: # # 1) The `conf.allow_only_compatible_base_modules` is False. This means that MBS should # not try to find any compatible base modules in its DB and simply use the buildrequired # base module as it is. # 2) The `conf.allow_only_compatible_base_modules` is True and DBResolver is used. This means # that MBS should try to find the compatible modules using its database. # The `get_base_module_mmds` finds out the list of compatible modules and returns mmds of # all of them. # 3) The `conf.allow_only_compatible_base_modules` is True and KojiResolver is used. This # means that MBS should *not* try to find any compatible base modules in its DB, but # instead just query Koji using KojiResolver later to find out the module to # reuse. The list of compatible base modules is defined by Koji tag inheritance directly # in Koji. # The `get_base_module_mmds` in this case returns just the buildrequired base module. if conf.allow_only_compatible_base_modules: log.debug("Checking for compatible base modules") base_mmds = get_base_module_mmds(db_session, mmd)["ready"] # Sort the base_mmds based on the stream version, higher version first. base_mmds.sort( key=lambda mmd: models.ModuleBuild.get_stream_version(mmd.get_stream_name(), False), reverse=True) else: log.debug("Skipping the check for compatible base modules") base_mmds = [] for br in module.buildrequires: if br.name in conf.base_module_names: base_mmds.append(br.mmd()) for base_mmd in base_mmds: previous_module_build = ( db_session.query(models.ModuleBuild) .filter_by(name=mmd.get_module_name()) .filter_by(stream=mmd.get_stream_name()) .filter_by(state=models.BUILD_STATES["ready"]) .filter(models.ModuleBuild.scmurl.isnot(None)) .order_by(models.ModuleBuild.time_completed.desc())) koji_resolver_enabled = base_mmd.get_xmd().get("mbs", {}).get("koji_tag_with_modules") if koji_resolver_enabled: # Find ModuleBuilds tagged in the Koji tag using KojiResolver. resolver = GenericResolver.create(db_session, conf, backend="koji") possible_modules_to_reuse = resolver.get_buildrequired_modules( module.name, module.stream, base_mmd) # Limit the query to these modules. possible_module_ids = [m.id for m in possible_modules_to_reuse] previous_module_build = previous_module_build.filter( models.ModuleBuild.id.in_(possible_module_ids)) # Limit the query to modules sharing the same `build_context_no_bms`. That means they # have the same buildrequirements. previous_module_build = previous_module_build.filter_by( build_context_no_bms=module.build_context_no_bms) else: # Recompute the build_context with compatible base module stream. mbs_xmd = mmd.get_xmd()["mbs"] if base_mmd.get_module_name() not in mbs_xmd["buildrequires"]: previous_module_build = None continue mbs_xmd["buildrequires"][base_mmd.get_module_name()]["stream"] \ = base_mmd.get_stream_name() build_context = module.calculate_build_context(mbs_xmd["buildrequires"]) # Limit the query to find only modules sharing the same build_context. previous_module_build = previous_module_build.filter_by(build_context=build_context) # If we are rebuilding with the "changed-and-after" option, then we can't reuse # components from modules that were built more liberally if module.rebuild_strategy == "changed-and-after": previous_module_build = previous_module_build.filter( models.ModuleBuild.rebuild_strategy.in_(["all", "changed-and-after"]) ) previous_module_build = previous_module_build.first() if previous_module_build: break # The component can't be reused if there isn't a previous build in the done # or ready state if not previous_module_build: log.info("Cannot re-use. %r is the first module build." % module) return None module.reused_module_id = previous_module_build.id db_session.commit() return previous_module_build def attempt_to_reuse_all_components(builder, module): """ Tries to reuse all the components in a build. The components are also tagged to the tags using the `builder`. Returns True if all components could be reused, otherwise False. When False is returned, no component has been reused. """ previous_module_build = get_reusable_module(module) if not previous_module_build: return False mmd = module.mmd() old_mmd = previous_module_build.mmd() # [(component, component_to_reuse), ...] component_pairs = [] # Find out if we can reuse all components and cache component and # component to reuse pairs. for c in module.component_builds: if c.package == "module-build-macros": continue component_to_reuse = get_reusable_component( module, c.package, previous_module_build=previous_module_build, mmd=mmd, old_mmd=old_mmd, ) if not component_to_reuse: return False component_pairs.append((c, component_to_reuse)) # Stores components we will tag to buildroot and final tag. components_to_tag = [] # Reuse all components. for c, component_to_reuse in component_pairs: # Set the module.batch to the last batch we have. if c.batch > module.batch: module.batch = c.batch # Reuse the component reuse_component(c, component_to_reuse, True) components_to_tag.append(c.nvr) # Tag them builder.buildroot_add_artifacts(components_to_tag, install=False) builder.tag_artifacts(components_to_tag, dest_tag=True) return True def get_reusable_components(module, component_names, previous_module_build=None): """ Returns the list of ComponentBuild instances belonging to previous module build which can be reused in the build of module `module`. The ComponentBuild instances in returned list are in the same order as their names in the component_names input list. In case some component cannot be reused, None is used instead of a ComponentBuild instance in the returned list. :param module: the ModuleBuild object of module being built. :param component_names: List of component names to be reused. :kwarg previous_module_build: the ModuleBuild instance of a module build which contains the components to reuse. If not passed, get_reusable_module is called to get the ModuleBuild instance. :return: List of ComponentBuild instances to reuse in the same order as `component_names` """ # We support components reusing only for koji and test backend. if conf.system not in ["koji", "test"]: return [None] * len(component_names) if not previous_module_build: previous_module_build = get_reusable_module(module) if not previous_module_build: return [None] * len(component_names) mmd = module.mmd() old_mmd = previous_module_build.mmd() ret = [] for component_name in component_names: ret.append( get_reusable_component( module, component_name, previous_module_build, mmd, old_mmd) ) return ret def get_reusable_component( module, component_name, previous_module_build=None, mmd=None, old_mmd=None ): """ Returns the component (RPM) build of a module that can be reused instead of needing to rebuild it :param module: the ModuleBuild object of module being built with a formatted mmd :param component_name: the name of the component (RPM) that you'd like to reuse a previous build of :param previous_module_build: the ModuleBuild instances of a module build which contains the components to reuse. If not passed, get_reusable_module is called to get the ModuleBuild instance. Consider passing the ModuleBuild instance in case you plan to call get_reusable_component repeatedly for the same module to make this method faster. :param mmd: Modulemd.ModuleStream of `module`. If not passed, it is taken from module.mmd(). Consider passing this arg in case you plan to call get_reusable_component repeatedly for the same module to make this method faster. :param old_mmd: Modulemd.ModuleStream of `previous_module_build`. If not passed, it is taken from previous_module_build.mmd(). Consider passing this arg in case you plan to call get_reusable_component repeatedly for the same module to make this method faster. :return: the component (RPM) build SQLAlchemy object, if one is not found, None is returned """ # We support component reusing only for koji and test backend. if conf.system not in ["koji", "test"]: return None # If the rebuild strategy is "all", that means that nothing can be reused if module.rebuild_strategy == "all": message = ("Cannot reuse the component {component_name} because the module " "rebuild strategy is \"all\".").format( component_name=component_name) module.log_message(db_session, message) return None if not previous_module_build: previous_module_build = get_reusable_module(module) if not previous_module_build: message = ("Cannot reuse because no previous build of " "module {module_name} found!").format( module_name=module.name) module.log_message(db_session, message) return None if not mmd: mmd = module.mmd() if not old_mmd: old_mmd = previous_module_build.mmd() # If the chosen component for some reason was not found in the database, # or the ref is missing, something has gone wrong and the component cannot # be reused new_module_build_component = models.ComponentBuild.from_component_name( db_session, component_name, module.id) if ( not new_module_build_component or not new_module_build_component.batch or not new_module_build_component.ref ): message = ("Cannot reuse the component {} because it can't be found in the " "database").format(component_name) module.log_message(db_session, message) return None prev_module_build_component = models.ComponentBuild.from_component_name( db_session, component_name, previous_module_build.id ) # If the component to reuse for some reason was not found in the database, # or the ref is missing, something has gone wrong and the component cannot # be reused if ( not prev_module_build_component or not prev_module_build_component.batch or not prev_module_build_component.ref ): message = ("Cannot reuse the component {} because a previous build of " "it can't be found in the database").format(component_name) new_module_build_component.log_message(db_session, message) return None # Make sure the ref for the component that is trying to be reused # hasn't changed since the last build if prev_module_build_component.ref != new_module_build_component.ref: message = ("Cannot reuse the component because the commit hash changed" " since the last build") new_module_build_component.log_message(db_session, message) return None # At this point we've determined that both module builds contain the component # and the components share the same commit hash if module.rebuild_strategy == "changed-and-after": # Make sure the batch number for the component that is trying to be reused # hasn't changed since the last build if prev_module_build_component.batch != new_module_build_component.batch: message = ("Cannot reuse the component because it is being built in " "a different batch than in the compatible module build") new_module_build_component.log_message(db_session, message) return None # If the mmd.buildopts.macros.rpms changed, we cannot reuse buildopts = mmd.get_buildopts() if buildopts: modulemd_macros = buildopts.get_rpm_macros() else: modulemd_macros = None old_buildopts = old_mmd.get_buildopts() if old_buildopts: old_modulemd_macros = old_buildopts.get_rpm_macros() else: old_modulemd_macros = None if modulemd_macros != old_modulemd_macros: message = ("Cannot reuse the component because the modulemd's macros are" " different than those of the compatible module build") new_module_build_component.log_message(db_session, message) return None # At this point we've determined that both module builds contain the component # with the same commit hash and they are in the same batch. We've also determined # that both module builds depend(ed) on the same exact module builds. Now it's time # to determine if the components before it have changed. # # Convert the component_builds to a list and sort them by batch new_component_builds = list(module.component_builds) new_component_builds.sort(key=lambda x: x.batch) prev_component_builds = list(previous_module_build.component_builds) prev_component_builds.sort(key=lambda x: x.batch) new_module_build_components = [] previous_module_build_components = [] # Create separate lists for the new and previous module build. These lists # will have an entry for every build batch *before* the component's # batch except for 1, which is reserved for the module-build-macros RPM. # Each batch entry will contain a set of "(name, ref, arches)" with the name, # ref (commit), and arches of the component. for i in range(new_module_build_component.batch - 1): # This is the first batch which we want to skip since it will always # contain only the module-build-macros RPM and it gets built every time if i == 0: continue new_module_build_components.append({ (value.package, value.ref, tuple(sorted(mmd.get_rpm_component(value.package).get_arches()))) for value in new_component_builds if value.batch == i + 1 }) previous_module_build_components.append({ (value.package, value.ref, tuple(sorted(old_mmd.get_rpm_component(value.package).get_arches()))) for value in prev_component_builds if value.batch == i + 1 }) # If the previous batches don't have the same ordering, hashes, and arches, then the # component can't be reused if previous_module_build_components != new_module_build_components: message = ("Cannot reuse the component because a component in a previous" " batch has been added, removed, or rebuilt") new_module_build_component.log_message(db_session, message) return None # check that arches have not changed pkg = mmd.get_rpm_component(component_name) if set(pkg.get_arches()) != set(old_mmd.get_rpm_component(component_name).get_arches()): message = ("Cannot reuse the component because its architectures" " have changed since the compatible module build").format(component_name) new_module_build_component.log_message(db_session, message) return None reusable_component = db_session.query(models.ComponentBuild).filter_by( package=component_name, module_id=previous_module_build.id).one() log.debug("Found reusable component!") return reusable_component