From 385045d778a7820b4edd20b788c3ade4c928550e Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Mon, 16 Jan 2017 14:46:02 -0500 Subject: [PATCH] Reuse components from previous module builds --- module_build_service/messaging.py | 7 +- .../migrations/versions/a1fc0736bca8_.py | 28 +++ module_build_service/models.py | 18 +- module_build_service/pdc.py | 30 ++- .../scheduler/handlers/components.py | 4 +- module_build_service/utils.py | 178 +++++++++++++++++- 6 files changed, 247 insertions(+), 18 deletions(-) create mode 100644 module_build_service/migrations/versions/a1fc0736bca8_.py diff --git a/module_build_service/messaging.py b/module_build_service/messaging.py index 85fc13ab..db54518c 100644 --- a/module_build_service/messaging.py +++ b/module_build_service/messaging.py @@ -219,9 +219,12 @@ class KojiBuildChange(BaseMessage): (e.g. golang-googlecode-tools) :param build_version: the version of the build (e.g. 6.06.06) :param build_release: the release of the build (e.g. 4.fc25) + :param module_build_id: the optional id of the module_build in the database + :param state_reason: the optional reason as to why the state changed """ def __init__(self, msg_id, build_id, task_id, build_new_state, build_name, - build_version, build_release): + build_version, build_release, module_build_id=None, + state_reason=None): super(KojiBuildChange, self).__init__(msg_id) self.build_id = build_id self.task_id = task_id @@ -229,6 +232,8 @@ class KojiBuildChange(BaseMessage): self.build_name = build_name self.build_version = build_version self.build_release = build_release + self.module_build_id = module_build_id + self.state_reason = state_reason class KojiRepoChange(BaseMessage): diff --git a/module_build_service/migrations/versions/a1fc0736bca8_.py b/module_build_service/migrations/versions/a1fc0736bca8_.py new file mode 100644 index 00000000..92a6da88 --- /dev/null +++ b/module_build_service/migrations/versions/a1fc0736bca8_.py @@ -0,0 +1,28 @@ +"""Add reused_component_id + +Revision ID: a1fc0736bca8 +Revises: 0ef60c3ed440 +Create Date: 2017-02-10 19:32:35.288596 + +""" + +# revision identifiers, used by Alembic. +revision = 'a1fc0736bca8' +down_revision = '0ef60c3ed440' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + with op.batch_alter_table('component_builds') as b: + b.add_column(sa.Column('reused_component_id', sa.Integer(), nullable=True)) + b.create_foreign_key('component_builds', 'component_builds', ['reused_component_id'], ['id']) + b.add_column(sa.Column('ref', sa.String(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table('component_builds') as b: + b.drop_constraint('component_builds', type_='foreignkey') + b.drop_column('reused_component_id') + b.drop_column('ref') diff --git a/module_build_service/models.py b/module_build_service/models.py index 04563d8f..d567a45c 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -294,6 +294,8 @@ class ComponentBuild(RidaBase): # XXX: Consider making this a proper ENUM format = db.Column(db.String, nullable=False) task_id = db.Column(db.Integer) # This is the id of the build in koji + # This is the commit hash that component was built with + ref = db.Column(db.String, nullable=True) # XXX: Consider making this a proper ENUM (or an int) state = db.Column(db.Integer) # Reason why the build failed @@ -309,15 +311,27 @@ class ComponentBuild(RidaBase): module_id = db.Column(db.Integer, db.ForeignKey('module_builds.id'), nullable=False) module_build = db.relationship('ModuleBuild', backref='component_builds', lazy=False) + reused_component_id = db.Column( + db.Integer, db.ForeignKey('component_builds.id')) @classmethod def from_component_event(cls, session, event): if type(event) == module_build_service.messaging.KojiBuildChange: - return session.query(cls).filter( - cls.task_id == event.task_id).first() + if event.module_build_id: + return session.query(cls).filter_by( + task_id=event.task_id, module_id=event.module_build_id)\ + .one() + else: + return session.query(cls).filter( + cls.task_id == event.task_id).first() else: raise ValueError("%r is not a koji message." % event['topic']) + @classmethod + def from_component_name(cls, session, component_name, module_id): + return session.query(cls).filter_by( + package=component_name, module_id=module_id).first() + def json(self): retval = { 'id': self.id, diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index a50b9a18..8e2c4111 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -243,13 +243,16 @@ def resolve_profiles(session, mmd, keys, seen=None): results = {} for key in keys: results[key] = set() - for name, stream in mmd.buildrequires.items(): + for module_name, module_info in mmd.xmd['mbs']['buildrequires'].items(): # First, guard against infinite recursion - if name in seen: + if module_name in seen: continue - # Find the latest of the dep in our db of built modules. - module_info = {'variant_id': name, 'variant_stream': stream} + # Find the dep in the built modules in PDC + module_info = { + 'variant_id': module_name, + 'variant_stream': module_info['stream'], + 'variant_release': module_info['version']} dep_mmd = get_module_modulemd(session, module_info, True) # Take note of what rpms are in this dep's profile. @@ -258,7 +261,7 @@ def resolve_profiles(session, mmd, keys, seen=None): results[key] |= dep_mmd.profiles[key].rpms # And recurse to all modules that are deps of our dep. - rec_results = resolve_profiles(session, dep_mmd, keys, seen + [name]) + rec_results = resolve_profiles(session, dep_mmd, keys, seen + [module_name]) for rec_key, rec_result in rec_results.items(): results[rec_key] |= rec_result @@ -332,11 +335,18 @@ def get_module_build_dependencies(session, module_info, strict=False): # XXX get definitive list of modules deps = [] - module_info = get_module(session, module_info, strict=strict) - if module_info and module_info.get('build_deps', None): - deps = [dict(variant_id=x['dependency'], variant_stream=x['stream']) - for x in module_info['build_deps']] - deps = module_depsolving_wrapper(session, deps, strict=strict) + queried_mmd = get_module_modulemd(session, module_info, strict=strict) + if not queried_mmd or not queried_mmd.xmd.get('mbs') or not \ + queried_mmd.xmd['mbs'].get('buildrequires'): + raise RuntimeError( + 'The module "{0!r}" did not contain its modulemd or did not have ' + 'its xmd attribute filled out in PDC'.format(module_info)) + + deps = [dict(name=dep_name, version=dep_info['stream'], + release=dep_info['version']) + for dep_name, dep_info in + queried_mmd.xmd['mbs']['buildrequires'].items()] + deps = module_depsolving_wrapper(session, deps, strict=strict) return deps diff --git a/module_build_service/scheduler/handlers/components.py b/module_build_service/scheduler/handlers/components.py index e84d9fa2..e3f9fac4 100644 --- a/module_build_service/scheduler/handlers/components.py +++ b/module_build_service/scheduler/handlers/components.py @@ -50,7 +50,9 @@ def _finalize(config, session, msg, state): log.debug("We have no record of %s" % nvr) return - if state != koji.BUILD_STATES['COMPLETE']: + if msg.state_reason: + state_reason = msg.state_reason + elif state != koji.BUILD_STATES['COMPLETE']: state_reason = "Failed to build artifact %s in Koji" % (msg.build_name) else: state_reason = "" diff --git a/module_build_service/utils.py b/module_build_service/utils.py index f4d4623c..5fc1ffc2 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -30,6 +30,7 @@ import tempfile import os import logging import copy +import kobo.rpmlib from six import iteritems import modulemd @@ -139,6 +140,8 @@ def start_build_batch(config, module, session, builder, components=None): log.info("Starting build of next batch %d, %s" % (module.batch, unbuilt_components)) + further_work = [] + def start_build_component(c): """ Submits single component build to builder. Called in thread @@ -146,10 +149,51 @@ def start_build_batch(config, module, session, builder, components=None): """ try: with models.make_session(conf) as s: - if at_concurrent_component_threshold(config, s): + previous_component_build = None + # Check to see if we can reuse a previous component build + # instead of rebuilding it if the builder is Koji + if conf.system == 'koji': + previous_component_build = get_reusable_component( + s, module, c.package) + # If a component build can't be reused, we need to check + # the concurrent threshold + if not previous_component_build and \ + at_concurrent_component_threshold(config, s): log.info('Concurrent build threshold met') return + if previous_component_build: + log.info( + 'Reusing component "{0}" from a previous module ' + 'build with the nvr "{1}"'.format( + c.package, previous_component_build.nvr)) + c.reused_component_id = previous_component_build.id + c.task_id = previous_component_build.task_id + c.state = previous_component_build.state + c.reused_component_id = previous_component_build.id + c.task_id = previous_component_build.task_id + c.state = previous_component_build.state + c.state_reason = \ + 'Reused component from previous module build' + c.nvr = previous_component_build.nvr + nvr_dict = kobo.rpmlib.parse_nvr(c.nvr) + # Add this message to further_work so that the reused + # component will be tagged properly + further_work.append( + module_build_service.messaging.KojiBuildChange( + msg_id='start_build_batch: fake msg', + build_id=None, + task_id=c.task_id, + build_new_state=c.state, + build_name=c.package, + build_version=nvr_dict['version'], + build_release=nvr_dict['release'], + module_build_id=c.module_id, + state_reason=c.state_reason + ) + ) + return + c.task_id, c.state, c.state_reason, c.nvr = builder.build( artifact_name=c.package, source=c.scmurl) except Exception as e: @@ -168,8 +212,6 @@ def start_build_batch(config, module, session, builder, components=None): futures = {executor.submit(start_build_component, c): c for c in unbuilt_components} concurrent.futures.wait(futures) - further_work = [] - # If all components in this batch are already done, it can mean that they # have been built in the past and have been skipped in this module build. # We therefore have to generate fake KojiRepoChange message, because the @@ -476,7 +518,8 @@ def record_component_builds(scm, mmd, module, initial_batch = 1): package=pkg.name, format="rpms", scmurl=full_url, - batch=batch + batch=batch, + ref=pkg.ref ) db.session.add(build) @@ -582,3 +625,130 @@ def module_build_state_from_msg(msg): 'state=%s(%s) is not in %s' % (state, type(state), list(models.BUILD_STATES.values()))) return state + +def get_reusable_component(session, module, component_name): + """ + Returns the component (RPM) build of a module that can be reused + instead of needing to rebuild it + :param session: SQLAlchemy database session + :param module: the ModuleBuild object of module being built with a formatted + mmd + :param component_name: the name of the component (RPM) that you'd like to + reuse a previous build of + :return: the component (RPM) build SQLAlchemy object, if one is not found, + None is returned + """ + mmd = module.mmd() + # Find the latest module that is in the done or ready state + previous_module_build = session.query(models.ModuleBuild)\ + .filter_by(name=mmd.name)\ + .filter(models.ModuleBuild.state.in_([3, 5]))\ + .order_by(models.ModuleBuild.time_completed.desc())\ + .first() + # The component can't be reused if there isn't a previous build in the done + # or ready state + if not previous_module_build: + return None + old_mmd = previous_module_build.mmd() + + # Perform a sanity check to make sure that the buildrequires are the same + # as the buildrequires in xmd for the passed in mmd + if mmd.buildrequires.keys() != mmd.xmd['mbs']['buildrequires'].keys(): + log.error( + 'The submitted module "{0}" has different keys in mmd.buildrequires' + ' than in mmd.xmd[\'mbs\'][\'buildrequires\']'.format(mmd.name)) + return None + # Perform a sanity check to make sure that the buildrequires are the same + # as the buildrequires in xmd for the mmd of the previous module build + if old_mmd.buildrequires.keys() != \ + old_mmd.xmd['mbs']['buildrequires'].keys(): + log.error( + 'Version "{0}" of the module "{1}" has different keys in ' + 'mmd.buildrequires than in mmd.xmd[\'mbs\'][\'buildrequires\']' + .format(previous_module_build.version, previous_module_build.name)) + return None + + # If the module buildrequires are different, then we can't reuse the + # component + if mmd.buildrequires.keys() != old_mmd.buildrequires.keys(): + return None + + # Make sure that the module buildrequires commit hashes are exactly the same + for br_module_name, br_module in \ + mmd.xmd['mbs']['buildrequires'].items(): + # Assumes that the streams have been replaced with commit hashes, so we + # can compare to see if they have changed. Since a build is unique to + # a commit hash, this is a safe test. + if br_module['ref'] != \ + old_mmd.xmd['mbs']['buildrequires'][br_module_name]['ref']: + return None + + # At this point we've determined that both module builds depend(ed) on the + # same exact module builds. Now it's time to determine if the batch of the + # components have changed + # + # If the chosen component for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + new_module_build_component = models.ComponentBuild.from_component_name( + session, component_name, module.id) + if not new_module_build_component or not new_module_build_component.batch \ + or not new_module_build_component.ref: + return None + + prev_module_build_component = models.ComponentBuild.from_component_name( + session, component_name, previous_module_build.id) + # If the component to reuse for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + if not prev_module_build_component or not prev_module_build_component.batch\ + or not prev_module_build_component.ref: + return None + + # Make sure the batch number for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.batch != new_module_build_component.batch: + return None + + # Make sure the ref for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.ref != new_module_build_component.ref: + return None + + # Convert the component_builds to a list and sort them by batch + new_component_builds = list(module.component_builds) + new_component_builds.sort(key=lambda x: x.batch) + prev_component_builds = list(previous_module_build.component_builds) + prev_component_builds.sort(key=lambda x: x.batch) + + new_module_build_components = [] + previous_module_build_components = [] + # Create separate lists for the new and previous module build. These lists + # will have an entry for every build batch *before* the component's + # batch except for 1, which is reserved for the module-build-macros RPM. + # Each batch entry will contain a list of dicts with the name and ref + # (commit) of the component. + for i in range(new_module_build_component.batch - 1): + # This is the first batch which we want to skip since it will always + # contain only the module-build-macros RPM and it gets built every time + if i == 0: + continue + + new_module_build_components.append([ + {'name': value.package, 'ref': value.ref} for value in + new_component_builds if value.batch == i + 1 + ]) + + previous_module_build_components.append([ + {'name': value.package, 'ref': value.ref} for value in + prev_component_builds if value.batch == i + 1 + ]) + + # If the previous batches have the same ordering and hashes, then the + # component can be reused + if previous_module_build_components == new_module_build_components: + reusable_component = models.ComponentBuild.query.filter_by( + package=component_name, module_id=previous_module_build.id).one() + return reusable_component + + return None