diff --git a/module_build_service/resolver/PDCResolver.py b/module_build_service/resolver/PDCResolver.py index 6560d836..df52fc1a 100644 --- a/module_build_service/resolver/PDCResolver.py +++ b/module_build_service/resolver/PDCResolver.py @@ -33,7 +33,6 @@ from module_build_service.errors import UnprocessableEntity from module_build_service.resolver.base import GenericResolver import inspect -import pprint import logging import kobo.rpmlib log = logging.getLogger() diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 603f9540..5cdd034d 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -37,14 +37,14 @@ import datetime from module_build_service import log, conf from module_build_service.errors import ( Forbidden, ValidationError, UnprocessableEntity, ProgrammingError) -import module_build_service.utils +from module_build_service.utils.general import scm_url_schemes, retry class SCM(object): "SCM abstraction class" # Assuming git for HTTP schemas - types = module_build_service.utils.scm_url_schemes() + types = scm_url_schemes() def __init__(self, url, branch=None, allowed_scm=None, allow_local=False): """Initialize the SCM object using the specified scmurl. @@ -129,7 +129,7 @@ class SCM(object): return None @staticmethod - @module_build_service.utils.retry( + @retry( timeout=conf.scm_net_timeout, interval=conf.scm_net_retry_interval, wait_on=UnprocessableEntity) diff --git a/module_build_service/utils.py b/module_build_service/utils.py deleted file mode 100644 index e2eb1aac..00000000 --- a/module_build_service/utils.py +++ /dev/null @@ -1,1974 +0,0 @@ -# Copyright (c) 2016 Red Hat, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Written by Ralph Bean -# Matt Prahl - -""" Utility functions for module_build_service. """ -import re -import copy -import functools -import time -import shutil -import tempfile -import os -import kobo.rpmlib -import inspect -import hashlib -from functools import wraps - -from flask import request, url_for, Response -from datetime import datetime -from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean - -from module_build_service import log, models, Modulemd, api_version -from module_build_service.errors import (ValidationError, UnprocessableEntity, - ProgrammingError, NotFound, StreamAmbigous) -from module_build_service import conf, db -from module_build_service.errors import (Forbidden, Conflict) -import module_build_service.messaging -from multiprocessing.dummy import Pool as ThreadPool -import module_build_service.resolver -from module_build_service import glib -from module_build_service.mmd_resolver import MMDResolver - -import concurrent.futures - - -def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): - """ A decorator that allows to retry a section of code... - ...until success or timeout. - """ - def wrapper(function): - @functools.wraps(function) - def inner(*args, **kwargs): - start = time.time() - while True: - try: - return function(*args, **kwargs) - except wait_on as e: - log.warn("Exception %r raised from %r. Retry in %rs" % ( - e, function, interval)) - time.sleep(interval) - if (time.time() - start) >= timeout: - raise # This re-raises the last exception. - return inner - return wrapper - - -def at_concurrent_component_threshold(config, session): - """ - Determines if the number of concurrent component builds has reached - the configured threshold - :param config: Module Build Service configuration object - :param session: SQLAlchemy database session - :return: boolean representing if there are too many concurrent builds at - this time - """ - - # We must not check it for "mock" backend. - # It would lead to multiple calls of continue_batch_build method and - # creation of multiple worker threads there. Mock backend uses thread-id - # to create and identify mock buildroot and for mock backend, we must - # build whole module in this single continue_batch_build call to keep - # the number of created buildroots low. The concurrent build limit - # for mock backend is secured by setting max_workers in - # ThreadPoolExecutor to num_concurrent_builds. - if conf.system == "mock": - return False - - import koji # Placed here to avoid py2/py3 conflicts... - - if config.num_concurrent_builds and config.num_concurrent_builds <= \ - session.query(models.ComponentBuild).filter_by( - state=koji.BUILD_STATES['BUILDING'], - # Components which are reused should not be counted in, because - # we do not submit new build for them. They are in BUILDING state - # just internally in MBS to be handled by - # scheduler.handlers.components.complete. - reused_component_id=None).count(): - return True - - return False - - -def start_build_component(builder, c): - """ - Submits single component build to builder. Called in thread - by QueueBasedThreadPool in continue_batch_build. - """ - import koji - try: - c.task_id, c.state, c.state_reason, c.nvr = builder.build( - artifact_name=c.package, source=c.scmurl) - except Exception as e: - c.state = koji.BUILD_STATES['FAILED'] - c.state_reason = "Failed to build artifact %s: %s" % (c.package, str(e)) - log.exception(e) - return - - if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']: - c.state = koji.BUILD_STATES['FAILED'] - c.state_reason = ("Failed to build artifact %s: " - "Builder did not return task ID" % (c.package)) - return - - -def continue_batch_build(config, module, session, builder, components=None): - """ - Continues building current batch. Submits next components in the batch - until it hits concurrent builds limit. - - Returns list of BaseMessage instances which should be scheduled by the - scheduler. - """ - import koji # Placed here to avoid py2/py3 conflicts... - - # The user can either pass in a list of components to 'seed' the batch, or - # if none are provided then we just select everything that hasn't - # successfully built yet or isn't currently being built. - unbuilt_components = components or [ - c for c in module.component_builds - if (c.state != koji.BUILD_STATES['COMPLETE'] and - c.state != koji.BUILD_STATES['BUILDING'] and - c.state != koji.BUILD_STATES['FAILED'] and - c.batch == module.batch) - ] - - if not unbuilt_components: - log.debug("Cannot continue building module %s. No component to build." % module) - return [] - - # Get the list of components to be built in this batch. We are not building - # all `unbuilt_components`, because we can meet the num_concurrent_builds - # threshold - further_work = [] - components_to_build = [] - # Sort the unbuilt_components so that the components that take the longest to build are - # first - unbuilt_components.sort(key=lambda c: c.weight, reverse=True) - - # Check for builds that exist in the build system but MBS doesn't know about - for component in unbuilt_components: - # Only evaluate new components - if component.state is not None: - continue - msgs = builder.recover_orphaned_artifact(component) - further_work += msgs - - for c in unbuilt_components: - # If a previous build of the component was found, then the state will be marked as - # COMPLETE so we should skip this - if c.state == koji.BUILD_STATES['COMPLETE']: - continue - # Check the concurrent build threshold. - if at_concurrent_component_threshold(config, session): - log.info('Concurrent build threshold met') - break - - # We set state to "BUILDING" here because at this point we are committed - # to build the component and at_concurrent_component_threshold() works by - # counting the number of components in the "BUILDING" state. - c.state = koji.BUILD_STATES['BUILDING'] - components_to_build.append(c) - - # Start build of components in this batch. - max_workers = 1 - if config.num_concurrent_builds > 0: - max_workers = config.num_concurrent_builds - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = {executor.submit(start_build_component, builder, c): - c for c in components_to_build} - concurrent.futures.wait(futures) - # In case there has been an excepion generated directly in the - # start_build_component, the future.result() will re-raise it in the - # main thread so it is not lost. - for future in futures: - future.result() - - session.commit() - return further_work - - -def start_next_batch_build(config, module, session, builder, components=None): - """ - Tries to start the build of next batch. In case there are still unbuilt - components in a batch, tries to submit more components until it hits - concurrent builds limit. Otherwise Increments module.batch and submits component - builds from the next batch. - - :return: a list of BaseMessage instances to be handled by the MBSConsumer. - """ - import koji # Placed here to avoid py2/py3 conflicts... - - # Check the status of the module build and current batch so we can - # later decide if we can start new batch or not. - has_unbuilt_components = False - has_unbuilt_components_in_batch = False - has_building_components_in_batch = False - has_failed_components = False - # This is used to determine if it's worth checking if a component can be reused - # later on in the code - all_reused_in_prev_batch = True - for c in module.component_builds: - if c.state in [None, koji.BUILD_STATES['BUILDING']]: - has_unbuilt_components = True - - if c.batch == module.batch: - if not c.state: - has_unbuilt_components_in_batch = True - elif c.state == koji.BUILD_STATES['BUILDING']: - has_building_components_in_batch = True - elif (c.state in [koji.BUILD_STATES['FAILED'], - koji.BUILD_STATES['CANCELED']]): - has_failed_components = True - - if c.batch == module.batch and not c.reused_component_id: - all_reused_in_prev_batch = False - - # Do not start new batch if there are no components to build. - if not has_unbuilt_components: - log.debug("Not starting new batch, there is no component to build " - "for module %s" % module) - return [] - - # Check that there is something to build in current batch before starting - # the new one. If there is, continue building current batch. - if has_unbuilt_components_in_batch: - log.info("Continuing building batch %d", module.batch) - return continue_batch_build( - config, module, session, builder, components) - - # Check that there are no components in BUILDING state in current batch. - # If there are, wait until they are built. - if has_building_components_in_batch: - log.debug("Not starting new batch, there are still components in " - "BUILDING state in current batch for module %s", module) - return [] - - # Check that there are no failed components in this batch. If there are, - # do not start the new batch. - if has_failed_components: - log.info("Not starting new batch, there are failed components for " - "module %s", module) - return [] - - # Identify active tasks which might contain relicts of previous builds - # and fail the module build if this^ happens. - active_tasks = builder.list_tasks_for_components(module.component_builds, - state='active') - if isinstance(active_tasks, list) and active_tasks: - state_reason = ("Cannot start a batch, because some components are already" - " in 'building' state.") - state_reason += " See tasks (ID): {}".format( - ', '.join([str(t['id']) for t in active_tasks]) - ) - module.transition(config, state=models.BUILD_STATES['failed'], - state_reason=state_reason) - session.commit() - return [] - - else: - log.debug("Builder {} doesn't provide information about active tasks." - .format(builder)) - - # Find out if there is repo regeneration in progress for this module. - # If there is, wait until the repo is regenerated before starting a new - # batch. - artifacts = [c.nvr for c in module.current_batch()] - if not builder.buildroot_ready(artifacts): - log.info("Not starting new batch, not all of %r are in the buildroot. " - "Waiting." % artifacts) - return [] - - # Although this variable isn't necessary, it is easier to read code later on with it - prev_batch = module.batch - module.batch += 1 - - # The user can either pass in a list of components to 'seed' the batch, or - # if none are provided then we just select everything that hasn't - # successfully built yet or isn't currently being built. - unbuilt_components = components or [ - c for c in module.component_builds - if (c.state != koji.BUILD_STATES['COMPLETE'] and - c.state != koji.BUILD_STATES['BUILDING'] and - c.state != koji.BUILD_STATES['FAILED'] and - c.batch == module.batch) - ] - - # If there are no components to build, skip the batch and start building - # the new one. This can happen when resubmitting the failed module build. - if not unbuilt_components and not components: - log.info("Skipping build of batch %d, no component to build.", - module.batch) - return start_next_batch_build(config, module, session, builder) - - log.info("Starting build of next batch %d, %s" % (module.batch, - unbuilt_components)) - - # Attempt to reuse any components possible in the batch before attempting to build any - further_work = [] - unbuilt_components_after_reuse = [] - components_reused = False - should_try_reuse = True - # If the rebuild strategy is "changed-and-after", try to figure out if it's worth checking if - # the components can be reused to save on resources - if module.rebuild_strategy == 'changed-and-after': - # Check to see if the previous batch had all their builds reused except for when the - # previous batch was 1 because that always has the module-build-macros component built - should_try_reuse = all_reused_in_prev_batch or prev_batch == 1 - if should_try_reuse: - component_names = [c.package for c in unbuilt_components] - reusable_components = get_reusable_components( - session, module, component_names) - for c, reusable_c in zip(unbuilt_components, reusable_components): - if reusable_c: - components_reused = True - further_work += reuse_component(c, reusable_c) - else: - unbuilt_components_after_reuse.append(c) - # Commit the changes done by reuse_component - if components_reused: - session.commit() - - # If all the components were reused in the batch then make a KojiRepoChange - # message and return - if components_reused and not unbuilt_components_after_reuse: - further_work.append(module_build_service.messaging.KojiRepoChange( - 'start_build_batch: fake msg', builder.module_build_tag['name'])) - return further_work - - return further_work + continue_batch_build( - config, module, session, builder, unbuilt_components_after_reuse) - - -def pagination_metadata(p_query, api_version, request_args): - """ - Returns a dictionary containing metadata about the paginated query. - This must be run as part of a Flask request. - :param p_query: flask_sqlalchemy.Pagination object - :param api_version: an int of the API version - :param request_args: a dictionary of the arguments that were part of the - Flask request - :return: a dictionary containing metadata about the paginated query - """ - request_args_wo_page = dict(copy.deepcopy(request_args)) - # Remove pagination related args because those are handled elsewhere - # Also, remove any args that url_for accepts in case the user entered - # those in - for key in ['page', 'per_page', 'endpoint']: - if key in request_args_wo_page: - request_args_wo_page.pop(key) - for key in request_args: - if key.startswith('_'): - request_args_wo_page.pop(key) - - pagination_data = { - 'page': p_query.page, - 'pages': p_query.pages, - 'per_page': p_query.per_page, - 'prev': None, - 'next': None, - 'total': p_query.total, - 'first': url_for(request.endpoint, api_version=api_version, page=1, - per_page=p_query.per_page, _external=True, **request_args_wo_page), - 'last': url_for(request.endpoint, api_version=api_version, page=p_query.pages, - per_page=p_query.per_page, _external=True, - **request_args_wo_page) - } - - if p_query.has_prev: - pagination_data['prev'] = url_for(request.endpoint, api_version=api_version, - page=p_query.prev_num, per_page=p_query.per_page, - _external=True, **request_args_wo_page) - if p_query.has_next: - pagination_data['next'] = url_for(request.endpoint, api_version=api_version, - page=p_query.next_num, per_page=p_query.per_page, - _external=True, **request_args_wo_page) - - return pagination_data - - -def _add_order_by_clause(flask_request, query, column_source): - """ - Orders the given SQLAlchemy query based on the GET arguments provided - :param flask_request: a Flask request object - :param query: a SQLAlchemy query object - :param column_source: a SQLAlchemy database model - :return: a SQLAlchemy query object - """ - colname = "id" - descending = True - order_desc_by = flask_request.args.get("order_desc_by", None) - if order_desc_by: - colname = order_desc_by - else: - order_by = flask_request.args.get("order_by", None) - if order_by: - colname = order_by - descending = False - - column = getattr(column_source, colname, None) - if not column: - raise ValidationError('An invalid order_by or order_desc_by key ' - 'was supplied') - if descending: - column = column.desc() - return query.order_by(column) - - -def str_to_bool(value): - """ - Parses a string to determine its boolean value - :param value: a string - :return: a boolean - """ - return value.lower() in ["true", "1"] - - -def filter_component_builds(flask_request): - """ - Returns a flask_sqlalchemy.Pagination object based on the request parameters - :param request: Flask request object - :return: flask_sqlalchemy.Pagination - """ - search_query = dict() - for key in request.args.keys(): - # Only filter on valid database columns - if key in models.ComponentBuild.__table__.columns.keys(): - if isinstance(models.ComponentBuild.__table__.columns[key].type, sqlalchemy_boolean): - search_query[key] = str_to_bool(flask_request.args[key]) - else: - search_query[key] = flask_request.args[key] - - state = flask_request.args.get('state', None) - if state: - if state.isdigit(): - search_query['state'] = state - else: - try: - import koji - except ImportError: - raise ValidationError('Cannot filter by state names because koji isn\'t installed') - - if state.upper() in koji.BUILD_STATES: - search_query['state'] = koji.BUILD_STATES[state.upper()] - else: - raise ValidationError('An invalid state was supplied') - - # Allow the user to specify the module build ID with a more intuitive key name - if 'module_build' in flask_request.args: - search_query['module_id'] = flask_request.args['module_build'] - - query = models.ComponentBuild.query - - if search_query: - query = query.filter_by(**search_query) - - query = _add_order_by_clause(flask_request, query, models.ComponentBuild) - - page = flask_request.args.get('page', 1, type=int) - per_page = flask_request.args.get('per_page', 10, type=int) - return query.paginate(page, per_page, False) - - -def filter_module_builds(flask_request): - """ - Returns a flask_sqlalchemy.Pagination object based on the request parameters - :param request: Flask request object - :return: flask_sqlalchemy.Pagination - """ - search_query = dict() - special_columns = ['time_submitted', 'time_modified', 'time_completed', 'state'] - for key in request.args.keys(): - # Only filter on valid database columns but skip columns that are treated specially or - # ignored - if key not in special_columns and key in models.ModuleBuild.__table__.columns.keys(): - search_query[key] = flask_request.args[key] - - state = flask_request.args.get('state', None) - if state: - if state.isdigit(): - search_query['state'] = state - else: - if state in models.BUILD_STATES: - search_query['state'] = models.BUILD_STATES[state] - else: - raise ValidationError('An invalid state was supplied') - - query = models.ModuleBuild.query - - if search_query: - query = query.filter_by(**search_query) - - # This is used when filtering the date request parameters, but it is here to avoid recompiling - utc_iso_datetime_regex = re.compile( - r'^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?' - r'(?:Z|[-+]00(?::00)?)?$') - - # Filter the query based on date request parameters - for item in ('submitted', 'modified', 'completed'): - for context in ('before', 'after'): - request_arg = '%s_%s' % (item, context) # i.e. submitted_before - iso_datetime_arg = request.args.get(request_arg, None) - - if iso_datetime_arg: - iso_datetime_matches = re.match(utc_iso_datetime_regex, iso_datetime_arg) - - if not iso_datetime_matches or not iso_datetime_matches.group('datetime'): - raise ValidationError(('An invalid Zulu ISO 8601 timestamp was provided' - ' for the "%s" parameter') - % request_arg) - # Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter - item_datetime = datetime.strptime(iso_datetime_matches.group('datetime'), - '%Y-%m-%dT%H:%M:%S') - # Get the database column to filter against - column = getattr(models.ModuleBuild, 'time_' + item) - - if context == 'after': - query = query.filter(column >= item_datetime) - elif context == 'before': - query = query.filter(column <= item_datetime) - - query = _add_order_by_clause(flask_request, query, models.ModuleBuild) - - page = flask_request.args.get('page', 1, type=int) - per_page = flask_request.args.get('per_page', 10, type=int) - return query.paginate(page, per_page, False) - - -def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): - # Import it here, because SCM uses utils methods - # and fails to import them because of dep-chain. - import module_build_service.scm - - td = None - scm = None - try: - log.debug('Verifying modulemd') - td = tempfile.mkdtemp() - if whitelist_url: - scm = module_build_service.scm.SCM(url, branch, [url], allow_local_url) - else: - scm = module_build_service.scm.SCM(url, branch, conf.scmurls, allow_local_url) - scm.checkout(td) - scm.verify() - cofn = scm.get_module_yaml() - mmd = load_mmd(cofn, is_file=True) - finally: - try: - if td is not None: - shutil.rmtree(td) - except Exception as e: - log.warning( - "Failed to remove temporary directory {!r}: {}".format( - td, str(e))) - - # If the name was set in the modulemd, make sure it matches what the scmurl - # says it should be - if mmd.get_name() and mmd.get_name() != scm.name: - raise ValidationError('The name "{0}" that is stored in the modulemd ' - 'is not valid'.format(mmd.get_name())) - else: - mmd.set_name(scm.name) - - # If the stream was set in the modulemd, make sure it matches what the repo - # branch is - if mmd.get_stream() and mmd.get_stream() != scm.branch: - raise ValidationError('The stream "{0}" that is stored in the modulemd ' - 'does not match the branch "{1}"'.format( - mmd.get_stream(), scm.branch)) - else: - mmd.set_stream(str(scm.branch)) - - # If the version is in the modulemd, throw an exception since the version - # is generated by pdc-updater - if mmd.get_version(): - raise ValidationError('The version "{0}" is already defined in the ' - 'modulemd but it shouldn\'t be since the version ' - 'is generated based on the commit time'.format( - mmd.get_version())) - else: - mmd.set_version(int(scm.version)) - - return mmd, scm - - -def load_mmd(yaml, is_file=False): - try: - if is_file: - mmd = Modulemd.Module().new_from_file(yaml) - else: - mmd = Modulemd.Module().new_from_string(yaml) - # If the modulemd was v1, it will be upgraded to v2 - mmd.upgrade() - except Exception: - error = 'The following invalid modulemd was encountered: {0}'.format(yaml) - log.error(error) - raise UnprocessableEntity(error) - - return mmd - - -def _scm_get_latest(pkg): - try: - # If the modulemd specifies that the 'f25' branch is what - # we want to pull from, we need to resolve that f25 branch - # to the specific commit available at the time of - # submission (now). - pkgref = module_build_service.scm.SCM( - pkg.get_repository()).get_latest(pkg.get_ref()) - except Exception as e: - log.exception(e) - return {'error': "Failed to get the latest commit for %s#%s" % ( - pkg.get_repository(), pkg.get_ref())} - - return { - 'pkg_name': pkg.get_name(), - 'pkg_ref': pkgref, - 'error': None - } - - -def load_local_builds(local_build_nsvs, session=None): - """ - Loads previously finished local module builds from conf.mock_resultsdir - and imports them to database. - - :param local_build_nsvs: List of NSV separated by ':' defining the modules - to load from the mock_resultsdir. - """ - if not local_build_nsvs: - return - - if not session: - session = db.session - - if type(local_build_nsvs) != list: - local_build_nsvs = [local_build_nsvs] - - # Get the list of all available local module builds. - builds = [] - try: - for d in os.listdir(conf.mock_resultsdir): - m = re.match('^module-(.*)-([^-]*)-([0-9]+)$', d) - if m: - builds.append((m.group(1), m.group(2), int(m.group(3)), d)) - except OSError: - pass - - # Sort with the biggest version first - try: - # py27 - builds.sort(lambda a, b: -cmp(a[2], b[2])) - except TypeError: - # py3 - builds.sort(key=lambda a: a[2], reverse=True) - - for build_id in local_build_nsvs: - parts = build_id.split(':') - if len(parts) < 1 or len(parts) > 3: - raise RuntimeError( - 'The local build "{0}" couldn\'t be be parsed into ' - 'NAME[:STREAM[:VERSION]]'.format(build_id)) - - name = parts[0] - stream = parts[1] if len(parts) > 1 else None - version = int(parts[2]) if len(parts) > 2 else None - - found_build = None - for build in builds: - if name != build[0]: - continue - if stream is not None and stream != build[1]: - continue - if version is not None and version != build[2]: - continue - - found_build = build - break - - if not found_build: - raise RuntimeError( - 'The local build "{0}" couldn\'t be found in "{1}"'.format( - build_id, conf.mock_resultsdir)) - - # Load the modulemd metadata. - path = os.path.join(conf.mock_resultsdir, found_build[3], 'results') - mmd = load_mmd(os.path.join(path, 'modules.yaml'), is_file=True) - - # Create ModuleBuild in database. - module = models.ModuleBuild.create( - session, - conf, - name=mmd.get_name(), - stream=mmd.get_stream(), - version=str(mmd.get_version()), - modulemd=mmd.dumps(), - scmurl="", - username="mbs", - publish_msg=False) - module.koji_tag = path - module.state = models.BUILD_STATES['ready'] - session.commit() - - if (found_build[0] != module.name or found_build[1] != module.stream or - str(found_build[2]) != module.version): - raise RuntimeError( - 'Parsed metadata results for "{0}" don\'t match the directory name' - .format(found_build[3])) - log.info("Loaded local module build %r", module) - - -def format_mmd(mmd, scmurl, session=None): - """ - Prepares the modulemd for the MBS. This does things such as replacing the - branches of components with commit hashes and adding metadata in the xmd - dictionary. - :param mmd: the ModuleMetadata object to format - :param scmurl: the url to the modulemd - """ - # Import it here, because SCM uses utils methods and fails to import - # them because of dep-chain. - from module_build_service.scm import SCM - - if not session: - session = db.session - - xmd = glib.from_variant_dict(mmd.get_xmd()) - if 'mbs' not in xmd: - xmd['mbs'] = {} - if 'scmurl' not in xmd['mbs']: - xmd['mbs']['scmurl'] = scmurl or '' - if 'commit' not in xmd['mbs']: - xmd['mbs']['commit'] = '' - - local_modules = models.ModuleBuild.local_modules(session) - local_modules = {m.name + "-" + m.stream: m for m in local_modules} - - # If module build was submitted via yaml file, there is no scmurl - if scmurl: - scm = SCM(scmurl) - # If a commit hash is provided, add that information to the modulemd - if scm.commit: - # We want to make sure we have the full commit hash for consistency - if SCM.is_full_commit_hash(scm.scheme, scm.commit): - full_scm_hash = scm.commit - else: - full_scm_hash = scm.get_full_commit_hash() - - xmd['mbs']['commit'] = full_scm_hash - # If a commit hash wasn't provided then just get the latest from master - else: - xmd['mbs']['commit'] = scm.get_latest() - - if mmd.get_rpm_components() or mmd.get_module_components(): - if 'rpms' not in xmd['mbs']: - xmd['mbs']['rpms'] = {} - # Add missing data in RPM components - for pkgname, pkg in mmd.get_rpm_components().items(): - if pkg.get_repository() and not conf.rpms_allow_repository: - raise Forbidden( - "Custom component repositories aren't allowed. " - "%r bears repository %r" % (pkgname, pkg.get_repository())) - if pkg.get_cache() and not conf.rpms_allow_cache: - raise Forbidden( - "Custom component caches aren't allowed. " - "%r bears cache %r" % (pkgname, pkg.cache)) - if not pkg.get_repository(): - pkg.set_repository(conf.rpms_default_repository + pkgname) - if not pkg.get_cache(): - pkg.set_cache(conf.rpms_default_cache + pkgname) - if not pkg.get_ref(): - pkg.set_ref('master') - - # Add missing data in included modules components - for modname, mod in mmd.get_module_components().items(): - if mod.get_repository() and not conf.modules_allow_repository: - raise Forbidden( - "Custom module repositories aren't allowed. " - "%r bears repository %r" % (modname, mod.get_repository())) - if not mod.get_repository(): - mod.set_repository(conf.modules_default_repository + modname) - if not mod.get_ref(): - mod.set_ref('master') - - # Check that SCM URL is valid and replace potential branches in pkg refs - # by real SCM hash and store the result to our private xmd place in modulemd. - pool = ThreadPool(20) - pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values()) - err_msg = "" - for pkg_dict in pkg_dicts: - if pkg_dict["error"]: - err_msg += pkg_dict["error"] + "\n" - else: - pkg_name = pkg_dict["pkg_name"] - pkg_ref = pkg_dict["pkg_ref"] - xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref} - if err_msg: - raise UnprocessableEntity(err_msg) - - # Set the modified xmd back to the modulemd - mmd.set_xmd(glib.dict_values(xmd)) - - -def validate_mmd(mmd): - for modname, mod in mmd.get_module_components().items(): - if mod.get_repository() and not conf.modules_allow_repository: - raise Forbidden( - "Custom module repositories aren't allowed. " - "%r bears repository %r" % (modname, mod.get_repository())) - - -def merge_included_mmd(mmd, included_mmd): - """ - Merges two modulemds. This merges only metadata which are needed in - the `main` when it includes another module defined by `included_mmd` - """ - included_xmd = glib.from_variant_dict(included_mmd.get_xmd()) - if 'rpms' in included_xmd['mbs']: - xmd = glib.from_variant_dict(mmd.get_xmd()) - if 'rpms' not in xmd['mbs']: - xmd['mbs']['rpms'] = included_xmd['mbs']['rpms'] - else: - xmd['mbs']['rpms'].update(included_xmd['mbs']['rpms']) - # Set the modified xmd back to the modulemd - mmd.set_xmd(glib.dict_values(xmd)) - - -def record_component_builds(mmd, module, initial_batch=1, - previous_buildorder=None, main_mmd=None, session=None): - # Imported here to allow import of utils in GenericBuilder. - import module_build_service.builder - - if not session: - session = db.session - - # Format the modulemd by putting in defaults and replacing streams that - # are branches with commit hashes - format_mmd(mmd, module.scmurl, session=session) - - # When main_mmd is set, merge the metadata from this mmd to main_mmd, - # otherwise our current mmd is main_mmd. - if main_mmd: - # Check for components that are in both MMDs before merging since MBS - # currently can't handle that situation. - duplicate_components = [rpm for rpm in main_mmd.get_rpm_components().keys() - if rpm in mmd.get_rpm_components()] - if duplicate_components: - error_msg = ( - 'The included module "{0}" in "{1}" have the following ' - 'conflicting components: {2}'.format( - mmd.get_name(), main_mmd.get_name(), ', '.join(duplicate_components))) - raise UnprocessableEntity(error_msg) - merge_included_mmd(main_mmd, mmd) - else: - main_mmd = mmd - - # If the modulemd yaml specifies components, then submit them for build - rpm_components = mmd.get_rpm_components().values() - module_components = mmd.get_module_components().values() - all_components = rpm_components + module_components - if not all_components: - return - - rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights( - [c.get_name() for c in rpm_components]) - all_components.sort(key=lambda x: x.get_buildorder()) - # We do not start with batch = 0 here, because the first batch is - # reserved for module-build-macros. First real components must be - # planned for batch 2 and following. - batch = initial_batch - - for component in all_components: - # Increment the batch number when buildorder increases. - if previous_buildorder != component.get_buildorder(): - previous_buildorder = component.get_buildorder() - batch += 1 - - # If the component is another module, we fetch its modulemd file - # and record its components recursively with the initial_batch - # set to our current batch, so the components of this module - # are built in the right global order. - if isinstance(component, Modulemd.ComponentModule): - full_url = component.get_repository() + "?#" + component.get_ref() - # It is OK to whitelist all URLs here, because the validity - # of every URL have been already checked in format_mmd(...). - included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0] - batch = record_component_builds(included_mmd, module, batch, - previous_buildorder, main_mmd, session=session) - continue - - component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref'] - full_url = component.get_repository() + "?#" + component_ref - build = models.ComponentBuild( - module_id=module.id, - package=component.get_name(), - format="rpms", - scmurl=full_url, - batch=batch, - ref=component_ref, - weight=rpm_weights[component.get_name()] - ) - session.add(build) - - return batch - - -def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False, - optional_params=None): - yaml_file = handle.read() - mmd = load_mmd(yaml_file) - - # Mimic the way how default values are generated for modules that are stored in SCM - # We can take filename as the module name as opposed to repo name, - # and also we can take numeric representation of current datetime - # as opposed to datetime of the last commit - dt = datetime.utcfromtimestamp(int(time.time())) - def_name = str(os.path.splitext(os.path.basename(handle.filename))[0]) - def_version = int(dt.strftime("%Y%m%d%H%M%S")) - mmd.set_name(mmd.get_name() or def_name) - mmd.set_stream(stream or mmd.get_stream() or "master") - mmd.set_version(mmd.get_version() or def_version) - if skiptests: - buildopts = mmd.get_rpm_buildopts() - buildopts["macros"] = buildopts.get("macros", "") + "\n\n%__spec_check_pre exit 0\n" - mmd.set_rpm_buildopts(buildopts) - return submit_module_build(username, None, mmd, None, optional_params) - - -_url_check_re = re.compile(r"^[^:/]+:.*$") - - -def submit_module_build_from_scm(username, url, branch, allow_local_url=False, - optional_params=None): - # Translate local paths into file:// URL - if allow_local_url and not _url_check_re.match(url): - log.info( - "'{}' is not a valid URL, assuming local path".format(url)) - url = os.path.abspath(url) - url = "file://" + url - mmd, scm = _fetch_mmd(url, branch, allow_local_url) - - return submit_module_build(username, url, mmd, scm, optional_params) - - -def generate_expanded_mmds(session, mmd, raise_if_stream_ambigous=False, default_streams=None): - """ - Returns list with MMDs with buildrequires and requires set according - to module stream expansion rules. These module metadata can be directly - built using MBS. - - :param session: SQLAlchemy DB session. - :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - """ - if not session: - session = db.session - - if not default_streams: - default_streams = {} - - # Create local copy of mmd, because we will expand its dependencies, - # which would change the module. - # TODO: Use copy method once its in released libmodulemd: - # https://github.com/fedora-modularity/libmodulemd/pull/20 - current_mmd = Modulemd.Module.new_from_string(mmd.dumps()) - - # MMDResolver expects the input MMD to have no context. - current_mmd.set_context(None) - - # Expands the MSE streams. This mainly handles '-' prefix in MSE streams. - expand_mse_streams(session, current_mmd, default_streams, raise_if_stream_ambigous) - - # Get the list of all MMDs which this module can be possibly built against - # and add them to MMDResolver. - mmd_resolver = MMDResolver() - mmds_for_resolving = get_mmds_required_by_module_recursively( - session, current_mmd, default_streams, raise_if_stream_ambigous) - for m in mmds_for_resolving: - mmd_resolver.add_modules(m) - - # Show log.info message with the NSVCs we have added to mmd_resolver. - nsvcs_to_solve = [ - ":".join([m.get_name(), m.get_stream(), str(m.get_version()), str(m.get_context())]) - for m in mmds_for_resolving] - log.info("Starting resolving with following input modules: %r", nsvcs_to_solve) - - # Resolve the dependencies between modules and get the list of all valid - # combinations in which we can build this module. - requires_combinations = mmd_resolver.solve(current_mmd) - log.info("Resolving done, possible requires: %r", requires_combinations) - - # This is where we are going to store the generated MMDs. - mmds = [] - for requires in requires_combinations: - # Each generated MMD must be new Module object... - # TODO: Use copy method once its in released libmodulemd: - # https://github.com/fedora-modularity/libmodulemd/pull/20 - mmd_copy = Modulemd.Module.new_from_string(mmd.dumps()) - xmd = glib.from_variant_dict(mmd_copy.get_xmd()) - - # Requires contain the NSVC representing the input mmd. - # The 'context' of this NSVC defines the id of buildrequires/requires - # pair in the mmd.get_dependencies(). - dependencies_id = None - - # We don't want to depend on ourselves, so store the NSVC of the current_mmd - # to be able to ignore it later. - self_nsvca = None - - # Dict to store name:stream pairs from nsvca, so we are able to access it - # easily later. - req_name_stream = {} - - # Get the values for dependencies_id, self_nsvca and req_name_stream variables. - for nsvca in requires: - req_name, req_stream, _ = nsvca.split(":", 2) - if req_name == current_mmd.get_name() and req_stream == current_mmd.get_stream(): - dependencies_id = int(nsvca.split(":")[3]) - self_nsvca = nsvca - continue - req_name_stream[req_name] = req_stream - if dependencies_id is None or self_nsvca is None: - raise RuntimeError( - "%s:%s not found in requires %r" % ( - current_mmd.get_name(), current_mmd.get_stream(), requires)) - - # The name:[streams, ...] pairs do not have to be the same in both - # buildrequires/requires. In case they are the same, we replace the streams - # in requires section with a single stream against which we will build this MMD. - # In case they are not the same, we have to keep the streams as they are in requires - # section. We always replace stream(s) for build-requirement with the one we - # will build this MMD against. - new_dep = Modulemd.Dependencies() - dep = mmd_copy.get_dependencies()[dependencies_id] - dep_requires = dep.get_requires() - dep_buildrequires = dep.get_buildrequires() - for req_name, req_streams in dep_requires.items(): - if (req_name not in dep_buildrequires or - set(req_streams.get()) != set(dep_buildrequires[req_name].get())): - # Streams in runtime section are not the same as in buildtime section, - # so just copy this runtime requirement to new_dep. - new_dep.add_requires(req_name, req_streams.get()) - else: - # This runtime requirement has the same streams in both runtime/buildtime - # requires sections, so replace streams in both sections by the one we - # really used in this resolved variant. - new_dep.add_requires(req_name, [req_name_stream[req_name]]) - new_dep.add_buildrequires(req_name, [req_name_stream[req_name]]) - mmd_copy.set_dependencies((new_dep, )) - - # The Modulemd.Dependencies() stores only streams, but to really build this - # module, we need NSVC of buildrequires, so we have to store this data in XMD. - # We also need additional data like for example list of filtered_rpms. We will - # get them using module_build_service.resolver.GenericResolver.resolve_requires, - # so prepare list with NSVCs of buildrequires as an input for this method. - br_list = [] - for nsvca in requires: - if nsvca == self_nsvca: - continue - # Remove the arch from nsvca - nsvc = ":".join(nsvca.split(":")[:-1]) - br_list.append(nsvc) - - # Resolve the buildrequires and store the result in XMD. - if 'mbs' not in xmd: - xmd['mbs'] = {} - resolver = module_build_service.resolver.GenericResolver.create(conf) - xmd['mbs']['buildrequires'] = resolver.resolve_requires(br_list) - xmd['mbs']['mse'] = True - - mmd_copy.set_xmd(glib.dict_values(xmd)) - - # Now we have all the info to actually compute context of this module. - build_context, runtime_context = models.ModuleBuild.contexts_from_mmd(mmd_copy.dumps()) - context = models.ModuleBuild.context_from_contexts(build_context, runtime_context) - mmd_copy.set_context(context) - - mmds.append(mmd_copy) - - return mmds - - -def submit_module_build(username, url, mmd, scm, optional_params=None): - """ - Submits new module build. - - :param str username: Username of the build's owner. - :param str url: SCM URL of submitted build. - :param Modulemd.Module mmd: Modulemd defining the build. - :param scm.SCM scm: SCM class representing the cloned git repo. - :param dict optional_params: Dict with optional params for a build: - - "local_build" (bool): The module is being built locally (the MBS is - not running in infra, but on local developer's machine). - - "default_streams" (dict): Dict with name:stream mapping defining the stream - to choose for given module name if multiple streams are available to choose from. - - Any optional ModuleBuild class field (str). - :rtype: list with ModuleBuild - :return: List with submitted module builds. - """ - import koji # Placed here to avoid py2/py3 conflicts... - - # For local builds, we want the user to choose the exact stream using the default_streams - # in case there are multiple streams to choose from and raise an exception otherwise. - if optional_params and "local_build" in optional_params: - raise_if_stream_ambigous = True - del optional_params["local_build"] - else: - raise_if_stream_ambigous = False - - # Get the default_streams if set. - if optional_params and "default_streams" in optional_params: - default_streams = optional_params["default_streams"] - del optional_params["default_streams"] - else: - default_streams = {} - - validate_mmd(mmd) - mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams) - modules = [] - - for mmd in mmds: - log.debug('Checking whether module build already exists: %s.', - ":".join([mmd.get_name(), mmd.get_stream(), - str(mmd.get_version()), mmd.get_context()])) - module = models.ModuleBuild.get_build_from_nsvc( - db.session, mmd.get_name(), mmd.get_stream(), str(mmd.get_version()), - mmd.get_context()) - if module: - if module.state != models.BUILD_STATES['failed']: - err_msg = ('Module (state=%s) already exists. Only a new build or resubmission of ' - 'a failed build is allowed.' % module.state) - log.error(err_msg) - raise Conflict(err_msg) - if optional_params: - rebuild_strategy = optional_params.get('rebuild_strategy') - if rebuild_strategy and module.rebuild_strategy != rebuild_strategy: - raise ValidationError( - 'You cannot change the module\'s "rebuild_strategy" when ' - 'resuming a module build') - log.debug('Resuming existing module build %r' % module) - # Reset all component builds that didn't complete - for component in module.component_builds: - if component.state and component.state != koji.BUILD_STATES['COMPLETE']: - component.state = None - component.state_reason = None - db.session.add(component) - module.username = username - prev_state = module.previous_non_failed_state - if prev_state == models.BUILD_STATES['init']: - transition_to = models.BUILD_STATES['init'] - else: - transition_to = models.BUILD_STATES['wait'] - module.batch = 0 - module.transition(conf, transition_to, "Resubmitted by %s" % username) - log.info("Resumed existing module build in previous state %s" % module.state) - else: - log.debug('Creating new module build') - module = models.ModuleBuild.create( - db.session, - conf, - name=mmd.get_name(), - stream=mmd.get_stream(), - version=str(mmd.get_version()), - modulemd=mmd.dumps(), - scmurl=url, - username=username, - **(optional_params or {}) - ) - module.build_context, module.runtime_context = \ - module.contexts_from_mmd(module.modulemd) - - db.session.add(module) - db.session.commit() - modules.append(module) - log.info("%s submitted build of %s, stream=%s, version=%s, context=%s", username, - mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context()) - return modules - - -def scm_url_schemes(terse=False): - """ - Definition of URL schemes supported by both frontend and scheduler. - - NOTE: only git URLs in the following formats are supported atm: - git:// - git+http:// - git+https:// - git+rsync:// - http:// - https:// - file:// - - :param terse=False: Whether to return terse list of unique URL schemes - even without the "://". - """ - - scm_types = { - "git": ("git://", "git+http://", "git+https://", - "git+rsync://", "http://", "https://", "file://") - } - - if not terse: - return scm_types - else: - scheme_list = [] - for scm_type, scm_schemes in scm_types.items(): - scheme_list.extend([scheme[:-3] for scheme in scm_schemes]) - return list(set(scheme_list)) - - -def get_scm_url_re(): - schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) - return re.compile( - r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" - r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") - - -def module_build_state_from_msg(msg): - state = int(msg.module_build_state) - # TODO better handling - assert state in models.BUILD_STATES.values(), ( - 'state=%s(%s) is not in %s' - % (state, type(state), list(models.BUILD_STATES.values()))) - return state - - -def reuse_component(component, previous_component_build, - change_state_now=False): - """ - Reuses component build `previous_component_build` instead of building - component `component` - - Returns the list of BaseMessage instances to be handled later by the - scheduler. - """ - - import koji - - log.info( - 'Reusing component "{0}" from a previous module ' - 'build with the nvr "{1}"'.format( - component.package, previous_component_build.nvr)) - component.reused_component_id = previous_component_build.id - component.task_id = previous_component_build.task_id - if change_state_now: - component.state = previous_component_build.state - else: - # Use BUILDING state here, because we want the state to change to - # COMPLETE by the fake KojiBuildChange message we are generating - # few lines below. If we would set it to the right state right - # here, we would miss the code path handling the KojiBuildChange - # which works only when switching from BUILDING to COMPLETE. - component.state = koji.BUILD_STATES['BUILDING'] - component.state_reason = \ - 'Reused component from previous module build' - component.nvr = previous_component_build.nvr - nvr_dict = kobo.rpmlib.parse_nvr(component.nvr) - # Add this message to further_work so that the reused - # component will be tagged properly - return [ - module_build_service.messaging.KojiBuildChange( - msg_id='reuse_component: fake msg', - build_id=None, - task_id=component.task_id, - build_new_state=previous_component_build.state, - build_name=component.package, - build_version=nvr_dict['version'], - build_release=nvr_dict['release'], - module_build_id=component.module_id, - state_reason=component.state_reason - ) - ] - - -def _get_reusable_module(session, module): - """ - Returns previous module build of the module `module` in case it can be - used as a source module to get the components to reuse from. - - In case there is no such module, returns None. - - :param session: SQLAlchemy database session - :param module: the ModuleBuild object of module being built. - :return: ModuleBuild object which can be used for component reuse. - """ - mmd = module.mmd() - - # Find the latest module that is in the done or ready state - previous_module_build = session.query(models.ModuleBuild)\ - .filter_by(name=mmd.get_name())\ - .filter_by(stream=mmd.get_stream())\ - .filter(models.ModuleBuild.state.in_([3, 5]))\ - .filter(models.ModuleBuild.scmurl.isnot(None))\ - .order_by(models.ModuleBuild.time_completed.desc()) - # If we are rebuilding with the "changed-and-after" option, then we can't reuse - # components from modules that were built more liberally - if module.rebuild_strategy == 'changed-and-after': - previous_module_build = previous_module_build.filter( - models.ModuleBuild.rebuild_strategy.in_(['all', 'changed-and-after'])) - previous_module_build = previous_module_build.filter_by( - build_context=module.build_context) - previous_module_build = previous_module_build.first() - # The component can't be reused if there isn't a previous build in the done - # or ready state - if not previous_module_build: - log.info("Cannot re-use. %r is the first module build." % module) - return None - - return previous_module_build - - -def attempt_to_reuse_all_components(builder, session, module): - """ - Tries to reuse all the components in a build. The components are also - tagged to the tags using the `builder`. - - Returns True if all components could be reused, otherwise False. When - False is returned, no component has been reused. - """ - - previous_module_build = _get_reusable_module(session, module) - if not previous_module_build: - return False - - mmd = module.mmd() - old_mmd = previous_module_build.mmd() - - # [(component, component_to_reuse), ...] - component_pairs = [] - - # Find out if we can reuse all components and cache component and - # component to reuse pairs. - for c in module.component_builds: - if c.package == "module-build-macros": - continue - component_to_reuse = get_reusable_component( - session, module, c.package, - previous_module_build=previous_module_build, mmd=mmd, - old_mmd=old_mmd) - if not component_to_reuse: - return False - - component_pairs.append((c, component_to_reuse)) - - # Stores components we will tag to buildroot and final tag. - components_to_tag = [] - - # Reuse all components. - for c, component_to_reuse in component_pairs: - # Set the module.batch to the last batch we have. - if c.batch > module.batch: - module.batch = c.batch - - # Reuse the component - reuse_component(c, component_to_reuse, True) - components_to_tag.append(c.nvr) - - # Tag them - builder.buildroot_add_artifacts(components_to_tag, install=False) - builder.tag_artifacts(components_to_tag, dest_tag=True) - - return True - - -def get_reusable_components(session, module, component_names): - """ - Returns the list of ComponentBuild instances belonging to previous module - build which can be reused in the build of module `module`. - - The ComponentBuild instances in returned list are in the same order as - their names in the component_names input list. - - In case some component cannot be reused, None is used instead of a - ComponentBuild instance in the returned list. - - :param session: SQLAlchemy database session - :param module: the ModuleBuild object of module being built. - :param component_names: List of component names to be reused. - :return: List of ComponentBuild instances to reuse in the same - order as `component_names` - """ - # We support components reusing only for koji and test backend. - if conf.system not in ['koji', 'test']: - return [None] * len(component_names) - - previous_module_build = _get_reusable_module(session, module) - if not previous_module_build: - return [None] * len(component_names) - - mmd = module.mmd() - old_mmd = previous_module_build.mmd() - - ret = [] - for component_name in component_names: - ret.append(get_reusable_component( - session, module, component_name, previous_module_build, mmd, - old_mmd)) - - return ret - - -def get_reusable_component(session, module, component_name, - previous_module_build=None, mmd=None, old_mmd=None): - """ - Returns the component (RPM) build of a module that can be reused - instead of needing to rebuild it - :param session: SQLAlchemy database session - :param module: the ModuleBuild object of module being built with a formatted - mmd - :param component_name: the name of the component (RPM) that you'd like to - reuse a previous build of - :param previous_module_build: the ModuleBuild instances of a module build - which contains the components to reuse. If not passed, _get_reusable_module - is called to get the ModuleBuild instance. Consider passing the ModuleBuild - instance in case you plan to call get_reusable_component repeatedly for the - same module to make this method faster. - :param mmd: ModuleMd.Module of `module`. If not passed, it is taken from - module.mmd(). Consider passing this arg in case you plan to call - get_reusable_component repeatedly for the same module to make this method faster. - :param old_mmd: ModuleMd.Module of `previous_module_build`. If not passed, - it is taken from previous_module_build.mmd(). Consider passing this arg in - case you plan to call get_reusable_component repeatedly for the same - module to make this method faster. - :return: the component (RPM) build SQLAlchemy object, if one is not found, - None is returned - """ - - # We support component reusing only for koji and test backend. - if conf.system not in ['koji', 'test']: - return None - - # If the rebuild strategy is "all", that means that nothing can be reused - if module.rebuild_strategy == 'all': - log.info('Cannot re-use the component because the rebuild strategy is "all".') - return None - - if not previous_module_build: - previous_module_build = _get_reusable_module(session, module) - if not previous_module_build: - return None - - if not mmd: - mmd = module.mmd() - if not old_mmd: - old_mmd = previous_module_build.mmd() - - # If the chosen component for some reason was not found in the database, - # or the ref is missing, something has gone wrong and the component cannot - # be reused - new_module_build_component = models.ComponentBuild.from_component_name( - session, component_name, module.id) - if not new_module_build_component or not new_module_build_component.batch \ - or not new_module_build_component.ref: - log.info('Cannot re-use. New component not found in the db.') - return None - - prev_module_build_component = models.ComponentBuild.from_component_name( - session, component_name, previous_module_build.id) - # If the component to reuse for some reason was not found in the database, - # or the ref is missing, something has gone wrong and the component cannot - # be reused - if not prev_module_build_component or not prev_module_build_component.batch\ - or not prev_module_build_component.ref: - log.info('Cannot re-use. Previous component not found in the db.') - return None - - # Make sure the ref for the component that is trying to be reused - # hasn't changed since the last build - if prev_module_build_component.ref != new_module_build_component.ref: - log.info('Cannot re-use. Component commit hashes do not match.') - return None - - # At this point we've determined that both module builds contain the component - # and the components share the same commit hash - if module.rebuild_strategy == 'changed-and-after': - # Make sure the batch number for the component that is trying to be reused - # hasn't changed since the last build - if prev_module_build_component.batch != new_module_build_component.batch: - log.info('Cannot re-use. Batch numbers do not match.') - return None - - # If the mmd.buildopts.macros.rpms changed, we cannot reuse - if mmd.get_rpm_buildopts().get('macros') != old_mmd.get_rpm_buildopts().get('macros'): - log.info('Cannot re-use. Old modulemd macros do not match the new.') - return None - - # At this point we've determined that both module builds contain the component - # with the same commit hash and they are in the same batch. We've also determined - # that both module builds depend(ed) on the same exact module builds. Now it's time - # to determine if the components before it have changed. - # - # Convert the component_builds to a list and sort them by batch - new_component_builds = list(module.component_builds) - new_component_builds.sort(key=lambda x: x.batch) - prev_component_builds = list(previous_module_build.component_builds) - prev_component_builds.sort(key=lambda x: x.batch) - - new_module_build_components = [] - previous_module_build_components = [] - # Create separate lists for the new and previous module build. These lists - # will have an entry for every build batch *before* the component's - # batch except for 1, which is reserved for the module-build-macros RPM. - # Each batch entry will contain a set of "(name, ref)" with the name and - # ref (commit) of the component. - for i in range(new_module_build_component.batch - 1): - # This is the first batch which we want to skip since it will always - # contain only the module-build-macros RPM and it gets built every time - if i == 0: - continue - - new_module_build_components.append(set([ - (value.package, value.ref) for value in - new_component_builds if value.batch == i + 1 - ])) - - previous_module_build_components.append(set([ - (value.package, value.ref) for value in - prev_component_builds if value.batch == i + 1 - ])) - - # If the previous batches don't have the same ordering and hashes, then the - # component can't be reused - if previous_module_build_components != new_module_build_components: - log.info('Cannot re-use. Ordering or commit hashes of ' - 'previous batches differ.') - return None - - reusable_component = models.ComponentBuild.query.filter_by( - package=component_name, module_id=previous_module_build.id).one() - log.debug('Found reusable component!') - return reusable_component - - -def _expand_mse_streams(session, name, streams, default_streams, raise_if_stream_ambigous): - """ - Helper method for `expand_mse_stream()` expanding single name:[streams]. - Returns list of expanded streams. - - :param session: SQLAlchemy DB session. - :param str name: Name of the module which will be expanded. - :param list streams: List of streams to expand. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - """ - default_streams = default_streams or {} - # Stream can be prefixed with '-' sign to define that this stream should - # not appear in a resulting list of streams. There can be two situations: - # a) all streams have '-' prefix. In this case, we treat list of streams - # as blacklist and we find all the valid streams and just remove those with - # '-' prefix. - # b) there is at least one stream without '-' prefix. In this case, we can - # ignore all the streams with '-' prefix and just add those without - # '-' prefix to the list of valid streams. - streams_is_blacklist = all(stream.startswith("-") for stream in streams.get()) - if streams_is_blacklist or len(streams.get()) == 0: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous( - "There are multiple streams to choose from for module %s." % name) - else: - builds = models.ModuleBuild.get_last_build_in_all_streams( - session, name) - expanded_streams = [build.stream for build in builds] - else: - expanded_streams = [] - for stream in streams.get(): - if stream.startswith("-"): - if streams_is_blacklist and stream[1:] in expanded_streams: - expanded_streams.remove(stream[1:]) - else: - expanded_streams.append(stream) - - if len(expanded_streams) > 1: - if name in default_streams: - expanded_streams = [default_streams[name]] - elif raise_if_stream_ambigous: - raise StreamAmbigous("There are multiple streams %r to choose from for module %s." - % (expanded_streams, name)) - - return expanded_streams - - -def expand_mse_streams(session, mmd, default_streams=None, raise_if_stream_ambigous=False): - """ - Expands streams in both buildrequires/requires sections of MMD. - - :param session: SQLAlchemy DB session. - :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - """ - for deps in mmd.get_dependencies(): - expanded = {} - for name, streams in deps.get_requires().items(): - streams_set = Modulemd.SimpleSet() - streams_set.set(_expand_mse_streams( - session, name, streams, default_streams, raise_if_stream_ambigous)) - expanded[name] = streams_set - deps.set_requires(expanded) - - expanded = {} - for name, streams in deps.get_buildrequires().items(): - streams_set = Modulemd.SimpleSet() - streams_set.set(_expand_mse_streams( - session, name, streams, default_streams, raise_if_stream_ambigous)) - expanded[name] = streams_set - deps.set_buildrequires(expanded) - - -def _get_mmds_from_requires(session, requires, mmds, recursive=False, - default_streams=None, raise_if_stream_ambigous=False): - """ - Helper method for get_mmds_required_by_module_recursively returning - the list of module metadata objects defined by `requires` dict. - - :param session: SQLAlchemy DB session. - :param requires: Modulemetadata requires or buildrequires. - :param mmds: Dictionary with already handled name:streams as a keys and lists - of resulting mmds as values. - :param recursive: If True, the requires are checked recursively. - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :return: Dict with name:stream as a key and list with mmds as value. - """ - default_streams = default_streams or {} - # To be able to call itself recursively, we need to store list of mmds - # we have added to global mmds list in this particular call. - added_mmds = {} - resolver = module_build_service.resolver.GenericResolver.create(conf) - - for name, streams in requires.items(): - streams_to_try = streams.get() - if name in default_streams: - streams_to_try = [default_streams[name]] - elif len(streams_to_try) > 1 and raise_if_stream_ambigous: - raise StreamAmbigous("There are multiple streams %r to choose from for module %s." - % (streams_to_try, name)) - - # For each valid stream, find the last build in a stream and also all - # its contexts and add mmds of these builds to `mmds` and `added_mmds`. - # Of course only do that if we have not done that already in some - # previous call of this method. - for stream in streams.get(): - ns = "%s:%s" % (name, stream) - if ns in mmds: - continue - - mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) - added_mmds[ns] = mmds[ns] - - # Get the requires recursively. - if recursive: - for mmd_list in added_mmds.values(): - for mmd in mmd_list: - for deps in mmd.get_dependencies(): - mmds = _get_mmds_from_requires(session, deps.get_requires(), mmds, True) - - return mmds - - -def get_mmds_required_by_module_recursively( - session, mmd, default_streams=None, raise_if_stream_ambigous=False): - """ - Returns the list of Module metadata objects of all modules required while - building the module defined by `mmd` module metadata. This presumes the - module metadata streams are expanded using `expand_mse_streams(...)` - method. - - This method finds out latest versions of all the build-requires of - the `mmd` module and then also all contexts of these latest versions. - - For each build-required name:stream:version:context module, it checks - recursively all the "requires" and finds the latest version of each - required module and also all contexts of these latest versions. - - :param dict default_streams: Dict in {module_name: module_stream, ...} format defining - the default stream to choose for module in case when there are multiple streams to - choose from. - :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case - there are multiple streams for some dependency of module and the module name is not - defined in `default_streams`, so it is not clear which stream should be used. - :rtype: list of Modulemd metadata - :return: List of all modulemd metadata of all modules required to build - the module `mmd`. - """ - # We use dict with name:stream as a key and list with mmds as value. - # That way, we can ensure we won't have any duplicate mmds in a resulting - # list and we also don't waste resources on getting the modules we already - # handled from DB. - mmds = {} - - # At first get all the buildrequires of the module of interest. - for deps in mmd.get_dependencies(): - mmds = _get_mmds_from_requires( - session, deps.get_buildrequires(), mmds, False, default_streams, - raise_if_stream_ambigous) - - # Now get the requires of buildrequires recursively. - for mmd_key in list(mmds.keys()): - for mmd in mmds[mmd_key]: - for deps in mmd.get_dependencies(): - mmds = _get_mmds_from_requires( - session, deps.get_requires(), mmds, True, default_streams, - raise_if_stream_ambigous) - - # Make single list from dict of lists. - res = [] - for mmds_list in mmds.values(): - res += mmds_list - return res - - -def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'): - """ - Used as a decorator validates koji tag arg(s)' value(s) - against configurable list of koji tag prefixes. - Supported arg value types are: dict, list, str - - :param tag_arg_names: Str or list of parameters to validate. - :param pre: Prepend this optional string (e.g. '.' in case of disttag - validation) to each koji tag prefix. - :param post: Append this string/delimiter ('-' by default) to each koji - tag prefix. - :param dict_key: In case of a dict arg, inspect this key ('name' by default). - """ - - if not isinstance(tag_arg_names, list): - tag_arg_names = [tag_arg_names] - - def validation_decorator(function): - def wrapper(*args, **kwargs): - call_args = inspect.getcallargs(function, *args, **kwargs) - - for tag_arg_name in tag_arg_names: - err_subject = "Koji tag validation:" - - # If any of them don't appear in the function, then fail. - if tag_arg_name not in call_args: - raise ProgrammingError( - '{} Inspected argument {} is not within function args.' - ' The function was: {}.' - .format(err_subject, tag_arg_name, function.__name__)) - - tag_arg_val = call_args[tag_arg_name] - - # First, check that we have some value - if not tag_arg_val: - raise ValidationError('{} Can not validate {}. No value provided.' - .format(err_subject, tag_arg_name)) - - # If any of them are a dict, then use the provided dict_key - if isinstance(tag_arg_val, dict): - if dict_key not in tag_arg_val: - raise ProgrammingError( - '{} Inspected dict arg {} does not contain {} key.' - ' The function was: {}.' - .format(err_subject, tag_arg_name, dict_key, function.__name__)) - tag_list = [tag_arg_val[dict_key]] - elif isinstance(tag_arg_val, list): - tag_list = tag_arg_val - else: - tag_list = [tag_arg_val] - - # Check to make sure the provided values match our whitelist. - for allowed_prefix in conf.koji_tag_prefixes: - if all([t.startswith(pre + allowed_prefix + post) for t in tag_list]): - break - else: - # Only raise this error if the given tags don't start with - # *any* of our allowed prefixes. - raise ValidationError( - 'Koji tag validation: {} does not satisfy any of allowed prefixes: {}' - .format(tag_list, - [pre + p + post for p in conf.koji_tag_prefixes])) - - # Finally.. after all that validation, call the original function - # and return its value. - return function(*args, **kwargs) - - # We're replacing the original function with our synthetic wrapper, - # but dress it up to make it look more like the original function. - wrapper.__name__ = function.__name__ - wrapper.__doc__ = function.__doc__ - return wrapper - - return validation_decorator - - -def get_rpm_release(module_build): - """ - Generates the dist tag for the specified module - :param module_build: a models.ModuleBuild object - :return: a string of the module's dist tag - """ - dist_str = '.'.join([module_build.name, module_build.stream, str(module_build.version), - str(module_build.context)]).encode('utf-8') - dist_hash = hashlib.sha1(dist_str).hexdigest()[:8] - return "{prefix}{index}+{dist_hash}".format( - prefix=conf.default_dist_tag_prefix, - index=module_build.id or 0, - dist_hash=dist_hash, - ) - - -def create_dogpile_key_generator_func(skip_first_n_args=0): - """ - Creates dogpile key_generator function with additional features: - - - when models.ModuleBuild is an argument of method cached by dogpile-cache, - the ModuleBuild.id is used as a key. Therefore it is possible to cache - data per particular module build, while normally, it would be per - ModuleBuild.__str__() output, which contains also batch and other data - which changes during the build of a module. - - it is able to skip first N arguments of a cached method. This is useful - when the db.session or PDCClient instance is part of cached method call, - and the caching should work no matter what session instance is passed - to cached method argument. - """ - def key_generator(namespace, fn): - fname = fn.__name__ - - def generate_key(*arg, **kwarg): - key_template = fname + "_" - for s in arg[skip_first_n_args:]: - if type(s) == models.ModuleBuild: - key_template += str(s.id) - else: - key_template += str(s) + "_" - return key_template - - return generate_key - return key_generator - - -def cors_header(allow='*'): - """ - A decorator that sets the Access-Control-Allow-Origin header to the desired value on a Flask - route - :param allow: a string of the domain to allow. This defaults to '*'. - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - rv = func(*args, **kwargs) - if rv: - # If a tuple was provided, then the Flask Response should be the first object - if isinstance(rv, tuple): - response = rv[0] - else: - response = rv - # Make sure we are dealing with a Flask Response object - if isinstance(response, Response): - response.headers.add('Access-Control-Allow-Origin', allow) - return rv - return wrapper - return decorator - - -def validate_api_version(): - """ - A decorator that validates the requested API version on a route - """ - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - req_api_version = kwargs.get('api_version', 1) - if req_api_version > api_version or req_api_version < 1: - raise NotFound('The requested API version is not available') - return func(*args, **kwargs) - return wrapper - return decorator - - -def import_mmd(session, mmd): - """ - Imports new module build defined by `mmd` to MBS database using `session`. - If it already exists, it is updated. - - The ModuleBuild.koji_tag is set according to xmd['mbs]['koji_tag']. - The ModuleBuild.state is set to "ready". - The ModuleBuild.rebuild_strategy is set to "all". - The ModuleBuild.owner is set to "mbs_import". - - TODO: The "context" is not stored directly in database. We only store - build_context and runtime_context and compute context, but when importing - the module, we have no idea what build_context or runtime_context is - we only - know the resulting "context", but there is no way to store it into do DB. - By now, we just ignore mmd.get_context() and use default 00000000 context instead. - """ - mmd.set_context("00000000") - name = mmd.get_name() - stream = mmd.get_stream() - version = str(mmd.get_version()) - context = mmd.get_context() - - # NSVC is used for logging purpose later. - nsvc = ":".join([name, stream, version, context]) - - # Get the koji_tag. - xmd = mmd.get_xmd() - if "mbs" in xmd.keys() and "koji_tag" in xmd["mbs"].keys(): - koji_tag = xmd["mbs"]["koji_tag"] - else: - log.warn("'koji_tag' is not set in xmd['mbs'] for module %s", nsvc) - koji_tag = "" - - # Get the ModuleBuild from DB. - build = models.ModuleBuild.get_build_from_nsvc( - session, name, stream, version, context) - if build: - log.info("Updating existing module build %s.", nsvc) - else: - build = models.ModuleBuild() - - build.name = name - build.stream = stream - build.version = version - build.koji_tag = koji_tag - build.state = models.BUILD_STATES['ready'] - build.modulemd = mmd.dumps() - build.owner = "mbs_import" - build.rebuild_strategy = 'all' - build.time_submitted = datetime.utcnow() - build.time_modified = datetime.utcnow() - build.time_completed = datetime.utcnow() - session.add(build) - session.commit() - log.info("Module %s imported", nsvc) diff --git a/module_build_service/utils/__init__.py b/module_build_service/utils/__init__.py new file mode 100644 index 00000000..22fbd17e --- /dev/null +++ b/module_build_service/utils/__init__.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Jan Kaluza + +from module_build_service.utils.general import * # noqa +from module_build_service.utils.mse import * # noqa +from module_build_service.utils.views import * # noqa +from module_build_service.utils.reuse import * # noqa +from module_build_service.utils.submit import * # noqa +from module_build_service.utils.batches import * # noqa diff --git a/module_build_service/utils/batches.py b/module_build_service/utils/batches.py new file mode 100644 index 00000000..6288dd44 --- /dev/null +++ b/module_build_service/utils/batches.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +import concurrent.futures + +from module_build_service import conf, log, models +import module_build_service.messaging +from .reuse import get_reusable_components, reuse_component + + +def at_concurrent_component_threshold(config, session): + """ + Determines if the number of concurrent component builds has reached + the configured threshold + :param config: Module Build Service configuration object + :param session: SQLAlchemy database session + :return: boolean representing if there are too many concurrent builds at + this time + """ + + # We must not check it for "mock" backend. + # It would lead to multiple calls of continue_batch_build method and + # creation of multiple worker threads there. Mock backend uses thread-id + # to create and identify mock buildroot and for mock backend, we must + # build whole module in this single continue_batch_build call to keep + # the number of created buildroots low. The concurrent build limit + # for mock backend is secured by setting max_workers in + # ThreadPoolExecutor to num_concurrent_builds. + if conf.system == "mock": + return False + + import koji # Placed here to avoid py2/py3 conflicts... + + if config.num_concurrent_builds and config.num_concurrent_builds <= \ + session.query(models.ComponentBuild).filter_by( + state=koji.BUILD_STATES['BUILDING'], + # Components which are reused should not be counted in, because + # we do not submit new build for them. They are in BUILDING state + # just internally in MBS to be handled by + # scheduler.handlers.components.complete. + reused_component_id=None).count(): + return True + + return False + + +def start_build_component(builder, c): + """ + Submits single component build to builder. Called in thread + by QueueBasedThreadPool in continue_batch_build. + """ + import koji + try: + c.task_id, c.state, c.state_reason, c.nvr = builder.build( + artifact_name=c.package, source=c.scmurl) + except Exception as e: + c.state = koji.BUILD_STATES['FAILED'] + c.state_reason = "Failed to build artifact %s: %s" % (c.package, str(e)) + log.exception(e) + return + + if not c.task_id and c.state == koji.BUILD_STATES['BUILDING']: + c.state = koji.BUILD_STATES['FAILED'] + c.state_reason = ("Failed to build artifact %s: " + "Builder did not return task ID" % (c.package)) + return + + +def continue_batch_build(config, module, session, builder, components=None): + """ + Continues building current batch. Submits next components in the batch + until it hits concurrent builds limit. + + Returns list of BaseMessage instances which should be scheduled by the + scheduler. + """ + import koji # Placed here to avoid py2/py3 conflicts... + + # The user can either pass in a list of components to 'seed' the batch, or + # if none are provided then we just select everything that hasn't + # successfully built yet or isn't currently being built. + unbuilt_components = components or [ + c for c in module.component_builds + if (c.state != koji.BUILD_STATES['COMPLETE'] and + c.state != koji.BUILD_STATES['BUILDING'] and + c.state != koji.BUILD_STATES['FAILED'] and + c.batch == module.batch) + ] + + if not unbuilt_components: + log.debug("Cannot continue building module %s. No component to build." % module) + return [] + + # Get the list of components to be built in this batch. We are not building + # all `unbuilt_components`, because we can meet the num_concurrent_builds + # threshold + further_work = [] + components_to_build = [] + # Sort the unbuilt_components so that the components that take the longest to build are + # first + unbuilt_components.sort(key=lambda c: c.weight, reverse=True) + + # Check for builds that exist in the build system but MBS doesn't know about + for component in unbuilt_components: + # Only evaluate new components + if component.state is not None: + continue + msgs = builder.recover_orphaned_artifact(component) + further_work += msgs + + for c in unbuilt_components: + # If a previous build of the component was found, then the state will be marked as + # COMPLETE so we should skip this + if c.state == koji.BUILD_STATES['COMPLETE']: + continue + # Check the concurrent build threshold. + if at_concurrent_component_threshold(config, session): + log.info('Concurrent build threshold met') + break + + # We set state to "BUILDING" here because at this point we are committed + # to build the component and at_concurrent_component_threshold() works by + # counting the number of components in the "BUILDING" state. + c.state = koji.BUILD_STATES['BUILDING'] + components_to_build.append(c) + + # Start build of components in this batch. + max_workers = 1 + if config.num_concurrent_builds > 0: + max_workers = config.num_concurrent_builds + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(start_build_component, builder, c): + c for c in components_to_build} + concurrent.futures.wait(futures) + # In case there has been an excepion generated directly in the + # start_build_component, the future.result() will re-raise it in the + # main thread so it is not lost. + for future in futures: + future.result() + + session.commit() + return further_work + + +def start_next_batch_build(config, module, session, builder, components=None): + """ + Tries to start the build of next batch. In case there are still unbuilt + components in a batch, tries to submit more components until it hits + concurrent builds limit. Otherwise Increments module.batch and submits component + builds from the next batch. + + :return: a list of BaseMessage instances to be handled by the MBSConsumer. + """ + import koji # Placed here to avoid py2/py3 conflicts... + + # Check the status of the module build and current batch so we can + # later decide if we can start new batch or not. + has_unbuilt_components = False + has_unbuilt_components_in_batch = False + has_building_components_in_batch = False + has_failed_components = False + # This is used to determine if it's worth checking if a component can be reused + # later on in the code + all_reused_in_prev_batch = True + for c in module.component_builds: + if c.state in [None, koji.BUILD_STATES['BUILDING']]: + has_unbuilt_components = True + + if c.batch == module.batch: + if not c.state: + has_unbuilt_components_in_batch = True + elif c.state == koji.BUILD_STATES['BUILDING']: + has_building_components_in_batch = True + elif (c.state in [koji.BUILD_STATES['FAILED'], + koji.BUILD_STATES['CANCELED']]): + has_failed_components = True + + if c.batch == module.batch and not c.reused_component_id: + all_reused_in_prev_batch = False + + # Do not start new batch if there are no components to build. + if not has_unbuilt_components: + log.debug("Not starting new batch, there is no component to build " + "for module %s" % module) + return [] + + # Check that there is something to build in current batch before starting + # the new one. If there is, continue building current batch. + if has_unbuilt_components_in_batch: + log.info("Continuing building batch %d", module.batch) + return continue_batch_build( + config, module, session, builder, components) + + # Check that there are no components in BUILDING state in current batch. + # If there are, wait until they are built. + if has_building_components_in_batch: + log.debug("Not starting new batch, there are still components in " + "BUILDING state in current batch for module %s", module) + return [] + + # Check that there are no failed components in this batch. If there are, + # do not start the new batch. + if has_failed_components: + log.info("Not starting new batch, there are failed components for " + "module %s", module) + return [] + + # Identify active tasks which might contain relicts of previous builds + # and fail the module build if this^ happens. + active_tasks = builder.list_tasks_for_components(module.component_builds, + state='active') + if isinstance(active_tasks, list) and active_tasks: + state_reason = ("Cannot start a batch, because some components are already" + " in 'building' state.") + state_reason += " See tasks (ID): {}".format( + ', '.join([str(t['id']) for t in active_tasks]) + ) + module.transition(config, state=models.BUILD_STATES['failed'], + state_reason=state_reason) + session.commit() + return [] + + else: + log.debug("Builder {} doesn't provide information about active tasks." + .format(builder)) + + # Find out if there is repo regeneration in progress for this module. + # If there is, wait until the repo is regenerated before starting a new + # batch. + artifacts = [c.nvr for c in module.current_batch()] + if not builder.buildroot_ready(artifacts): + log.info("Not starting new batch, not all of %r are in the buildroot. " + "Waiting." % artifacts) + return [] + + # Although this variable isn't necessary, it is easier to read code later on with it + prev_batch = module.batch + module.batch += 1 + + # The user can either pass in a list of components to 'seed' the batch, or + # if none are provided then we just select everything that hasn't + # successfully built yet or isn't currently being built. + unbuilt_components = components or [ + c for c in module.component_builds + if (c.state != koji.BUILD_STATES['COMPLETE'] and + c.state != koji.BUILD_STATES['BUILDING'] and + c.state != koji.BUILD_STATES['FAILED'] and + c.batch == module.batch) + ] + + # If there are no components to build, skip the batch and start building + # the new one. This can happen when resubmitting the failed module build. + if not unbuilt_components and not components: + log.info("Skipping build of batch %d, no component to build.", + module.batch) + return start_next_batch_build(config, module, session, builder) + + log.info("Starting build of next batch %d, %s" % (module.batch, + unbuilt_components)) + + # Attempt to reuse any components possible in the batch before attempting to build any + further_work = [] + unbuilt_components_after_reuse = [] + components_reused = False + should_try_reuse = True + # If the rebuild strategy is "changed-and-after", try to figure out if it's worth checking if + # the components can be reused to save on resources + if module.rebuild_strategy == 'changed-and-after': + # Check to see if the previous batch had all their builds reused except for when the + # previous batch was 1 because that always has the module-build-macros component built + should_try_reuse = all_reused_in_prev_batch or prev_batch == 1 + if should_try_reuse: + component_names = [c.package for c in unbuilt_components] + reusable_components = get_reusable_components( + session, module, component_names) + for c, reusable_c in zip(unbuilt_components, reusable_components): + if reusable_c: + components_reused = True + further_work += reuse_component(c, reusable_c) + else: + unbuilt_components_after_reuse.append(c) + # Commit the changes done by reuse_component + if components_reused: + session.commit() + + # If all the components were reused in the batch then make a KojiRepoChange + # message and return + if components_reused and not unbuilt_components_after_reuse: + further_work.append(module_build_service.messaging.KojiRepoChange( + 'start_build_batch: fake msg', builder.module_build_tag['name'])) + return further_work + + return further_work + continue_batch_build( + config, module, session, builder, unbuilt_components_after_reuse) diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py new file mode 100644 index 00000000..ce96ff1a --- /dev/null +++ b/module_build_service/utils/general.py @@ -0,0 +1,272 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +import functools +import inspect +import hashlib +import time +from datetime import datetime + +from module_build_service import conf, log, models +from module_build_service.errors import ValidationError, ProgrammingError + + +def scm_url_schemes(terse=False): + """ + Definition of URL schemes supported by both frontend and scheduler. + + NOTE: only git URLs in the following formats are supported atm: + git:// + git+http:// + git+https:// + git+rsync:// + http:// + https:// + file:// + + :param terse=False: Whether to return terse list of unique URL schemes + even without the "://". + """ + + scm_types = { + "git": ("git://", "git+http://", "git+https://", + "git+rsync://", "http://", "https://", "file://") + } + + if not terse: + return scm_types + else: + scheme_list = [] + for scm_type, scm_schemes in scm_types.items(): + scheme_list.extend([scheme[:-3] for scheme in scm_schemes]) + return list(set(scheme_list)) + + +def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception): + """ A decorator that allows to retry a section of code... + ...until success or timeout. + """ + def wrapper(function): + @functools.wraps(function) + def inner(*args, **kwargs): + start = time.time() + while True: + try: + return function(*args, **kwargs) + except wait_on as e: + log.warn("Exception %r raised from %r. Retry in %rs" % ( + e, function, interval)) + time.sleep(interval) + if (time.time() - start) >= timeout: + raise # This re-raises the last exception. + return inner + return wrapper + + +def module_build_state_from_msg(msg): + state = int(msg.module_build_state) + # TODO better handling + assert state in models.BUILD_STATES.values(), ( + 'state=%s(%s) is not in %s' + % (state, type(state), list(models.BUILD_STATES.values()))) + return state + + +def validate_koji_tag(tag_arg_names, pre='', post='-', dict_key='name'): + """ + Used as a decorator validates koji tag arg(s)' value(s) + against configurable list of koji tag prefixes. + Supported arg value types are: dict, list, str + + :param tag_arg_names: Str or list of parameters to validate. + :param pre: Prepend this optional string (e.g. '.' in case of disttag + validation) to each koji tag prefix. + :param post: Append this string/delimiter ('-' by default) to each koji + tag prefix. + :param dict_key: In case of a dict arg, inspect this key ('name' by default). + """ + + if not isinstance(tag_arg_names, list): + tag_arg_names = [tag_arg_names] + + def validation_decorator(function): + def wrapper(*args, **kwargs): + call_args = inspect.getcallargs(function, *args, **kwargs) + + for tag_arg_name in tag_arg_names: + err_subject = "Koji tag validation:" + + # If any of them don't appear in the function, then fail. + if tag_arg_name not in call_args: + raise ProgrammingError( + '{} Inspected argument {} is not within function args.' + ' The function was: {}.' + .format(err_subject, tag_arg_name, function.__name__)) + + tag_arg_val = call_args[tag_arg_name] + + # First, check that we have some value + if not tag_arg_val: + raise ValidationError('{} Can not validate {}. No value provided.' + .format(err_subject, tag_arg_name)) + + # If any of them are a dict, then use the provided dict_key + if isinstance(tag_arg_val, dict): + if dict_key not in tag_arg_val: + raise ProgrammingError( + '{} Inspected dict arg {} does not contain {} key.' + ' The function was: {}.' + .format(err_subject, tag_arg_name, dict_key, function.__name__)) + tag_list = [tag_arg_val[dict_key]] + elif isinstance(tag_arg_val, list): + tag_list = tag_arg_val + else: + tag_list = [tag_arg_val] + + # Check to make sure the provided values match our whitelist. + for allowed_prefix in conf.koji_tag_prefixes: + if all([t.startswith(pre + allowed_prefix + post) for t in tag_list]): + break + else: + # Only raise this error if the given tags don't start with + # *any* of our allowed prefixes. + raise ValidationError( + 'Koji tag validation: {} does not satisfy any of allowed prefixes: {}' + .format(tag_list, + [pre + p + post for p in conf.koji_tag_prefixes])) + + # Finally.. after all that validation, call the original function + # and return its value. + return function(*args, **kwargs) + + # We're replacing the original function with our synthetic wrapper, + # but dress it up to make it look more like the original function. + wrapper.__name__ = function.__name__ + wrapper.__doc__ = function.__doc__ + return wrapper + + return validation_decorator + + +def get_rpm_release(module_build): + """ + Generates the dist tag for the specified module + :param module_build: a models.ModuleBuild object + :return: a string of the module's dist tag + """ + dist_str = '.'.join([module_build.name, module_build.stream, str(module_build.version), + str(module_build.context)]).encode('utf-8') + dist_hash = hashlib.sha1(dist_str).hexdigest()[:8] + return "{prefix}{index}+{dist_hash}".format( + prefix=conf.default_dist_tag_prefix, + index=module_build.id or 0, + dist_hash=dist_hash, + ) + + +def create_dogpile_key_generator_func(skip_first_n_args=0): + """ + Creates dogpile key_generator function with additional features: + + - when models.ModuleBuild is an argument of method cached by dogpile-cache, + the ModuleBuild.id is used as a key. Therefore it is possible to cache + data per particular module build, while normally, it would be per + ModuleBuild.__str__() output, which contains also batch and other data + which changes during the build of a module. + - it is able to skip first N arguments of a cached method. This is useful + when the db.session or PDCClient instance is part of cached method call, + and the caching should work no matter what session instance is passed + to cached method argument. + """ + def key_generator(namespace, fn): + fname = fn.__name__ + + def generate_key(*arg, **kwarg): + key_template = fname + "_" + for s in arg[skip_first_n_args:]: + if type(s) == models.ModuleBuild: + key_template += str(s.id) + else: + key_template += str(s) + "_" + return key_template + + return generate_key + return key_generator + + +def import_mmd(session, mmd): + """ + Imports new module build defined by `mmd` to MBS database using `session`. + If it already exists, it is updated. + + The ModuleBuild.koji_tag is set according to xmd['mbs]['koji_tag']. + The ModuleBuild.state is set to "ready". + The ModuleBuild.rebuild_strategy is set to "all". + The ModuleBuild.owner is set to "mbs_import". + + TODO: The "context" is not stored directly in database. We only store + build_context and runtime_context and compute context, but when importing + the module, we have no idea what build_context or runtime_context is - we only + know the resulting "context", but there is no way to store it into do DB. + By now, we just ignore mmd.get_context() and use default 00000000 context instead. + """ + mmd.set_context("00000000") + name = mmd.get_name() + stream = mmd.get_stream() + version = str(mmd.get_version()) + context = mmd.get_context() + + # NSVC is used for logging purpose later. + nsvc = ":".join([name, stream, version, context]) + + # Get the koji_tag. + xmd = mmd.get_xmd() + if "mbs" in xmd.keys() and "koji_tag" in xmd["mbs"].keys(): + koji_tag = xmd["mbs"]["koji_tag"] + else: + log.warn("'koji_tag' is not set in xmd['mbs'] for module %s", nsvc) + koji_tag = "" + + # Get the ModuleBuild from DB. + build = models.ModuleBuild.get_build_from_nsvc( + session, name, stream, version, context) + if build: + log.info("Updating existing module build %s.", nsvc) + else: + build = models.ModuleBuild() + + build.name = name + build.stream = stream + build.version = version + build.koji_tag = koji_tag + build.state = models.BUILD_STATES['ready'] + build.modulemd = mmd.dumps() + build.owner = "mbs_import" + build.rebuild_strategy = 'all' + build.time_submitted = datetime.utcnow() + build.time_modified = datetime.utcnow() + build.time_completed = datetime.utcnow() + session.add(build) + session.commit() + log.info("Module %s imported", nsvc) diff --git a/module_build_service/utils/mse.py b/module_build_service/utils/mse.py new file mode 100644 index 00000000..b0454bb2 --- /dev/null +++ b/module_build_service/utils/mse.py @@ -0,0 +1,364 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +from module_build_service import log, models, Modulemd, conf, db +from module_build_service.errors import StreamAmbigous +from module_build_service.mmd_resolver import MMDResolver +from module_build_service import glib +import module_build_service.resolver + + +def _expand_mse_streams(session, name, streams, default_streams, raise_if_stream_ambigous): + """ + Helper method for `expand_mse_stream()` expanding single name:[streams]. + Returns list of expanded streams. + + :param session: SQLAlchemy DB session. + :param str name: Name of the module which will be expanded. + :param list streams: List of streams to expand. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + """ + default_streams = default_streams or {} + # Stream can be prefixed with '-' sign to define that this stream should + # not appear in a resulting list of streams. There can be two situations: + # a) all streams have '-' prefix. In this case, we treat list of streams + # as blacklist and we find all the valid streams and just remove those with + # '-' prefix. + # b) there is at least one stream without '-' prefix. In this case, we can + # ignore all the streams with '-' prefix and just add those without + # '-' prefix to the list of valid streams. + streams_is_blacklist = all(stream.startswith("-") for stream in streams.get()) + if streams_is_blacklist or len(streams.get()) == 0: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous( + "There are multiple streams to choose from for module %s." % name) + else: + builds = models.ModuleBuild.get_last_build_in_all_streams( + session, name) + expanded_streams = [build.stream for build in builds] + else: + expanded_streams = [] + for stream in streams.get(): + if stream.startswith("-"): + if streams_is_blacklist and stream[1:] in expanded_streams: + expanded_streams.remove(stream[1:]) + else: + expanded_streams.append(stream) + + if len(expanded_streams) > 1: + if name in default_streams: + expanded_streams = [default_streams[name]] + elif raise_if_stream_ambigous: + raise StreamAmbigous("There are multiple streams %r to choose from for module %s." + % (expanded_streams, name)) + + return expanded_streams + + +def expand_mse_streams(session, mmd, default_streams=None, raise_if_stream_ambigous=False): + """ + Expands streams in both buildrequires/requires sections of MMD. + + :param session: SQLAlchemy DB session. + :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + """ + for deps in mmd.get_dependencies(): + expanded = {} + for name, streams in deps.get_requires().items(): + streams_set = Modulemd.SimpleSet() + streams_set.set(_expand_mse_streams( + session, name, streams, default_streams, raise_if_stream_ambigous)) + expanded[name] = streams_set + deps.set_requires(expanded) + + expanded = {} + for name, streams in deps.get_buildrequires().items(): + streams_set = Modulemd.SimpleSet() + streams_set.set(_expand_mse_streams( + session, name, streams, default_streams, raise_if_stream_ambigous)) + expanded[name] = streams_set + deps.set_buildrequires(expanded) + + +def _get_mmds_from_requires(session, requires, mmds, recursive=False, + default_streams=None, raise_if_stream_ambigous=False): + """ + Helper method for get_mmds_required_by_module_recursively returning + the list of module metadata objects defined by `requires` dict. + + :param session: SQLAlchemy DB session. + :param requires: Modulemetadata requires or buildrequires. + :param mmds: Dictionary with already handled name:streams as a keys and lists + of resulting mmds as values. + :param recursive: If True, the requires are checked recursively. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :return: Dict with name:stream as a key and list with mmds as value. + """ + default_streams = default_streams or {} + # To be able to call itself recursively, we need to store list of mmds + # we have added to global mmds list in this particular call. + added_mmds = {} + resolver = module_build_service.resolver.GenericResolver.create(conf) + + for name, streams in requires.items(): + streams_to_try = streams.get() + if name in default_streams: + streams_to_try = [default_streams[name]] + elif len(streams_to_try) > 1 and raise_if_stream_ambigous: + raise StreamAmbigous("There are multiple streams %r to choose from for module %s." + % (streams_to_try, name)) + + # For each valid stream, find the last build in a stream and also all + # its contexts and add mmds of these builds to `mmds` and `added_mmds`. + # Of course only do that if we have not done that already in some + # previous call of this method. + for stream in streams.get(): + ns = "%s:%s" % (name, stream) + if ns in mmds: + continue + + mmds[ns] = resolver.get_module_modulemds(name, stream, strict=True) + added_mmds[ns] = mmds[ns] + + # Get the requires recursively. + if recursive: + for mmd_list in added_mmds.values(): + for mmd in mmd_list: + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires(session, deps.get_requires(), mmds, True) + + return mmds + + +def get_mmds_required_by_module_recursively( + session, mmd, default_streams=None, raise_if_stream_ambigous=False): + """ + Returns the list of Module metadata objects of all modules required while + building the module defined by `mmd` module metadata. This presumes the + module metadata streams are expanded using `expand_mse_streams(...)` + method. + + This method finds out latest versions of all the build-requires of + the `mmd` module and then also all contexts of these latest versions. + + For each build-required name:stream:version:context module, it checks + recursively all the "requires" and finds the latest version of each + required module and also all contexts of these latest versions. + + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :rtype: list of Modulemd metadata + :return: List of all modulemd metadata of all modules required to build + the module `mmd`. + """ + # We use dict with name:stream as a key and list with mmds as value. + # That way, we can ensure we won't have any duplicate mmds in a resulting + # list and we also don't waste resources on getting the modules we already + # handled from DB. + mmds = {} + + # At first get all the buildrequires of the module of interest. + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires( + session, deps.get_buildrequires(), mmds, False, default_streams, + raise_if_stream_ambigous) + + # Now get the requires of buildrequires recursively. + for mmd_key in list(mmds.keys()): + for mmd in mmds[mmd_key]: + for deps in mmd.get_dependencies(): + mmds = _get_mmds_from_requires( + session, deps.get_requires(), mmds, True, default_streams, + raise_if_stream_ambigous) + + # Make single list from dict of lists. + res = [] + for mmds_list in mmds.values(): + res += mmds_list + return res + + +def generate_expanded_mmds(session, mmd, raise_if_stream_ambigous=False, default_streams=None): + """ + Returns list with MMDs with buildrequires and requires set according + to module stream expansion rules. These module metadata can be directly + built using MBS. + + :param session: SQLAlchemy DB session. + :param Modulemd.Module mmd: Modulemd metadata with original unexpanded module. + :param bool raise_if_stream_ambigous: When True, raises a StreamAmbigous exception in case + there are multiple streams for some dependency of module and the module name is not + defined in `default_streams`, so it is not clear which stream should be used. + :param dict default_streams: Dict in {module_name: module_stream, ...} format defining + the default stream to choose for module in case when there are multiple streams to + choose from. + """ + if not session: + session = db.session + + if not default_streams: + default_streams = {} + + # Create local copy of mmd, because we will expand its dependencies, + # which would change the module. + # TODO: Use copy method once its in released libmodulemd: + # https://github.com/fedora-modularity/libmodulemd/pull/20 + current_mmd = Modulemd.Module.new_from_string(mmd.dumps()) + + # MMDResolver expects the input MMD to have no context. + current_mmd.set_context(None) + + # Expands the MSE streams. This mainly handles '-' prefix in MSE streams. + expand_mse_streams(session, current_mmd, default_streams, raise_if_stream_ambigous) + + # Get the list of all MMDs which this module can be possibly built against + # and add them to MMDResolver. + mmd_resolver = MMDResolver() + mmds_for_resolving = get_mmds_required_by_module_recursively( + session, current_mmd, default_streams, raise_if_stream_ambigous) + for m in mmds_for_resolving: + mmd_resolver.add_modules(m) + + # Show log.info message with the NSVCs we have added to mmd_resolver. + nsvcs_to_solve = [ + ":".join([m.get_name(), m.get_stream(), str(m.get_version()), str(m.get_context())]) + for m in mmds_for_resolving] + log.info("Starting resolving with following input modules: %r", nsvcs_to_solve) + + # Resolve the dependencies between modules and get the list of all valid + # combinations in which we can build this module. + requires_combinations = mmd_resolver.solve(current_mmd) + log.info("Resolving done, possible requires: %r", requires_combinations) + + # This is where we are going to store the generated MMDs. + mmds = [] + for requires in requires_combinations: + # Each generated MMD must be new Module object... + # TODO: Use copy method once its in released libmodulemd: + # https://github.com/fedora-modularity/libmodulemd/pull/20 + mmd_copy = Modulemd.Module.new_from_string(mmd.dumps()) + xmd = glib.from_variant_dict(mmd_copy.get_xmd()) + + # Requires contain the NSVC representing the input mmd. + # The 'context' of this NSVC defines the id of buildrequires/requires + # pair in the mmd.get_dependencies(). + dependencies_id = None + + # We don't want to depend on ourselves, so store the NSVC of the current_mmd + # to be able to ignore it later. + self_nsvca = None + + # Dict to store name:stream pairs from nsvca, so we are able to access it + # easily later. + req_name_stream = {} + + # Get the values for dependencies_id, self_nsvca and req_name_stream variables. + for nsvca in requires: + req_name, req_stream, _ = nsvca.split(":", 2) + if req_name == current_mmd.get_name() and req_stream == current_mmd.get_stream(): + dependencies_id = int(nsvca.split(":")[3]) + self_nsvca = nsvca + continue + req_name_stream[req_name] = req_stream + if dependencies_id is None or self_nsvca is None: + raise RuntimeError( + "%s:%s not found in requires %r" % ( + current_mmd.get_name(), current_mmd.get_stream(), requires)) + + # The name:[streams, ...] pairs do not have to be the same in both + # buildrequires/requires. In case they are the same, we replace the streams + # in requires section with a single stream against which we will build this MMD. + # In case they are not the same, we have to keep the streams as they are in requires + # section. We always replace stream(s) for build-requirement with the one we + # will build this MMD against. + new_dep = Modulemd.Dependencies() + dep = mmd_copy.get_dependencies()[dependencies_id] + dep_requires = dep.get_requires() + dep_buildrequires = dep.get_buildrequires() + for req_name, req_streams in dep_requires.items(): + if (req_name not in dep_buildrequires or + set(req_streams.get()) != set(dep_buildrequires[req_name].get())): + # Streams in runtime section are not the same as in buildtime section, + # so just copy this runtime requirement to new_dep. + new_dep.add_requires(req_name, req_streams.get()) + else: + # This runtime requirement has the same streams in both runtime/buildtime + # requires sections, so replace streams in both sections by the one we + # really used in this resolved variant. + new_dep.add_requires(req_name, [req_name_stream[req_name]]) + new_dep.add_buildrequires(req_name, [req_name_stream[req_name]]) + mmd_copy.set_dependencies((new_dep, )) + + # The Modulemd.Dependencies() stores only streams, but to really build this + # module, we need NSVC of buildrequires, so we have to store this data in XMD. + # We also need additional data like for example list of filtered_rpms. We will + # get them using module_build_service.resolver.GenericResolver.resolve_requires, + # so prepare list with NSVCs of buildrequires as an input for this method. + br_list = [] + for nsvca in requires: + if nsvca == self_nsvca: + continue + # Remove the arch from nsvca + nsvc = ":".join(nsvca.split(":")[:-1]) + br_list.append(nsvc) + + # Resolve the buildrequires and store the result in XMD. + if 'mbs' not in xmd: + xmd['mbs'] = {} + resolver = module_build_service.resolver.GenericResolver.create(conf) + xmd['mbs']['buildrequires'] = resolver.resolve_requires(br_list) + xmd['mbs']['mse'] = True + + mmd_copy.set_xmd(glib.dict_values(xmd)) + + # Now we have all the info to actually compute context of this module. + build_context, runtime_context = models.ModuleBuild.contexts_from_mmd(mmd_copy.dumps()) + context = models.ModuleBuild.context_from_contexts(build_context, runtime_context) + mmd_copy.set_context(context) + + mmds.append(mmd_copy) + + return mmds diff --git a/module_build_service/utils/reuse.py b/module_build_service/utils/reuse.py new file mode 100644 index 00000000..efbbc0ac --- /dev/null +++ b/module_build_service/utils/reuse.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +import kobo.rpmlib + +import module_build_service.messaging +from module_build_service import log, models, conf + + +def reuse_component(component, previous_component_build, + change_state_now=False): + """ + Reuses component build `previous_component_build` instead of building + component `component` + + Returns the list of BaseMessage instances to be handled later by the + scheduler. + """ + + import koji + + log.info( + 'Reusing component "{0}" from a previous module ' + 'build with the nvr "{1}"'.format( + component.package, previous_component_build.nvr)) + component.reused_component_id = previous_component_build.id + component.task_id = previous_component_build.task_id + if change_state_now: + component.state = previous_component_build.state + else: + # Use BUILDING state here, because we want the state to change to + # COMPLETE by the fake KojiBuildChange message we are generating + # few lines below. If we would set it to the right state right + # here, we would miss the code path handling the KojiBuildChange + # which works only when switching from BUILDING to COMPLETE. + component.state = koji.BUILD_STATES['BUILDING'] + component.state_reason = \ + 'Reused component from previous module build' + component.nvr = previous_component_build.nvr + nvr_dict = kobo.rpmlib.parse_nvr(component.nvr) + # Add this message to further_work so that the reused + # component will be tagged properly + return [ + module_build_service.messaging.KojiBuildChange( + msg_id='reuse_component: fake msg', + build_id=None, + task_id=component.task_id, + build_new_state=previous_component_build.state, + build_name=component.package, + build_version=nvr_dict['version'], + build_release=nvr_dict['release'], + module_build_id=component.module_id, + state_reason=component.state_reason + ) + ] + + +def _get_reusable_module(session, module): + """ + Returns previous module build of the module `module` in case it can be + used as a source module to get the components to reuse from. + + In case there is no such module, returns None. + + :param session: SQLAlchemy database session + :param module: the ModuleBuild object of module being built. + :return: ModuleBuild object which can be used for component reuse. + """ + mmd = module.mmd() + + # Find the latest module that is in the done or ready state + previous_module_build = session.query(models.ModuleBuild)\ + .filter_by(name=mmd.get_name())\ + .filter_by(stream=mmd.get_stream())\ + .filter(models.ModuleBuild.state.in_([3, 5]))\ + .filter(models.ModuleBuild.scmurl.isnot(None))\ + .order_by(models.ModuleBuild.time_completed.desc()) + # If we are rebuilding with the "changed-and-after" option, then we can't reuse + # components from modules that were built more liberally + if module.rebuild_strategy == 'changed-and-after': + previous_module_build = previous_module_build.filter( + models.ModuleBuild.rebuild_strategy.in_(['all', 'changed-and-after'])) + previous_module_build = previous_module_build.filter_by( + build_context=module.build_context) + previous_module_build = previous_module_build.first() + # The component can't be reused if there isn't a previous build in the done + # or ready state + if not previous_module_build: + log.info("Cannot re-use. %r is the first module build." % module) + return None + + return previous_module_build + + +def attempt_to_reuse_all_components(builder, session, module): + """ + Tries to reuse all the components in a build. The components are also + tagged to the tags using the `builder`. + + Returns True if all components could be reused, otherwise False. When + False is returned, no component has been reused. + """ + + previous_module_build = _get_reusable_module(session, module) + if not previous_module_build: + return False + + mmd = module.mmd() + old_mmd = previous_module_build.mmd() + + # [(component, component_to_reuse), ...] + component_pairs = [] + + # Find out if we can reuse all components and cache component and + # component to reuse pairs. + for c in module.component_builds: + if c.package == "module-build-macros": + continue + component_to_reuse = get_reusable_component( + session, module, c.package, + previous_module_build=previous_module_build, mmd=mmd, + old_mmd=old_mmd) + if not component_to_reuse: + return False + + component_pairs.append((c, component_to_reuse)) + + # Stores components we will tag to buildroot and final tag. + components_to_tag = [] + + # Reuse all components. + for c, component_to_reuse in component_pairs: + # Set the module.batch to the last batch we have. + if c.batch > module.batch: + module.batch = c.batch + + # Reuse the component + reuse_component(c, component_to_reuse, True) + components_to_tag.append(c.nvr) + + # Tag them + builder.buildroot_add_artifacts(components_to_tag, install=False) + builder.tag_artifacts(components_to_tag, dest_tag=True) + + return True + + +def get_reusable_components(session, module, component_names): + """ + Returns the list of ComponentBuild instances belonging to previous module + build which can be reused in the build of module `module`. + + The ComponentBuild instances in returned list are in the same order as + their names in the component_names input list. + + In case some component cannot be reused, None is used instead of a + ComponentBuild instance in the returned list. + + :param session: SQLAlchemy database session + :param module: the ModuleBuild object of module being built. + :param component_names: List of component names to be reused. + :return: List of ComponentBuild instances to reuse in the same + order as `component_names` + """ + # We support components reusing only for koji and test backend. + if conf.system not in ['koji', 'test']: + return [None] * len(component_names) + + previous_module_build = _get_reusable_module(session, module) + if not previous_module_build: + return [None] * len(component_names) + + mmd = module.mmd() + old_mmd = previous_module_build.mmd() + + ret = [] + for component_name in component_names: + ret.append(get_reusable_component( + session, module, component_name, previous_module_build, mmd, + old_mmd)) + + return ret + + +def get_reusable_component(session, module, component_name, + previous_module_build=None, mmd=None, old_mmd=None): + """ + Returns the component (RPM) build of a module that can be reused + instead of needing to rebuild it + :param session: SQLAlchemy database session + :param module: the ModuleBuild object of module being built with a formatted + mmd + :param component_name: the name of the component (RPM) that you'd like to + reuse a previous build of + :param previous_module_build: the ModuleBuild instances of a module build + which contains the components to reuse. If not passed, _get_reusable_module + is called to get the ModuleBuild instance. Consider passing the ModuleBuild + instance in case you plan to call get_reusable_component repeatedly for the + same module to make this method faster. + :param mmd: ModuleMd.Module of `module`. If not passed, it is taken from + module.mmd(). Consider passing this arg in case you plan to call + get_reusable_component repeatedly for the same module to make this method faster. + :param old_mmd: ModuleMd.Module of `previous_module_build`. If not passed, + it is taken from previous_module_build.mmd(). Consider passing this arg in + case you plan to call get_reusable_component repeatedly for the same + module to make this method faster. + :return: the component (RPM) build SQLAlchemy object, if one is not found, + None is returned + """ + + # We support component reusing only for koji and test backend. + if conf.system not in ['koji', 'test']: + return None + + # If the rebuild strategy is "all", that means that nothing can be reused + if module.rebuild_strategy == 'all': + log.info('Cannot re-use the component because the rebuild strategy is "all".') + return None + + if not previous_module_build: + previous_module_build = _get_reusable_module(session, module) + if not previous_module_build: + return None + + if not mmd: + mmd = module.mmd() + if not old_mmd: + old_mmd = previous_module_build.mmd() + + # If the chosen component for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + new_module_build_component = models.ComponentBuild.from_component_name( + session, component_name, module.id) + if not new_module_build_component or not new_module_build_component.batch \ + or not new_module_build_component.ref: + log.info('Cannot re-use. New component not found in the db.') + return None + + prev_module_build_component = models.ComponentBuild.from_component_name( + session, component_name, previous_module_build.id) + # If the component to reuse for some reason was not found in the database, + # or the ref is missing, something has gone wrong and the component cannot + # be reused + if not prev_module_build_component or not prev_module_build_component.batch\ + or not prev_module_build_component.ref: + log.info('Cannot re-use. Previous component not found in the db.') + return None + + # Make sure the ref for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.ref != new_module_build_component.ref: + log.info('Cannot re-use. Component commit hashes do not match.') + return None + + # At this point we've determined that both module builds contain the component + # and the components share the same commit hash + if module.rebuild_strategy == 'changed-and-after': + # Make sure the batch number for the component that is trying to be reused + # hasn't changed since the last build + if prev_module_build_component.batch != new_module_build_component.batch: + log.info('Cannot re-use. Batch numbers do not match.') + return None + + # If the mmd.buildopts.macros.rpms changed, we cannot reuse + if mmd.get_rpm_buildopts().get('macros') != old_mmd.get_rpm_buildopts().get('macros'): + log.info('Cannot re-use. Old modulemd macros do not match the new.') + return None + + # At this point we've determined that both module builds contain the component + # with the same commit hash and they are in the same batch. We've also determined + # that both module builds depend(ed) on the same exact module builds. Now it's time + # to determine if the components before it have changed. + # + # Convert the component_builds to a list and sort them by batch + new_component_builds = list(module.component_builds) + new_component_builds.sort(key=lambda x: x.batch) + prev_component_builds = list(previous_module_build.component_builds) + prev_component_builds.sort(key=lambda x: x.batch) + + new_module_build_components = [] + previous_module_build_components = [] + # Create separate lists for the new and previous module build. These lists + # will have an entry for every build batch *before* the component's + # batch except for 1, which is reserved for the module-build-macros RPM. + # Each batch entry will contain a set of "(name, ref)" with the name and + # ref (commit) of the component. + for i in range(new_module_build_component.batch - 1): + # This is the first batch which we want to skip since it will always + # contain only the module-build-macros RPM and it gets built every time + if i == 0: + continue + + new_module_build_components.append(set([ + (value.package, value.ref) for value in + new_component_builds if value.batch == i + 1 + ])) + + previous_module_build_components.append(set([ + (value.package, value.ref) for value in + prev_component_builds if value.batch == i + 1 + ])) + + # If the previous batches don't have the same ordering and hashes, then the + # component can't be reused + if previous_module_build_components != new_module_build_components: + log.info('Cannot re-use. Ordering or commit hashes of ' + 'previous batches differ.') + return None + + reusable_component = models.ComponentBuild.query.filter_by( + package=component_name, module_id=previous_module_build.id).one() + log.debug('Found reusable component!') + return reusable_component diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py new file mode 100644 index 00000000..8d209cc0 --- /dev/null +++ b/module_build_service/utils/submit.py @@ -0,0 +1,551 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +import re +import time +import shutil +import tempfile +import os +from multiprocessing.dummy import Pool as ThreadPool +from datetime import datetime + +from module_build_service import conf, db, log, models, Modulemd +from module_build_service.errors import ( + ValidationError, UnprocessableEntity, Forbidden, Conflict) +from module_build_service import glib +import module_build_service.scm +from .mse import generate_expanded_mmds + + +def _scm_get_latest(pkg): + try: + # If the modulemd specifies that the 'f25' branch is what + # we want to pull from, we need to resolve that f25 branch + # to the specific commit available at the time of + # submission (now). + pkgref = module_build_service.scm.SCM( + pkg.get_repository()).get_latest(pkg.get_ref()) + except Exception as e: + log.exception(e) + return {'error': "Failed to get the latest commit for %s#%s" % ( + pkg.get_repository(), pkg.get_ref())} + + return { + 'pkg_name': pkg.get_name(), + 'pkg_ref': pkgref, + 'error': None + } + + +def format_mmd(mmd, scmurl, session=None): + """ + Prepares the modulemd for the MBS. This does things such as replacing the + branches of components with commit hashes and adding metadata in the xmd + dictionary. + :param mmd: the ModuleMetadata object to format + :param scmurl: the url to the modulemd + """ + # Import it here, because SCM uses utils methods and fails to import + # them because of dep-chain. + from module_build_service.scm import SCM + + if not session: + session = db.session + + xmd = glib.from_variant_dict(mmd.get_xmd()) + if 'mbs' not in xmd: + xmd['mbs'] = {} + if 'scmurl' not in xmd['mbs']: + xmd['mbs']['scmurl'] = scmurl or '' + if 'commit' not in xmd['mbs']: + xmd['mbs']['commit'] = '' + + local_modules = models.ModuleBuild.local_modules(session) + local_modules = {m.name + "-" + m.stream: m for m in local_modules} + + # If module build was submitted via yaml file, there is no scmurl + if scmurl: + scm = SCM(scmurl) + # If a commit hash is provided, add that information to the modulemd + if scm.commit: + # We want to make sure we have the full commit hash for consistency + if SCM.is_full_commit_hash(scm.scheme, scm.commit): + full_scm_hash = scm.commit + else: + full_scm_hash = scm.get_full_commit_hash() + + xmd['mbs']['commit'] = full_scm_hash + # If a commit hash wasn't provided then just get the latest from master + else: + xmd['mbs']['commit'] = scm.get_latest() + + if mmd.get_rpm_components() or mmd.get_module_components(): + if 'rpms' not in xmd['mbs']: + xmd['mbs']['rpms'] = {} + # Add missing data in RPM components + for pkgname, pkg in mmd.get_rpm_components().items(): + if pkg.get_repository() and not conf.rpms_allow_repository: + raise Forbidden( + "Custom component repositories aren't allowed. " + "%r bears repository %r" % (pkgname, pkg.get_repository())) + if pkg.get_cache() and not conf.rpms_allow_cache: + raise Forbidden( + "Custom component caches aren't allowed. " + "%r bears cache %r" % (pkgname, pkg.cache)) + if not pkg.get_repository(): + pkg.set_repository(conf.rpms_default_repository + pkgname) + if not pkg.get_cache(): + pkg.set_cache(conf.rpms_default_cache + pkgname) + if not pkg.get_ref(): + pkg.set_ref('master') + + # Add missing data in included modules components + for modname, mod in mmd.get_module_components().items(): + if mod.get_repository() and not conf.modules_allow_repository: + raise Forbidden( + "Custom module repositories aren't allowed. " + "%r bears repository %r" % (modname, mod.get_repository())) + if not mod.get_repository(): + mod.set_repository(conf.modules_default_repository + modname) + if not mod.get_ref(): + mod.set_ref('master') + + # Check that SCM URL is valid and replace potential branches in pkg refs + # by real SCM hash and store the result to our private xmd place in modulemd. + pool = ThreadPool(20) + pkg_dicts = pool.map(_scm_get_latest, mmd.get_rpm_components().values()) + err_msg = "" + for pkg_dict in pkg_dicts: + if pkg_dict["error"]: + err_msg += pkg_dict["error"] + "\n" + else: + pkg_name = pkg_dict["pkg_name"] + pkg_ref = pkg_dict["pkg_ref"] + xmd['mbs']['rpms'][pkg_name] = {'ref': pkg_ref} + if err_msg: + raise UnprocessableEntity(err_msg) + + # Set the modified xmd back to the modulemd + mmd.set_xmd(glib.dict_values(xmd)) + + +def validate_mmd(mmd): + for modname, mod in mmd.get_module_components().items(): + if mod.get_repository() and not conf.modules_allow_repository: + raise Forbidden( + "Custom module repositories aren't allowed. " + "%r bears repository %r" % (modname, mod.get_repository())) + + +def merge_included_mmd(mmd, included_mmd): + """ + Merges two modulemds. This merges only metadata which are needed in + the `main` when it includes another module defined by `included_mmd` + """ + included_xmd = glib.from_variant_dict(included_mmd.get_xmd()) + if 'rpms' in included_xmd['mbs']: + xmd = glib.from_variant_dict(mmd.get_xmd()) + if 'rpms' not in xmd['mbs']: + xmd['mbs']['rpms'] = included_xmd['mbs']['rpms'] + else: + xmd['mbs']['rpms'].update(included_xmd['mbs']['rpms']) + # Set the modified xmd back to the modulemd + mmd.set_xmd(glib.dict_values(xmd)) + + +def record_component_builds(mmd, module, initial_batch=1, + previous_buildorder=None, main_mmd=None, session=None): + # Imported here to allow import of utils in GenericBuilder. + import module_build_service.builder + + if not session: + session = db.session + + # Format the modulemd by putting in defaults and replacing streams that + # are branches with commit hashes + format_mmd(mmd, module.scmurl, session=session) + + # When main_mmd is set, merge the metadata from this mmd to main_mmd, + # otherwise our current mmd is main_mmd. + if main_mmd: + # Check for components that are in both MMDs before merging since MBS + # currently can't handle that situation. + duplicate_components = [rpm for rpm in main_mmd.get_rpm_components().keys() + if rpm in mmd.get_rpm_components()] + if duplicate_components: + error_msg = ( + 'The included module "{0}" in "{1}" have the following ' + 'conflicting components: {2}'.format( + mmd.get_name(), main_mmd.get_name(), ', '.join(duplicate_components))) + raise UnprocessableEntity(error_msg) + merge_included_mmd(main_mmd, mmd) + else: + main_mmd = mmd + + # If the modulemd yaml specifies components, then submit them for build + rpm_components = mmd.get_rpm_components().values() + module_components = mmd.get_module_components().values() + all_components = rpm_components + module_components + if not all_components: + return + + rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights( + [c.get_name() for c in rpm_components]) + all_components.sort(key=lambda x: x.get_buildorder()) + # We do not start with batch = 0 here, because the first batch is + # reserved for module-build-macros. First real components must be + # planned for batch 2 and following. + batch = initial_batch + + for component in all_components: + # Increment the batch number when buildorder increases. + if previous_buildorder != component.get_buildorder(): + previous_buildorder = component.get_buildorder() + batch += 1 + + # If the component is another module, we fetch its modulemd file + # and record its components recursively with the initial_batch + # set to our current batch, so the components of this module + # are built in the right global order. + if isinstance(component, Modulemd.ComponentModule): + full_url = component.get_repository() + "?#" + component.get_ref() + # It is OK to whitelist all URLs here, because the validity + # of every URL have been already checked in format_mmd(...). + included_mmd = _fetch_mmd(full_url, whitelist_url=True)[0] + batch = record_component_builds(included_mmd, module, batch, + previous_buildorder, main_mmd, session=session) + continue + + component_ref = mmd.get_xmd()['mbs']['rpms'][component.get_name()]['ref'] + full_url = component.get_repository() + "?#" + component_ref + build = models.ComponentBuild( + module_id=module.id, + package=component.get_name(), + format="rpms", + scmurl=full_url, + batch=batch, + ref=component_ref, + weight=rpm_weights[component.get_name()] + ) + session.add(build) + + return batch + + +def submit_module_build_from_yaml(username, handle, stream=None, skiptests=False, + optional_params=None): + yaml_file = handle.read() + mmd = load_mmd(yaml_file) + + # Mimic the way how default values are generated for modules that are stored in SCM + # We can take filename as the module name as opposed to repo name, + # and also we can take numeric representation of current datetime + # as opposed to datetime of the last commit + dt = datetime.utcfromtimestamp(int(time.time())) + def_name = str(os.path.splitext(os.path.basename(handle.filename))[0]) + def_version = int(dt.strftime("%Y%m%d%H%M%S")) + mmd.set_name(mmd.get_name() or def_name) + mmd.set_stream(stream or mmd.get_stream() or "master") + mmd.set_version(mmd.get_version() or def_version) + if skiptests: + buildopts = mmd.get_rpm_buildopts() + buildopts["macros"] = buildopts.get("macros", "") + "\n\n%__spec_check_pre exit 0\n" + mmd.set_rpm_buildopts(buildopts) + return submit_module_build(username, None, mmd, None, optional_params) + + +_url_check_re = re.compile(r"^[^:/]+:.*$") + + +def submit_module_build_from_scm(username, url, branch, allow_local_url=False, + optional_params=None): + # Translate local paths into file:// URL + if allow_local_url and not _url_check_re.match(url): + log.info( + "'{}' is not a valid URL, assuming local path".format(url)) + url = os.path.abspath(url) + url = "file://" + url + mmd, scm = _fetch_mmd(url, branch, allow_local_url) + + return submit_module_build(username, url, mmd, scm, optional_params) + + +def submit_module_build(username, url, mmd, scm, optional_params=None): + """ + Submits new module build. + + :param str username: Username of the build's owner. + :param str url: SCM URL of submitted build. + :param Modulemd.Module mmd: Modulemd defining the build. + :param scm.SCM scm: SCM class representing the cloned git repo. + :param dict optional_params: Dict with optional params for a build: + - "local_build" (bool): The module is being built locally (the MBS is + not running in infra, but on local developer's machine). + - "default_streams" (dict): Dict with name:stream mapping defining the stream + to choose for given module name if multiple streams are available to choose from. + - Any optional ModuleBuild class field (str). + :rtype: list with ModuleBuild + :return: List with submitted module builds. + """ + import koji # Placed here to avoid py2/py3 conflicts... + + # For local builds, we want the user to choose the exact stream using the default_streams + # in case there are multiple streams to choose from and raise an exception otherwise. + if optional_params and "local_build" in optional_params: + raise_if_stream_ambigous = True + del optional_params["local_build"] + else: + raise_if_stream_ambigous = False + + # Get the default_streams if set. + if optional_params and "default_streams" in optional_params: + default_streams = optional_params["default_streams"] + del optional_params["default_streams"] + else: + default_streams = {} + + validate_mmd(mmd) + mmds = generate_expanded_mmds(db.session, mmd, raise_if_stream_ambigous, default_streams) + modules = [] + + for mmd in mmds: + log.debug('Checking whether module build already exists: %s.', + ":".join([mmd.get_name(), mmd.get_stream(), + str(mmd.get_version()), mmd.get_context()])) + module = models.ModuleBuild.get_build_from_nsvc( + db.session, mmd.get_name(), mmd.get_stream(), str(mmd.get_version()), + mmd.get_context()) + if module: + if module.state != models.BUILD_STATES['failed']: + err_msg = ('Module (state=%s) already exists. Only a new build or resubmission of ' + 'a failed build is allowed.' % module.state) + log.error(err_msg) + raise Conflict(err_msg) + if optional_params: + rebuild_strategy = optional_params.get('rebuild_strategy') + if rebuild_strategy and module.rebuild_strategy != rebuild_strategy: + raise ValidationError( + 'You cannot change the module\'s "rebuild_strategy" when ' + 'resuming a module build') + log.debug('Resuming existing module build %r' % module) + # Reset all component builds that didn't complete + for component in module.component_builds: + if component.state and component.state != koji.BUILD_STATES['COMPLETE']: + component.state = None + component.state_reason = None + db.session.add(component) + module.username = username + prev_state = module.previous_non_failed_state + if prev_state == models.BUILD_STATES['init']: + transition_to = models.BUILD_STATES['init'] + else: + transition_to = models.BUILD_STATES['wait'] + module.batch = 0 + module.transition(conf, transition_to, "Resubmitted by %s" % username) + log.info("Resumed existing module build in previous state %s" % module.state) + else: + log.debug('Creating new module build') + module = models.ModuleBuild.create( + db.session, + conf, + name=mmd.get_name(), + stream=mmd.get_stream(), + version=str(mmd.get_version()), + modulemd=mmd.dumps(), + scmurl=url, + username=username, + **(optional_params or {}) + ) + module.build_context, module.runtime_context = \ + module.contexts_from_mmd(module.modulemd) + + db.session.add(module) + db.session.commit() + modules.append(module) + log.info("%s submitted build of %s, stream=%s, version=%s, context=%s", username, + mmd.get_name(), mmd.get_stream(), mmd.get_version(), mmd.get_context()) + return modules + + +def _fetch_mmd(url, branch=None, allow_local_url=False, whitelist_url=False): + # Import it here, because SCM uses utils methods + # and fails to import them because of dep-chain. + import module_build_service.scm + + td = None + scm = None + try: + log.debug('Verifying modulemd') + td = tempfile.mkdtemp() + if whitelist_url: + scm = module_build_service.scm.SCM(url, branch, [url], allow_local_url) + else: + scm = module_build_service.scm.SCM(url, branch, conf.scmurls, allow_local_url) + scm.checkout(td) + scm.verify() + cofn = scm.get_module_yaml() + mmd = load_mmd(cofn, is_file=True) + finally: + try: + if td is not None: + shutil.rmtree(td) + except Exception as e: + log.warning( + "Failed to remove temporary directory {!r}: {}".format( + td, str(e))) + + # If the name was set in the modulemd, make sure it matches what the scmurl + # says it should be + if mmd.get_name() and mmd.get_name() != scm.name: + raise ValidationError('The name "{0}" that is stored in the modulemd ' + 'is not valid'.format(mmd.get_name())) + else: + mmd.set_name(scm.name) + + # If the stream was set in the modulemd, make sure it matches what the repo + # branch is + if mmd.get_stream() and mmd.get_stream() != scm.branch: + raise ValidationError('The stream "{0}" that is stored in the modulemd ' + 'does not match the branch "{1}"'.format( + mmd.get_stream(), scm.branch)) + else: + mmd.set_stream(str(scm.branch)) + + # If the version is in the modulemd, throw an exception since the version + # is generated by pdc-updater + if mmd.get_version(): + raise ValidationError('The version "{0}" is already defined in the ' + 'modulemd but it shouldn\'t be since the version ' + 'is generated based on the commit time'.format( + mmd.get_version())) + else: + mmd.set_version(int(scm.version)) + + return mmd, scm + + +def load_mmd(yaml, is_file=False): + try: + if is_file: + mmd = Modulemd.Module().new_from_file(yaml) + else: + mmd = Modulemd.Module().new_from_string(yaml) + # If the modulemd was v1, it will be upgraded to v2 + mmd.upgrade() + except Exception: + error = 'The following invalid modulemd was encountered: {0}'.format(yaml) + log.error(error) + raise UnprocessableEntity(error) + + return mmd + + +def load_local_builds(local_build_nsvs, session=None): + """ + Loads previously finished local module builds from conf.mock_resultsdir + and imports them to database. + + :param local_build_nsvs: List of NSV separated by ':' defining the modules + to load from the mock_resultsdir. + """ + if not local_build_nsvs: + return + + if not session: + session = db.session + + if type(local_build_nsvs) != list: + local_build_nsvs = [local_build_nsvs] + + # Get the list of all available local module builds. + builds = [] + try: + for d in os.listdir(conf.mock_resultsdir): + m = re.match('^module-(.*)-([^-]*)-([0-9]+)$', d) + if m: + builds.append((m.group(1), m.group(2), int(m.group(3)), d)) + except OSError: + pass + + # Sort with the biggest version first + try: + # py27 + builds.sort(lambda a, b: -cmp(a[2], b[2])) + except TypeError: + # py3 + builds.sort(key=lambda a: a[2], reverse=True) + + for build_id in local_build_nsvs: + parts = build_id.split(':') + if len(parts) < 1 or len(parts) > 3: + raise RuntimeError( + 'The local build "{0}" couldn\'t be be parsed into ' + 'NAME[:STREAM[:VERSION]]'.format(build_id)) + + name = parts[0] + stream = parts[1] if len(parts) > 1 else None + version = int(parts[2]) if len(parts) > 2 else None + + found_build = None + for build in builds: + if name != build[0]: + continue + if stream is not None and stream != build[1]: + continue + if version is not None and version != build[2]: + continue + + found_build = build + break + + if not found_build: + raise RuntimeError( + 'The local build "{0}" couldn\'t be found in "{1}"'.format( + build_id, conf.mock_resultsdir)) + + # Load the modulemd metadata. + path = os.path.join(conf.mock_resultsdir, found_build[3], 'results') + mmd = load_mmd(os.path.join(path, 'modules.yaml'), is_file=True) + + # Create ModuleBuild in database. + module = models.ModuleBuild.create( + session, + conf, + name=mmd.get_name(), + stream=mmd.get_stream(), + version=str(mmd.get_version()), + modulemd=mmd.dumps(), + scmurl="", + username="mbs", + publish_msg=False) + module.koji_tag = path + module.state = models.BUILD_STATES['ready'] + session.commit() + + if (found_build[0] != module.name or found_build[1] != module.stream or + str(found_build[2]) != module.version): + raise RuntimeError( + 'Parsed metadata results for "{0}" don\'t match the directory name' + .format(found_build[3])) + log.info("Loaded local module build %r", module) diff --git a/module_build_service/utils/views.py b/module_build_service/utils/views.py new file mode 100644 index 00000000..00b46ae5 --- /dev/null +++ b/module_build_service/utils/views.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Ralph Bean +# Matt Prahl +# Jan Kaluza +import re +import copy +from functools import wraps +from datetime import datetime + +from flask import request, url_for, Response +from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean + +from module_build_service import models, api_version +from module_build_service.errors import ValidationError, NotFound +from .general import scm_url_schemes + + +def get_scm_url_re(): + schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) + return re.compile( + r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" + r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") + + +def pagination_metadata(p_query, api_version, request_args): + """ + Returns a dictionary containing metadata about the paginated query. + This must be run as part of a Flask request. + :param p_query: flask_sqlalchemy.Pagination object + :param api_version: an int of the API version + :param request_args: a dictionary of the arguments that were part of the + Flask request + :return: a dictionary containing metadata about the paginated query + """ + request_args_wo_page = dict(copy.deepcopy(request_args)) + # Remove pagination related args because those are handled elsewhere + # Also, remove any args that url_for accepts in case the user entered + # those in + for key in ['page', 'per_page', 'endpoint']: + if key in request_args_wo_page: + request_args_wo_page.pop(key) + for key in request_args: + if key.startswith('_'): + request_args_wo_page.pop(key) + + pagination_data = { + 'page': p_query.page, + 'pages': p_query.pages, + 'per_page': p_query.per_page, + 'prev': None, + 'next': None, + 'total': p_query.total, + 'first': url_for(request.endpoint, api_version=api_version, page=1, + per_page=p_query.per_page, _external=True, **request_args_wo_page), + 'last': url_for(request.endpoint, api_version=api_version, page=p_query.pages, + per_page=p_query.per_page, _external=True, + **request_args_wo_page) + } + + if p_query.has_prev: + pagination_data['prev'] = url_for(request.endpoint, api_version=api_version, + page=p_query.prev_num, per_page=p_query.per_page, + _external=True, **request_args_wo_page) + if p_query.has_next: + pagination_data['next'] = url_for(request.endpoint, api_version=api_version, + page=p_query.next_num, per_page=p_query.per_page, + _external=True, **request_args_wo_page) + + return pagination_data + + +def _add_order_by_clause(flask_request, query, column_source): + """ + Orders the given SQLAlchemy query based on the GET arguments provided + :param flask_request: a Flask request object + :param query: a SQLAlchemy query object + :param column_source: a SQLAlchemy database model + :return: a SQLAlchemy query object + """ + colname = "id" + descending = True + order_desc_by = flask_request.args.get("order_desc_by", None) + if order_desc_by: + colname = order_desc_by + else: + order_by = flask_request.args.get("order_by", None) + if order_by: + colname = order_by + descending = False + + column = getattr(column_source, colname, None) + if not column: + raise ValidationError('An invalid order_by or order_desc_by key ' + 'was supplied') + if descending: + column = column.desc() + return query.order_by(column) + + +def str_to_bool(value): + """ + Parses a string to determine its boolean value + :param value: a string + :return: a boolean + """ + return value.lower() in ["true", "1"] + + +def filter_component_builds(flask_request): + """ + Returns a flask_sqlalchemy.Pagination object based on the request parameters + :param request: Flask request object + :return: flask_sqlalchemy.Pagination + """ + search_query = dict() + for key in request.args.keys(): + # Only filter on valid database columns + if key in models.ComponentBuild.__table__.columns.keys(): + if isinstance(models.ComponentBuild.__table__.columns[key].type, sqlalchemy_boolean): + search_query[key] = str_to_bool(flask_request.args[key]) + else: + search_query[key] = flask_request.args[key] + + state = flask_request.args.get('state', None) + if state: + if state.isdigit(): + search_query['state'] = state + else: + try: + import koji + except ImportError: + raise ValidationError('Cannot filter by state names because koji isn\'t installed') + + if state.upper() in koji.BUILD_STATES: + search_query['state'] = koji.BUILD_STATES[state.upper()] + else: + raise ValidationError('An invalid state was supplied') + + # Allow the user to specify the module build ID with a more intuitive key name + if 'module_build' in flask_request.args: + search_query['module_id'] = flask_request.args['module_build'] + + query = models.ComponentBuild.query + + if search_query: + query = query.filter_by(**search_query) + + query = _add_order_by_clause(flask_request, query, models.ComponentBuild) + + page = flask_request.args.get('page', 1, type=int) + per_page = flask_request.args.get('per_page', 10, type=int) + return query.paginate(page, per_page, False) + + +def filter_module_builds(flask_request): + """ + Returns a flask_sqlalchemy.Pagination object based on the request parameters + :param request: Flask request object + :return: flask_sqlalchemy.Pagination + """ + search_query = dict() + special_columns = ['time_submitted', 'time_modified', 'time_completed', 'state'] + for key in request.args.keys(): + # Only filter on valid database columns but skip columns that are treated specially or + # ignored + if key not in special_columns and key in models.ModuleBuild.__table__.columns.keys(): + search_query[key] = flask_request.args[key] + + state = flask_request.args.get('state', None) + if state: + if state.isdigit(): + search_query['state'] = state + else: + if state in models.BUILD_STATES: + search_query['state'] = models.BUILD_STATES[state] + else: + raise ValidationError('An invalid state was supplied') + + query = models.ModuleBuild.query + + if search_query: + query = query.filter_by(**search_query) + + # This is used when filtering the date request parameters, but it is here to avoid recompiling + utc_iso_datetime_regex = re.compile( + r'^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?' + r'(?:Z|[-+]00(?::00)?)?$') + + # Filter the query based on date request parameters + for item in ('submitted', 'modified', 'completed'): + for context in ('before', 'after'): + request_arg = '%s_%s' % (item, context) # i.e. submitted_before + iso_datetime_arg = request.args.get(request_arg, None) + + if iso_datetime_arg: + iso_datetime_matches = re.match(utc_iso_datetime_regex, iso_datetime_arg) + + if not iso_datetime_matches or not iso_datetime_matches.group('datetime'): + raise ValidationError(('An invalid Zulu ISO 8601 timestamp was provided' + ' for the "%s" parameter') + % request_arg) + # Converts the ISO 8601 string to a datetime object for SQLAlchemy to use to filter + item_datetime = datetime.strptime(iso_datetime_matches.group('datetime'), + '%Y-%m-%dT%H:%M:%S') + # Get the database column to filter against + column = getattr(models.ModuleBuild, 'time_' + item) + + if context == 'after': + query = query.filter(column >= item_datetime) + elif context == 'before': + query = query.filter(column <= item_datetime) + + query = _add_order_by_clause(flask_request, query, models.ModuleBuild) + + page = flask_request.args.get('page', 1, type=int) + per_page = flask_request.args.get('per_page', 10, type=int) + return query.paginate(page, per_page, False) + + +def cors_header(allow='*'): + """ + A decorator that sets the Access-Control-Allow-Origin header to the desired value on a Flask + route + :param allow: a string of the domain to allow. This defaults to '*'. + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + rv = func(*args, **kwargs) + if rv: + # If a tuple was provided, then the Flask Response should be the first object + if isinstance(rv, tuple): + response = rv[0] + else: + response = rv + # Make sure we are dealing with a Flask Response object + if isinstance(response, Response): + response.headers.add('Access-Control-Allow-Origin', allow) + return rv + return wrapper + return decorator + + +def validate_api_version(): + """ + A decorator that validates the requested API version on a route + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + req_api_version = kwargs.get('api_version', 1) + if req_api_version > api_version or req_api_version < 1: + raise NotFound('The requested API version is not available') + return func(*args, **kwargs) + return wrapper + return decorator diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 8c3ac7b1..812c8f50 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -1111,7 +1111,7 @@ class TestBuild: '620ec77321b2ea7b0d67d82992dda3e1d67055b4') stop = module_build_service.scheduler.make_simple_stop_condition(db.session) - with patch('module_build_service.utils.format_mmd') as mock_format_mmd: + with patch('module_build_service.utils.submit.format_mmd') as mock_format_mmd: mock_format_mmd.side_effect = Forbidden( 'Custom component repositories aren\'t allowed.') rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps( @@ -1210,7 +1210,7 @@ class TestBuild: module = db.session.query(models.ModuleBuild).get(module_build_id) return module.batch == 2 or module.state >= models.BUILD_STATES['done'] - with patch('module_build_service.utils.at_concurrent_component_threshold') as mock_acct: + with patch('module_build_service.utils.batches.at_concurrent_component_threshold') as mock_acct: # Once we get to batch 2, then simulate the concurrent threshold being met def _at_concurrent_component_threshold(config, session): return db.session.query(models.ModuleBuild).get(module_build_id).batch == 2 diff --git a/tests/test_scheduler/test_poller.py b/tests/test_scheduler/test_poller.py index 4ec11b50..cc97a534 100644 --- a/tests/test_scheduler/test_poller.py +++ b/tests/test_scheduler/test_poller.py @@ -43,7 +43,7 @@ class TestPoller: clean_database() @pytest.mark.parametrize('fresh', [True, False]) - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') def test_process_paused_module_builds(self, start_build_component, create_builder, koji_get_session, global_consumer, dbg, fresh): diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index d2a56e26..ad33537e 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -197,7 +197,7 @@ class TestUtilsComponentReuse: db.session, second_module_build, 'tangerine') assert tangerine_rv is None - @patch("module_build_service.utils.submit_module_build") + @patch("module_build_service.utils.submit.submit_module_build") def test_submit_module_build_from_yaml_with_skiptests(self, mock_submit): """ Tests local module build from a yaml file with the skiptests option @@ -619,7 +619,7 @@ class TestBatches: # Check that packages have been tagged just once. assert len(DummyModuleBuilder.TAGGED_COMPONENTS) == 2 - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') def test_start_next_batch_build_reuse_some(self, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build: @@ -660,7 +660,7 @@ class TestBatches: assert plc_component.reused_component_id is None mock_sbc.assert_called_once() - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') @patch('module_build_service.config.Config.rebuild_strategy', new_callable=mock.PropertyMock, return_value='all') def test_start_next_batch_build_rebuild_strategy_all( @@ -685,7 +685,7 @@ class TestBatches: # Make sure that both components in the batch were submitted assert len(mock_sbc.mock_calls) == 2 - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') @patch('module_build_service.config.Config.rebuild_strategy', new_callable=mock.PropertyMock, return_value='only-changed') def test_start_next_batch_build_rebuild_strategy_only_changed( @@ -749,7 +749,7 @@ class TestBatches: assert component_build.reused_component_id is not None mock_sbc.assert_not_called() - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') def test_start_next_batch_build_smart_scheduling(self, mock_sbc, default_buildroot_groups): """ Tests that components with the longest build time will be scheduled first @@ -788,7 +788,7 @@ class TestBatches: expected_calls = [mock.call(builder, plc_component), mock.call(builder, pt_component)] assert mock_sbc.mock_calls == expected_calls - @patch('module_build_service.utils.start_build_component') + @patch('module_build_service.utils.batches.start_build_component') def test_start_next_batch_continue(self, mock_sbc, default_buildroot_groups): """ Tests that start_next_batch_build does not start new batch when