From a5cc4eb280dc2a98d57ffea9cfb5d655e8922f63 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Wed, 1 Feb 2017 15:39:13 -0500 Subject: [PATCH] Add a module's commit hash, scmurl, and the buildrequires' commit hashes, stream, and version in the modulemd --- module_build_service/pdc.py | 31 ++++++++++++++++ module_build_service/scm.py | 52 ++++++++++++++++++++++++++ module_build_service/utils.py | 70 +++++++++++++++++++++++++++++++++-- module_build_service/views.py | 13 ++----- requirements.txt | 2 +- 5 files changed, 155 insertions(+), 13 deletions(-) diff --git a/module_build_service/pdc.py b/module_build_service/pdc.py index f6ae7339..a50b9a18 100644 --- a/module_build_service/pdc.py +++ b/module_build_service/pdc.py @@ -339,3 +339,34 @@ def get_module_build_dependencies(session, module_info, strict=False): deps = module_depsolving_wrapper(session, deps, strict=strict) return deps + +def get_module_commit_hash_and_version(session, module_info): + """ + Gets the commit hash and version of a module stored in PDC + :param module_info: a dict containing filters for PDC + :param session: a PDC session instance + :return: a tuple containing the string of the commit hash and the version + of the module stored in PDC. If a value is not found, None is + returned for the values that aren't found. + """ + commit_hash = None + version = None + module = get_module(session, module_info) + if module: + if module.get('modulemd'): + mmd = modulemd.ModuleMetadata() + mmd.loads(module['modulemd']) + if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'): + commit_hash = mmd.xmd['mbs']['commit'] + if module.get('variant_release'): + version = module['variant_release'] + if not commit_hash: + # TODO: Should this eventually be an exception? + log.warn( + 'The commit hash for {0!r} was not part of the modulemd in PDC' + .format(module_info)) + if not version: + # TODO: Should this eventually be an exception? + log.warn( + 'The version for {0!r} was not in PDC'.format(module_info)) + return commit_hash, version diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 5de3f446..4411093d 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -162,6 +162,58 @@ class SCM(object): else: raise RuntimeError("get_latest: Unhandled SCM scheme.") + def get_full_commit_hash(self, commit_hash=None): + """ + Takes a shortened commit hash and returns the full hash + :param commit_hash: a shortened commit hash. If not specified, the + one in the URL will be used + :return: string of the full commit hash + """ + if commit_hash: + commit_to_check = commit_hash + elif self.commit: + commit_to_check = self.commit + else: + raise RuntimeError('No commit hash was specified for "{0}"'.format( + self.url)) + + if self.scheme == 'git': + log.debug('Getting the full commit hash for "{0}"' + .format(self.repository)) + td = None + try: + td = tempfile.mkdtemp() + SCM._run(['git', 'clone', '-q', self.repository, td]) + output = SCM._run( + ['git', 'rev-parse', commit_to_check], chdir=td)[1] + finally: + if td and os.path.exists(td): + shutil.rmtree(td) + + if output: + return str(output.strip('\n')) + + raise RuntimeError( + 'The full commit hash of "{0}" for "{1}" could not be found' + .format(commit_hash, self.repository)) + else: + raise RuntimeError('get_full_commit_hash: Unhandled SCM scheme.') + + @staticmethod + def is_full_commit_hash(scheme, commit): + """ + Determines if a commit hash is the full commit hash. For instance, if + the scheme is git, it will determine if the commit is a full SHA1 hash + :param scheme: a string containing the SCM type (e.g. git) + :param commit: a string containing the commit + :return: boolean + """ + if scheme == 'git': + sha1_pattern = re.compile(r'^[0-9a-f]{40}$') + return bool(re.match(sha1_pattern, commit)) + else: + raise RuntimeError('is_full_commit_hash: Unhandled SCM scheme.') + def is_available(self, strict=False): """Check whether the scmurl is available for checkout. diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 8dcf9d8c..f4d4623c 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -29,6 +29,8 @@ import shutil import tempfile import os import logging +import copy +from six import iteritems import modulemd @@ -41,6 +43,8 @@ from module_build_service import conf, db from module_build_service.errors import (Unauthorized, Conflict) import module_build_service.messaging from multiprocessing.dummy import Pool as ThreadPool +import module_build_service.pdc +from module_build_service.pdc import get_module_commit_hash_and_version import concurrent.futures @@ -328,7 +332,62 @@ def _scm_get_latest(pkg): return "Failed to get the latest commit for %s#%s" % (pkg.repository, pkg.ref) return None -def format_mmd(mmd): +def format_mmd(mmd, scmurl): + """ + Prepares the modulemd for the MBS. This does things such as replacing the + branches of components with commit hashes and adding metadata in the xmd + dictionary. + :param mmd: the ModuleMetadata object to format + :param scmurl: the url to the modulemd + """ + # Import it here, because SCM uses utils methods and fails to import + # them because of dep-chain. + from module_build_service.scm import SCM + + mmd.xmd['mbs'] = {'scmurl': scmurl} + + scm = SCM(scmurl) + # If a commit hash is provided, add that information to the modulemd + if scm.commit: + # We want to make sure we have the full commit hash for consistency + if SCM.is_full_commit_hash(scm.scheme, scm.commit): + full_scm_hash = scm.commit + else: + full_scm_hash = scm.get_full_commit_hash() + + mmd.xmd['mbs']['commit'] = full_scm_hash + # If a commit hash wasn't provided then just get the latest from master + else: + scm = SCM(scmurl) + mmd.xmd['mbs']['commit'] = scm.get_latest() + + + # If the modulemd yaml specifies module buildrequires, replace the streams + # with commit hashes + if mmd.buildrequires: + mmd.xmd['mbs']['buildrequires'] = copy.deepcopy(mmd.buildrequires) + pdc = module_build_service.pdc.get_pdc_client_session(conf) + for module_name, module_stream in \ + mmd.xmd['mbs']['buildrequires'].items(): + # Assumes that module_stream is the stream and not the commit hash + module_info = { + 'name': module_name, + 'version': module_stream} + commit_hash, version = get_module_commit_hash_and_version( + pdc, module_info) + if commit_hash and version: + mmd.xmd['mbs']['buildrequires'][module_name] = { + 'ref': commit_hash, + 'stream': mmd.buildrequires[module_name], + 'version': version + } + else: + raise RuntimeError( + 'The module "{0}" didn\'t contain either a commit hash or a' + ' version in PDC'.format(module_name)) + else: + mmd.xmd['mbs']['buildrequires'] = {} + if mmd.components: # Add missing data in components for pkgname, pkg in mmd.components.rpms.items(): @@ -353,13 +412,12 @@ def format_mmd(mmd): for err_msg in err_msgs: if err_msg: raise UnprocessableEntity(err_msg) - return mmd def record_component_builds(scm, mmd, module, initial_batch = 1): # Format the modulemd by putting in defaults and replacing streams that # are branches with commit hashes try: - mmd = format_mmd(mmd) + format_mmd(mmd, module.scmurl) except Exception: module.transition(conf, models.BUILD_STATES["failed"]) db.session.add(module) @@ -511,6 +569,12 @@ def scm_url_schemes(terse=False): scheme_list.extend([scheme[:-3] for scheme in scm_schemes]) return list(set(scheme_list)) +def get_scm_url_re(): + schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) + return re.compile( + r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" + r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") + def module_build_state_from_msg(msg): state = int(msg.module_build_state) # TODO better handling diff --git a/module_build_service/views.py b/module_build_service/views.py index 219eb4d8..65fc3187 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -28,15 +28,14 @@ This is the implementation of the orchestrator's public RESTful API. import json import module_build_service.auth -import re - from flask import request, jsonify from flask.views import MethodView from module_build_service import app, conf, log from module_build_service import models, db -from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build_from_scm, \ - submit_module_build_from_yaml, scm_url_schemes +from module_build_service.utils import ( + pagination_metadata, filter_module_builds, submit_module_build_from_scm, + submit_module_build_from_yaml, scm_url_schemes, get_scm_url_re) from module_build_service.errors import ( ValidationError, Unauthorized, NotFound) @@ -121,11 +120,7 @@ class ModuleBuildAPI(MethodView): log.error("The submitted scmurl %r is not allowed" % url) raise Unauthorized("The submitted scmurl %s is not allowed" % url) - schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True))) - scmurl_re = re.compile( - r"(?P(?:(?P(" + schemes_re + r"))://(?P[^/]+))?" - r"(?P/[^\?]+))\?(?P[^#]*)#(?P.+)") - if not scmurl_re.match(url): + if not get_scm_url_re().match(url): log.error("The submitted scmurl %r is not valid" % url) raise Unauthorized("The submitted scmurl %s is not valid" % url) diff --git a/requirements.txt b/requirements.txt index 4fb569c7..5281b57b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ kobo m2crypto m2ext mock -modulemd>=1.0.0,<2.0.0 +modulemd>=1.1.0,<2.0.0 munch pdc-client pyOpenSSL