Add a module's commit hash, scmurl, and the buildrequires' commit hashes, stream, and version in the modulemd

module_build_service/pdc.py

@@ -339,3 +339,34 @@ def get_module_build_dependencies(session, module_info, strict=False):
         deps = module_depsolving_wrapper(session, deps, strict=strict)
 
     return deps
+
+def get_module_commit_hash_and_version(session, module_info):
+    """
+    Gets the commit hash and version of a module stored in PDC
+    :param module_info: a dict containing filters for PDC
+    :param session: a PDC session instance
+    :return: a tuple containing the string of the commit hash and the version
+    of the module stored in PDC. If a value is not found, None is
+    returned for the values that aren't found.
+    """
+    commit_hash = None
+    version = None
+    module = get_module(session, module_info)
+    if module:
+        if module.get('modulemd'):
+            mmd = modulemd.ModuleMetadata()
+            mmd.loads(module['modulemd'])
+            if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'):
+                commit_hash = mmd.xmd['mbs']['commit']
+        if module.get('variant_release'):
+            version = module['variant_release']
+    if not commit_hash:
+        # TODO: Should this eventually be an exception?
+        log.warn(
+            'The commit hash for {0!r} was not part of the modulemd in PDC'
+            .format(module_info))
+    if not version:
+        # TODO: Should this eventually be an exception?
+        log.warn(
+            'The version for {0!r} was not in PDC'.format(module_info))
+    return commit_hash, version

module_build_service/scm.py

@@ -162,6 +162,58 @@ class SCM(object):
         else:
             raise RuntimeError("get_latest: Unhandled SCM scheme.")
 
+    def get_full_commit_hash(self, commit_hash=None):
+        """
+        Takes a shortened commit hash and returns the full hash
+        :param commit_hash: a shortened commit hash. If not specified, the
+        one in the URL will be used
+        :return: string of the full commit hash
+        """
+        if commit_hash:
+            commit_to_check = commit_hash
+        elif self.commit:
+            commit_to_check = self.commit
+        else:
+            raise RuntimeError('No commit hash was specified for "{0}"'.format(
+                self.url))
+
+        if self.scheme == 'git':
+            log.debug('Getting the full commit hash for "{0}"'
+                      .format(self.repository))
+            td = None
+            try:
+                td = tempfile.mkdtemp()
+                SCM._run(['git', 'clone', '-q', self.repository, td])
+                output = SCM._run(
+                    ['git', 'rev-parse', commit_to_check], chdir=td)[1]
+            finally:
+                if td and os.path.exists(td):
+                    shutil.rmtree(td)
+
+            if output:
+                return str(output.strip('\n'))
+
+            raise RuntimeError(
+                'The full commit hash of "{0}" for "{1}" could not be found'
+                .format(commit_hash, self.repository))
+        else:
+            raise RuntimeError('get_full_commit_hash: Unhandled SCM scheme.')
+
+    @staticmethod
+    def is_full_commit_hash(scheme, commit):
+        """
+        Determines if a commit hash is the full commit hash. For instance, if
+        the scheme is git, it will determine if the commit is a full SHA1 hash
+        :param scheme: a string containing the SCM type (e.g. git)
+        :param commit: a string containing the commit
+        :return: boolean
+        """
+        if scheme == 'git':
+            sha1_pattern = re.compile(r'^[0-9a-f]{40}$')
+            return bool(re.match(sha1_pattern, commit))
+        else:
+            raise RuntimeError('is_full_commit_hash: Unhandled SCM scheme.')
+
     def is_available(self, strict=False):
         """Check whether the scmurl is available for checkout.
 

module_build_service/utils.py

@@ -29,6 +29,8 @@ import shutil
 import tempfile
 import os
 import logging
+import copy
+from six import iteritems
 
 import modulemd
 
@@ -41,6 +43,8 @@ from module_build_service import conf, db
 from module_build_service.errors import (Unauthorized, Conflict)
 import module_build_service.messaging
 from multiprocessing.dummy import Pool as ThreadPool
+import module_build_service.pdc
+from module_build_service.pdc import get_module_commit_hash_and_version
 
 import concurrent.futures
 
@@ -328,7 +332,62 @@ def _scm_get_latest(pkg):
         return "Failed to get the latest commit for %s#%s" % (pkg.repository, pkg.ref)
     return None
 
-def format_mmd(mmd):
+def format_mmd(mmd, scmurl):
+    """
+    Prepares the modulemd for the MBS. This does things such as replacing the
+    branches of components with commit hashes and adding metadata in the xmd
+    dictionary.
+    :param mmd: the ModuleMetadata object to format
+    :param scmurl: the url to the modulemd
+    """
+    # Import it here, because SCM uses utils methods and fails to import
+    # them because of dep-chain.
+    from module_build_service.scm import SCM
+
+    mmd.xmd['mbs'] = {'scmurl': scmurl}
+
+    scm = SCM(scmurl)
+    # If a commit hash is provided, add that information to the modulemd
+    if scm.commit:
+        # We want to make sure we have the full commit hash for consistency
+        if SCM.is_full_commit_hash(scm.scheme, scm.commit):
+            full_scm_hash = scm.commit
+        else:
+            full_scm_hash = scm.get_full_commit_hash()
+
+        mmd.xmd['mbs']['commit'] = full_scm_hash
+    # If a commit hash wasn't provided then just get the latest from master
+    else:
+        scm = SCM(scmurl)
+        mmd.xmd['mbs']['commit'] = scm.get_latest()
+
+
+    # If the modulemd yaml specifies module buildrequires, replace the streams
+    # with commit hashes
+    if mmd.buildrequires:
+        mmd.xmd['mbs']['buildrequires'] = copy.deepcopy(mmd.buildrequires)
+        pdc = module_build_service.pdc.get_pdc_client_session(conf)
+        for module_name, module_stream in \
+                mmd.xmd['mbs']['buildrequires'].items():
+            # Assumes that module_stream is the stream and not the commit hash
+            module_info = {
+                'name': module_name,
+                'version': module_stream}
+            commit_hash, version = get_module_commit_hash_and_version(
+                pdc, module_info)
+            if commit_hash and version:
+                mmd.xmd['mbs']['buildrequires'][module_name] = {
+                    'ref': commit_hash,
+                    'stream': mmd.buildrequires[module_name],
+                    'version': version
+                }
+            else:
+                raise RuntimeError(
+                    'The module "{0}" didn\'t contain either a commit hash or a'
+                    ' version in PDC'.format(module_name))
+    else:
+        mmd.xmd['mbs']['buildrequires'] = {}
+
     if mmd.components:
         # Add missing data in components
         for pkgname, pkg in mmd.components.rpms.items():
@@ -353,13 +412,12 @@ def format_mmd(mmd):
         for err_msg in err_msgs:
             if err_msg:
                 raise UnprocessableEntity(err_msg)
-    return mmd
 
 def record_component_builds(scm, mmd, module, initial_batch = 1):
     # Format the modulemd by putting in defaults and replacing streams that
     # are branches with commit hashes
     try:
-        mmd = format_mmd(mmd)
+        format_mmd(mmd, module.scmurl)
     except Exception:
         module.transition(conf, models.BUILD_STATES["failed"])
         db.session.add(module)
@@ -511,6 +569,12 @@ def scm_url_schemes(terse=False):
             scheme_list.extend([scheme[:-3] for scheme in scm_schemes])
         return list(set(scheme_list))
 
+def get_scm_url_re():
+    schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True)))
+    return re.compile(
+        r"(?P<giturl>(?:(?P<scheme>(" + schemes_re + r"))://(?P<host>[^/]+))?"
+        r"(?P<repopath>/[^\?]+))\?(?P<modpath>[^#]*)#(?P<revision>.+)")
+
 def module_build_state_from_msg(msg):
     state = int(msg.module_build_state)
     # TODO better handling

module_build_service/views.py

@@ -28,15 +28,14 @@ This is the implementation of the orchestrator's public RESTful API.
 
 import json
 import module_build_service.auth
-import re
-
 from flask import request, jsonify
 from flask.views import MethodView
 
 from module_build_service import app, conf, log
 from module_build_service import models, db
-from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build_from_scm, \
-                                       submit_module_build_from_yaml, scm_url_schemes
+from module_build_service.utils import (
+    pagination_metadata, filter_module_builds, submit_module_build_from_scm,
+    submit_module_build_from_yaml, scm_url_schemes, get_scm_url_re)
 from module_build_service.errors import (
     ValidationError, Unauthorized, NotFound)
 
@@ -121,11 +120,7 @@ class ModuleBuildAPI(MethodView):
             log.error("The submitted scmurl %r is not allowed" % url)
             raise Unauthorized("The submitted scmurl %s is not allowed" % url)
 
-        schemes_re = '|'.join(map(re.escape, scm_url_schemes(terse=True)))
-        scmurl_re = re.compile(
-            r"(?P<giturl>(?:(?P<scheme>(" + schemes_re + r"))://(?P<host>[^/]+))?"
-            r"(?P<repopath>/[^\?]+))\?(?P<modpath>[^#]*)#(?P<revision>.+)")
-        if not scmurl_re.match(url):
+        if not get_scm_url_re().match(url):
             log.error("The submitted scmurl %r is not valid" % url)
             raise Unauthorized("The submitted scmurl %s is not valid" % url)
 

requirements.txt

@@ -11,7 +11,7 @@ kobo
 m2crypto
 m2ext
 mock
-modulemd>=1.0.0,<2.0.0
+modulemd>=1.1.0,<2.0.0
 munch
 pdc-client
 pyOpenSSL