Merge #578 Allow CoprModuleBuilder to obtain SRPMs from copr-dist-git

module_build_service/builder/CoprModuleBuilder.py

@@ -24,9 +24,12 @@
 
 import logging
 import os
+import re
 import koji
 import tempfile
 import threading
+import subprocess
+import shutil
 
 from copr.client import CoprClient
 from copr.exceptions import CoprRequestException
@@ -39,7 +42,7 @@ import module_build_service.scheduler
 import module_build_service.scheduler.consumer
 
 from base import GenericBuilder
-from utils import build_from_scm, fake_repo_done_message
+from utils import execute_cmd, fake_repo_done_message
 from KojiModuleBuilder import KojiModuleBuilder
 
 logging.basicConfig(level=logging.DEBUG)
@@ -256,6 +259,15 @@ class CoprModuleBuilder(GenericBuilder):
 
         return response.data["ids"][0], koji.BUILD_STATES["BUILDING"], response.message, None
 
+    def build_scm(self, source):
+        # @TODO use this method once support on Copr side is finished
+        # Copr is currently able to create a build from fedora distgit,
+        # but not from custom distgit, such as copr-dist-git
+        url, branch = source.split("?#")
+        url = (url.replace("git://", "https://")
+                  .replace("pkgs.fedoraproject.org", "src.fedoraproject.org/git"))
+        self.client.create_new_build_distgit(self.copr.projectname, url, branch=branch, username=self.copr.username)
+
     def finalize(self):
         modulemd = tempfile.mktemp()
         self.module.mmd().dump(modulemd)
@@ -316,3 +328,78 @@ class CoprModuleBuilder(GenericBuilder):
         return koji_tag.replace("+", "-")
 
 
+def build_from_scm(artifact_name, source, config, build_srpm,
+                   data=None, stdout=None, stderr=None):
+    """
+    Builds the artifact from the SCM based source.
+
+    :param artifact_name: Name of the artifact.
+    :param source: SCM URL with artifact's sources (spec file).
+    :param config: Config instance.
+    :param build_srpm: Method to call to build the RPM from the generate SRPM.
+    :param data: Data to be passed to the build_srpm method.
+    :param stdout: Python file object to which the stdout of SRPM build
+                   command is logged.
+    :param stderr: Python file object to which the stderr of SRPM build
+                   command is logged.
+    """
+    ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM", None)
+    td = None
+
+    try:
+        log.debug('Cloning source URL: %s' % source)
+        url, commit = source.split("?#")
+        # Create temp dir and clone the repo there.
+        td = tempfile.mkdtemp()
+        scm = module_build_service.scm.SCM(source)
+        cod = scm.checkout(td)
+
+        cmd = config.mock_build_srpm_cmd.split(" ")
+        if is_from_copr(source):
+            branch = git_branch_contains(cod, commit)
+            git_checkout(cod, branch)
+            cmd = ["fedpkg-copr", "--release", branch, "srpm"]
+
+        # Use configured command to create SRPM out of the SCM repo.
+        log.debug("Creating SRPM in %s" % cod)
+        execute_cmd(cmd, stdout=stdout, stderr=stderr, cwd=cod)
+
+        # Find out the built SRPM and build it normally.
+        for f in os.listdir(cod):
+            if f.endswith(".src.rpm"):
+                log.info("Created SRPM %s" % f)
+                source = os.path.join(cod, f)
+                ret = build_srpm(artifact_name, source, data)
+                break
+    except Exception as e:
+        log.error("Error while generating SRPM for artifact %s: %s" % (
+            artifact_name, str(e)))
+        ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM %s" % str(e), None)
+    finally:
+        try:
+            if td is not None:
+                shutil.rmtree(td)
+        except Exception as e:
+            log.warning(
+                "Failed to remove temporary directory {!r}: {}".format(
+                    td, str(e)))
+
+    return ret
+
+
+def git_branch_contains(cod, commit):
+    cmd = ["git", "branch", "-r", "--contains", commit]
+    out, err = execute_cmd(cmd, cwd=cod, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    branch = out.strip().split("/")[1]
+    if " -> " in branch:
+        branch = branch.split(" -> ")[0]
+    return branch
+
+
+def is_from_copr(source):
+    return bool(re.match("https?://copr-dist-git(-dev)?\.fedorainfracloud\.org", source))
+
+
+def git_checkout(cod, branch):
+    cmd = ["git", "checkout", branch]
+    execute_cmd(cmd, cwd=cod)

module_build_service/builder/MockModuleBuilder.py

@@ -38,7 +38,7 @@ import module_build_service.scheduler
 import module_build_service.scheduler.consumer
 
 from base import GenericBuilder
-from utils import (build_from_scm, fake_repo_done_message,
+from utils import (fake_repo_done_message,
                    create_local_repo_from_koji_tag, execute_cmd,
                    find_srpm)
 from KojiModuleBuilder import KojiModuleBuilder

module_build_service/builder/utils.py

@@ -1,7 +1,5 @@
 import os
 import sys
-import koji
-import tempfile
 import shutil
 import subprocess
 import munch
@@ -17,59 +15,6 @@ from module_build_service import log, scm, messaging
 logging.basicConfig(level=logging.DEBUG)
 
 
-def build_from_scm(artifact_name, source, config, build_srpm,
-                   data=None, stdout=None, stderr=None):
-    """
-    Builds the artifact from the SCM based source.
-
-    :param artifact_name: Name of the artifact.
-    :param source: SCM URL with artifact's sources (spec file).
-    :param config: Config instance.
-    :param build_srpm: Method to call to build the RPM from the generate SRPM.
-    :param data: Data to be passed to the build_srpm method.
-    :param stdout: Python file object to which the stdout of SRPM build
-                   command is logged.
-    :param stderr: Python file object to which the stderr of SRPM build
-                   command is logged.
-    """
-    ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM", None)
-    td = None
-
-    try:
-        log.debug('Cloning source URL: %s' % source)
-        # Create temp dir and clone the repo there.
-        td = tempfile.mkdtemp()
-        scm = module_build_service.scm.SCM(source)
-        cod = scm.checkout(td)
-
-        # Use configured command to create SRPM out of the SCM repo.
-        log.debug("Creating SRPM in %s" % cod)
-        execute_cmd(config.mock_build_srpm_cmd.split(" "),
-                    stdout=stdout, stderr=stderr, cwd=cod)
-
-        # Find out the built SRPM and build it normally.
-        for f in os.listdir(cod):
-            if f.endswith(".src.rpm"):
-                log.info("Created SRPM %s" % f)
-                source = os.path.join(cod, f)
-                ret = build_srpm(artifact_name, source, data)
-                break
-    except Exception as e:
-        log.error("Error while generating SRPM for artifact %s: %s" % (
-            artifact_name, str(e)))
-        ret = (0, koji.BUILD_STATES["FAILED"], "Cannot create SRPM %s" % str(e), None)
-    finally:
-        try:
-            if td is not None:
-                shutil.rmtree(td)
-        except Exception as e:
-            log.warning(
-                "Failed to remove temporary directory {!r}: {}".format(
-                    td, str(e)))
-
-    return ret
-
-
 def find_srpm(cod):
     for f in os.listdir(cod):
         if f.endswith(".src.rpm"):
@@ -90,18 +35,19 @@ def execute_cmd(args, stdout=None, stderr=None, cwd=None):
     :raises RuntimeError: Raised when command exits with non-zero exit code.
     """
     out_log_msg = ""
-    if stdout:
+    if stdout and hasattr(stdout, "name"):
         out_log_msg += ", stdout log: %s" % stdout.name
-    if stderr:
+    if stderr and hasattr(stderr, "name"):
         out_log_msg += ", stderr log: %s" % stderr.name
 
     log.info("Executing command: %s%s" % (args, out_log_msg))
     proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd)
-    proc.communicate()
+    out, err = proc.communicate()
 
     if proc.returncode != 0:
         err_msg = "Command '%s' returned non-zero value %d%s" % (args, proc.returncode, out_log_msg)
         raise RuntimeError(err_msg)
+    return out, err
 
 
 def fake_repo_done_message(tag_name):