Streamlined cmd line options for local builds

Signed-off-by: Martin Curlej <mcurlej@redhat.com>

.gitignore

@@ -19,3 +19,4 @@ tests/vcr-request-data
 .vscode
 .ropeproject
 tests/test_module_build_service.db-journal
+*.swp

contrib/mbs-build

@@ -317,31 +317,39 @@ def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=Tru
     return -3, None
 
 
-def do_local_build(scm_url, branch, skiptests, local_builds_nsvs, log_flag=None):
+def do_local_build(scm_url, branch, skiptests, local_builds_nsvs, log_flag=None,
+                   yaml_file=None, stream=None):
     """
     Starts the local build using the 'mbs-manager build_module_locally'
     command. Returns exit code of that command or None when scm_url or
     branch are not set and cannot be obtained from the CWD.
     """
-    scm_url = get_scm_url(scm_url, None, local=True)
-    branch = get_scm_branch(branch)
-    if not scm_url or not branch:
-        return None
-
-    logging.info("Starting local build of %s, branch %s", scm_url, branch)
     command = ['mbs-manager']
-    if log_flag:
-        command.append(log_flag)
-    command.append('build_module_locally')
-    if skiptests:
-        command.append('--skiptests')
-        logging.info("Tests will be skipped due to --skiptests option.")
+    if yaml_file:
+        command.append('build_module_locally_from_file')
+        logging.info("Starting local build from yaml file %s" % yaml_file)
+        command.extend(["--file", yaml_file])
+        if stream:
+            command.extend(["--stream", stream])
+    else:
+        command.append('build_module_locally')
+        scm_url = get_scm_url(scm_url, None, local=True)
+        branch = get_scm_branch(branch)
+        if not scm_url or not branch:
+            return None
 
-    if local_builds_nsvs:
-        for build_id in local_builds_nsvs:
-            command += ['--add-local-build', build_id]
+        logging.info("Starting local build of %s, branch %s", scm_url, branch)
+        if log_flag:
+            command.append(log_flag)
+        if skiptests:
+            command.append('--skiptests')
+            logging.info("Tests will be skipped due to --skiptests option.")
 
-    command.extend([scm_url, branch])
+        if local_builds_nsvs:
+            for build_id in local_builds_nsvs:
+                command += ['--add-local-build', build_id]
+
+        command.extend([scm_url, branch])
 
     process = subprocess.Popen(command)
     process.communicate()
@@ -494,6 +502,11 @@ def main():
                               dest="local_builds_nsvs", metavar='BUILD_ID')
     parser_local.add_argument('--skiptests', dest='skiptests', action='store_true',
                               help="add macro for skipping check section/phase")
+    parser_local.add_argument('--file', dest='file', action='store',
+                              help="Path to the modulemd yaml file")
+    parser_local.add_argument('--stream', dest='stream', action='store',
+                              help=("Name of the stream of this build."
+                                    " (builds from files only)"))
 
     parser_overview = subparsers.add_parser(
         'overview', help="show overview of module builds",
@@ -540,7 +553,8 @@ def main():
             print("Submitted module build %r" % build_id)
     elif args.cmd_name == "local":
         sys.exit(do_local_build(args.scm_url, args.branch, args.skiptests,
-                                args.local_builds_nsvs, log_flag))
+                                args.local_builds_nsvs, log_flag, args.file,
+                                args.stream))
     elif args.cmd_name == "watch":
         # Watch the module build.
         try:

module_build_service/config.py

@@ -60,7 +60,9 @@ def init_config(app):
 
     # Load LocalBuildConfiguration section in case we are building modules
     # locally.
-    if "build_module_locally" in sys.argv:
+    local_build_cmds = ["build_module_locally", "build_module_locally_from_file"]
+    local = [cmd for cmd in sys.argv if cmd in local_build_cmds]
+    if local:
         config_section = "LocalBuildConfiguration"
 
     # try getting config_file from os.environ

module_build_service/manage.py

@@ -28,10 +28,12 @@ import logging
 import os
 import getpass
 
+from werkzeug.datastructures import FileStorage
 from module_build_service import app, conf, db, create_app
 from module_build_service import models
 from module_build_service.utils import (
     submit_module_build_from_scm,
+    submit_module_build_from_yaml,
     load_local_builds,
 )
 import module_build_service.messaging
@@ -92,7 +94,7 @@ def cleardb():
 @manager.option('url')
 @manager.option('--skiptests', action='store_true')
 @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs')
-def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False):
+def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False, yaml_file=None, stream=None):
     """ Performs local module build using Mock
     """
     if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']:
@@ -116,15 +118,28 @@ def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False):
         load_local_builds(local_build_nsvs)
 
         username = getpass.getuser()
-        submit_module_build_from_scm(username, url, branch, allow_local_url=True,
-                                     skiptests=skiptests)
-
+        if yaml_file and yaml_file.endswith(".yaml"):
+            yaml_file_path = os.path.abspath(yaml_file)
+            with open(yaml_file_path) as fd:
+                filename = yaml_file.split("/")[-1]
+                handle = FileStorage(fd)
+                handle.filename = filename
+                submit_module_build_from_yaml(username, handle, stream)
+        else:
+            submit_module_build_from_scm(username, url, branch, allow_local_url=True,
+                                         skiptests=skiptests)
         stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
 
         # Run the consumer until stop_condition returns True
         module_build_service.scheduler.main([], stop)
 
 
+@manager.option('--file', action='store', dest="yaml_file")
+@manager.option('--stream', action='store', dest="stream")
+def build_module_locally_from_file(yaml_file, stream=None):
+    build_module_locally(None, None, yaml_file=yaml_file, stream=str(stream))
+
+
 @console_script_help
 @manager.command
 def run(host=None, port=None, debug=None):

module_build_service/scm.py

@@ -86,8 +86,17 @@ class SCM(object):
                 self.name = self.name[:-4]
             self.commit = match.group("commit")
             self.branch = branch if branch else "master"
+            self.latest = False
+            # if not stated otherwise the default behaviour is that we work with
+            # non-local bare repositories
+            self.local = False
+            self.bare_repo = True
+            if url.startswith("file://") and allow_local:
+                self.local = True
+                self.bare_repo = self._is_bare_repo(self.repository[7:])
             if not self.commit:
                 self.commit = self.get_latest(self.branch)
+                self.latest = True
             self.version = None
         else:
             raise ValidationError("Unhandled SCM scheme: %s" % self.scheme)
@@ -170,7 +179,11 @@ class SCM(object):
                             "within the repository. Perhaps you forgot to push. "
                             "The original message was: %s" % e.message)
                     raise
-
+            # will patch the temp git repo with uncommited changes only if there
+            # is no commit repo present in repo definition and its a local dir
+            # and not a bare repo.
+            if self.latest and self.local and not self.bare_repo:
+                self.patch_with_uncommited_changes(self.sourcedir)
             timestamp = SCM._run(["git", "show", "-s", "--format=%ct"], chdir=self.sourcedir)[1]
             dt = datetime.datetime.utcfromtimestamp(int(timestamp))
             self.version = dt.strftime("%Y%m%d%H%M%S")
@@ -197,6 +210,12 @@ class SCM(object):
         else:
             raise RuntimeError("get_latest: Unhandled SCM scheme.")
 
+    def _is_bare_repo(self, repo_path):
+        """ Checks if the repository is a bare repo """
+        is_bare_repo_cmd = ["git", "config", "core.bare"]
+        _, is_bare, _ = SCM._run(is_bare_repo_cmd, chdir=repo_path)
+        return is_bare.rstrip() == "true"
+
     def get_full_commit_hash(self, commit_hash=None):
         """
         Takes a shortened commit hash and returns the full hash
@@ -253,6 +272,30 @@ class SCM(object):
                       "Couldn't access: %s" % path_to_yaml)
             raise UnprocessableEntity("The SCM repository doesn't contain a modulemd file")
 
+    def patch_with_uncommited_changes(self, source_dir):
+        """
+        This method patches the given tmp git repository with uncommented changes from it
+        origin git dir. Creates a patch file witch holds result for `git diff` command
+        executed in the origin repo.
+
+        source_dir (str): path to the temp git repo
+        """
+        module_diff = ['git', 'diff']
+        # striping the self.repository from 'file://'
+        _, diff, _ = SCM._run(module_diff, chdir=self.repository[7:])
+        if diff:
+            try:
+                log.debug("Working with local, non-bare repository. Applying uncommited changes.")
+                patch_file = source_dir + "/patch"
+                with open(patch_file, "w+") as fd:
+                    fd.write(diff)
+                module_patch = ['git', 'apply', 'patch']
+                SCM._run(module_patch, chdir=source_dir)
+            except Exception as e:
+                log.exception("Failed to update repo %s with uncommited changes."
+                              % source_dir)
+                raise
+
     @staticmethod
     def is_full_commit_hash(scheme, commit):
         """

module_build_service/utils.py

@@ -907,7 +907,7 @@ def record_component_builds(mmd, module, initial_batch=1,
         return batch
 
 
-def submit_module_build_from_yaml(username, handle, optional_params=None):
+def submit_module_build_from_yaml(username, handle, stream=None, **kwargs):
     yaml = handle.read()
     mmd = load_mmd(yaml)
 
@@ -920,16 +920,17 @@ def submit_module_build_from_yaml(username, handle, optional_params=None):
     def_version = int(dt.strftime("%Y%m%d%H%M%S"))
 
     mmd.name = mmd.name or def_name
-    mmd.stream = mmd.stream or "master"
+    mmd.stream = mmd.stream or stream or "master"
     mmd.version = mmd.version or def_version
-    return submit_module_build(username, None, mmd, None, optional_params)
+
+    return submit_module_build(username, None, mmd, None, yaml, **kwargs)
 
 
 _url_check_re = re.compile(r"^[^:/]+:.*$")
 
 
 def submit_module_build_from_scm(username, url, branch, allow_local_url=False,
-                                 skiptests=False, optional_params=None):
+                                 skiptests=False, **kwargs):
     # Translate local paths into file:// URL
     if allow_local_url and not _url_check_re.match(url):
         log.info(
@@ -939,12 +940,14 @@ def submit_module_build_from_scm(username, url, branch, allow_local_url=False,
     mmd, scm = _fetch_mmd(url, branch, allow_local_url)
     if skiptests:
         mmd.buildopts.rpms.macros += "\n\n%__spec_check_pre exit 0\n"
-    return submit_module_build(username, url, mmd, scm, optional_params)
+    return submit_module_build(username, url, mmd, scm, yaml, **kwargs)
 
 
 def submit_module_build(username, url, mmd, scm, optional_params=None):
     import koji  # Placed here to avoid py2/py3 conflicts...
 
+
+def submit_module_build(username, url, mmd, scm, yaml, **kwargs):
     # Import it here, because SCM uses utils methods
     # and fails to import them because of dep-chain.
     validate_mmd(mmd)
@@ -989,7 +992,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None):
             modulemd=mmd.dumps(),
             scmurl=url,
             username=username,
-            **(optional_params or {})
+            **(kwargs or {})
         )
 
     db.session.add(module)

module_build_service/views.py

@@ -317,8 +317,7 @@ class SCMHandler(BaseHandler):
             branch = branch.encode('utf-8')
 
         return submit_module_build_from_scm(self.username, url, branch,
-                                            allow_local_url=False,
-                                            optional_params=self.optional_params)
+                                            allow_local_url=False, **self.optional_params)
 
 
 class YAMLFileHandler(BaseHandler):
@@ -336,8 +335,7 @@ class YAMLFileHandler(BaseHandler):
 
     def post(self):
         handle = request.files["yaml"]
-        return submit_module_build_from_yaml(self.username, handle,
-                                             optional_params=self.optional_params)
+        return submit_module_build_from_yaml(self.username, handle, **self.optional_params)
 
 
 def register_api_v1():

tests/test_scm.py

@@ -23,8 +23,10 @@
 import os
 import shutil
 import tempfile
+import subprocess as sp
 
 import unittest
+from mock import patch
 from nose.tools import raises
 
 import module_build_service.scm
@@ -36,12 +38,17 @@ repo_path = 'file://' + os.path.dirname(__file__) + "/scm_data/testrepo"
 class TestSCMModule(unittest.TestCase):
 
     def setUp(self):
+        # this var holds path to a cloned repo. For some tests we need a working
+        # tree not only a bare repo
+        self.temp_cloned_repo = None
         self.tempdir = tempfile.mkdtemp()
         self.repodir = self.tempdir + '/testrepo'
 
     def tearDown(self):
         if os.path.exists(self.tempdir):
             shutil.rmtree(self.tempdir)
+        if self.temp_cloned_repo and os.path.exists(self.temp_cloned_repo):
+            shutil.rmtree(self.temp_cloned_repo)
 
     def test_simple_local_checkout(self):
         """ See if we can clone a local git repo. """
@@ -115,3 +122,80 @@ class TestSCMModule(unittest.TestCase):
         scm.checkout(self.tempdir)
         scm.verify()
         scm.get_module_yaml()
+
+    @raises(UnprocessableEntity)
+    def test_get_latest_incorect_component_branch(self):
+        scm = module_build_service.scm.SCM(repo_path)
+        scm.get_latest(branch='foobar')
+
+    def test_patch_with_uncommited_changes(self):
+        cloned_repo, repo_link = self._clone_from_bare_repo()
+        with open(cloned_repo + "/foo", "a") as fd:
+            fd.write("Winter is comming!")
+        scm = module_build_service.scm.SCM(repo_link, allow_local=True)
+        scm.checkout(self.tempdir)
+        with open(self.repodir + "/foo", "r") as fd:
+            foo = fd.read()
+
+        assert "Winter is comming!" in foo
+
+    def test_dont_patch_if_commit_ref(self):
+        target = '7035bd33614972ac66559ac1fdd019ff6027ad21'
+        cloned_repo, repo_link = self._clone_from_bare_repo()
+        scm = module_build_service.scm.SCM(repo_link + "?#" + target, "dev", allow_local=True)
+        with open(cloned_repo + "/foo", "a") as fd:
+            fd.write("Winter is comming!")
+        scm.checkout(self.tempdir)
+        with open(self.repodir + "/foo", "r") as fd:
+            foo = fd.read()
+
+        assert "Winter is comming!" not in foo
+
+    @patch("module_build_service.scm.open")
+    @patch("module_build_service.scm.log")
+    def test_patch_with_exception(self, mock_log, mock_open):
+        cloned_repo, repo_link = self._clone_from_bare_repo()
+        with open(cloned_repo + "/foo", "a") as fd:
+            fd.write("Winter is comming!")
+        mock_open.side_effect = Exception("Can't write to patch file!")
+        scm = module_build_service.scm.SCM(repo_link, allow_local=True)
+        with self.assertRaises(Exception) as ex:
+            scm.checkout(self.tempdir)
+            mock_open.assert_called_once_with(self.repodir + "/patch", "w+")
+            err_msg = "Failed to update repo %s with uncommited changes." % self.repodir
+            mock_log.assert_called_once_with(err_msg)
+            assert ex is mock_open.side_effect
+            assert 0
+
+    def test_is_bare_repo(self):
+        scm = module_build_service.scm.SCM(repo_path)
+        assert scm.bare_repo
+
+    def _clone_from_bare_repo(self):
+        """
+        Helper method which will clone the bare test repo. Also it will create
+        a dev branch and track it to the remote bare repo.
+
+        Returns:
+            str: returns the path to the cloned repo
+            str: returns the file link (file://) to the repo
+        """
+        self.temp_cloned_repo = tempfile.mkdtemp()
+        cloned_repo = self.temp_cloned_repo + "/testrepo"
+        clone_cmd = ["git", "clone", "-q", repo_path]
+        get_dev_branch_cmd = ["git", "branch", "--track", "dev", "origin/dev"]
+        proc = sp.Popen(clone_cmd, stdout=sp.PIPE, stderr=sp.PIPE,
+                        cwd=self.temp_cloned_repo)
+        stdout, stderr = proc.communicate()
+        if stderr:
+            raise Exception("Failed to clone repo: %s, err code: %s"
+                            % (stderr, proc.returncode))
+        proc = sp.Popen(get_dev_branch_cmd, stdout=sp.PIPE, stderr=sp.PIPE,
+                        cwd=cloned_repo)
+        stdout, stderr = proc.communicate()
+        if stderr:
+            raise Exception("Failed to create and track dev branch: %s, err code: %s"
+                            % (stderr, proc.returncode))
+        repo_link = "".join(["file://", cloned_repo])
+
+        return cloned_repo, repo_link