diff --git a/.gitignore b/.gitignore index 6aa992c2..60186579 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ tests/vcr-request-data .vscode .ropeproject tests/test_module_build_service.db-journal +*.swp diff --git a/contrib/mbs-build b/contrib/mbs-build index ec63f916..2a096990 100755 --- a/contrib/mbs-build +++ b/contrib/mbs-build @@ -317,31 +317,39 @@ def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=Tru return -3, None -def do_local_build(scm_url, branch, skiptests, local_builds_nsvs, log_flag=None): +def do_local_build(scm_url, branch, skiptests, local_builds_nsvs, log_flag=None, + yaml_file=None, stream=None): """ Starts the local build using the 'mbs-manager build_module_locally' command. Returns exit code of that command or None when scm_url or branch are not set and cannot be obtained from the CWD. """ - scm_url = get_scm_url(scm_url, None, local=True) - branch = get_scm_branch(branch) - if not scm_url or not branch: - return None - - logging.info("Starting local build of %s, branch %s", scm_url, branch) command = ['mbs-manager'] - if log_flag: - command.append(log_flag) - command.append('build_module_locally') - if skiptests: - command.append('--skiptests') - logging.info("Tests will be skipped due to --skiptests option.") + if yaml_file: + command.append('build_module_locally_from_file') + logging.info("Starting local build from yaml file %s" % yaml_file) + command.extend(["--file", yaml_file]) + if stream: + command.extend(["--stream", stream]) + else: + command.append('build_module_locally') + scm_url = get_scm_url(scm_url, None, local=True) + branch = get_scm_branch(branch) + if not scm_url or not branch: + return None - if local_builds_nsvs: - for build_id in local_builds_nsvs: - command += ['--add-local-build', build_id] + logging.info("Starting local build of %s, branch %s", scm_url, branch) + if log_flag: + command.append(log_flag) + if skiptests: + command.append('--skiptests') + logging.info("Tests will be skipped due to --skiptests option.") - command.extend([scm_url, branch]) + if local_builds_nsvs: + for build_id in local_builds_nsvs: + command += ['--add-local-build', build_id] + + command.extend([scm_url, branch]) process = subprocess.Popen(command) process.communicate() @@ -494,6 +502,11 @@ def main(): dest="local_builds_nsvs", metavar='BUILD_ID') parser_local.add_argument('--skiptests', dest='skiptests', action='store_true', help="add macro for skipping check section/phase") + parser_local.add_argument('--file', dest='file', action='store', + help="Path to the modulemd yaml file") + parser_local.add_argument('--stream', dest='stream', action='store', + help=("Name of the stream of this build." + " (builds from files only)")) parser_overview = subparsers.add_parser( 'overview', help="show overview of module builds", @@ -540,7 +553,8 @@ def main(): print("Submitted module build %r" % build_id) elif args.cmd_name == "local": sys.exit(do_local_build(args.scm_url, args.branch, args.skiptests, - args.local_builds_nsvs, log_flag)) + args.local_builds_nsvs, log_flag, args.file, + args.stream)) elif args.cmd_name == "watch": # Watch the module build. try: diff --git a/module_build_service/config.py b/module_build_service/config.py index 70da0dba..23735a16 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -60,7 +60,9 @@ def init_config(app): # Load LocalBuildConfiguration section in case we are building modules # locally. - if "build_module_locally" in sys.argv: + local_build_cmds = ["build_module_locally", "build_module_locally_from_file"] + local = [cmd for cmd in sys.argv if cmd in local_build_cmds] + if local: config_section = "LocalBuildConfiguration" # try getting config_file from os.environ diff --git a/module_build_service/manage.py b/module_build_service/manage.py index 6228fcd8..d1f7c7f4 100755 --- a/module_build_service/manage.py +++ b/module_build_service/manage.py @@ -28,10 +28,12 @@ import logging import os import getpass +from werkzeug.datastructures import FileStorage from module_build_service import app, conf, db, create_app from module_build_service import models from module_build_service.utils import ( submit_module_build_from_scm, + submit_module_build_from_yaml, load_local_builds, ) import module_build_service.messaging @@ -92,7 +94,7 @@ def cleardb(): @manager.option('url') @manager.option('--skiptests', action='store_true') @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs') -def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False): +def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False, yaml_file=None, stream=None): """ Performs local module build using Mock """ if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']: @@ -116,15 +118,28 @@ def build_module_locally(url, branch, local_build_nsvs=None, skiptests=False): load_local_builds(local_build_nsvs) username = getpass.getuser() - submit_module_build_from_scm(username, url, branch, allow_local_url=True, - skiptests=skiptests) - + if yaml_file and yaml_file.endswith(".yaml"): + yaml_file_path = os.path.abspath(yaml_file) + with open(yaml_file_path) as fd: + filename = yaml_file.split("/")[-1] + handle = FileStorage(fd) + handle.filename = filename + submit_module_build_from_yaml(username, handle, stream) + else: + submit_module_build_from_scm(username, url, branch, allow_local_url=True, + skiptests=skiptests) stop = module_build_service.scheduler.make_simple_stop_condition(db.session) # Run the consumer until stop_condition returns True module_build_service.scheduler.main([], stop) +@manager.option('--file', action='store', dest="yaml_file") +@manager.option('--stream', action='store', dest="stream") +def build_module_locally_from_file(yaml_file, stream=None): + build_module_locally(None, None, yaml_file=yaml_file, stream=str(stream)) + + @console_script_help @manager.command def run(host=None, port=None, debug=None): diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 4098197a..2f83dadf 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -86,8 +86,17 @@ class SCM(object): self.name = self.name[:-4] self.commit = match.group("commit") self.branch = branch if branch else "master" + self.latest = False + # if not stated otherwise the default behaviour is that we work with + # non-local bare repositories + self.local = False + self.bare_repo = True + if url.startswith("file://") and allow_local: + self.local = True + self.bare_repo = self._is_bare_repo(self.repository[7:]) if not self.commit: self.commit = self.get_latest(self.branch) + self.latest = True self.version = None else: raise ValidationError("Unhandled SCM scheme: %s" % self.scheme) @@ -170,7 +179,11 @@ class SCM(object): "within the repository. Perhaps you forgot to push. " "The original message was: %s" % e.message) raise - + # will patch the temp git repo with uncommited changes only if there + # is no commit repo present in repo definition and its a local dir + # and not a bare repo. + if self.latest and self.local and not self.bare_repo: + self.patch_with_uncommited_changes(self.sourcedir) timestamp = SCM._run(["git", "show", "-s", "--format=%ct"], chdir=self.sourcedir)[1] dt = datetime.datetime.utcfromtimestamp(int(timestamp)) self.version = dt.strftime("%Y%m%d%H%M%S") @@ -197,6 +210,12 @@ class SCM(object): else: raise RuntimeError("get_latest: Unhandled SCM scheme.") + def _is_bare_repo(self, repo_path): + """ Checks if the repository is a bare repo """ + is_bare_repo_cmd = ["git", "config", "core.bare"] + _, is_bare, _ = SCM._run(is_bare_repo_cmd, chdir=repo_path) + return is_bare.rstrip() == "true" + def get_full_commit_hash(self, commit_hash=None): """ Takes a shortened commit hash and returns the full hash @@ -253,6 +272,30 @@ class SCM(object): "Couldn't access: %s" % path_to_yaml) raise UnprocessableEntity("The SCM repository doesn't contain a modulemd file") + def patch_with_uncommited_changes(self, source_dir): + """ + This method patches the given tmp git repository with uncommented changes from it + origin git dir. Creates a patch file witch holds result for `git diff` command + executed in the origin repo. + + source_dir (str): path to the temp git repo + """ + module_diff = ['git', 'diff'] + # striping the self.repository from 'file://' + _, diff, _ = SCM._run(module_diff, chdir=self.repository[7:]) + if diff: + try: + log.debug("Working with local, non-bare repository. Applying uncommited changes.") + patch_file = source_dir + "/patch" + with open(patch_file, "w+") as fd: + fd.write(diff) + module_patch = ['git', 'apply', 'patch'] + SCM._run(module_patch, chdir=source_dir) + except Exception as e: + log.exception("Failed to update repo %s with uncommited changes." + % source_dir) + raise + @staticmethod def is_full_commit_hash(scheme, commit): """ diff --git a/module_build_service/utils.py b/module_build_service/utils.py index cd8a723e..31769a7f 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -907,7 +907,7 @@ def record_component_builds(mmd, module, initial_batch=1, return batch -def submit_module_build_from_yaml(username, handle, optional_params=None): +def submit_module_build_from_yaml(username, handle, stream=None, **kwargs): yaml = handle.read() mmd = load_mmd(yaml) @@ -920,16 +920,17 @@ def submit_module_build_from_yaml(username, handle, optional_params=None): def_version = int(dt.strftime("%Y%m%d%H%M%S")) mmd.name = mmd.name or def_name - mmd.stream = mmd.stream or "master" + mmd.stream = mmd.stream or stream or "master" mmd.version = mmd.version or def_version - return submit_module_build(username, None, mmd, None, optional_params) + + return submit_module_build(username, None, mmd, None, yaml, **kwargs) _url_check_re = re.compile(r"^[^:/]+:.*$") def submit_module_build_from_scm(username, url, branch, allow_local_url=False, - skiptests=False, optional_params=None): + skiptests=False, **kwargs): # Translate local paths into file:// URL if allow_local_url and not _url_check_re.match(url): log.info( @@ -939,12 +940,14 @@ def submit_module_build_from_scm(username, url, branch, allow_local_url=False, mmd, scm = _fetch_mmd(url, branch, allow_local_url) if skiptests: mmd.buildopts.rpms.macros += "\n\n%__spec_check_pre exit 0\n" - return submit_module_build(username, url, mmd, scm, optional_params) + return submit_module_build(username, url, mmd, scm, yaml, **kwargs) def submit_module_build(username, url, mmd, scm, optional_params=None): import koji # Placed here to avoid py2/py3 conflicts... + +def submit_module_build(username, url, mmd, scm, yaml, **kwargs): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. validate_mmd(mmd) @@ -989,7 +992,7 @@ def submit_module_build(username, url, mmd, scm, optional_params=None): modulemd=mmd.dumps(), scmurl=url, username=username, - **(optional_params or {}) + **(kwargs or {}) ) db.session.add(module) diff --git a/module_build_service/views.py b/module_build_service/views.py index 3c0f2fb2..48a124ef 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -317,8 +317,7 @@ class SCMHandler(BaseHandler): branch = branch.encode('utf-8') return submit_module_build_from_scm(self.username, url, branch, - allow_local_url=False, - optional_params=self.optional_params) + allow_local_url=False, **self.optional_params) class YAMLFileHandler(BaseHandler): @@ -336,8 +335,7 @@ class YAMLFileHandler(BaseHandler): def post(self): handle = request.files["yaml"] - return submit_module_build_from_yaml(self.username, handle, - optional_params=self.optional_params) + return submit_module_build_from_yaml(self.username, handle, **self.optional_params) def register_api_v1(): diff --git a/tests/test_scm.py b/tests/test_scm.py index 2f500a2e..016cccd5 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -23,8 +23,10 @@ import os import shutil import tempfile +import subprocess as sp import unittest +from mock import patch from nose.tools import raises import module_build_service.scm @@ -36,12 +38,17 @@ repo_path = 'file://' + os.path.dirname(__file__) + "/scm_data/testrepo" class TestSCMModule(unittest.TestCase): def setUp(self): + # this var holds path to a cloned repo. For some tests we need a working + # tree not only a bare repo + self.temp_cloned_repo = None self.tempdir = tempfile.mkdtemp() self.repodir = self.tempdir + '/testrepo' def tearDown(self): if os.path.exists(self.tempdir): shutil.rmtree(self.tempdir) + if self.temp_cloned_repo and os.path.exists(self.temp_cloned_repo): + shutil.rmtree(self.temp_cloned_repo) def test_simple_local_checkout(self): """ See if we can clone a local git repo. """ @@ -115,3 +122,80 @@ class TestSCMModule(unittest.TestCase): scm.checkout(self.tempdir) scm.verify() scm.get_module_yaml() + + @raises(UnprocessableEntity) + def test_get_latest_incorect_component_branch(self): + scm = module_build_service.scm.SCM(repo_path) + scm.get_latest(branch='foobar') + + def test_patch_with_uncommited_changes(self): + cloned_repo, repo_link = self._clone_from_bare_repo() + with open(cloned_repo + "/foo", "a") as fd: + fd.write("Winter is comming!") + scm = module_build_service.scm.SCM(repo_link, allow_local=True) + scm.checkout(self.tempdir) + with open(self.repodir + "/foo", "r") as fd: + foo = fd.read() + + assert "Winter is comming!" in foo + + def test_dont_patch_if_commit_ref(self): + target = '7035bd33614972ac66559ac1fdd019ff6027ad21' + cloned_repo, repo_link = self._clone_from_bare_repo() + scm = module_build_service.scm.SCM(repo_link + "?#" + target, "dev", allow_local=True) + with open(cloned_repo + "/foo", "a") as fd: + fd.write("Winter is comming!") + scm.checkout(self.tempdir) + with open(self.repodir + "/foo", "r") as fd: + foo = fd.read() + + assert "Winter is comming!" not in foo + + @patch("module_build_service.scm.open") + @patch("module_build_service.scm.log") + def test_patch_with_exception(self, mock_log, mock_open): + cloned_repo, repo_link = self._clone_from_bare_repo() + with open(cloned_repo + "/foo", "a") as fd: + fd.write("Winter is comming!") + mock_open.side_effect = Exception("Can't write to patch file!") + scm = module_build_service.scm.SCM(repo_link, allow_local=True) + with self.assertRaises(Exception) as ex: + scm.checkout(self.tempdir) + mock_open.assert_called_once_with(self.repodir + "/patch", "w+") + err_msg = "Failed to update repo %s with uncommited changes." % self.repodir + mock_log.assert_called_once_with(err_msg) + assert ex is mock_open.side_effect + assert 0 + + def test_is_bare_repo(self): + scm = module_build_service.scm.SCM(repo_path) + assert scm.bare_repo + + def _clone_from_bare_repo(self): + """ + Helper method which will clone the bare test repo. Also it will create + a dev branch and track it to the remote bare repo. + + Returns: + str: returns the path to the cloned repo + str: returns the file link (file://) to the repo + """ + self.temp_cloned_repo = tempfile.mkdtemp() + cloned_repo = self.temp_cloned_repo + "/testrepo" + clone_cmd = ["git", "clone", "-q", repo_path] + get_dev_branch_cmd = ["git", "branch", "--track", "dev", "origin/dev"] + proc = sp.Popen(clone_cmd, stdout=sp.PIPE, stderr=sp.PIPE, + cwd=self.temp_cloned_repo) + stdout, stderr = proc.communicate() + if stderr: + raise Exception("Failed to clone repo: %s, err code: %s" + % (stderr, proc.returncode)) + proc = sp.Popen(get_dev_branch_cmd, stdout=sp.PIPE, stderr=sp.PIPE, + cwd=cloned_repo) + stdout, stderr = proc.communicate() + if stderr: + raise Exception("Failed to create and track dev branch: %s, err code: %s" + % (stderr, proc.returncode)) + repo_link = "".join(["file://", cloned_repo]) + + return cloned_repo, repo_link