diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 5c4bbe00..bb39672b 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -33,6 +33,8 @@ import shutil import subprocess import tempfile import time +from builtins import str +from io import open import koji @@ -145,21 +147,16 @@ class KojiContentGenerator(object): sep = ';' fmt = sep.join(["%%{%s}" % tag for tag in tags]) cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt) - try: - # py3 - (status, output) = subprocess.getstatusoutput(cmd) - except AttributeError: - # py2 - with open('/dev/null', 'r+') as devnull: - p = subprocess.Popen(cmd, - shell=True, - stdin=devnull, - stdout=subprocess.PIPE, - stderr=devnull) + with open('/dev/null', 'r+') as devnull: + p = subprocess.Popen(cmd, + shell=True, + stdin=devnull, + stdout=subprocess.PIPE, + stderr=devnull) - (stdout, stderr) = p.communicate() - status = p.wait() - output = stdout.decode() + (stdout, stderr) = p.communicate() + status = p.wait() + output = stdout if status != 0: log.debug("%s: stderr output: %s", cmd, stderr) @@ -173,7 +170,7 @@ class KojiContentGenerator(object): tools = [u"modulemd"] ret = [] for tool in tools: - version = unicode(pkg_resources.get_distribution(tool).version) + version = str(pkg_resources.get_distribution(tool).version) ret.append({u"name": tool, u"version": version}) return ret @@ -236,15 +233,15 @@ class KojiContentGenerator(object): ret = { u"id": 1, u"host": { - u"arch": unicode(platform.machine()), + u"arch": str(platform.machine()), u'os': u"%s %s" % (distro[0], distro[1]) }, u"content_generator": { u"name": u"module-build-service", - u"version": unicode(version) + u"version": str(version) }, u"container": { - u"arch": unicode(platform.machine()), + u"arch": str(platform.machine()), u"type": u"none" }, u"components": self.__get_rpms(), @@ -281,7 +278,7 @@ class KojiContentGenerator(object): }, u'filesize': len(self.mmd), u'checksum_type': u'md5', - u'checksum': unicode(hashlib.md5(self.mmd).hexdigest()), + u'checksum': str(hashlib.md5(self.mmd.encode('utf-8')).hexdigest()), u'filename': u'modulemd.txt', u'components': components } @@ -290,7 +287,7 @@ class KojiContentGenerator(object): try: log_path = os.path.join(output_path, "build.log") with open(log_path) as build_log: - checksum = hashlib.md5(build_log.read()).hexdigest() + checksum = hashlib.md5(build_log.read().encode('utf-8')).hexdigest() stat = os.stat(log_path) ret.append( { diff --git a/module_build_service/builder/MockModuleBuilder.py b/module_build_service/builder/MockModuleBuilder.py index 0ce30fa0..0ebb4765 100644 --- a/module_build_service/builder/MockModuleBuilder.py +++ b/module_build_service/builder/MockModuleBuilder.py @@ -495,6 +495,9 @@ class MockModuleBuilder(GenericBuilder): def list_tasks_for_components(self, component_builds=None, state='active'): pass + def repo_from_tag(cls, config, tag_name, arch): + pass + class BaseBuilder(object): def __init__(self, config, resultsdir): diff --git a/module_build_service/models.py b/module_build_service/models.py index 9b459597..42866308 100644 --- a/module_build_service/models.py +++ b/module_build_service/models.py @@ -295,13 +295,14 @@ class ModuleBuild(MBSBase): mmd_formatted_property = { dep: info['ref'] for dep, info in mbs_xmd[property_name].items()} property_json = json.dumps(OrderedDict(sorted(mmd_formatted_property.items()))) - rv.append(hashlib.sha1(property_json).hexdigest()) + rv.append(hashlib.sha1(property_json.encode('utf-8')).hexdigest()) return tuple(rv) @property def context(self): if self.build_context and self.runtime_context: - combined_hashes = '{0}:{1}'.format(self.build_context, self.runtime_context) + combined_hashes = '{0}:{1}'.format( + self.build_context, self.runtime_context).encode('utf-8') return hashlib.sha1(combined_hashes).hexdigest()[:8] else: # We can't compute the context because the necessary data isn't there, so return a diff --git a/module_build_service/scm.py b/module_build_service/scm.py index 182a4bbb..603f9540 100644 --- a/module_build_service/scm.py +++ b/module_build_service/scm.py @@ -110,7 +110,7 @@ class SCM(object): found = False branches = SCM._run(["git", "branch", "-r", "--contains", self.commit], chdir=self.sourcedir)[1] - for branch in branches.split("\n"): + for branch in branches.decode('utf-8').split("\n"): branch = branch.strip() if branch[len("origin/"):] == self.branch: found = True @@ -203,7 +203,7 @@ class SCM(object): # So the dictionary ends up in the format of # {"master": "bf028e573e7c18533d89c7873a411de92d4d913e"...}. branches = {} - for branch_and_ref in output.strip().split("\n"): + for branch_and_ref in output.decode('utf-8').strip().split("\n"): # Only look at refs/heads/* and not refs/remotes/origin/* if "refs/heads/" not in branch_and_ref: continue @@ -253,7 +253,7 @@ class SCM(object): shutil.rmtree(td) if output: - return str(output.strip('\n')) + return str(output.decode('utf-8').strip('\n')) raise UnprocessableEntity( 'The full commit hash of "{0}" for "{1}" could not be found' diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 7c511164..1a8abfd4 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -33,7 +33,6 @@ import kobo.rpmlib import inspect import hashlib from functools import wraps - import modulemd import yaml @@ -68,8 +67,8 @@ def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Ex log.warn("Exception %r raised from %r. Retry in %rs" % ( e, function, interval)) time.sleep(interval) - if (time.time() - start) >= timeout: - raise # This re-raises the last exception. + if (time.time() - start) >= timeout: + raise # This re-raises the last exception. return inner return wrapper @@ -675,7 +674,12 @@ def load_local_builds(local_build_nsvs, session=None): pass # Sort with the biggest version first - builds.sort(lambda a, b: -cmp(a[2], b[2])) + try: + # py27 + builds.sort(lambda a, b: -cmp(a[2], b[2])) + except TypeError: + # py3 + builds.sort(key=lambda a: a[2], reverse=True) for build_id in local_build_nsvs: parts = build_id.split(':') @@ -1470,7 +1474,7 @@ def get_rpm_release(module_build): :return: a string of the module's dist tag """ dist_str = '.'.join([module_build.name, module_build.stream, str(module_build.version), - str(module_build.context)]) + str(module_build.context)]).encode('utf-8') dist_hash = hashlib.sha1(dist_str).hexdigest()[:8] return "{prefix}{index}+{dist_hash}".format( prefix=conf.default_dist_tag_prefix, diff --git a/module_build_service/views.py b/module_build_service/views.py index 52cdc839..45b1050a 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -30,6 +30,7 @@ import json import module_build_service.auth from flask import request, jsonify, url_for from flask.views import MethodView +from builtins import str from module_build_service import app, conf, log, models, db, version from module_build_service.utils import ( @@ -329,7 +330,7 @@ class SCMHandler(BaseHandler): branch = self.data["branch"] # python-modulemd expects this to be bytes, not unicode. - if isinstance(branch, unicode): + if isinstance(branch, str): branch = branch.encode('utf-8') return submit_module_build_from_scm(self.username, url, branch, diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 553567e5..3edcc24d 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -146,6 +146,9 @@ class FakeModuleBuilder(GenericBuilder): def buildroot_add_dependency(self, dependencies): pass + def repo_from_tag(self, config, tag_name, arch): + pass + def buildroot_add_artifacts(self, artifacts, install=False): if FakeModuleBuilder.on_buildroot_add_artifacts_cb: FakeModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install) @@ -484,17 +487,19 @@ class TestBuild: def test_submit_build_with_optional_params(self, mocked_get_user, conf_system, dbg): params = {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' 'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'} + not_existing_param = {"not_existing_param": "foo"} + copr_owner_param = {"copr_owner": "foo"} def submit(data): rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(data)) return json.loads(rv.data) - data = submit(dict(params.items() + {"not_existing_param": "foo"}.items())) + data = submit(dict(params, **not_existing_param)) assert "The request contains unspecified parameters:" in data["message"] assert "not_existing_param" in data["message"] assert data["status"] == 400 - data = submit(dict(params.items() + {"copr_owner": "foo"}.items())) + data = submit(dict(params, **copr_owner_param)) assert "The request contains parameters specific to Copr builder" in data["message"] @patch('module_build_service.auth.get_user', return_value=user) diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index c6f72bca..a2cd88c0 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -300,6 +300,7 @@ class TestKojiBuilder: # getTaskDescendents response [[{'1': [], '2': [], '3': [{'weight': 1.0}, {'weight': 1.0}]}]] ] + session.getAverageBuildDuration.return_value = None get_session.return_value = session weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"]) @@ -320,6 +321,7 @@ class TestKojiBuilder: # getTaskDescendents response [[{'1': [], '2': [], '3': [{'weight': 1.0}, {'weight': 1.0}]}]] ] + session.getAverageBuildDuration.return_value = None get_session.return_value = session weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"]) @@ -333,6 +335,7 @@ class TestKojiBuilder: session = MagicMock() session.getLoggedInUser.return_value = {"id": 123} session.multiCall.side_effect = [[[1], [2]], []] + session.getAverageBuildDuration.return_value = None get_session.return_value = session weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"]) @@ -349,6 +352,7 @@ class TestKojiBuilder: session = MagicMock() session.getLoggedInUser.return_value = {"id": 123} session.multiCall.side_effect = [[], []] + session.getAverageBuildDuration.return_value = None get_session.return_value = session weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"]) @@ -359,5 +363,8 @@ class TestKojiBuilder: @patch('module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder.get_session') def test_get_build_weights_getLoggedInUser_failed(self, get_session): + session = MagicMock() + session.getAverageBuildDuration.return_value = None + get_session.return_value = session weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"]) assert weights == {"httpd": 1.5, "apr": 1.5} diff --git a/tests/test_logger.py b/tests/test_logger.py index b7251183..f2452787 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -36,10 +36,20 @@ class TestLogger: def setup_method(self, test_method): init_data(1) - test_id = '.'.join([ - path.splitext(path.basename(__file__))[0], - test_method.im_class.__name__, - test_method.im_func.__name__]) + log.debug(test_method.__module__) + try: + # py2 + test_id = '.'.join([ + path.splitext(path.basename(__file__))[0], + test_method.im_class.__name__, + test_method.im_func.__name__]) + except AttributeError: + # py3 + test_id = '.'.join([ + path.splitext(path.basename(__file__))[0], + test_method.__self__.__class__.__name__, + test_method.__self__.__class__.__name__]) + self.base = tempfile.mkdtemp(prefix='mbs-', suffix='-%s' % test_id) self.name_format = "build-{id}.log" print("Storing build logs in %r" % self.base) diff --git a/tests/test_scm.py b/tests/test_scm.py index 59ac93a5..0b639f3a 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -140,7 +140,7 @@ class TestSCMModule: @patch.object(module_build_service.scm.SCM, '_run') def test_get_latest_ignore_origin(self, mock_run): - output = """\ + output = b"""\ 58379ef7887cbc91b215bacd32430628c92bc869\tHEAD 58379ef7887cbc91b215bacd32430628c92bc869\trefs/heads/master 10a651f39911a07d85fe87fcfe91999545e44ae0\trefs/remotes/origin/master diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index c05cf1c6..627c1192 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -559,6 +559,9 @@ class DummyModuleBuilder(GenericBuilder): def list_tasks_for_components(self, component_builds=None, state='active'): pass + def repo_from_tag(self, config, tag_name, arch): + pass + @patch("module_build_service.builder.GenericBuilder.default_buildroot_groups", return_value={'build': [], 'srpm-build': []}) diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index e68d2b62..159a77f2 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -103,7 +103,7 @@ class FakeSCM(object): return self.sourcedir def get_latest(self, ref='master'): - return hashlib.sha1(ref).hexdigest()[:10] + return hashlib.sha1(ref.encode('utf-8')).hexdigest()[:10] def get_module_yaml(self): return path.join(self.sourcedir, self.name + ".yaml") diff --git a/tox.ini b/tox.ini index 8b153b45..a8d9c8d1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = flake8, py27 +envlist = flake8, py27, py3 [flake8] max-line-length = 100