modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-01 18:01:40 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Code changes for Python 3 support

Browse Source
This commit is contained in:
Yash Nanavati
2018-03-01 13:32:16 -05:00
parent 680da620d7
commit ed25afd9ec
13 changed files with 71 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
module_build_service/builder/KojiContentGenerator.py
View File

@@ -33,6 +33,8 @@ import shutil
import subprocess
import tempfile
import time
from builtins import str
from io import open
import koji
@@ -145,21 +147,16 @@ class KojiContentGenerator(object):
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout
if status != 0:
log.debug("%s: stderr output: %s", cmd, stderr)
@@ -173,7 +170,7 @@ class KojiContentGenerator(object):
tools = [u"modulemd"]
ret = []
for tool in tools:
version = unicode(pkg_resources.get_distribution(tool).version)
version = str(pkg_resources.get_distribution(tool).version)
ret.append({u"name": tool,
u"version": version})
return ret
@@ -236,15 +233,15 @@ class KojiContentGenerator(object):
ret = {
u"id": 1,
u"host": {
u"arch": unicode(platform.machine()),
u"arch": str(platform.machine()),
u'os': u"%s %s" % (distro[0], distro[1])
},
u"content_generator": {
u"name": u"module-build-service",
u"version": unicode(version)
u"version": str(version)
},
u"container": {
u"arch": unicode(platform.machine()),
u"arch": str(platform.machine()),
u"type": u"none"
},
u"components": self.__get_rpms(),
@@ -281,7 +278,7 @@ class KojiContentGenerator(object):
},
u'filesize': len(self.mmd),
u'checksum_type': u'md5',
u'checksum': unicode(hashlib.md5(self.mmd).hexdigest()),
u'checksum': str(hashlib.md5(self.mmd.encode('utf-8')).hexdigest()),
u'filename': u'modulemd.txt',
u'components': components
}
@@ -290,7 +287,7 @@ class KojiContentGenerator(object):
try:
log_path = os.path.join(output_path, "build.log")
with open(log_path) as build_log:
checksum = hashlib.md5(build_log.read()).hexdigest()
checksum = hashlib.md5(build_log.read().encode('utf-8')).hexdigest()
stat = os.stat(log_path)
ret.append(
{

3
module_build_service/builder/MockModuleBuilder.py
View File

@@ -495,6 +495,9 @@ class MockModuleBuilder(GenericBuilder):
def list_tasks_for_components(self, component_builds=None, state='active'):
pass
def repo_from_tag(cls, config, tag_name, arch):
pass
class BaseBuilder(object):
def __init__(self, config, resultsdir):

5
module_build_service/models.py
View File

@@ -295,13 +295,14 @@ class ModuleBuild(MBSBase):
mmd_formatted_property = {
dep: info['ref'] for dep, info in mbs_xmd[property_name].items()}
property_json = json.dumps(OrderedDict(sorted(mmd_formatted_property.items())))
rv.append(hashlib.sha1(property_json).hexdigest())
rv.append(hashlib.sha1(property_json.encode('utf-8')).hexdigest())
return tuple(rv)
@property
def context(self):
if self.build_context and self.runtime_context:
combined_hashes = '{0}:{1}'.format(self.build_context, self.runtime_context)
combined_hashes = '{0}:{1}'.format(
self.build_context, self.runtime_context).encode('utf-8')
return hashlib.sha1(combined_hashes).hexdigest()[:8]
else:
# We can't compute the context because the necessary data isn't there, so return a

6
module_build_service/scm.py
View File

@@ -110,7 +110,7 @@ class SCM(object):
found = False
branches = SCM._run(["git", "branch", "-r", "--contains", self.commit],
chdir=self.sourcedir)[1]
for branch in branches.split("\n"):
for branch in branches.decode('utf-8').split("\n"):
branch = branch.strip()
if branch[len("origin/"):] == self.branch:
found = True
@@ -203,7 +203,7 @@ class SCM(object):
# So the dictionary ends up in the format of
# {"master": "bf028e573e7c18533d89c7873a411de92d4d913e"...}.
branches = {}
for branch_and_ref in output.strip().split("\n"):
for branch_and_ref in output.decode('utf-8').strip().split("\n"):
# Only look at refs/heads/* and not refs/remotes/origin/*
if "refs/heads/" not in branch_and_ref:
continue
@@ -253,7 +253,7 @@ class SCM(object):
shutil.rmtree(td)
if output:
return str(output.strip('\n'))
return str(output.decode('utf-8').strip('\n'))
raise UnprocessableEntity(
'The full commit hash of "{0}" for "{1}" could not be found'

14
module_build_service/utils.py
View File

@@ -33,7 +33,6 @@ import kobo.rpmlib
import inspect
import hashlib
from functools import wraps
import modulemd
import yaml
@@ -68,8 +67,8 @@ def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Ex
log.warn("Exception %r raised from %r. Retry in %rs" % (
e, function, interval))
time.sleep(interval)
if (time.time() - start) >= timeout:
raise # This re-raises the last exception.
if (time.time() - start) >= timeout:
raise # This re-raises the last exception.
return inner
return wrapper
@@ -675,7 +674,12 @@ def load_local_builds(local_build_nsvs, session=None):
pass
# Sort with the biggest version first
builds.sort(lambda a, b: -cmp(a[2], b[2]))
try:
# py27
builds.sort(lambda a, b: -cmp(a[2], b[2]))
except TypeError:
# py3
builds.sort(key=lambda a: a[2], reverse=True)
for build_id in local_build_nsvs:
parts = build_id.split(':')
@@ -1470,7 +1474,7 @@ def get_rpm_release(module_build):
:return: a string of the module's dist tag
"""
dist_str = '.'.join([module_build.name, module_build.stream, str(module_build.version),
str(module_build.context)])
str(module_build.context)]).encode('utf-8')
dist_hash = hashlib.sha1(dist_str).hexdigest()[:8]
return "{prefix}{index}+{dist_hash}".format(
prefix=conf.default_dist_tag_prefix,

3
module_build_service/views.py
View File

@@ -30,6 +30,7 @@ import json
import module_build_service.auth
from flask import request, jsonify, url_for
from flask.views import MethodView
from builtins import str
from module_build_service import app, conf, log, models, db, version
from module_build_service.utils import (
@@ -329,7 +330,7 @@ class SCMHandler(BaseHandler):
branch = self.data["branch"]
# python-modulemd expects this to be bytes, not unicode.
if isinstance(branch, unicode):
if isinstance(branch, str):
branch = branch.encode('utf-8')
return submit_module_build_from_scm(self.username, url, branch,

9
tests/test_build/test_build.py
View File

@@ -146,6 +146,9 @@ class FakeModuleBuilder(GenericBuilder):
def buildroot_add_dependency(self, dependencies):
pass
def repo_from_tag(self, config, tag_name, arch):
pass
def buildroot_add_artifacts(self, artifacts, install=False):
if FakeModuleBuilder.on_buildroot_add_artifacts_cb:
FakeModuleBuilder.on_buildroot_add_artifacts_cb(self, artifacts, install)
@@ -484,17 +487,19 @@ class TestBuild:
def test_submit_build_with_optional_params(self, mocked_get_user, conf_system, dbg):
params = {'branch': 'master', 'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
'testmodule.git?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'}
not_existing_param = {"not_existing_param": "foo"}
copr_owner_param = {"copr_owner": "foo"}
def submit(data):
rv = self.client.post('/module-build-service/1/module-builds/', data=json.dumps(data))
return json.loads(rv.data)
data = submit(dict(params.items() + {"not_existing_param": "foo"}.items()))
data = submit(dict(params, **not_existing_param))
assert "The request contains unspecified parameters:" in data["message"]
assert "not_existing_param" in data["message"]
assert data["status"] == 400
data = submit(dict(params.items() + {"copr_owner": "foo"}.items()))
data = submit(dict(params, **copr_owner_param))
assert "The request contains parameters specific to Copr builder" in data["message"]
@patch('module_build_service.auth.get_user', return_value=user)

7
tests/test_builder/test_koji.py
View File

@@ -300,6 +300,7 @@ class TestKojiBuilder:
# getTaskDescendents response
[[{'1': [], '2': [], '3': [{'weight': 1.0}, {'weight': 1.0}]}]]
]
session.getAverageBuildDuration.return_value = None
get_session.return_value = session
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
@@ -320,6 +321,7 @@ class TestKojiBuilder:
# getTaskDescendents response
[[{'1': [], '2': [], '3': [{'weight': 1.0}, {'weight': 1.0}]}]]
]
session.getAverageBuildDuration.return_value = None
get_session.return_value = session
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
@@ -333,6 +335,7 @@ class TestKojiBuilder:
session = MagicMock()
session.getLoggedInUser.return_value = {"id": 123}
session.multiCall.side_effect = [[[1], [2]], []]
session.getAverageBuildDuration.return_value = None
get_session.return_value = session
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
@@ -349,6 +352,7 @@ class TestKojiBuilder:
session = MagicMock()
session.getLoggedInUser.return_value = {"id": 123}
session.multiCall.side_effect = [[], []]
session.getAverageBuildDuration.return_value = None
get_session.return_value = session
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
@@ -359,5 +363,8 @@ class TestKojiBuilder:
@patch('module_build_service.builder.KojiModuleBuilder.KojiModuleBuilder.get_session')
def test_get_build_weights_getLoggedInUser_failed(self, get_session):
session = MagicMock()
session.getAverageBuildDuration.return_value = None
get_session.return_value = session
weights = KojiModuleBuilder.get_build_weights(["httpd", "apr"])
assert weights == {"httpd": 1.5, "apr": 1.5}

18
tests/test_logger.py
View File

@@ -36,10 +36,20 @@ class TestLogger:
def setup_method(self, test_method):
init_data(1)
test_id = '.'.join([
path.splitext(path.basename(__file__))[0],
test_method.im_class.__name__,
test_method.im_func.__name__])
log.debug(test_method.__module__)
try:
# py2
test_id = '.'.join([
path.splitext(path.basename(__file__))[0],
test_method.im_class.__name__,
test_method.im_func.__name__])
except AttributeError:
# py3
test_id = '.'.join([
path.splitext(path.basename(__file__))[0],
test_method.__self__.__class__.__name__,
test_method.__self__.__class__.__name__])
self.base = tempfile.mkdtemp(prefix='mbs-', suffix='-%s' % test_id)
self.name_format = "build-{id}.log"
print("Storing build logs in %r" % self.base)

2
tests/test_scm.py
View File

@@ -140,7 +140,7 @@ class TestSCMModule:
@patch.object(module_build_service.scm.SCM, '_run')
def test_get_latest_ignore_origin(self, mock_run):
output = """\
output = b"""\
58379ef7887cbc91b215bacd32430628c92bc869\tHEAD
58379ef7887cbc91b215bacd32430628c92bc869\trefs/heads/master
10a651f39911a07d85fe87fcfe91999545e44ae0\trefs/remotes/origin/master

3
tests/test_utils/test_utils.py
View File

@@ -559,6 +559,9 @@ class DummyModuleBuilder(GenericBuilder):
def list_tasks_for_components(self, component_builds=None, state='active'):
pass
def repo_from_tag(self, config, tag_name, arch):
pass
@patch("module_build_service.builder.GenericBuilder.default_buildroot_groups",
return_value={'build': [], 'srpm-build': []})

2
tests/test_views/test_views.py
View File

@@ -103,7 +103,7 @@ class FakeSCM(object):
return self.sourcedir
def get_latest(self, ref='master'):
return hashlib.sha1(ref).hexdigest()[:10]
return hashlib.sha1(ref.encode('utf-8')).hexdigest()[:10]
def get_module_yaml(self):
return path.join(self.sourcedir, self.name + ".yaml")

2
tox.ini
View File

@@ -4,7 +4,7 @@
# and then run "tox" from this directory.
[tox]
envlist = flake8, py27
envlist = flake8, py27, py3
[flake8]
max-line-length = 100