modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-03 10:48:03 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix the handling of modulemd files with unicode characters.

Browse Source 
This commit introduces new to_text_type helper method and calls it
for return value of all mmd.dumps() calls. That way, we always
end up with proper unicode string represntation on both python
major versions.

This commit also adds unicode character to description of all
the yaml files we use in the tests so we can be sure MBS can
handle unicode characters properly.

This might be temporary fix, depending on the result of discussion
at https://github.com/fedora-modularity/libmodulemd/issues/184.
This commit is contained in:
Jan Kaluza
2019-02-12 15:57:22 +01:00
committed by mprahl
parent 7052ea0a11
commit 40f23b65e4
37 changed files with 122 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
module_build_service/builder/KojiContentGenerator.py
View File

@@ -22,7 +22,6 @@
# Written by Stanislav Ochotnicky <sochotnicky@redhat.com>
# Jan Kaluza <jkaluza@redhat.com>
import calendar
import hashlib
import logging
@@ -46,6 +45,7 @@ import pungi.arch
from module_build_service import conf, log, build_logs, Modulemd, glib
from module_build_service.scm import SCM
from module_build_service.utils import to_text_type
logging.basicConfig(level=logging.DEBUG)
@@ -95,7 +95,7 @@ class KojiContentGenerator(object):
self.owner = module.owner
self.module = module
self.module_name = module.name
self.mmd = module.modulemd
self.mmd = to_text_type(module.modulemd)
self.config = config
self.devel = False
# List of architectures the module is built for.
@@ -407,7 +407,7 @@ class KojiContentGenerator(object):
mmd_path = os.path.join(output_path, mmd_filename)
try:
with open(mmd_path, 'rb') as mmd_f:
data = mmd_f.read().decode('utf-8')
data = to_text_type(mmd_f.read())
mmd = Modulemd.Module().new_from_string(data)
ret['filename'] = mmd_filename
ret['filesize'] = len(data)
@@ -710,7 +710,7 @@ class KojiContentGenerator(object):
# Fill in the list of built RPMs.
mmd = self._fill_in_rpms_list(mmd, arch)
return text_type(mmd.dumps())
return to_text_type(mmd.dumps())
def _download_source_modulemd(self, mmd, output_path):
"""
@@ -759,7 +759,7 @@ class KojiContentGenerator(object):
prepdir = tempfile.mkdtemp(prefix="koji-cg-import")
mmd_path = os.path.join(prepdir, "modulemd.txt")
log.info("Writing generic modulemd.yaml to %r" % mmd_path)
with open(mmd_path, "w") as mmd_f:
with open(mmd_path, "w", encoding="utf-8") as mmd_f:
mmd_f.write(self.mmd)
mmd_path = os.path.join(prepdir, "modulemd.src.txt")
@@ -769,7 +769,7 @@ class KojiContentGenerator(object):
mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch)
log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path))
mmd = self._finalize_mmd(arch)
with open(mmd_path, "w") as mmd_f:
with open(mmd_path, "w", encoding="utf-8") as mmd_f:
mmd_f.write(mmd)
log_path = os.path.join(prepdir, "build.log")

3
module_build_service/scheduler/handlers/modules.py
View File

@@ -38,6 +38,7 @@ from module_build_service.errors import UnprocessableEntity, Forbidden, Validati
from module_build_service.utils.ursine import handle_stream_collision_modules
from requests.exceptions import ConnectionError
from module_build_service.utils import to_text_type
import koji
@@ -155,7 +156,7 @@ def init(config, session, msg):
record_component_builds(mmd, build, session=session)
handle_stream_collision_modules(mmd)
mmd = record_filtered_rpms(mmd)
build.modulemd = mmd.dumps()
build.modulemd = to_text_type(mmd.dumps())
build.transition(conf, models.BUILD_STATES["wait"])
# Catch custom exceptions that we can expose to the user
except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e:

13
module_build_service/utils/general.py
View File

@@ -27,12 +27,23 @@ import inspect
import hashlib
import time
from datetime import datetime
from six import text_type
from module_build_service import conf, log, models
from module_build_service.errors import (
ValidationError, ProgrammingError, UnprocessableEntity)
def to_text_type(s):
"""
Converts `s` to `text_type`. In case it fails, returns `s`.
"""
try:
return text_type(s, "utf-8")
except TypeError:
return s
def scm_url_schemes(terse=False):
"""
Definition of URL schemes supported by both frontend and scheduler.
@@ -323,7 +334,7 @@ def import_mmd(session, mmd):
build.version = version
build.koji_tag = koji_tag
build.state = models.BUILD_STATES['ready']
build.modulemd = mmd.dumps()
build.modulemd = to_text_type(mmd.dumps())
build.context = context
build.owner = "mbs_import"
build.rebuild_strategy = 'all'

5
module_build_service/utils/submit.py
View File

@@ -29,6 +29,7 @@ import tempfile
import os
from multiprocessing.dummy import Pool as ThreadPool
from datetime import datetime
from module_build_service.utils import to_text_type
import kobo.rpmlib
import requests
@@ -562,7 +563,7 @@ def submit_module_build(username, url, mmd, optional_params=None):
name=mmd.get_name(),
stream=mmd.get_stream(),
version=version_str,
modulemd=mmd.dumps(),
modulemd=to_text_type(mmd.dumps()),
scmurl=url,
username=username,
**(optional_params or {})
@@ -767,7 +768,7 @@ def load_local_builds(local_build_nsvs, session=None):
stream=mmd.get_stream(),
version=str(mmd.get_version()),
context=mmd.get_context(),
modulemd=mmd.dumps(),
modulemd=to_text_type(mmd.dumps()),
scmurl="",
username="mbs",
publish_msg=False)

21
tests/__init__.py
View File

@@ -26,6 +26,7 @@ from mock import patch
import time
import hashlib
from traceback import extract_stack
from module_build_service.utils import to_text_type
import koji
import module_build_service
@@ -54,7 +55,7 @@ def read_staged_data(yaml_name):
if not os.path.exists(filename):
raise ValueError('Staged data {}.yaml does not exist.'.format(yaml_name))
with open(filename, 'r') as mmd:
return mmd.read()
return to_text_type(mmd.read())
def patch_config():
@@ -153,8 +154,7 @@ def _populate_data(session, data_size=10, contexts=False):
build_one.ref_build_context = unique_hash
combined_hashes = '{0}:{1}'.format(unique_hash, unique_hash)
build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8]
with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
build_one.modulemd = mmd.read()
build_one.modulemd = read_staged_data('nginx_mmd')
build_one.koji_tag = 'module-nginx-1.2'
build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?'
'#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')
@@ -343,7 +343,7 @@ def scheduler_init_data(tangerine_state=None):
build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35)
build_one.rebuild_strategy = 'changed-and-after'
build_one.modulemd = mmd.dumps()
build_one.modulemd = to_text_type(mmd.dumps())
build_one_component_release = get_rpm_release(build_one)
build_one.buildrequires.append(platform_br)
@@ -456,7 +456,7 @@ def reuse_component_init_data():
xmd['mbs']['scmurl'] = build_one.scmurl
xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba'
mmd.set_xmd(glib.dict_values(xmd))
build_one.modulemd = mmd.dumps()
build_one.modulemd = to_text_type(mmd.dumps())
build_one.buildrequires.append(platform_br)
component_one_build_one = module_build_service.models.ComponentBuild()
@@ -541,7 +541,7 @@ def reuse_component_init_data():
xmd['mbs']['scmurl'] = build_one.scmurl
xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8'
mmd.set_xmd(glib.dict_values(xmd))
build_two.modulemd = mmd.dumps()
build_two.modulemd = to_text_type(mmd.dumps())
build_two.buildrequires.append(platform_br)
component_one_build_two = module_build_service.models.ComponentBuild()
@@ -620,7 +620,7 @@ def reuse_shared_userspace_init_data():
build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'
build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'
build_one.state = BUILD_STATES['ready']
build_one.modulemd = mmd.dumps()
build_one.modulemd = to_text_type(mmd.dumps())
build_one.koji_tag = 'module-shared-userspace-f26-20170601141014-75f92abb'
build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.'
'git?#7fea453')
@@ -671,7 +671,7 @@ def reuse_shared_userspace_init_data():
build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe'
build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a'
build_one.state = BUILD_STATES['done']
build_one.modulemd = mmd2.dumps()
build_one.modulemd = to_text_type(mmd2.dumps())
build_one.koji_tag = 'module-shared-userspace-f26-20170605091544-75f92abb'
build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.'
'git?#7fea453')
@@ -741,7 +741,8 @@ def make_module(nsvc, requires_list=None, build_requires_list=None, base_module=
mmd.set_version(int(version))
mmd.set_context(context)
mmd.set_summary("foo")
mmd.set_description("foo")
# Test unicode in mmd.
mmd.set_description(u"foo \u2019s")
licenses = Modulemd.SimpleSet()
licenses.add("GPL")
mmd.set_module_licenses(licenses)
@@ -802,7 +803,7 @@ def make_module(nsvc, requires_list=None, build_requires_list=None, base_module=
module_build.build_context = context
module_build.stream_build_context = context
module_build.runtime_context = context
module_build.modulemd = mmd.dumps()
module_build.modulemd = to_text_type(mmd.dumps())
if base_module:
module_build.buildrequires.append(base_module)
if 'koji_tag' in xmd['mbs']:

2
tests/staged_data/bad.yaml
View File

@@ -6,7 +6,7 @@ data:
version: 5
summary: A fake module containing the bash shell
description: >
A fake module used for testing
A fake module used for testing ’
license:
module:
- MIT

2
tests/staged_data/fakemodule.yaml
View File

@@ -3,7 +3,7 @@ version: 1
data:
summary: A fake module containing the bash shell
description: >
A fake module used for testing
A fake module used for testing ’
license:
module:
- MIT

2
tests/staged_data/formatted_python3-no-components.yaml
View File

@@ -19,7 +19,7 @@ data:
Programmers can write new built-in modules for Python in C or C++.
Python can be used as an extension language for applications that
need a programmable interface.
need a programmable interface. ’
license:
module:
- MIT

2
tests/staged_data/formatted_testmodule-more-components.yaml
View File

@@ -7,7 +7,7 @@ data:
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.
be used for testing the build and release pipeline. ’
license:
module:
- MIT

2
tests/staged_data/formatted_testmodule.yaml
View File

@@ -7,7 +7,7 @@ data:
context: 9c690d0e
summary: A test module in all its beautiful beauty
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.
be used for testing the build and release pipeline. ’
license:
module:
- MIT

2
tests/staged_data/includedmodules.yaml
View File

@@ -3,7 +3,7 @@ version: 1
data:
summary: A fake module containing the bash shell
description: >
A fake module used for testing
A fake module used for testing ’
license:
module:
- MIT

2
tests/staged_data/nginx_mmd.yaml
View File

@@ -20,7 +20,7 @@ data:
summary: An example nginx module
# A verbose description of the module, required
description: >
A module for the tests of module build service
A module for the tests of module build service ’
# Module and content licenses in the Fedora license identifier
# format, required
license:

2
tests/staged_data/platform.yaml
View File

@@ -1,7 +1,7 @@
document: modulemd
version: 1
data:
description: Fedora 28 traditional base
description: Fedora 28 traditional base ’
name: platform
license:
module: [MIT]

2
tests/staged_data/python3-no-components.yaml
View File

@@ -19,7 +19,7 @@ data:
Programmers can write new built-in modules for Python in C or C++.
Python can be used as an extension language for applications that
need a programmable interface.
need a programmable interface. ’
license:
module:
- MIT

2
tests/staged_data/testmodule-more-components.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline.
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule-no-base-module.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
testing the build and release pipeline. ’
license:
module:
- MIT

2
tests/staged_data/testmodule-no-deps.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline.
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule-version-set.yaml
View File

@@ -2,7 +2,7 @@ document: modulemd
version: 1
data:
summary: A test module in all its beauty
description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline.
description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’
version: 0123456789
license:
module: [ MIT ]

2
tests/staged_data/testmodule-with-filters.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline.
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule-wrong-stream.yaml
View File

@@ -3,7 +3,7 @@ version: 1
data:
stream: wrong_stream
summary: A test module in all its beauty
description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline.
description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline.
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule_dependencies.yaml
View File

@@ -23,7 +23,7 @@ data:
silly: puddy
glass: water
description: This module demonstrates how to write simple modulemd files And can
be used for testing the build and release pipeline.
be used for testing the build and release pipeline. 😄
filter: {}
license:
module: [MIT]

2
tests/staged_data/testmodule_init.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And
can be used for testing the build and release pipeline.
can be used for testing the build and release pipeline. ’
license:
module: [ MIT ]
dependencies:

2
tests/staged_data/testmodule_mse.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
testing the build and release pipeline. ’
license:
module:
- MIT

2
tests/staged_data/testmodule_platform_f290000.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
testing the build and release pipeline. ’
license:
module:
- MIT

2
tests/staged_data/testmodule_v2.yaml
View File

@@ -4,7 +4,7 @@ data:
summary: A test module in all its beautiful beauty
description: >-
This module demonstrates how to write simple modulemd files And can be used for
testing the build and release pipeline.
testing the build and release pipeline. ’
license:
module:
- MIT

7
tests/test_build/test_build.py
View File

@@ -28,6 +28,7 @@ from os.path import dirname
from shutil import copyfile
from datetime import datetime, timedelta
from random import randint
from module_build_service.utils import to_text_type
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
@@ -443,7 +444,7 @@ class TestBuild:
testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml')
with open(testmodule) as f:
yaml = f.read()
yaml = to_text_type(f.read())
with patch.object(module_build_service.config.Config, 'yaml_submit_allowed',
new_callable=PropertyMock, return_value=False):
@@ -885,7 +886,7 @@ class TestBuild:
formatted_testmodule_yml_path = os.path.join(
current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')
with open(formatted_testmodule_yml_path, 'r') as f:
build_one.modulemd = f.read()
build_one.modulemd = to_text_type(f.read())
build_one.koji_tag = 'module-testmodule-master-20180205135154-9c690d0e'
build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'
build_one.batch = 2
@@ -1012,7 +1013,7 @@ class TestBuild:
formatted_testmodule_yml_path = os.path.join(
current_dir, '..', 'staged_data', 'formatted_testmodule.yaml')
with open(formatted_testmodule_yml_path, 'r') as f:
build_one.modulemd = f.read()
build_one.modulemd = to_text_type(f.read())
build_one.koji_tag = 'module-testmodule-master-20180205135154-6ef9a711'
build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453'
build_one.batch = 2

13
tests/test_builder/test_koji.py
View File

@@ -31,6 +31,7 @@ try:
except ImportError:
import xmlrpc.client as xmlrpclib
from collections import OrderedDict
from module_build_service.utils import to_text_type
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos
@@ -266,7 +267,7 @@ class TestKojiBuilder:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd["mbs_options"] = {"blocked_packages": ["foo", "bar", "new"]}
mmd.set_xmd(glib.dict_values(xmd))
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
builder = FakeKojiModuleBuilder(owner=self.module.owner,
module=self.module,
@@ -448,14 +449,14 @@ class TestKojiBuilder:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}
mmd.set_xmd(glib.dict_values(xmd))
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
if custom_whitelist:
mmd = self.module.mmd()
opts = mmd.get_buildopts()
opts.set_rpm_whitelist(['custom1', 'custom2'])
mmd.set_buildopts(opts)
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
if repo_include_all is False:
mmd = self.module.mmd()
@@ -464,7 +465,7 @@ class TestKojiBuilder:
mbs_options["repo_include_all"] = False
xmd["mbs_options"] = mbs_options
mmd.set_xmd(glib.dict_values(xmd))
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
if override_arches:
mmd = self.module.mmd()
@@ -473,7 +474,7 @@ class TestKojiBuilder:
mbs_options["buildrequires"] = {"platform": {"stream": "xx"}}
xmd["mbs"] = mbs_options
mmd.set_xmd(glib.dict_values(xmd))
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
builder = FakeKojiModuleBuilder(
owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo',
@@ -532,7 +533,7 @@ class TestKojiBuilder:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]}
mmd.set_xmd(glib.dict_values(xmd))
self.module.modulemd = mmd.dumps()
self.module.modulemd = to_text_type(mmd.dumps())
builder = FakeKojiModuleBuilder(
owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo',

3
tests/test_builder/test_mock.py
View File

@@ -4,6 +4,7 @@ import koji
import tempfile
import shutil
from textwrap import dedent
from module_build_service.utils import to_text_type
import kobo.rpmlib
@@ -92,7 +93,7 @@ class TestMockModuleBuilder:
name="mbs-testmodule",
stream="test",
version="20171027111452",
modulemd=mmd.dumps(),
modulemd=to_text_type(mmd.dumps()),
scmurl="file:///testdir",
username="test",
)

40
tests/test_content_generator.py
View File

@@ -21,11 +21,13 @@
# Written by Stanislav Ochotnicky <sochotnicky@redhat.com>
# Jan Kaluza <jkaluza@redhat.com>
import io
import pytest
import json
import os
from os import path
from module_build_service.utils import to_text_type
import module_build_service.messaging
import module_build_service.scheduler.handlers.repos # noqa
@@ -192,22 +194,22 @@ class TestBuild:
def test_prepare_file_directory(self):
""" Test preparation of directory with output files """
dir_path = self.cg._prepare_file_directory()
with open(path.join(dir_path, "modulemd.txt")) as mmd:
assert len(mmd.read()) == 1134
with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 1136
def test_prepare_file_directory_per_arch_mmds(self):
""" Test preparation of directory with output files """
self.cg.arches = ["x86_64", "i686"]
dir_path = self.cg._prepare_file_directory()
with open(path.join(dir_path, "modulemd.txt")) as mmd:
assert len(mmd.read()) == 1134
with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 1136
with open(path.join(dir_path, "modulemd.x86_64.txt")) as mmd:
with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 259
with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 257
with open(path.join(dir_path, "modulemd.i686.txt")) as mmd:
assert len(mmd.read()) == 255
@patch.dict("sys.modules", krbV=Mock())
@patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession")
def test_tag_cg_build(self, ClientSession):
@@ -274,17 +276,17 @@ class TestBuild:
@patch("module_build_service.builder.KojiContentGenerator.open", create=True)
def test_get_arch_mmd_output(self, patched_open):
patched_open.return_value = mock_open(
read_data=self.cg.mmd.encode("utf-8")).return_value
read_data=self.cg.mmd).return_value
ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64")
assert ret == {
'arch': 'x86_64',
'buildroot_id': 1,
'checksum': 'bf1615b15f6a0fee485abe94af6b56b6',
'checksum': '96b7739ffa3918e6ac3e3bd422b064ea',
'checksum_type': 'md5',
'components': [],
'extra': {'typeinfo': {'module': {}}},
'filename': 'modulemd.x86_64.txt',
'filesize': 1134,
'filesize': 1136,
'type': 'file'
}
@@ -294,7 +296,7 @@ class TestBuild:
rpm_artifacts = mmd.get_rpm_artifacts()
rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64")
mmd.set_rpm_artifacts(rpm_artifacts)
mmd_data = mmd.dumps().encode("utf-8")
mmd_data = to_text_type(mmd.dumps())
patched_open.return_value = mock_open(
read_data=mmd_data).return_value
@@ -323,7 +325,7 @@ class TestBuild:
assert ret == {
'arch': 'x86_64',
'buildroot_id': 1,
'checksum': '1bcc38b6f19285b3656b84a0443f46d2',
'checksum': '502e46889affec24d98a281289104d4d',
'checksum_type': 'md5',
'components': [{u'arch': 'x86_64',
u'epoch': '12',
@@ -334,7 +336,7 @@ class TestBuild:
u'version': '4.3.5'}],
'extra': {'typeinfo': {'module': {}}},
'filename': 'modulemd.x86_64.txt',
'filesize': 315,
'filesize': 317,
'type': 'file'
}
@@ -546,8 +548,8 @@ class TestBuild:
component.set_multilib(multilib_set)
mmd.add_rpm_component(component)
self.cg.module.modulemd = mmd.dumps()
self.cg.modulemd = mmd.dumps()
self.cg.module.modulemd = to_text_type(mmd.dumps())
self.cg.modulemd = to_text_type(mmd.dumps())
@pytest.mark.parametrize("devel", (False, True))
def test_fill_in_rpms_list(self, devel):
@@ -855,10 +857,10 @@ class TestBuild:
mmd.set_xmd(glib.dict_values({"mbs": {
"commit": "foo",
"scmurl": "git://localhost/modules/foo.git#master"}}))
self.cg.module.modulemd = mmd.dumps()
self.cg.module.modulemd = to_text_type(mmd.dumps())
file_dir = self.cg._prepare_file_directory()
with open(path.join(file_dir, "modulemd.src.txt")) as mmd:
assert len(mmd.read()) == 1337
with io.open(path.join(file_dir, "modulemd.src.txt"), encoding="utf-8") as mmd:
assert len(mmd.read()) == 1339
def test_finalize_mmd_devel(self):
self.cg.devel = True

6
tests/test_get_generator_json_expected_output.json
View File

@@ -625,8 +625,8 @@
}
],
"arch": "noarch",
"filesize": 1134,
"checksum": "bf1615b15f6a0fee485abe94af6b56b6",
"filesize": 1136,
"checksum": "96b7739ffa3918e6ac3e3bd422b064ea",
"checksum_type": "md5",
"type": "file",
"extra": {
@@ -652,7 +652,7 @@
"version": "2",
"module_build_service_id": 2,
"content_koji_tag": "module-nginx-1.2",
"modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n"
"modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service ’\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n"
}
}
},

6
tests/test_get_generator_json_expected_output_with_log.json
View File

@@ -625,8 +625,8 @@
}
],
"arch": "noarch",
"filesize": 1134,
"checksum": "bf1615b15f6a0fee485abe94af6b56b6",
"filesize": 1136,
"checksum": "96b7739ffa3918e6ac3e3bd422b064ea",
"checksum_type": "md5",
"type": "file",
"extra": {
@@ -661,7 +661,7 @@
"version": "2",
"module_build_service_id": 2,
"content_koji_tag": "module-nginx-1.2",
"modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n"
"modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service ’\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n"
}
}
},

5
tests/test_models/test_models.py
View File

@@ -21,6 +21,7 @@
# Written by Ralph Bean <rbean@redhat.com>
import os
from module_build_service.utils import to_text_type
from tests.test_models import init_data, module_build_from_modulemd
from tests import (init_data as init_data_contexts, clean_database)
@@ -66,7 +67,7 @@ class TestModels:
os.path.dirname(__file__), '..', 'staged_data', 'testmodule_dependencies.yaml')
mmd = Modulemd.Module.new_from_file(yaml_path)
mmd.upgrade()
build.modulemd = mmd.dumps()
build.modulemd = to_text_type(mmd.dumps())
(build.ref_build_context, build.build_context, build.runtime_context,
build.context) = ModuleBuild.contexts_from_mmd(build.modulemd)
assert build.ref_build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7'
@@ -85,7 +86,7 @@ class TestModels:
mmd.upgrade()
with make_session(conf) as session:
for i in range(3):
build = module_build_from_modulemd(mmd.dumps())
build = module_build_from_modulemd(to_text_type(mmd.dumps()))
build.build_context = 'f6e2aeec7576196241b9afa0b6b22acf2b6873d' + str(i)
build.runtime_context = 'bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c' + str(i)
session.add(build)

7
tests/test_resolver/test_db.py
View File

@@ -25,6 +25,7 @@ import os
from datetime import datetime
from mock import patch, PropertyMock
import pytest
from module_build_service.utils import to_text_type
import module_build_service.resolver as mbs_resolver
from module_build_service import app, db, models, glib, utils, Modulemd
@@ -65,7 +66,7 @@ class TestDBModule:
time_submitted=datetime(2018, 11, 15, 16, 8, 18),
time_modified=datetime(2018, 11, 15, 16, 19, 35),
rebuild_strategy='changed-and-after',
modulemd=mmd.dumps()
modulemd=to_text_type(mmd.dumps())
)
build.buildrequires.append(platform_f300103)
db.session.add(build)
@@ -105,7 +106,7 @@ class TestDBModule:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires'] = {}
mmd.set_xmd(glib.dict_values(xmd))
module.modulemd = mmd.dumps()
module.modulemd = to_text_type(mmd.dumps())
db.session.add(module)
db.session.commit()
resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db')
@@ -134,7 +135,7 @@ class TestDBModule:
'version': '20180205135154'
}
mmd.set_xmd(glib.dict_values(xmd))
module.modulemd = mmd.dumps()
module.modulemd = to_text_type(mmd.dumps())
module.name = 'testmodule2'
module.version = str(mmd.get_version())
module.koji_tag = 'module-ae2adf69caf0e1b6'

7
tests/test_scheduler/test_module_init.py
View File

@@ -24,6 +24,7 @@ import yaml
from mock import patch, PropertyMock
from gi.repository import GLib
from module_build_service.utils import to_text_type
from tests import conf, clean_database
from tests.test_views.test_views import FakeSCM
@@ -42,7 +43,7 @@ class TestModuleInit:
testmodule_yml_path = os.path.join(
self.staged_data_dir, 'testmodule_init.yaml')
with open(testmodule_yml_path, 'r') as f:
yaml = f.read()
yaml = to_text_type(f.read())
scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77'
clean_database()
with make_session(conf) as session:
@@ -78,7 +79,7 @@ class TestModuleInit:
filter_list.add("foo")
filter_list.add("bar")
mmd.set_rpm_filter(filter_list)
platform_build.modulemd = mmd.dumps()
platform_build.modulemd = to_text_type(mmd.dumps())
db.session.commit()
msg = module_build_service.messaging.MBSModule(
@@ -137,7 +138,7 @@ class TestModuleInit:
includedmodules_yml_path = os.path.join(
self.staged_data_dir, 'includedmodules.yaml')
with open(includedmodules_yml_path, 'r') as f:
yaml = f.read()
yaml = to_text_type(f.read())
scmurl = 'git://pkgs.domain.local/modules/includedmodule?#da95886'
with make_session(conf) as session:
ModuleBuild.create(

24
tests/test_utils/test_utils.py
View File

@@ -18,12 +18,14 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import io
import tempfile
from os import path, mkdir
from shutil import copyfile, rmtree
from datetime import datetime
from werkzeug.datastructures import FileStorage
from mock import patch
from module_build_service.utils import to_text_type
import module_build_service.utils
import module_build_service.scm
from module_build_service import models, conf
@@ -93,7 +95,7 @@ class TestUtilsComponentReuse:
mmd = second_module_build.mmd()
mmd.get_rpm_components()['tangerine'].set_ref(
'00ea1da4192a2030f9ae023de3b3143ed647bbab')
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
second_module_changed_component = models.ComponentBuild.query.filter_by(
package=changed_component, module_id=3).one()
second_module_changed_component.ref = '00ea1da4192a2030f9ae023de3b3143ed647bbab'
@@ -133,7 +135,7 @@ class TestUtilsComponentReuse:
second_module_build = models.ModuleBuild.query.filter_by(id=3).one()
mmd = second_module_build.mmd()
mmd.set_rpm_buildopts({'macros': '%my_macro 1'})
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
db.session.commit()
plc_rv = module_build_service.utils.get_reusable_component(
@@ -153,7 +155,7 @@ class TestUtilsComponentReuse:
arches = Modulemd.SimpleSet()
arches.set(['i686'])
mmd.get_rpm_components()['tangerine'].set_arches(arches)
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
if set_database_arch: # set architecture for build in database
second_module_changed_component = models.ComponentBuild.query.filter_by(
package='tangerine', module_id=2).one()
@@ -161,7 +163,7 @@ class TestUtilsComponentReuse:
arches = Modulemd.SimpleSet()
arches.set(['i686'])
mmd.get_rpm_components()['tangerine'].set_arches(arches)
second_module_changed_component.module_build.modulemd = mmd.dumps()
second_module_changed_component.module_build.modulemd = to_text_type(mmd.dumps())
db.session.add(second_module_changed_component)
db.session.commit()
@@ -179,7 +181,7 @@ class TestUtilsComponentReuse:
xmd['mbs']['buildrequires']['platform']['ref'] = \
'da39a3ee5e6b4b0d3255bfef95601890afd80709'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
@@ -209,7 +211,7 @@ class TestUtilsComponentReuse:
xmd = glib.from_variant_dict(mmd.get_xmd())
xmd['mbs']['buildrequires']['platform']['stream'] = 'different'
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
second_module_build.rebuild_strategy = rebuild_strategy
db.session.commit()
@@ -240,7 +242,7 @@ class TestUtilsComponentReuse:
}
}
mmd.set_xmd(glib.dict_values(xmd))
second_module_build.modulemd = mmd.dumps()
second_module_build.modulemd = to_text_type(mmd.dumps())
second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17'
db.session.commit()
@@ -268,13 +270,13 @@ class TestUtilsComponentReuse:
module_dir = tempfile.mkdtemp()
module = models.ModuleBuild.query.filter_by(id=3).one()
mmd = module.mmd()
modulemd_yaml = mmd.dumps()
modulemd_yaml = to_text_type(mmd.dumps())
modulemd_file_path = path.join(module_dir, "testmodule.yaml")
username = "test"
stream = "dev"
with open(modulemd_file_path, "w") as fd:
with io.open(modulemd_file_path, "w", encoding='utf-8') as fd:
fd.write(modulemd_yaml)
with open(modulemd_file_path, "rb") as fd:
@@ -530,7 +532,7 @@ class TestUtils:
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = 'changed-and-after'
module_build.modulemd = mmd.dumps()
module_build.modulemd = to_text_type(mmd.dumps())
db.session.add(module_build)
db.session.commit()
# Rename the the modulemd to include
@@ -578,7 +580,7 @@ class TestUtils:
module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18)
module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35)
module_build.rebuild_strategy = 'changed-and-after'
module_build.modulemd = mmd.dumps()
module_build.modulemd = to_text_type(mmd.dumps())
db.session.add(module_build)
db.session.commit()

5
tests/test_views/test_views.py
View File

@@ -32,6 +32,7 @@ from os.path import dirname
from requests.utils import quote
import hashlib
import pytest
from module_build_service.utils import to_text_type
from tests import app, init_data, clean_database, reuse_component_init_data
from tests import read_staged_data
@@ -183,7 +184,7 @@ class TestViews:
assert data['runtime_context'] is None
assert data['id'] == 2
with open(path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd:
assert data['modulemd'] == mmd.read()
assert data['modulemd'] == to_text_type(mmd.read())
assert data['name'] == 'nginx'
assert data['owner'] == 'Moe Szyslak'
assert data['scmurl'] == ('git://pkgs.domain.local/modules/nginx'
@@ -1614,7 +1615,7 @@ class TestViews:
buildrequires['modulea'] = br_modulea
buildrequires['moduleb'] = br_moduleb
mmd.set_xmd(dict_values(xmd))
build.modulemd = mmd.dumps()
build.modulemd = to_text_type(mmd.dumps())
session.commit()
rv = self.client.get('/module-build-service/1/module-builds/{}'.format(build.id))