diff --git a/module_build_service/builder/KojiContentGenerator.py b/module_build_service/builder/KojiContentGenerator.py index 27142e61..28f45034 100644 --- a/module_build_service/builder/KojiContentGenerator.py +++ b/module_build_service/builder/KojiContentGenerator.py @@ -22,7 +22,6 @@ # Written by Stanislav Ochotnicky # Jan Kaluza - import calendar import hashlib import logging @@ -46,6 +45,7 @@ import pungi.arch from module_build_service import conf, log, build_logs, Modulemd, glib from module_build_service.scm import SCM +from module_build_service.utils import to_text_type logging.basicConfig(level=logging.DEBUG) @@ -95,7 +95,7 @@ class KojiContentGenerator(object): self.owner = module.owner self.module = module self.module_name = module.name - self.mmd = module.modulemd + self.mmd = to_text_type(module.modulemd) self.config = config self.devel = False # List of architectures the module is built for. @@ -407,7 +407,7 @@ class KojiContentGenerator(object): mmd_path = os.path.join(output_path, mmd_filename) try: with open(mmd_path, 'rb') as mmd_f: - data = mmd_f.read().decode('utf-8') + data = to_text_type(mmd_f.read()) mmd = Modulemd.Module().new_from_string(data) ret['filename'] = mmd_filename ret['filesize'] = len(data) @@ -710,7 +710,7 @@ class KojiContentGenerator(object): # Fill in the list of built RPMs. mmd = self._fill_in_rpms_list(mmd, arch) - return text_type(mmd.dumps()) + return to_text_type(mmd.dumps()) def _download_source_modulemd(self, mmd, output_path): """ @@ -759,7 +759,7 @@ class KojiContentGenerator(object): prepdir = tempfile.mkdtemp(prefix="koji-cg-import") mmd_path = os.path.join(prepdir, "modulemd.txt") log.info("Writing generic modulemd.yaml to %r" % mmd_path) - with open(mmd_path, "w") as mmd_f: + with open(mmd_path, "w", encoding="utf-8") as mmd_f: mmd_f.write(self.mmd) mmd_path = os.path.join(prepdir, "modulemd.src.txt") @@ -769,7 +769,7 @@ class KojiContentGenerator(object): mmd_path = os.path.join(prepdir, "modulemd.%s.txt" % arch) log.info("Writing %s modulemd.yaml to %r" % (arch, mmd_path)) mmd = self._finalize_mmd(arch) - with open(mmd_path, "w") as mmd_f: + with open(mmd_path, "w", encoding="utf-8") as mmd_f: mmd_f.write(mmd) log_path = os.path.join(prepdir, "build.log") diff --git a/module_build_service/scheduler/handlers/modules.py b/module_build_service/scheduler/handlers/modules.py index 1536ec32..f18f43d0 100644 --- a/module_build_service/scheduler/handlers/modules.py +++ b/module_build_service/scheduler/handlers/modules.py @@ -38,6 +38,7 @@ from module_build_service.errors import UnprocessableEntity, Forbidden, Validati from module_build_service.utils.ursine import handle_stream_collision_modules from requests.exceptions import ConnectionError +from module_build_service.utils import to_text_type import koji @@ -155,7 +156,7 @@ def init(config, session, msg): record_component_builds(mmd, build, session=session) handle_stream_collision_modules(mmd) mmd = record_filtered_rpms(mmd) - build.modulemd = mmd.dumps() + build.modulemd = to_text_type(mmd.dumps()) build.transition(conf, models.BUILD_STATES["wait"]) # Catch custom exceptions that we can expose to the user except (UnprocessableEntity, Forbidden, ValidationError, RuntimeError) as e: diff --git a/module_build_service/utils/general.py b/module_build_service/utils/general.py index 99e4f879..f7300d98 100644 --- a/module_build_service/utils/general.py +++ b/module_build_service/utils/general.py @@ -27,12 +27,23 @@ import inspect import hashlib import time from datetime import datetime +from six import text_type from module_build_service import conf, log, models from module_build_service.errors import ( ValidationError, ProgrammingError, UnprocessableEntity) +def to_text_type(s): + """ + Converts `s` to `text_type`. In case it fails, returns `s`. + """ + try: + return text_type(s, "utf-8") + except TypeError: + return s + + def scm_url_schemes(terse=False): """ Definition of URL schemes supported by both frontend and scheduler. @@ -323,7 +334,7 @@ def import_mmd(session, mmd): build.version = version build.koji_tag = koji_tag build.state = models.BUILD_STATES['ready'] - build.modulemd = mmd.dumps() + build.modulemd = to_text_type(mmd.dumps()) build.context = context build.owner = "mbs_import" build.rebuild_strategy = 'all' diff --git a/module_build_service/utils/submit.py b/module_build_service/utils/submit.py index e9fac58e..0ede68a3 100644 --- a/module_build_service/utils/submit.py +++ b/module_build_service/utils/submit.py @@ -29,6 +29,7 @@ import tempfile import os from multiprocessing.dummy import Pool as ThreadPool from datetime import datetime +from module_build_service.utils import to_text_type import kobo.rpmlib import requests @@ -562,7 +563,7 @@ def submit_module_build(username, url, mmd, optional_params=None): name=mmd.get_name(), stream=mmd.get_stream(), version=version_str, - modulemd=mmd.dumps(), + modulemd=to_text_type(mmd.dumps()), scmurl=url, username=username, **(optional_params or {}) @@ -767,7 +768,7 @@ def load_local_builds(local_build_nsvs, session=None): stream=mmd.get_stream(), version=str(mmd.get_version()), context=mmd.get_context(), - modulemd=mmd.dumps(), + modulemd=to_text_type(mmd.dumps()), scmurl="", username="mbs", publish_msg=False) diff --git a/tests/__init__.py b/tests/__init__.py index fa73ebe0..ffaeb297 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,6 +26,7 @@ from mock import patch import time import hashlib from traceback import extract_stack +from module_build_service.utils import to_text_type import koji import module_build_service @@ -54,7 +55,7 @@ def read_staged_data(yaml_name): if not os.path.exists(filename): raise ValueError('Staged data {}.yaml does not exist.'.format(yaml_name)) with open(filename, 'r') as mmd: - return mmd.read() + return to_text_type(mmd.read()) def patch_config(): @@ -153,8 +154,7 @@ def _populate_data(session, data_size=10, contexts=False): build_one.ref_build_context = unique_hash combined_hashes = '{0}:{1}'.format(unique_hash, unique_hash) build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8] - with open(os.path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd: - build_one.modulemd = mmd.read() + build_one.modulemd = read_staged_data('nginx_mmd') build_one.koji_tag = 'module-nginx-1.2' build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?' '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9') @@ -343,7 +343,7 @@ def scheduler_init_data(tangerine_state=None): build_one.time_submitted = datetime(2017, 2, 15, 16, 8, 18) build_one.time_modified = datetime(2017, 2, 15, 16, 19, 35) build_one.rebuild_strategy = 'changed-and-after' - build_one.modulemd = mmd.dumps() + build_one.modulemd = to_text_type(mmd.dumps()) build_one_component_release = get_rpm_release(build_one) build_one.buildrequires.append(platform_br) @@ -456,7 +456,7 @@ def reuse_component_init_data(): xmd['mbs']['scmurl'] = build_one.scmurl xmd['mbs']['commit'] = 'ff1ea79fc952143efeed1851aa0aa006559239ba' mmd.set_xmd(glib.dict_values(xmd)) - build_one.modulemd = mmd.dumps() + build_one.modulemd = to_text_type(mmd.dumps()) build_one.buildrequires.append(platform_br) component_one_build_one = module_build_service.models.ComponentBuild() @@ -541,7 +541,7 @@ def reuse_component_init_data(): xmd['mbs']['scmurl'] = build_one.scmurl xmd['mbs']['commit'] = '55f4a0a2e6cc255c88712a905157ab39315b8fd8' mmd.set_xmd(glib.dict_values(xmd)) - build_two.modulemd = mmd.dumps() + build_two.modulemd = to_text_type(mmd.dumps()) build_two.buildrequires.append(platform_br) component_one_build_two = module_build_service.models.ComponentBuild() @@ -620,7 +620,7 @@ def reuse_shared_userspace_init_data(): build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe' build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a' build_one.state = BUILD_STATES['ready'] - build_one.modulemd = mmd.dumps() + build_one.modulemd = to_text_type(mmd.dumps()) build_one.koji_tag = 'module-shared-userspace-f26-20170601141014-75f92abb' build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.' 'git?#7fea453') @@ -671,7 +671,7 @@ def reuse_shared_userspace_init_data(): build_one.build_context = 'e046b867a400a06a3571f3c71142d497895fefbe' build_one.runtime_context = '50dd3eb5dde600d072e45d4120e1548ce66bc94a' build_one.state = BUILD_STATES['done'] - build_one.modulemd = mmd2.dumps() + build_one.modulemd = to_text_type(mmd2.dumps()) build_one.koji_tag = 'module-shared-userspace-f26-20170605091544-75f92abb' build_one.scmurl = ('https://src.stg.fedoraproject.org/modules/testmodule.' 'git?#7fea453') @@ -741,7 +741,8 @@ def make_module(nsvc, requires_list=None, build_requires_list=None, base_module= mmd.set_version(int(version)) mmd.set_context(context) mmd.set_summary("foo") - mmd.set_description("foo") + # Test unicode in mmd. + mmd.set_description(u"foo \u2019s") licenses = Modulemd.SimpleSet() licenses.add("GPL") mmd.set_module_licenses(licenses) @@ -802,7 +803,7 @@ def make_module(nsvc, requires_list=None, build_requires_list=None, base_module= module_build.build_context = context module_build.stream_build_context = context module_build.runtime_context = context - module_build.modulemd = mmd.dumps() + module_build.modulemd = to_text_type(mmd.dumps()) if base_module: module_build.buildrequires.append(base_module) if 'koji_tag' in xmd['mbs']: diff --git a/tests/staged_data/bad.yaml b/tests/staged_data/bad.yaml index 8432c7e8..be142e58 100644 --- a/tests/staged_data/bad.yaml +++ b/tests/staged_data/bad.yaml @@ -6,7 +6,7 @@ data: version: 5 summary: A fake module containing the bash shell description: > - A fake module used for testing + A fake module used for testing ’ license: module: - MIT diff --git a/tests/staged_data/fakemodule.yaml b/tests/staged_data/fakemodule.yaml index ba9b9b2b..2b721f8d 100644 --- a/tests/staged_data/fakemodule.yaml +++ b/tests/staged_data/fakemodule.yaml @@ -3,7 +3,7 @@ version: 1 data: summary: A fake module containing the bash shell description: > - A fake module used for testing + A fake module used for testing ’ license: module: - MIT diff --git a/tests/staged_data/formatted_python3-no-components.yaml b/tests/staged_data/formatted_python3-no-components.yaml index 57a2d726..d9982258 100644 --- a/tests/staged_data/formatted_python3-no-components.yaml +++ b/tests/staged_data/formatted_python3-no-components.yaml @@ -19,7 +19,7 @@ data: Programmers can write new built-in modules for Python in C or C++. Python can be used as an extension language for applications that - need a programmable interface. + need a programmable interface. ’ license: module: - MIT diff --git a/tests/staged_data/formatted_testmodule-more-components.yaml b/tests/staged_data/formatted_testmodule-more-components.yaml index fbd7901a..27b12ff2 100644 --- a/tests/staged_data/formatted_testmodule-more-components.yaml +++ b/tests/staged_data/formatted_testmodule-more-components.yaml @@ -7,7 +7,7 @@ data: context: 9c690d0e summary: A test module in all its beautiful beauty description: This module demonstrates how to write simple modulemd files And can - be used for testing the build and release pipeline. + be used for testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/staged_data/formatted_testmodule.yaml b/tests/staged_data/formatted_testmodule.yaml index 6aa91645..edb60b9b 100644 --- a/tests/staged_data/formatted_testmodule.yaml +++ b/tests/staged_data/formatted_testmodule.yaml @@ -7,7 +7,7 @@ data: context: 9c690d0e summary: A test module in all its beautiful beauty description: This module demonstrates how to write simple modulemd files And can - be used for testing the build and release pipeline. + be used for testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/staged_data/includedmodules.yaml b/tests/staged_data/includedmodules.yaml index 8e4bdc4d..9446a907 100644 --- a/tests/staged_data/includedmodules.yaml +++ b/tests/staged_data/includedmodules.yaml @@ -3,7 +3,7 @@ version: 1 data: summary: A fake module containing the bash shell description: > - A fake module used for testing + A fake module used for testing ’ license: module: - MIT diff --git a/tests/staged_data/nginx_mmd.yaml b/tests/staged_data/nginx_mmd.yaml index 98911d0b..edeeb25d 100644 --- a/tests/staged_data/nginx_mmd.yaml +++ b/tests/staged_data/nginx_mmd.yaml @@ -20,7 +20,7 @@ data: summary: An example nginx module # A verbose description of the module, required description: > - A module for the tests of module build service + A module for the tests of module build service ’ # Module and content licenses in the Fedora license identifier # format, required license: diff --git a/tests/staged_data/platform.yaml b/tests/staged_data/platform.yaml index 690ad23f..4d364f2a 100644 --- a/tests/staged_data/platform.yaml +++ b/tests/staged_data/platform.yaml @@ -1,7 +1,7 @@ document: modulemd version: 1 data: - description: Fedora 28 traditional base + description: Fedora 28 traditional base ’ name: platform license: module: [MIT] diff --git a/tests/staged_data/python3-no-components.yaml b/tests/staged_data/python3-no-components.yaml index ab354be5..264b0dde 100644 --- a/tests/staged_data/python3-no-components.yaml +++ b/tests/staged_data/python3-no-components.yaml @@ -19,7 +19,7 @@ data: Programmers can write new built-in modules for Python in C or C++. Python can be used as an extension language for applications that - need a programmable interface. + need a programmable interface. ’ license: module: - MIT diff --git a/tests/staged_data/testmodule-more-components.yaml b/tests/staged_data/testmodule-more-components.yaml index 60980990..6bd2289c 100644 --- a/tests/staged_data/testmodule-more-components.yaml +++ b/tests/staged_data/testmodule-more-components.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And - can be used for testing the build and release pipeline. + can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule-no-base-module.yaml b/tests/staged_data/testmodule-no-base-module.yaml index 9abaec0f..92244c15 100644 --- a/tests/staged_data/testmodule-no-base-module.yaml +++ b/tests/staged_data/testmodule-no-base-module.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And can be used for - testing the build and release pipeline. + testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/staged_data/testmodule-no-deps.yaml b/tests/staged_data/testmodule-no-deps.yaml index 1a61405c..e33e1c7d 100644 --- a/tests/staged_data/testmodule-no-deps.yaml +++ b/tests/staged_data/testmodule-no-deps.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And - can be used for testing the build and release pipeline. + can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule-version-set.yaml b/tests/staged_data/testmodule-version-set.yaml index 246f6911..95e1aba6 100644 --- a/tests/staged_data/testmodule-version-set.yaml +++ b/tests/staged_data/testmodule-version-set.yaml @@ -2,7 +2,7 @@ document: modulemd version: 1 data: summary: A test module in all its beauty - description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. + description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’ version: 0123456789 license: module: [ MIT ] diff --git a/tests/staged_data/testmodule-with-filters.yaml b/tests/staged_data/testmodule-with-filters.yaml index f59e1523..bcccbe81 100644 --- a/tests/staged_data/testmodule-with-filters.yaml +++ b/tests/staged_data/testmodule-with-filters.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And - can be used for testing the build and release pipeline. + can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule-wrong-stream.yaml b/tests/staged_data/testmodule-wrong-stream.yaml index 87705aa2..ed164958 100644 --- a/tests/staged_data/testmodule-wrong-stream.yaml +++ b/tests/staged_data/testmodule-wrong-stream.yaml @@ -3,7 +3,7 @@ version: 1 data: stream: wrong_stream summary: A test module in all its beauty - description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. + description: This module demonstrates how to write simple modulemd files And can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule.yaml b/tests/staged_data/testmodule.yaml index aef4b997..1e404daa 100644 --- a/tests/staged_data/testmodule.yaml +++ b/tests/staged_data/testmodule.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And - can be used for testing the build and release pipeline. + can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule_dependencies.yaml b/tests/staged_data/testmodule_dependencies.yaml index a7ccd9a4..2c72d311 100644 --- a/tests/staged_data/testmodule_dependencies.yaml +++ b/tests/staged_data/testmodule_dependencies.yaml @@ -23,7 +23,7 @@ data: silly: puddy glass: water description: This module demonstrates how to write simple modulemd files And can - be used for testing the build and release pipeline. + be used for testing the build and release pipeline. 😄 filter: {} license: module: [MIT] diff --git a/tests/staged_data/testmodule_init.yaml b/tests/staged_data/testmodule_init.yaml index 9ca7fe86..37195d0f 100644 --- a/tests/staged_data/testmodule_init.yaml +++ b/tests/staged_data/testmodule_init.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And - can be used for testing the build and release pipeline. + can be used for testing the build and release pipeline. ’ license: module: [ MIT ] dependencies: diff --git a/tests/staged_data/testmodule_mse.yaml b/tests/staged_data/testmodule_mse.yaml index f48df1c8..4df624e9 100644 --- a/tests/staged_data/testmodule_mse.yaml +++ b/tests/staged_data/testmodule_mse.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And can be used for - testing the build and release pipeline. + testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/staged_data/testmodule_platform_f290000.yaml b/tests/staged_data/testmodule_platform_f290000.yaml index 84abc87e..3470b696 100644 --- a/tests/staged_data/testmodule_platform_f290000.yaml +++ b/tests/staged_data/testmodule_platform_f290000.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And can be used for - testing the build and release pipeline. + testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/staged_data/testmodule_v2.yaml b/tests/staged_data/testmodule_v2.yaml index 463e1dcc..c3c5cc07 100644 --- a/tests/staged_data/testmodule_v2.yaml +++ b/tests/staged_data/testmodule_v2.yaml @@ -4,7 +4,7 @@ data: summary: A test module in all its beautiful beauty description: >- This module demonstrates how to write simple modulemd files And can be used for - testing the build and release pipeline. + testing the build and release pipeline. ’ license: module: - MIT diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index c99adffc..a29d1c23 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -28,6 +28,7 @@ from os.path import dirname from shutil import copyfile from datetime import datetime, timedelta from random import randint +from module_build_service.utils import to_text_type import module_build_service.messaging import module_build_service.scheduler.handlers.repos @@ -443,7 +444,7 @@ class TestBuild: testmodule = os.path.join(base_dir, 'staged_data', 'testmodule.yaml') with open(testmodule) as f: - yaml = f.read() + yaml = to_text_type(f.read()) with patch.object(module_build_service.config.Config, 'yaml_submit_allowed', new_callable=PropertyMock, return_value=False): @@ -885,7 +886,7 @@ class TestBuild: formatted_testmodule_yml_path = os.path.join( current_dir, '..', 'staged_data', 'formatted_testmodule.yaml') with open(formatted_testmodule_yml_path, 'r') as f: - build_one.modulemd = f.read() + build_one.modulemd = to_text_type(f.read()) build_one.koji_tag = 'module-testmodule-master-20180205135154-9c690d0e' build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453' build_one.batch = 2 @@ -1012,7 +1013,7 @@ class TestBuild: formatted_testmodule_yml_path = os.path.join( current_dir, '..', 'staged_data', 'formatted_testmodule.yaml') with open(formatted_testmodule_yml_path, 'r') as f: - build_one.modulemd = f.read() + build_one.modulemd = to_text_type(f.read()) build_one.koji_tag = 'module-testmodule-master-20180205135154-6ef9a711' build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#7fea453' build_one.batch = 2 diff --git a/tests/test_builder/test_koji.py b/tests/test_builder/test_koji.py index 321cc291..b4619956 100644 --- a/tests/test_builder/test_koji.py +++ b/tests/test_builder/test_koji.py @@ -31,6 +31,7 @@ try: except ImportError: import xmlrpc.client as xmlrpclib from collections import OrderedDict +from module_build_service.utils import to_text_type import module_build_service.messaging import module_build_service.scheduler.handlers.repos @@ -266,7 +267,7 @@ class TestKojiBuilder: xmd = glib.from_variant_dict(mmd.get_xmd()) xmd["mbs_options"] = {"blocked_packages": ["foo", "bar", "new"]} mmd.set_xmd(glib.dict_values(xmd)) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) builder = FakeKojiModuleBuilder(owner=self.module.owner, module=self.module, @@ -448,14 +449,14 @@ class TestKojiBuilder: xmd = glib.from_variant_dict(mmd.get_xmd()) xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]} mmd.set_xmd(glib.dict_values(xmd)) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) if custom_whitelist: mmd = self.module.mmd() opts = mmd.get_buildopts() opts.set_rpm_whitelist(['custom1', 'custom2']) mmd.set_buildopts(opts) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) if repo_include_all is False: mmd = self.module.mmd() @@ -464,7 +465,7 @@ class TestKojiBuilder: mbs_options["repo_include_all"] = False xmd["mbs_options"] = mbs_options mmd.set_xmd(glib.dict_values(xmd)) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) if override_arches: mmd = self.module.mmd() @@ -473,7 +474,7 @@ class TestKojiBuilder: mbs_options["buildrequires"] = {"platform": {"stream": "xx"}} xmd["mbs"] = mbs_options mmd.set_xmd(glib.dict_values(xmd)) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) builder = FakeKojiModuleBuilder( owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo', @@ -532,7 +533,7 @@ class TestKojiBuilder: xmd = glib.from_variant_dict(mmd.get_xmd()) xmd["mbs_options"] = {"blocked_packages": ["foo", "nginx"]} mmd.set_xmd(glib.dict_values(xmd)) - self.module.modulemd = mmd.dumps() + self.module.modulemd = to_text_type(mmd.dumps()) builder = FakeKojiModuleBuilder( owner=self.module.owner, module=self.module, config=conf, tag_name='module-foo', diff --git a/tests/test_builder/test_mock.py b/tests/test_builder/test_mock.py index b564f66a..3ef3e0ac 100644 --- a/tests/test_builder/test_mock.py +++ b/tests/test_builder/test_mock.py @@ -4,6 +4,7 @@ import koji import tempfile import shutil from textwrap import dedent +from module_build_service.utils import to_text_type import kobo.rpmlib @@ -92,7 +93,7 @@ class TestMockModuleBuilder: name="mbs-testmodule", stream="test", version="20171027111452", - modulemd=mmd.dumps(), + modulemd=to_text_type(mmd.dumps()), scmurl="file:///testdir", username="test", ) diff --git a/tests/test_content_generator.py b/tests/test_content_generator.py index 03816581..63d1e10e 100644 --- a/tests/test_content_generator.py +++ b/tests/test_content_generator.py @@ -21,11 +21,13 @@ # Written by Stanislav Ochotnicky # Jan Kaluza +import io import pytest import json import os from os import path +from module_build_service.utils import to_text_type import module_build_service.messaging import module_build_service.scheduler.handlers.repos # noqa @@ -192,22 +194,22 @@ class TestBuild: def test_prepare_file_directory(self): """ Test preparation of directory with output files """ dir_path = self.cg._prepare_file_directory() - with open(path.join(dir_path, "modulemd.txt")) as mmd: - assert len(mmd.read()) == 1134 + with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd: + assert len(mmd.read()) == 1136 def test_prepare_file_directory_per_arch_mmds(self): """ Test preparation of directory with output files """ self.cg.arches = ["x86_64", "i686"] dir_path = self.cg._prepare_file_directory() - with open(path.join(dir_path, "modulemd.txt")) as mmd: - assert len(mmd.read()) == 1134 + with io.open(path.join(dir_path, "modulemd.txt"), encoding="utf-8") as mmd: + assert len(mmd.read()) == 1136 - with open(path.join(dir_path, "modulemd.x86_64.txt")) as mmd: + with io.open(path.join(dir_path, "modulemd.x86_64.txt"), encoding="utf-8") as mmd: + assert len(mmd.read()) == 259 + + with io.open(path.join(dir_path, "modulemd.i686.txt"), encoding="utf-8") as mmd: assert len(mmd.read()) == 257 - with open(path.join(dir_path, "modulemd.i686.txt")) as mmd: - assert len(mmd.read()) == 255 - @patch.dict("sys.modules", krbV=Mock()) @patch("module_build_service.builder.KojiModuleBuilder.KojiClientSession") def test_tag_cg_build(self, ClientSession): @@ -274,17 +276,17 @@ class TestBuild: @patch("module_build_service.builder.KojiContentGenerator.open", create=True) def test_get_arch_mmd_output(self, patched_open): patched_open.return_value = mock_open( - read_data=self.cg.mmd.encode("utf-8")).return_value + read_data=self.cg.mmd).return_value ret = self.cg._get_arch_mmd_output("./fake-dir", "x86_64") assert ret == { 'arch': 'x86_64', 'buildroot_id': 1, - 'checksum': 'bf1615b15f6a0fee485abe94af6b56b6', + 'checksum': '96b7739ffa3918e6ac3e3bd422b064ea', 'checksum_type': 'md5', 'components': [], 'extra': {'typeinfo': {'module': {}}}, 'filename': 'modulemd.x86_64.txt', - 'filesize': 1134, + 'filesize': 1136, 'type': 'file' } @@ -294,7 +296,7 @@ class TestBuild: rpm_artifacts = mmd.get_rpm_artifacts() rpm_artifacts.add("dhcp-libs-12:4.3.5-5.module_2118aef6.x86_64") mmd.set_rpm_artifacts(rpm_artifacts) - mmd_data = mmd.dumps().encode("utf-8") + mmd_data = to_text_type(mmd.dumps()) patched_open.return_value = mock_open( read_data=mmd_data).return_value @@ -323,7 +325,7 @@ class TestBuild: assert ret == { 'arch': 'x86_64', 'buildroot_id': 1, - 'checksum': '1bcc38b6f19285b3656b84a0443f46d2', + 'checksum': '502e46889affec24d98a281289104d4d', 'checksum_type': 'md5', 'components': [{u'arch': 'x86_64', u'epoch': '12', @@ -334,7 +336,7 @@ class TestBuild: u'version': '4.3.5'}], 'extra': {'typeinfo': {'module': {}}}, 'filename': 'modulemd.x86_64.txt', - 'filesize': 315, + 'filesize': 317, 'type': 'file' } @@ -546,8 +548,8 @@ class TestBuild: component.set_multilib(multilib_set) mmd.add_rpm_component(component) - self.cg.module.modulemd = mmd.dumps() - self.cg.modulemd = mmd.dumps() + self.cg.module.modulemd = to_text_type(mmd.dumps()) + self.cg.modulemd = to_text_type(mmd.dumps()) @pytest.mark.parametrize("devel", (False, True)) def test_fill_in_rpms_list(self, devel): @@ -855,10 +857,10 @@ class TestBuild: mmd.set_xmd(glib.dict_values({"mbs": { "commit": "foo", "scmurl": "git://localhost/modules/foo.git#master"}})) - self.cg.module.modulemd = mmd.dumps() + self.cg.module.modulemd = to_text_type(mmd.dumps()) file_dir = self.cg._prepare_file_directory() - with open(path.join(file_dir, "modulemd.src.txt")) as mmd: - assert len(mmd.read()) == 1337 + with io.open(path.join(file_dir, "modulemd.src.txt"), encoding="utf-8") as mmd: + assert len(mmd.read()) == 1339 def test_finalize_mmd_devel(self): self.cg.devel = True diff --git a/tests/test_get_generator_json_expected_output.json b/tests/test_get_generator_json_expected_output.json index d0188a17..07825958 100644 --- a/tests/test_get_generator_json_expected_output.json +++ b/tests/test_get_generator_json_expected_output.json @@ -625,8 +625,8 @@ } ], "arch": "noarch", - "filesize": 1134, - "checksum": "bf1615b15f6a0fee485abe94af6b56b6", + "filesize": 1136, + "checksum": "96b7739ffa3918e6ac3e3bd422b064ea", "checksum_type": "md5", "type": "file", "extra": { @@ -652,7 +652,7 @@ "version": "2", "module_build_service_id": 2, "content_koji_tag": "module-nginx-1.2", - "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" + "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service ’\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" } } }, diff --git a/tests/test_get_generator_json_expected_output_with_log.json b/tests/test_get_generator_json_expected_output_with_log.json index 7bf13f0b..01a3727d 100644 --- a/tests/test_get_generator_json_expected_output_with_log.json +++ b/tests/test_get_generator_json_expected_output_with_log.json @@ -625,8 +625,8 @@ } ], "arch": "noarch", - "filesize": 1134, - "checksum": "bf1615b15f6a0fee485abe94af6b56b6", + "filesize": 1136, + "checksum": "96b7739ffa3918e6ac3e3bd422b064ea", "checksum_type": "md5", "type": "file", "extra": { @@ -661,7 +661,7 @@ "version": "2", "module_build_service_id": 2, "content_koji_tag": "module-nginx-1.2", - "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" + "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n # Module name, optional\n # Typically filled in by the buildsystem, using the VCS repository\n # name as the name of the module.\n name: nginx\n # Module update stream, optional\n # Typically filled in by the buildsystem, using the VCS branch name\n # as the name of the stream.\n stream: 1\n # Module version, integer, optional, cannot be negative\n # Typically filled in by the buildsystem, using the VCS commit\n # timestamp. Module version defines upgrade path for the particular\n # update stream.\n version: 2\n # A short summary describing the module, required\n summary: An example nginx module\n # A verbose description of the module, required\n description: >\n A module for the tests of module build service ’\n # Module and content licenses in the Fedora license identifier\n # format, required\n license:\n # Module license, required\n # This list covers licenses used for the module metadata, SPEC\n # files or extra patches\n module:\n - MIT\n" } } }, diff --git a/tests/test_models/test_models.py b/tests/test_models/test_models.py index 4cb64c48..2b1c9863 100644 --- a/tests/test_models/test_models.py +++ b/tests/test_models/test_models.py @@ -21,6 +21,7 @@ # Written by Ralph Bean import os +from module_build_service.utils import to_text_type from tests.test_models import init_data, module_build_from_modulemd from tests import (init_data as init_data_contexts, clean_database) @@ -66,7 +67,7 @@ class TestModels: os.path.dirname(__file__), '..', 'staged_data', 'testmodule_dependencies.yaml') mmd = Modulemd.Module.new_from_file(yaml_path) mmd.upgrade() - build.modulemd = mmd.dumps() + build.modulemd = to_text_type(mmd.dumps()) (build.ref_build_context, build.build_context, build.runtime_context, build.context) = ModuleBuild.contexts_from_mmd(build.modulemd) assert build.ref_build_context == 'f6e2aeec7576196241b9afa0b6b22acf2b6873d7' @@ -85,7 +86,7 @@ class TestModels: mmd.upgrade() with make_session(conf) as session: for i in range(3): - build = module_build_from_modulemd(mmd.dumps()) + build = module_build_from_modulemd(to_text_type(mmd.dumps())) build.build_context = 'f6e2aeec7576196241b9afa0b6b22acf2b6873d' + str(i) build.runtime_context = 'bbc84c7b817ab3dd54916c0bcd6c6bdf512f7f9c' + str(i) session.add(build) diff --git a/tests/test_resolver/test_db.py b/tests/test_resolver/test_db.py index 782ca3be..c1359bc3 100644 --- a/tests/test_resolver/test_db.py +++ b/tests/test_resolver/test_db.py @@ -25,6 +25,7 @@ import os from datetime import datetime from mock import patch, PropertyMock import pytest +from module_build_service.utils import to_text_type import module_build_service.resolver as mbs_resolver from module_build_service import app, db, models, glib, utils, Modulemd @@ -65,7 +66,7 @@ class TestDBModule: time_submitted=datetime(2018, 11, 15, 16, 8, 18), time_modified=datetime(2018, 11, 15, 16, 19, 35), rebuild_strategy='changed-and-after', - modulemd=mmd.dumps() + modulemd=to_text_type(mmd.dumps()) ) build.buildrequires.append(platform_f300103) db.session.add(build) @@ -105,7 +106,7 @@ class TestDBModule: xmd = glib.from_variant_dict(mmd.get_xmd()) xmd['mbs']['buildrequires'] = {} mmd.set_xmd(glib.dict_values(xmd)) - module.modulemd = mmd.dumps() + module.modulemd = to_text_type(mmd.dumps()) db.session.add(module) db.session.commit() resolver = mbs_resolver.GenericResolver.create(tests.conf, backend='db') @@ -134,7 +135,7 @@ class TestDBModule: 'version': '20180205135154' } mmd.set_xmd(glib.dict_values(xmd)) - module.modulemd = mmd.dumps() + module.modulemd = to_text_type(mmd.dumps()) module.name = 'testmodule2' module.version = str(mmd.get_version()) module.koji_tag = 'module-ae2adf69caf0e1b6' diff --git a/tests/test_scheduler/test_module_init.py b/tests/test_scheduler/test_module_init.py index 97264438..aa4aafce 100644 --- a/tests/test_scheduler/test_module_init.py +++ b/tests/test_scheduler/test_module_init.py @@ -24,6 +24,7 @@ import yaml from mock import patch, PropertyMock from gi.repository import GLib +from module_build_service.utils import to_text_type from tests import conf, clean_database from tests.test_views.test_views import FakeSCM @@ -42,7 +43,7 @@ class TestModuleInit: testmodule_yml_path = os.path.join( self.staged_data_dir, 'testmodule_init.yaml') with open(testmodule_yml_path, 'r') as f: - yaml = f.read() + yaml = to_text_type(f.read()) scmurl = 'git://pkgs.domain.local/modules/testmodule?#620ec77' clean_database() with make_session(conf) as session: @@ -78,7 +79,7 @@ class TestModuleInit: filter_list.add("foo") filter_list.add("bar") mmd.set_rpm_filter(filter_list) - platform_build.modulemd = mmd.dumps() + platform_build.modulemd = to_text_type(mmd.dumps()) db.session.commit() msg = module_build_service.messaging.MBSModule( @@ -137,7 +138,7 @@ class TestModuleInit: includedmodules_yml_path = os.path.join( self.staged_data_dir, 'includedmodules.yaml') with open(includedmodules_yml_path, 'r') as f: - yaml = f.read() + yaml = to_text_type(f.read()) scmurl = 'git://pkgs.domain.local/modules/includedmodule?#da95886' with make_session(conf) as session: ModuleBuild.create( diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py index 75954ca9..d3b8e944 100644 --- a/tests/test_utils/test_utils.py +++ b/tests/test_utils/test_utils.py @@ -18,12 +18,14 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import io import tempfile from os import path, mkdir from shutil import copyfile, rmtree from datetime import datetime from werkzeug.datastructures import FileStorage from mock import patch +from module_build_service.utils import to_text_type import module_build_service.utils import module_build_service.scm from module_build_service import models, conf @@ -93,7 +95,7 @@ class TestUtilsComponentReuse: mmd = second_module_build.mmd() mmd.get_rpm_components()['tangerine'].set_ref( '00ea1da4192a2030f9ae023de3b3143ed647bbab') - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) second_module_changed_component = models.ComponentBuild.query.filter_by( package=changed_component, module_id=3).one() second_module_changed_component.ref = '00ea1da4192a2030f9ae023de3b3143ed647bbab' @@ -133,7 +135,7 @@ class TestUtilsComponentReuse: second_module_build = models.ModuleBuild.query.filter_by(id=3).one() mmd = second_module_build.mmd() mmd.set_rpm_buildopts({'macros': '%my_macro 1'}) - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) db.session.commit() plc_rv = module_build_service.utils.get_reusable_component( @@ -153,7 +155,7 @@ class TestUtilsComponentReuse: arches = Modulemd.SimpleSet() arches.set(['i686']) mmd.get_rpm_components()['tangerine'].set_arches(arches) - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) if set_database_arch: # set architecture for build in database second_module_changed_component = models.ComponentBuild.query.filter_by( package='tangerine', module_id=2).one() @@ -161,7 +163,7 @@ class TestUtilsComponentReuse: arches = Modulemd.SimpleSet() arches.set(['i686']) mmd.get_rpm_components()['tangerine'].set_arches(arches) - second_module_changed_component.module_build.modulemd = mmd.dumps() + second_module_changed_component.module_build.modulemd = to_text_type(mmd.dumps()) db.session.add(second_module_changed_component) db.session.commit() @@ -179,7 +181,7 @@ class TestUtilsComponentReuse: xmd['mbs']['buildrequires']['platform']['ref'] = \ 'da39a3ee5e6b4b0d3255bfef95601890afd80709' mmd.set_xmd(glib.dict_values(xmd)) - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17' second_module_build.rebuild_strategy = rebuild_strategy db.session.commit() @@ -209,7 +211,7 @@ class TestUtilsComponentReuse: xmd = glib.from_variant_dict(mmd.get_xmd()) xmd['mbs']['buildrequires']['platform']['stream'] = 'different' mmd.set_xmd(glib.dict_values(xmd)) - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) second_module_build.build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17' second_module_build.rebuild_strategy = rebuild_strategy db.session.commit() @@ -240,7 +242,7 @@ class TestUtilsComponentReuse: } } mmd.set_xmd(glib.dict_values(xmd)) - second_module_build.modulemd = mmd.dumps() + second_module_build.modulemd = to_text_type(mmd.dumps()) second_module_build.ref_build_context = '37c6c57bedf4305ef41249c1794760b5cb8fad17' db.session.commit() @@ -268,13 +270,13 @@ class TestUtilsComponentReuse: module_dir = tempfile.mkdtemp() module = models.ModuleBuild.query.filter_by(id=3).one() mmd = module.mmd() - modulemd_yaml = mmd.dumps() + modulemd_yaml = to_text_type(mmd.dumps()) modulemd_file_path = path.join(module_dir, "testmodule.yaml") username = "test" stream = "dev" - with open(modulemd_file_path, "w") as fd: + with io.open(modulemd_file_path, "w", encoding='utf-8') as fd: fd.write(modulemd_yaml) with open(modulemd_file_path, "rb") as fd: @@ -530,7 +532,7 @@ class TestUtils: module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = 'changed-and-after' - module_build.modulemd = mmd.dumps() + module_build.modulemd = to_text_type(mmd.dumps()) db.session.add(module_build) db.session.commit() # Rename the the modulemd to include @@ -578,7 +580,7 @@ class TestUtils: module_build.time_submitted = datetime(2017, 2, 15, 16, 8, 18) module_build.time_modified = datetime(2017, 2, 15, 16, 19, 35) module_build.rebuild_strategy = 'changed-and-after' - module_build.modulemd = mmd.dumps() + module_build.modulemd = to_text_type(mmd.dumps()) db.session.add(module_build) db.session.commit() diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index db554d98..b7fc11ae 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -32,6 +32,7 @@ from os.path import dirname from requests.utils import quote import hashlib import pytest +from module_build_service.utils import to_text_type from tests import app, init_data, clean_database, reuse_component_init_data from tests import read_staged_data @@ -183,7 +184,7 @@ class TestViews: assert data['runtime_context'] is None assert data['id'] == 2 with open(path.join(base_dir, "staged_data", "nginx_mmd.yaml")) as mmd: - assert data['modulemd'] == mmd.read() + assert data['modulemd'] == to_text_type(mmd.read()) assert data['name'] == 'nginx' assert data['owner'] == 'Moe Szyslak' assert data['scmurl'] == ('git://pkgs.domain.local/modules/nginx' @@ -1614,7 +1615,7 @@ class TestViews: buildrequires['modulea'] = br_modulea buildrequires['moduleb'] = br_moduleb mmd.set_xmd(dict_values(xmd)) - build.modulemd = mmd.dumps() + build.modulemd = to_text_type(mmd.dumps()) session.commit() rv = self.client.get('/module-build-service/1/module-builds/{}'.format(build.id))