From 1591b9c5e7b47914374a5b1128a1f064cdd3bef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Fri, 17 Feb 2017 00:14:30 +0100 Subject: [PATCH 1/9] Create method for loading modulemd object from yaml --- module_build_service/utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index a11a16ea..82504c1c 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -287,12 +287,7 @@ def _fetch_mmd(url, allow_local_url = False): "Failed to remove temporary directory {!r}: {}".format( td, str(e))) - mmd = modulemd.ModuleMetadata() - try: - mmd.loads(yaml) - except Exception as e: - log.error('Invalid modulemd: %s' % str(e)) - raise UnprocessableEntity('Invalid modulemd: %s' % str(e)) + mmd = load_mmd(yaml) # If undefined, set the name field to VCS repo name. if not mmd.name and scm: @@ -308,6 +303,17 @@ def _fetch_mmd(url, allow_local_url = False): return mmd, scm, yaml + +def load_mmd(yaml): + mmd = modulemd.ModuleMetadata() + try: + mmd.loads(yaml) + except Exception as e: + log.error('Invalid modulemd: %s' % str(e)) + raise UnprocessableEntity('Invalid modulemd: %s' % str(e)) + return mmd + + def _scm_get_latest(pkg): try: # If the modulemd specifies that the 'f25' branch is what From 43acbdbdbade34ecca7e414c52a21a1a5d53b496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Fri, 17 Feb 2017 00:23:19 +0100 Subject: [PATCH 2/9] Implement possibility to submit yaml files (See #310) --- module_build_service/utils.py | 15 ++++++++++++--- module_build_service/views.py | 23 ++++++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 82504c1c..e1dc28d1 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -418,13 +418,22 @@ def record_component_builds(scm, mmd, module, initial_batch = 1): return batch -def submit_module_build(username, url, allow_local_url = False): + +def submit_module_build_from_yaml(username, yaml): + mmd = load_mmd(yaml) + return submit_module_build(username, None, mmd, None, yaml) + + +def submit_module_build_from_scm(username, url, allow_local_url=False): + mmd, scm, yaml = _fetch_mmd(url, allow_local_url) + return submit_module_build(username, url, mmd, scm, yaml) + + +def submit_module_build(username, url, mmd, scm, yaml): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. import module_build_service.scm - mmd, scm, yaml = _fetch_mmd(url, allow_local_url) - module = models.ModuleBuild.query.filter_by( name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first() if module: diff --git a/module_build_service/views.py b/module_build_service/views.py index 24ccba4b..8847404f 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -35,7 +35,8 @@ from flask.views import MethodView from module_build_service import app, conf, log from module_build_service import models, db -from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build, scm_url_schemes +from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build_from_scm, \ + submit_module_build_from_yaml, scm_url_schemes from module_build_service.errors import ( ValidationError, Unauthorized, NotFound) @@ -97,6 +98,14 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) + if "multipart/form-data" in request.headers.get("Content-Type"): + module = self.post_file(username) + else: + module = self.post_scm(username) + + return jsonify(module.json()), 201 + + def post_scm(self, username): try: r = json.loads(request.get_data().decode("utf-8")) except: @@ -120,8 +129,16 @@ class ModuleBuildAPI(MethodView): log.error("The submitted scmurl %r is not valid" % url) raise Unauthorized("The submitted scmurl %s is not valid" % url) - module = submit_module_build(username, url, allow_local_url=False) - return jsonify(module.json()), 201 + return submit_module_build_from_scm(username, url, allow_local_url=False) + + def post_file(self, username): + try: + r = request.files["yaml"] + except: + log.error('Invalid file submitted') + raise ValidationError('Invalid file submitted') + + return submit_module_build_from_yaml(username, r.read()) def patch(self, id): username, groups = module_build_service.auth.get_user(request) From 43b653097abd7d9167f53da4d7c4ccee946b8b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Fri, 17 Feb 2017 00:31:09 +0100 Subject: [PATCH 3/9] Add config option for allowing of direct submitting yaml files to mbs --- conf/config.py | 1 + module_build_service/views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/config.py b/conf/config.py index a4c691ef..b8ff8a5a 100644 --- a/conf/config.py +++ b/conf/config.py @@ -36,6 +36,7 @@ class BaseConfiguration(object): PDC_INSECURE = True PDC_DEVELOP = True SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"] + SUBMITTING_YAML = False # How often should we resort to polling, in seconds # Set to zero to disable polling diff --git a/module_build_service/views.py b/module_build_service/views.py index 8847404f..372dc8a3 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -98,7 +98,7 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) - if "multipart/form-data" in request.headers.get("Content-Type"): + if "multipart/form-data" in request.headers.get("Content-Type") and conf.submitting_yaml: module = self.post_file(username) else: module = self.post_scm(username) From 3e2ee93dd82ce20e2613fe3d27c3fb3005b221ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Sun, 19 Feb 2017 22:29:19 +0100 Subject: [PATCH 4/9] Rename conf option to YAML_SUBMIT_ALLOWED --- conf/config.py | 2 +- module_build_service/config.py | 4 ++++ module_build_service/views.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/config.py b/conf/config.py index b8ff8a5a..c08b2b2c 100644 --- a/conf/config.py +++ b/conf/config.py @@ -36,7 +36,7 @@ class BaseConfiguration(object): PDC_INSECURE = True PDC_DEVELOP = True SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"] - SUBMITTING_YAML = False + YAML_SUBMIT_ALLOWED = False # How often should we resort to polling, in seconds # Set to zero to disable polling diff --git a/module_build_service/config.py b/module_build_service/config.py index 3b906618..6786f7ec 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -268,6 +268,10 @@ class Config(object): 'type': list, 'default': [], 'desc': 'Allowed SCM URLs.'}, + 'yaml_submit_allowed': { + 'type': bool, + 'default': False, + 'desc': 'Is it allowed to directly submit modulemd yaml file?'}, 'num_consecutive_builds': { 'type': int, 'default': 0, diff --git a/module_build_service/views.py b/module_build_service/views.py index 372dc8a3..8fa556a7 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -98,7 +98,7 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) - if "multipart/form-data" in request.headers.get("Content-Type") and conf.submitting_yaml: + if "multipart/form-data" in request.headers.get("Content-Type") and conf.yaml_submit_allowed: module = self.post_file(username) else: module = self.post_scm(username) From 1630c5f14f9614c919e2d5479e8bf6085004ba98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Sun, 19 Feb 2017 22:38:43 +0100 Subject: [PATCH 5/9] Raise user friendly exception when yaml submission is not allowed --- module_build_service/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/module_build_service/views.py b/module_build_service/views.py index 8fa556a7..219eb4d8 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -98,7 +98,7 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) - if "multipart/form-data" in request.headers.get("Content-Type") and conf.yaml_submit_allowed: + if "multipart/form-data" in request.headers.get("Content-Type"): module = self.post_file(username) else: module = self.post_scm(username) @@ -132,6 +132,8 @@ class ModuleBuildAPI(MethodView): return submit_module_build_from_scm(username, url, allow_local_url=False) def post_file(self, username): + if not conf.yaml_submit_allowed: + raise Unauthorized("YAML submission is not enabled") try: r = request.files["yaml"] except: From defabcfc28d476c7f5f0630494d291dbf13a41e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Mon, 20 Feb 2017 20:54:43 +0100 Subject: [PATCH 6/9] Add test for submitting yaml file --- tests/test_build/test_build.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 53c2e7a7..7d99bc0b 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -36,7 +36,10 @@ from module_build_service import db, models, conf from mock import patch from tests import app, init_data +import os import json +import requests +from tempfile import mkdtemp from module_build_service.builder import KojiModuleBuilder, GenericBuilder import module_build_service.scheduler.consumer @@ -282,6 +285,35 @@ class TestBuild(unittest.TestCase): self.assertEqual(tag_groups, []) self.assertEqual(buildroot_groups, []) + @timed(30) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_from_yaml(self, mocked_scm, mocked_get_user): + MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + tmp = mkdtemp() + url = 'http://pkgs.stg.fedoraproject.org/cgit/modules/testmodule.git/plain/testmodule.yaml' + yaml = requests.get(url).content + + testmodule = os.path.join(tmp, 'testmodule.yaml') + with open(testmodule, 'w') as f: + f.write(yaml) + + def submit(): + rv = self.client.post('/module-build-service/1/module-builds/', + content_type='multipart/form-data', + data={'yaml': (testmodule, yaml)}) + return json.loads(rv.data) + + conf.set_item("yaml_submit_allowed", True) + data = submit() + self.assertEqual(data['id'], 1) + + conf.set_item("yaml_submit_allowed", False) + data = submit() + self.assertEqual(data['status'], 401) + self.assertEqual(data['message'], 'YAML submission is not enabled') + @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') From 2b55cfc81df83a393eb86c2b7494905414ad3b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Feb 2017 08:02:33 +0100 Subject: [PATCH 7/9] Use locally stored testmodule.yaml file --- tests/test_build/test_build.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 7d99bc0b..6b4ce5d7 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -38,8 +38,6 @@ from mock import patch from tests import app, init_data import os import json -import requests -from tempfile import mkdtemp from module_build_service.builder import KojiModuleBuilder, GenericBuilder import module_build_service.scheduler.consumer @@ -291,13 +289,10 @@ class TestBuild(unittest.TestCase): def test_submit_build_from_yaml(self, mocked_scm, mocked_get_user): MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") - tmp = mkdtemp() - url = 'http://pkgs.stg.fedoraproject.org/cgit/modules/testmodule.git/plain/testmodule.yaml' - yaml = requests.get(url).content - - testmodule = os.path.join(tmp, 'testmodule.yaml') - with open(testmodule, 'w') as f: - f.write(yaml) + here = os.path.dirname(os.path.abspath(__file__)) + testmodule = os.path.join(here, 'testmodule.yaml') + with open(testmodule) as f: + yaml = f.read() def submit(): rv = self.client.post('/module-build-service/1/module-builds/', From 6b598b1bf7bc56550c7eb836febb2f5a1481ad63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Feb 2017 10:41:31 +0100 Subject: [PATCH 8/9] Describe yaml file submission --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index f3f28420..87459e54 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,10 @@ The response, in case of a successful submission, would include the task ID. id: 42 } + +When ``YAML_SUBMIT_ALLOWED`` is enabled, it is also possible to submit raw modulemd yaml file by sending +``multipart/form-data`` request with input file named as ``yaml``. + Module build state query ------------------------ From 09391e759e7c1cd35428275e085fefdf92356383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kadl=C4=8D=C3=ADk?= Date: Tue, 21 Feb 2017 14:33:32 +0100 Subject: [PATCH 9/9] Add vcr-request-data from test_submit_build_from_yaml --- ...uild.TestBuild.test_submit_build_from_yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml diff --git a/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml b/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml new file mode 100644 index 00000000..629c4677 --- /dev/null +++ b/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml @@ -0,0 +1,46 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [python-requests/2.10.0] + method: GET + uri: http://pkgs.stg.fedoraproject.org/cgit/modules/testmodule.git/plain/testmodule.yaml + response: + body: {string: !!python/unicode "document: modulemd\nversion: 1\ndata:\n summary:\ + \ A test module in all its beautiful beauty\n description: This module\ + \ demonstrates how to write simple modulemd files And can be used for testing\ + \ the build and release pipeline.\n license:\n module: [ MIT ]\n\ + \ dependencies:\n buildrequires:\n base-runtime: master\n\ + \ requires:\n base-runtime: master\n references:\n \ + \ community: https://fedoraproject.org/wiki/Modularity\n documentation:\ + \ https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules\n\ + \ tracker: https://taiga.fedorainfracloud.org/project/modularity\n\ + \ profiles:\n default:\n rpms:\n - tangerine\n\ + \ api:\n rpms:\n - perl-Tangerine\n - tangerine\n\ + \ components:\n rpms:\n perl-List-Compare:\n \ + \ rationale: A dependency of tangerine.\n ref: f25\n\ + \ perl-Tangerine:\n rationale: Provides API for\ + \ this module and is a dependency of tangerine.\n ref: f25\n\ + \ tangerine:\n rationale: Provides API for this\ + \ module.\n buildorder: 10\n ref: f25\n"} + headers: + appserver: [pkgs01.stg.phx2.fedoraproject.org] + apptime: [D=736979] + connection: [Keep-Alive] + content-disposition: [inline; filename="testmodule.yaml"] + content-length: ['1204'] + content-security-policy: [default-src 'none'] + content-type: [text/plain; charset=UTF-8] + date: ['Mon, 20 Feb 2017 19:18:02 GMT'] + etag: ['"4a3ac897696788dde43baefec432690e102ec0ad"'] + expires: ['Mon, 20 Feb 2017 19:21:35 GMT'] + keep-alive: ['timeout=5, max=100'] + last-modified: ['Mon, 20 Feb 2017 19:16:35 GMT'] + server: [Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.1e-fips mod_auth_gssapi/1.4.0 + mod_wsgi/3.4 Python/2.7.5] + x-content-type-options: [nosniff] + status: {code: 200, message: OK} +version: 1