diff --git a/README.rst b/README.rst index f3f28420..87459e54 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,10 @@ The response, in case of a successful submission, would include the task ID. id: 42 } + +When ``YAML_SUBMIT_ALLOWED`` is enabled, it is also possible to submit raw modulemd yaml file by sending +``multipart/form-data`` request with input file named as ``yaml``. + Module build state query ------------------------ diff --git a/conf/config.py b/conf/config.py index c0c90571..33f1b033 100644 --- a/conf/config.py +++ b/conf/config.py @@ -36,6 +36,7 @@ class BaseConfiguration(object): PDC_INSECURE = True PDC_DEVELOP = True SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"] + YAML_SUBMIT_ALLOWED = False # How often should we resort to polling, in seconds # Set to zero to disable polling diff --git a/module_build_service/config.py b/module_build_service/config.py index 33bd2fc2..1f4e7ba7 100644 --- a/module_build_service/config.py +++ b/module_build_service/config.py @@ -268,6 +268,10 @@ class Config(object): 'type': list, 'default': [], 'desc': 'Allowed SCM URLs.'}, + 'yaml_submit_allowed': { + 'type': bool, + 'default': False, + 'desc': 'Is it allowed to directly submit modulemd yaml file?'}, 'num_consecutive_builds': { 'type': int, 'default': 0, diff --git a/module_build_service/utils.py b/module_build_service/utils.py index 7f3e0bee..69dddd9b 100644 --- a/module_build_service/utils.py +++ b/module_build_service/utils.py @@ -289,12 +289,7 @@ def _fetch_mmd(url, allow_local_url = False): "Failed to remove temporary directory {!r}: {}".format( td, str(e))) - mmd = modulemd.ModuleMetadata() - try: - mmd.loads(yaml) - except Exception as e: - log.error('Invalid modulemd: %s' % str(e)) - raise UnprocessableEntity('Invalid modulemd: %s' % str(e)) + mmd = load_mmd(yaml) # If undefined, set the name field to VCS repo name. if not mmd.name and scm: @@ -310,6 +305,17 @@ def _fetch_mmd(url, allow_local_url = False): return mmd, scm, yaml + +def load_mmd(yaml): + mmd = modulemd.ModuleMetadata() + try: + mmd.loads(yaml) + except Exception as e: + log.error('Invalid modulemd: %s' % str(e)) + raise UnprocessableEntity('Invalid modulemd: %s' % str(e)) + return mmd + + def _scm_get_latest(pkg): try: # If the modulemd specifies that the 'f25' branch is what @@ -414,13 +420,22 @@ def record_component_builds(scm, mmd, module, initial_batch = 1): return batch -def submit_module_build(username, url, allow_local_url = False): + +def submit_module_build_from_yaml(username, yaml): + mmd = load_mmd(yaml) + return submit_module_build(username, None, mmd, None, yaml) + + +def submit_module_build_from_scm(username, url, allow_local_url=False): + mmd, scm, yaml = _fetch_mmd(url, allow_local_url) + return submit_module_build(username, url, mmd, scm, yaml) + + +def submit_module_build(username, url, mmd, scm, yaml): # Import it here, because SCM uses utils methods # and fails to import them because of dep-chain. import module_build_service.scm - mmd, scm, yaml = _fetch_mmd(url, allow_local_url) - module = models.ModuleBuild.query.filter_by( name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first() if module: diff --git a/module_build_service/views.py b/module_build_service/views.py index 24ccba4b..219eb4d8 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -35,7 +35,8 @@ from flask.views import MethodView from module_build_service import app, conf, log from module_build_service import models, db -from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build, scm_url_schemes +from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build_from_scm, \ + submit_module_build_from_yaml, scm_url_schemes from module_build_service.errors import ( ValidationError, Unauthorized, NotFound) @@ -97,6 +98,14 @@ class ModuleBuildAPI(MethodView): raise Unauthorized("%s is not in any of %r, only %r" % ( username, conf.allowed_groups, groups)) + if "multipart/form-data" in request.headers.get("Content-Type"): + module = self.post_file(username) + else: + module = self.post_scm(username) + + return jsonify(module.json()), 201 + + def post_scm(self, username): try: r = json.loads(request.get_data().decode("utf-8")) except: @@ -120,8 +129,18 @@ class ModuleBuildAPI(MethodView): log.error("The submitted scmurl %r is not valid" % url) raise Unauthorized("The submitted scmurl %s is not valid" % url) - module = submit_module_build(username, url, allow_local_url=False) - return jsonify(module.json()), 201 + return submit_module_build_from_scm(username, url, allow_local_url=False) + + def post_file(self, username): + if not conf.yaml_submit_allowed: + raise Unauthorized("YAML submission is not enabled") + try: + r = request.files["yaml"] + except: + log.error('Invalid file submitted') + raise ValidationError('Invalid file submitted') + + return submit_module_build_from_yaml(username, r.read()) def patch(self, id): username, groups = module_build_service.auth.get_user(request) diff --git a/tests/test_build/test_build.py b/tests/test_build/test_build.py index 53c2e7a7..6b4ce5d7 100644 --- a/tests/test_build/test_build.py +++ b/tests/test_build/test_build.py @@ -36,6 +36,7 @@ from module_build_service import db, models, conf from mock import patch from tests import app, init_data +import os import json from module_build_service.builder import KojiModuleBuilder, GenericBuilder @@ -282,6 +283,32 @@ class TestBuild(unittest.TestCase): self.assertEqual(tag_groups, []) self.assertEqual(buildroot_groups, []) + @timed(30) + @patch('module_build_service.auth.get_user', return_value=user) + @patch('module_build_service.scm.SCM') + def test_submit_build_from_yaml(self, mocked_scm, mocked_get_user): + MockedSCM(mocked_scm, "testmodule", "testmodule.yaml") + + here = os.path.dirname(os.path.abspath(__file__)) + testmodule = os.path.join(here, 'testmodule.yaml') + with open(testmodule) as f: + yaml = f.read() + + def submit(): + rv = self.client.post('/module-build-service/1/module-builds/', + content_type='multipart/form-data', + data={'yaml': (testmodule, yaml)}) + return json.loads(rv.data) + + conf.set_item("yaml_submit_allowed", True) + data = submit() + self.assertEqual(data['id'], 1) + + conf.set_item("yaml_submit_allowed", False) + data = submit() + self.assertEqual(data['status'], 401) + self.assertEqual(data['message'], 'YAML submission is not enabled') + @timed(30) @patch('module_build_service.auth.get_user', return_value=user) @patch('module_build_service.scm.SCM') diff --git a/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml b/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml new file mode 100644 index 00000000..629c4677 --- /dev/null +++ b/tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml @@ -0,0 +1,46 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + User-Agent: [python-requests/2.10.0] + method: GET + uri: http://pkgs.stg.fedoraproject.org/cgit/modules/testmodule.git/plain/testmodule.yaml + response: + body: {string: !!python/unicode "document: modulemd\nversion: 1\ndata:\n summary:\ + \ A test module in all its beautiful beauty\n description: This module\ + \ demonstrates how to write simple modulemd files And can be used for testing\ + \ the build and release pipeline.\n license:\n module: [ MIT ]\n\ + \ dependencies:\n buildrequires:\n base-runtime: master\n\ + \ requires:\n base-runtime: master\n references:\n \ + \ community: https://fedoraproject.org/wiki/Modularity\n documentation:\ + \ https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules\n\ + \ tracker: https://taiga.fedorainfracloud.org/project/modularity\n\ + \ profiles:\n default:\n rpms:\n - tangerine\n\ + \ api:\n rpms:\n - perl-Tangerine\n - tangerine\n\ + \ components:\n rpms:\n perl-List-Compare:\n \ + \ rationale: A dependency of tangerine.\n ref: f25\n\ + \ perl-Tangerine:\n rationale: Provides API for\ + \ this module and is a dependency of tangerine.\n ref: f25\n\ + \ tangerine:\n rationale: Provides API for this\ + \ module.\n buildorder: 10\n ref: f25\n"} + headers: + appserver: [pkgs01.stg.phx2.fedoraproject.org] + apptime: [D=736979] + connection: [Keep-Alive] + content-disposition: [inline; filename="testmodule.yaml"] + content-length: ['1204'] + content-security-policy: [default-src 'none'] + content-type: [text/plain; charset=UTF-8] + date: ['Mon, 20 Feb 2017 19:18:02 GMT'] + etag: ['"4a3ac897696788dde43baefec432690e102ec0ad"'] + expires: ['Mon, 20 Feb 2017 19:21:35 GMT'] + keep-alive: ['timeout=5, max=100'] + last-modified: ['Mon, 20 Feb 2017 19:16:35 GMT'] + server: [Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.1e-fips mod_auth_gssapi/1.4.0 + mod_wsgi/3.4 Python/2.7.5] + x-content-type-options: [nosniff] + status: {code: 200, message: OK} +version: 1