Merge #323 Possibility to submit yaml file

README.rst

@@ -52,6 +52,10 @@ The response, in case of a successful submission, would include the task ID.
         id: 42
     }
 
+
+When ``YAML_SUBMIT_ALLOWED`` is enabled, it is also possible to submit raw modulemd yaml file by sending
+``multipart/form-data`` request with input file named as ``yaml``.
+
 Module build state query
 ------------------------
 

conf/config.py

@@ -36,6 +36,7 @@ class BaseConfiguration(object):
     PDC_INSECURE = True
     PDC_DEVELOP = True
     SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"]
+    YAML_SUBMIT_ALLOWED = False
 
     # How often should we resort to polling, in seconds
     # Set to zero to disable polling

module_build_service/config.py

@@ -268,6 +268,10 @@ class Config(object):
             'type': list,
             'default': [],
             'desc': 'Allowed SCM URLs.'},
+        'yaml_submit_allowed': {
+            'type': bool,
+            'default': False,
+            'desc': 'Is it allowed to directly submit modulemd yaml file?'},
         'num_consecutive_builds': {
             'type': int,
             'default': 0,

module_build_service/utils.py

@@ -289,12 +289,7 @@ def _fetch_mmd(url, allow_local_url = False):
                 "Failed to remove temporary directory {!r}: {}".format(
                     td, str(e)))
 
-    mmd = modulemd.ModuleMetadata()
-    try:
-        mmd.loads(yaml)
-    except Exception as e:
-        log.error('Invalid modulemd: %s' % str(e))
-        raise UnprocessableEntity('Invalid modulemd: %s' % str(e))
+    mmd = load_mmd(yaml)
 
     # If undefined, set the name field to VCS repo name.
     if not mmd.name and scm:
@@ -310,6 +305,17 @@ def _fetch_mmd(url, allow_local_url = False):
 
     return mmd, scm, yaml
 
+
+def load_mmd(yaml):
+    mmd = modulemd.ModuleMetadata()
+    try:
+        mmd.loads(yaml)
+    except Exception as e:
+        log.error('Invalid modulemd: %s' % str(e))
+        raise UnprocessableEntity('Invalid modulemd: %s' % str(e))
+    return mmd
+
+
 def _scm_get_latest(pkg):
     try:
         # If the modulemd specifies that the 'f25' branch is what
@@ -414,13 +420,22 @@ def record_component_builds(scm, mmd, module, initial_batch = 1):
 
         return batch
 
-def submit_module_build(username, url, allow_local_url = False):
+
+def submit_module_build_from_yaml(username, yaml):
+    mmd = load_mmd(yaml)
+    return submit_module_build(username, None, mmd, None, yaml)
+
+
+def submit_module_build_from_scm(username, url, allow_local_url=False):
+    mmd, scm, yaml = _fetch_mmd(url, allow_local_url)
+    return submit_module_build(username, url, mmd, scm, yaml)
+
+
+def submit_module_build(username, url, mmd, scm, yaml):
     # Import it here, because SCM uses utils methods
     # and fails to import them because of dep-chain.
     import module_build_service.scm
 
-    mmd, scm, yaml = _fetch_mmd(url, allow_local_url)
-
     module = models.ModuleBuild.query.filter_by(
         name=mmd.name, stream=mmd.stream, version=str(mmd.version)).first()
     if module:

module_build_service/views.py

@@ -35,7 +35,8 @@ from flask.views import MethodView
 
 from module_build_service import app, conf, log
 from module_build_service import models, db
-from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build, scm_url_schemes
+from module_build_service.utils import pagination_metadata, filter_module_builds, submit_module_build_from_scm, \
+                                       submit_module_build_from_yaml, scm_url_schemes
 from module_build_service.errors import (
     ValidationError, Unauthorized, NotFound)
 
@@ -97,6 +98,14 @@ class ModuleBuildAPI(MethodView):
             raise Unauthorized("%s is not in any of  %r, only %r" % (
                 username, conf.allowed_groups, groups))
 
+        if "multipart/form-data" in request.headers.get("Content-Type"):
+            module = self.post_file(username)
+        else:
+            module = self.post_scm(username)
+
+        return jsonify(module.json()), 201
+
+    def post_scm(self, username):
         try:
             r = json.loads(request.get_data().decode("utf-8"))
         except:
@@ -120,8 +129,18 @@ class ModuleBuildAPI(MethodView):
             log.error("The submitted scmurl %r is not valid" % url)
             raise Unauthorized("The submitted scmurl %s is not valid" % url)
 
-        module = submit_module_build(username, url, allow_local_url=False)
-        return jsonify(module.json()), 201
+        return submit_module_build_from_scm(username, url, allow_local_url=False)
+
+    def post_file(self, username):
+        if not conf.yaml_submit_allowed:
+            raise Unauthorized("YAML submission is not enabled")
+        try:
+            r = request.files["yaml"]
+        except:
+            log.error('Invalid file submitted')
+            raise ValidationError('Invalid file submitted')
+
+        return submit_module_build_from_yaml(username, r.read())
 
     def patch(self, id):
         username, groups = module_build_service.auth.get_user(request)

tests/test_build/test_build.py

@@ -36,6 +36,7 @@ from module_build_service import db, models, conf
 from mock import patch
 
 from tests import app, init_data
+import os
 import json
 
 from module_build_service.builder import KojiModuleBuilder, GenericBuilder
@@ -282,6 +283,32 @@ class TestBuild(unittest.TestCase):
         self.assertEqual(tag_groups, [])
         self.assertEqual(buildroot_groups, [])
 
+    @timed(30)
+    @patch('module_build_service.auth.get_user', return_value=user)
+    @patch('module_build_service.scm.SCM')
+    def test_submit_build_from_yaml(self, mocked_scm, mocked_get_user):
+        MockedSCM(mocked_scm, "testmodule", "testmodule.yaml")
+
+        here = os.path.dirname(os.path.abspath(__file__))
+        testmodule = os.path.join(here, 'testmodule.yaml')
+        with open(testmodule) as f:
+            yaml = f.read()
+
+        def submit():
+            rv = self.client.post('/module-build-service/1/module-builds/',
+                                  content_type='multipart/form-data',
+                                  data={'yaml': (testmodule, yaml)})
+            return json.loads(rv.data)
+
+        conf.set_item("yaml_submit_allowed", True)
+        data = submit()
+        self.assertEqual(data['id'], 1)
+
+        conf.set_item("yaml_submit_allowed", False)
+        data = submit()
+        self.assertEqual(data['status'], 401)
+        self.assertEqual(data['message'], 'YAML submission is not enabled')
+
     @timed(30)
     @patch('module_build_service.auth.get_user', return_value=user)
     @patch('module_build_service.scm.SCM')

tests/vcr-request-data/tests.test_build.test_build.TestBuild.test_submit_build_from_yaml

@@ -0,0 +1,46 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept: ['*/*']
+      Accept-Encoding: ['gzip, deflate']
+      Connection: [keep-alive]
+      User-Agent: [python-requests/2.10.0]
+    method: GET
+    uri: http://pkgs.stg.fedoraproject.org/cgit/modules/testmodule.git/plain/testmodule.yaml
+  response:
+    body: {string: !!python/unicode "document: modulemd\nversion: 1\ndata:\n    summary:\
+        \ A test module in all its beautiful beauty\n    description: This module\
+        \ demonstrates how to write simple modulemd files And can be used for testing\
+        \ the build and release pipeline.\n    license:\n        module: [ MIT ]\n\
+        \    dependencies:\n        buildrequires:\n            base-runtime: master\n\
+        \        requires:\n            base-runtime: master\n    references:\n  \
+        \      community: https://fedoraproject.org/wiki/Modularity\n        documentation:\
+        \ https://fedoraproject.org/wiki/Fedora_Packaging_Guidelines_for_Modules\n\
+        \        tracker: https://taiga.fedorainfracloud.org/project/modularity\n\
+        \    profiles:\n        default:\n            rpms:\n                - tangerine\n\
+        \    api:\n        rpms:\n            - perl-Tangerine\n            - tangerine\n\
+        \    components:\n        rpms:\n            perl-List-Compare:\n        \
+        \        rationale: A dependency of tangerine.\n                ref: f25\n\
+        \            perl-Tangerine:\n                rationale: Provides API for\
+        \ this module and is a dependency of tangerine.\n                ref: f25\n\
+        \            tangerine:\n                rationale: Provides API for this\
+        \ module.\n                buildorder: 10\n                ref: f25\n"}
+    headers:
+      appserver: [pkgs01.stg.phx2.fedoraproject.org]
+      apptime: [D=736979]
+      connection: [Keep-Alive]
+      content-disposition: [inline; filename="testmodule.yaml"]
+      content-length: ['1204']
+      content-security-policy: [default-src 'none']
+      content-type: [text/plain; charset=UTF-8]
+      date: ['Mon, 20 Feb 2017 19:18:02 GMT']
+      etag: ['"4a3ac897696788dde43baefec432690e102ec0ad"']
+      expires: ['Mon, 20 Feb 2017 19:21:35 GMT']
+      keep-alive: ['timeout=5, max=100']
+      last-modified: ['Mon, 20 Feb 2017 19:16:35 GMT']
+      server: [Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.1e-fips mod_auth_gssapi/1.4.0
+          mod_wsgi/3.4 Python/2.7.5]
+      x-content-type-options: [nosniff]
+    status: {code: 200, message: OK}
+version: 1