Check the SCM availability of components of modulemd file in parallel in multiple threads to save time.

rida/views.py

@@ -43,6 +43,7 @@ from rida import models
 from rida.utils import pagination_metadata, filter_module_builds
 from rida.errors import (
     ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound)
+from multiprocessing.dummy import Pool as ThreadPool
 
 
 class ModuleBuildAPI(MethodView):
@@ -134,6 +135,10 @@ class ModuleBuildAPI(MethodView):
             username=username
         )
 
+        # List of (pkg_name, git_url) tuples to be used to check
+        # the availability of git URLs paralelly later.
+        full_urls = []
+
         for pkgname, pkg in mmd.components.rpms.packages.items():
             try:
                 if pkg.get("repository") and not conf.rpms_allow_repository:
@@ -159,9 +164,19 @@ class ModuleBuildAPI(MethodView):
                 raise
 
             full_url = pkg["repository"] + "?#" + pkg["commit"]
+            full_urls.append((pkgname, full_url))
 
-            if not rida.scm.SCM(full_url).is_available():
-                raise UnprocessableEntity("Cannot checkout %s" % pkgname)
+        # Checks the availability of SCM urls.
+        pool = ThreadPool(10)
+        err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0])
+                            if not rida.scm.SCM(data[1]).is_available()
+                            else None, full_urls)
+        for err_msg in err_msgs:
+            if err_msg:
+                raise UnprocessableEntity(err_msg)
+
+        for pkgname, pkg in mmd.components.rpms.packages.items():
+            full_url = pkg["repository"] + "?#" + pkg["commit"]
 
             build = models.ComponentBuild(
                 module_id=module.id,

tests/test_views/base-runtime.yaml

@@ -0,0 +1,182 @@
+document: modulemd
+version: 0
+data:
+    name: base-runtime
+    version: 0.25
+    release: 1
+    summary: The base application runtime and hardware enablement layer
+    description: >
+        A project closely linked to the Modularity Initiative, base-runtime
+        is about the defining the common shared package and feature set of
+        the operating system, providing both the hardware enablement layer
+        and the minimal application runtime environment other modules can
+        build upon.
+    license:
+        module: [ MIT ]
+    dependencies:
+        # Build-require self for now.
+        # TODO: Don't forget to update this once we're no longer self-hosted
+        buildrequires:
+            base-runtime: ~
+    references:
+        community: https://fedoraproject.org/wiki/BaseRuntime
+        documentation: https://pagure.io/base-runtime
+        tracker: https://pagure.io/base-runtime/issues
+    profiles:
+        # The default profile currently consists of the Linux kernel, the
+        # systemd init system and all the POSIX userspace utilities that
+        # are available in Fedora.
+        default:
+            rpms:
+                # at, batch
+                - at
+                # sh
+                - bash
+                # bc
+                - bc
+                # ar, nm, strings, strip
+                - binutils
+                # yacc
+                - byacc
+                # cflow
+                - cflow
+                # basename, cat, chgrp, chmod, chown, cksum, comm, cp, csplit,
+                # cut, date, dd, df, dirname, du, echo, env, expand, expr,
+                # false, fold, head, id, join, link, ln, logname, ls, mkdir,
+                # mkfifo, mv, nice, nl, nohup, od, paste, pathchk, pr, printf,
+                # pwd, rm, rmdir, sleep, sort, split, stty, talk, tee, test,
+                # touch, tr, true, tsort, tty, uname, unexpand, uniq, unlink,
+                # wc, who
+                - coreutils
+                # crontab
+                - cronie
+                # ctags
+                - ctags
+                # lp
+                - cups-client
+                # cmp, diff
+                - diffutils
+                # ed
+                - ed
+                # file
+                - file
+                # flex
+                - flex
+                # find, xargs
+                - findutils
+                # awk
+                - gawk
+                # cc, c89, c99
+                - gcc
+                # f95, gfortran
+                - gcc-gfortran
+                # gencat, getconf, iconv, locale, localedef
+                - glibc-common
+                # grep
+                - grep
+                # zcat
+                - gzip
+                # the Linux kernel
+                - kernel
+                # m4
+                - m4
+                # mailx
+                - mailx
+                # make
+                - make
+                # man
+                - man-db
+                # compress, uncompress
+                - ncompress
+                # tabs, tput
+                - ncurses
+                # patch
+                - patch
+                # fuser
+                - psmisc
+                # ps
+                - procps-ng
+                # sed
+                - sed
+                # newgrp
+                - shadow-utils
+                # uudecode, uuencode
+                - sharutils
+                # pax
+                - spax
+                # the chosen init system
+                - systemd
+                # talk
+                - talk
+                # time
+                - time
+                # qmove, qmsg, qrerun, qsig
+                - torque-client
+                # cal, ipcrm, ipcs, kill, logger, mesg, more, renice, write
+                - util-linux
+                # uucp, uustat, uux
+                - uucp
+                # ex, vi
+                - vim-minimal
+    components:
+        rpms:
+            # The same as the default installation profile for now
+            api:
+                - at
+                - bash
+                - bc
+                - binutils
+                - byacc
+                - cflow
+                - coreutils
+                - cronie
+                - ctags
+                - cups-client
+                - diffutils
+                - ed
+                - file
+                - flex
+                - findutils
+                - gawk
+                - gcc
+                - gcc-gfortran
+                - glibc-common
+                - grep
+                - gzip
+                - kernel
+                - m4
+                - mailx
+                - make
+                - man-db
+                - ncompress
+                - ncurses
+                - patch
+                - psmisc
+                - procps-ng
+                - sed
+                - shadow-utils
+                - sharutils
+                - spax
+                - systemd
+                - talk
+                - time
+                - torque-client
+                - util-linux
+                - uucp
+                - vim-minimal
+            packages:
+                CUnit2:
+                    rationale: Part of the first base-runtime prototype.
+                    commit: 2fb6e8f0529a1196a4bd0897e9bfcd93719e686c
+                Canna:
+                    rationale: Part of the first base-runtime prototype.
+                    commit: 8d636177ba858cfabf02df029b2089cf192d683e
+                Cython:
+                    rationale: Part of the first base-runtime prototype.
+                    commit: 6c9d7cad8ac0a840b18b495c4fe152967d808600
+                DevIL:
+                    rationale: Part of the first base-runtime prototype.
+                    commit: de58614c0febb5032b710739f34f7344679668f6
+                GConf2:
+                    rationale: Part of the first base-runtime prototype.
+                    commit: a13087720a27b17b61bd0a1cbd6042a1d0ab98b7

tests/test_views/test_views.py

@@ -22,6 +22,7 @@
 
 import unittest
 import json
+import time
 from mock import patch
 from shutil import copyfile
 from os import path, mkdir
@@ -244,3 +245,79 @@ class TestViews(unittest.TestCase):
             data['message'], 'Invalid modulemd')
         self.assertEquals(data['status'], 422)
         self.assertEquals(data['error'], 'Unprocessable Entity')
+
+    @patch('rida.auth.get_username', return_value='Homer J. Simpson')
+    @patch('rida.auth.assert_is_packager')
+    @patch('rida.scm.SCM')
+    def test_submit_build_scm_parallalization(self, mocked_scm,
+                          mocked_assert_is_packager, mocked_get_username):
+        def mocked_scm_checkout(temp_dir):
+            scm_dir = path.join(temp_dir, 'base-runtime')
+            mkdir(scm_dir)
+            base_dir = path.abspath(path.dirname(__file__))
+            copyfile(path.join(base_dir, 'base-runtime.yaml'),
+                     path.join(scm_dir, 'base-runtime.yaml'))
+
+            return scm_dir
+
+        def mocked_scm_is_available():
+            time.sleep(1)
+            return True
+
+        start = time.time()
+        mocked_scm.return_value.checkout = mocked_scm_checkout
+        mocked_scm.return_value.name = 'base-runtime'
+        mocked_scm.return_value.is_available = mocked_scm_is_available
+        rv = self.client.post('/rida/1/module-builds/', data=json.dumps(
+            {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
+                'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
+        data = json.loads(rv.data)
+
+        self.assertEquals(len(data['component_builds']), 5)
+        self.assertEquals(data['name'], 'base-runtime')
+        self.assertEquals(data['scmurl'],
+                          ('git://pkgs.stg.fedoraproject.org/modules/testmodule'
+                          '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'))
+        self.assertTrue(data['time_submitted'] is not None)
+        self.assertTrue(data['time_modified'] is not None)
+        self.assertEquals(data['time_completed'], None)
+        self.assertEquals(data['owner'], 'Homer J. Simpson')
+        self.assertEquals(data['id'], 31)
+        self.assertEquals(data['state_name'], 'wait')
+
+        # SCM availability check is parallelized, so 5 components should not
+        # take longer than 3 second, because each takes 1 second, but they
+        # are execute in 10 threads. They should take around 1 or 2 seconds
+        # max to complete.
+        self.assertTrue(time.time() - start < 3)
+
+    @patch('rida.auth.get_username', return_value='Homer J. Simpson')
+    @patch('rida.auth.assert_is_packager')
+    @patch('rida.scm.SCM')
+    def test_submit_build_scm_non_available(self, mocked_scm,
+                          mocked_assert_is_packager, mocked_get_username):
+        def mocked_scm_checkout(temp_dir):
+            scm_dir = path.join(temp_dir, 'base-runtime')
+            mkdir(scm_dir)
+            base_dir = path.abspath(path.dirname(__file__))
+            copyfile(path.join(base_dir, 'base-runtime.yaml'),
+                     path.join(scm_dir, 'base-runtime.yaml'))
+
+            return scm_dir
+
+        def mocked_scm_is_available():
+            return False
+
+        mocked_scm.return_value.checkout = mocked_scm_checkout
+        mocked_scm.return_value.name = 'base-runtime'
+        mocked_scm.return_value.is_available = mocked_scm_is_available
+        rv = self.client.post('/rida/1/module-builds/', data=json.dumps(
+            {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/'
+                'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'}))
+        data = json.loads(rv.data)
+        print(data)
+
+        self.assertEquals(data['status'], 422)
+        self.assertEquals(data['message'][:15], "Cannot checkout")
+        self.assertEquals(data['error'], "Unprocessable Entity")
+