diff --git a/rida/views.py b/rida/views.py index dab7ab90..076dae82 100644 --- a/rida/views.py +++ b/rida/views.py @@ -43,6 +43,7 @@ from rida import models from rida.utils import pagination_metadata, filter_module_builds from rida.errors import ( ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound) +from multiprocessing.dummy import Pool as ThreadPool class ModuleBuildAPI(MethodView): @@ -134,6 +135,10 @@ class ModuleBuildAPI(MethodView): username=username ) + # List of (pkg_name, git_url) tuples to be used to check + # the availability of git URLs paralelly later. + full_urls = [] + for pkgname, pkg in mmd.components.rpms.packages.items(): try: if pkg.get("repository") and not conf.rpms_allow_repository: @@ -159,9 +164,19 @@ class ModuleBuildAPI(MethodView): raise full_url = pkg["repository"] + "?#" + pkg["commit"] + full_urls.append((pkgname, full_url)) - if not rida.scm.SCM(full_url).is_available(): - raise UnprocessableEntity("Cannot checkout %s" % pkgname) + # Checks the availability of SCM urls. + pool = ThreadPool(10) + err_msgs = pool.map(lambda data: "Cannot checkout {}".format(data[0]) + if not rida.scm.SCM(data[1]).is_available() + else None, full_urls) + for err_msg in err_msgs: + if err_msg: + raise UnprocessableEntity(err_msg) + + for pkgname, pkg in mmd.components.rpms.packages.items(): + full_url = pkg["repository"] + "?#" + pkg["commit"] build = models.ComponentBuild( module_id=module.id, diff --git a/tests/test_views/base-runtime.yaml b/tests/test_views/base-runtime.yaml new file mode 100644 index 00000000..c2c6de8f --- /dev/null +++ b/tests/test_views/base-runtime.yaml @@ -0,0 +1,182 @@ +document: modulemd +version: 0 +data: + name: base-runtime + version: 0.25 + release: 1 + summary: The base application runtime and hardware enablement layer + description: > + A project closely linked to the Modularity Initiative, base-runtime + is about the defining the common shared package and feature set of + the operating system, providing both the hardware enablement layer + and the minimal application runtime environment other modules can + build upon. + license: + module: [ MIT ] + dependencies: + # Build-require self for now. + # TODO: Don't forget to update this once we're no longer self-hosted + buildrequires: + base-runtime: ~ + references: + community: https://fedoraproject.org/wiki/BaseRuntime + documentation: https://pagure.io/base-runtime + tracker: https://pagure.io/base-runtime/issues + profiles: + # The default profile currently consists of the Linux kernel, the + # systemd init system and all the POSIX userspace utilities that + # are available in Fedora. + default: + rpms: + # at, batch + - at + # sh + - bash + # bc + - bc + # ar, nm, strings, strip + - binutils + # yacc + - byacc + # cflow + - cflow + # basename, cat, chgrp, chmod, chown, cksum, comm, cp, csplit, + # cut, date, dd, df, dirname, du, echo, env, expand, expr, + # false, fold, head, id, join, link, ln, logname, ls, mkdir, + # mkfifo, mv, nice, nl, nohup, od, paste, pathchk, pr, printf, + # pwd, rm, rmdir, sleep, sort, split, stty, talk, tee, test, + # touch, tr, true, tsort, tty, uname, unexpand, uniq, unlink, + # wc, who + - coreutils + # crontab + - cronie + # ctags + - ctags + # lp + - cups-client + # cmp, diff + - diffutils + # ed + - ed + # file + - file + # flex + - flex + # find, xargs + - findutils + # awk + - gawk + # cc, c89, c99 + - gcc + # f95, gfortran + - gcc-gfortran + # gencat, getconf, iconv, locale, localedef + - glibc-common + # grep + - grep + # zcat + - gzip + # the Linux kernel + - kernel + # m4 + - m4 + # mailx + - mailx + # make + - make + # man + - man-db + # compress, uncompress + - ncompress + # tabs, tput + - ncurses + # patch + - patch + # fuser + - psmisc + # ps + - procps-ng + # sed + - sed + # newgrp + - shadow-utils + # uudecode, uuencode + - sharutils + # pax + - spax + # the chosen init system + - systemd + # talk + - talk + # time + - time + # qmove, qmsg, qrerun, qsig + - torque-client + # cal, ipcrm, ipcs, kill, logger, mesg, more, renice, write + - util-linux + # uucp, uustat, uux + - uucp + # ex, vi + - vim-minimal + components: + rpms: + # The same as the default installation profile for now + api: + - at + - bash + - bc + - binutils + - byacc + - cflow + - coreutils + - cronie + - ctags + - cups-client + - diffutils + - ed + - file + - flex + - findutils + - gawk + - gcc + - gcc-gfortran + - glibc-common + - grep + - gzip + - kernel + - m4 + - mailx + - make + - man-db + - ncompress + - ncurses + - patch + - psmisc + - procps-ng + - sed + - shadow-utils + - sharutils + - spax + - systemd + - talk + - time + - torque-client + - util-linux + - uucp + - vim-minimal + packages: + CUnit2: + rationale: Part of the first base-runtime prototype. + commit: 2fb6e8f0529a1196a4bd0897e9bfcd93719e686c + Canna: + rationale: Part of the first base-runtime prototype. + commit: 8d636177ba858cfabf02df029b2089cf192d683e + Cython: + rationale: Part of the first base-runtime prototype. + commit: 6c9d7cad8ac0a840b18b495c4fe152967d808600 + DevIL: + rationale: Part of the first base-runtime prototype. + commit: de58614c0febb5032b710739f34f7344679668f6 + GConf2: + rationale: Part of the first base-runtime prototype. + commit: a13087720a27b17b61bd0a1cbd6042a1d0ab98b7 diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 4e13ad8f..8a0c2040 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -22,6 +22,7 @@ import unittest import json +import time from mock import patch from shutil import copyfile from os import path, mkdir @@ -244,3 +245,79 @@ class TestViews(unittest.TestCase): data['message'], 'Invalid modulemd') self.assertEquals(data['status'], 422) self.assertEquals(data['error'], 'Unprocessable Entity') + + @patch('rida.auth.get_username', return_value='Homer J. Simpson') + @patch('rida.auth.assert_is_packager') + @patch('rida.scm.SCM') + def test_submit_build_scm_parallalization(self, mocked_scm, + mocked_assert_is_packager, mocked_get_username): + def mocked_scm_checkout(temp_dir): + scm_dir = path.join(temp_dir, 'base-runtime') + mkdir(scm_dir) + base_dir = path.abspath(path.dirname(__file__)) + copyfile(path.join(base_dir, 'base-runtime.yaml'), + path.join(scm_dir, 'base-runtime.yaml')) + + return scm_dir + + def mocked_scm_is_available(): + time.sleep(1) + return True + + start = time.time() + mocked_scm.return_value.checkout = mocked_scm_checkout + mocked_scm.return_value.name = 'base-runtime' + mocked_scm.return_value.is_available = mocked_scm_is_available + rv = self.client.post('/rida/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + data = json.loads(rv.data) + + self.assertEquals(len(data['component_builds']), 5) + self.assertEquals(data['name'], 'base-runtime') + self.assertEquals(data['scmurl'], + ('git://pkgs.stg.fedoraproject.org/modules/testmodule' + '.git?#68932c90de214d9d13feefbd35246a81b6cb8d49')) + self.assertTrue(data['time_submitted'] is not None) + self.assertTrue(data['time_modified'] is not None) + self.assertEquals(data['time_completed'], None) + self.assertEquals(data['owner'], 'Homer J. Simpson') + self.assertEquals(data['id'], 31) + self.assertEquals(data['state_name'], 'wait') + + # SCM availability check is parallelized, so 5 components should not + # take longer than 3 second, because each takes 1 second, but they + # are execute in 10 threads. They should take around 1 or 2 seconds + # max to complete. + self.assertTrue(time.time() - start < 3) + + @patch('rida.auth.get_username', return_value='Homer J. Simpson') + @patch('rida.auth.assert_is_packager') + @patch('rida.scm.SCM') + def test_submit_build_scm_non_available(self, mocked_scm, + mocked_assert_is_packager, mocked_get_username): + def mocked_scm_checkout(temp_dir): + scm_dir = path.join(temp_dir, 'base-runtime') + mkdir(scm_dir) + base_dir = path.abspath(path.dirname(__file__)) + copyfile(path.join(base_dir, 'base-runtime.yaml'), + path.join(scm_dir, 'base-runtime.yaml')) + + return scm_dir + + def mocked_scm_is_available(): + return False + + mocked_scm.return_value.checkout = mocked_scm_checkout + mocked_scm.return_value.name = 'base-runtime' + mocked_scm.return_value.is_available = mocked_scm_is_available + rv = self.client.post('/rida/1/module-builds/', data=json.dumps( + {'scmurl': 'git://pkgs.stg.fedoraproject.org/modules/' + 'testmodule.git?#68932c90de214d9d13feefbd35246a81b6cb8d49'})) + data = json.loads(rv.data) + print(data) + + self.assertEquals(data['status'], 422) + self.assertEquals(data['message'][:15], "Cannot checkout") + self.assertEquals(data['error'], "Unprocessable Entity") +