From c0ec0f82c10edc41d028629a360c87bad495c4af Mon Sep 17 00:00:00 2001
From: mprahl <mprahl@redhat.com>
Date: Tue, 2 Jan 2018 16:16:17 -0500
Subject: [PATCH] Remove urlgrabber as a dependency

---
 Dockerfile                               |   1 -
 Vagrantfile                              |   1 -
 module_build_service/builder/utils.py    |  61 ++++++----
 requirements.txt                         |   3 +-
 tests/test_builder/test_builder_utils.py | 141 +++++++++++++++++++++++
 5 files changed, 179 insertions(+), 28 deletions(-)
 create mode 100644 tests/test_builder/test_builder_utils.py

diff --git a/Dockerfile b/Dockerfile
index fc7b7cd7..495aa101 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,6 @@ RUN dnf install -y \
         python-pip \
         python-qpid \
         python-futures \
-        python-urlgrabber \
         python2-cffi \
         python2-cryptography \
         python2-pdc-client \
diff --git a/Vagrantfile b/Vagrantfile
index b9b22cf7..f9167247 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -24,7 +24,6 @@ $script = <<SCRIPT
         python-m2ext \
         python-mock \
         python-qpid \
-        python-urlgrabber \
         python-virtualenv \
         python-futures \
         redhat-rpm-config \
diff --git a/module_build_service/builder/utils.py b/module_build_service/builder/utils.py
index 96977585..a1e29a53 100644
--- a/module_build_service/builder/utils.py
+++ b/module_build_service/builder/utils.py
@@ -1,12 +1,13 @@
 import os
-import sys
 import shutil
 import subprocess
 import munch
 import errno
 import logging
-import urlgrabber.grabber as grabber
-import urlgrabber.progress as progress
+from multiprocessing.dummy import Pool as ThreadPool
+
+import requests
+
 from module_build_service import log
 
 
@@ -85,8 +86,11 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
     # Prepare pathinfo we will use to generate the URL.
     pathinfo = koji.PathInfo(topdir=session.opts["topurl"])
 
+    # When True, we want to run the createrepo_c.
+    repo_changed = False
+
     # Prepare the list of URLs to download
-    urls = []
+    download_args = []
     for rpm in rpms:
         build_info = builds[rpm['build_id']]
 
@@ -96,10 +100,18 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
             continue
 
         fname = pathinfo.rpm(rpm)
-        url = pathinfo.build(build_info) + '/' + fname
-        urls.append((url, os.path.basename(fname), rpm['size']))
+        relpath = os.path.basename(fname)
+        local_fn = os.path.join(repo_dir, relpath)
+        # Download only when the RPM is not downloaded or the size does not match.
+        if not os.path.exists(local_fn) or os.path.getsize(local_fn) != rpm['size']:
+            if os.path.exists(local_fn):
+                os.remove(local_fn)
+            repo_changed = True
+            url = pathinfo.build(build_info) + '/' + fname
+            download_args.append((url, local_fn))
 
-    log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir))
+    log.info(
+        "Downloading %d packages from Koji tag %s to %s" % (len(download_args), tag, repo_dir))
 
     # Create the output directory
     try:
@@ -108,25 +120,26 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
         if exception.errno != errno.EEXIST:
             raise
 
-    # When True, we want to run the createrepo_c.
-    repo_changed = False
+    def _download_file(url_and_dest):
+        """
+        Download a file in a memory efficient manner
+        :param url_and_dest: a tuple containing the URL and the destination to download to
+        :return: None
+        """
+        log.info("Downloading {0}...".format(url_and_dest[0]))
+        if len(url_and_dest) != 2:
+            raise ValueError("url_and_dest must have two values")
 
-    # Donload the RPMs.
-    pg = progress.TextMeter(sys.stdout)
-    multi_pg = progress.TextMultiFileMeter(sys.stdout)
-    for url, relpath, size in urls:
-        local_fn = os.path.join(repo_dir, relpath)
+        rv = requests.get(url_and_dest[0], stream=True, timeout=60)
+        with open(url_and_dest[1], "wb") as f:
+            for chunk in rv.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
 
-        # Download only when RPM is missing or the size does not match.
-        if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size:
-            if os.path.exists(local_fn):
-                os.remove(local_fn)
-            repo_changed = True
-            grabber.urlgrab(url, filename=local_fn, progress_obj=pg,
-                            multi_progress_obj=multi_pg, async=(tag, 5),
-                            text=relpath)
-
-    grabber.parallel_wait()
+    # Download the RPMs four at a time.
+    pool = ThreadPool(4)
+    pool.map(_download_file, download_args)
+    pool.close()
 
     # If we downloaded something, run the createrepo_c.
     if repo_changed:
diff --git a/requirements.txt b/requirements.txt
index 1b27a3f2..f24d0dba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,9 +21,8 @@ psutil
 pyOpenSSL
 python-fedora
 qpid-python
-requests # Client only
+requests
 requests_kerberos # Client only
 six
 sqlalchemy
 tabulate
-urlgrabber
diff --git a/tests/test_builder/test_builder_utils.py b/tests/test_builder/test_builder_utils.py
new file mode 100644
index 00000000..112fa2bc
--- /dev/null
+++ b/tests/test_builder/test_builder_utils.py
@@ -0,0 +1,141 @@
+# Copyright (c) 2017  Red Hat, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Written by Matt Prahl <mprahl@redhat.com>
+
+import unittest
+import tempfile
+import shutil
+
+from mock import patch, Mock, call
+
+from module_build_service.builder import utils
+from tests import conf
+
+
+class TestBuilderUtils(unittest.TestCase):
+
+    @patch('requests.get')
+    @patch('koji.ClientSession')
+    @patch('module_build_service.builder.utils.execute_cmd')
+    def test_create_local_repo_from_koji_tag(self, mock_exec_cmd, mock_koji_session, mock_get):
+        session = Mock()
+        rpms = [
+            {
+                'arch': 'src',
+                'build_id': 875991,
+                'name': 'module-build-macros',
+                'release': '1.module_92011fe6',
+                'size': 6890,
+                'version': '0.1'
+            },
+            {
+                'arch': 'noarch',
+                'build_id': 875991,
+                'name': 'module-build-macros',
+                'release': '1.module_92011fe6',
+                'size': 6890,
+                'version': '0.1'
+            },
+            {
+                'arch': 'x86_64',
+                'build_id': 875636,
+                'name': 'ed-debuginfo',
+                'release': '2.module_bd6e0eb1',
+                'size': 81438,
+                'version': '1.14.1'
+            },
+            {
+                'arch': 'x86_64',
+                'build_id': 875636,
+                'name': 'ed',
+                'release': '2.module_bd6e0eb1',
+                'size': 80438,
+                'version': '1.14.1'
+            },
+            {
+                'arch': 'x86_64',
+                'build_id': 875640,
+                'name': 'mksh-debuginfo',
+                'release': '2.module_bd6e0eb1',
+                'size': 578774,
+                'version': '54'
+            },
+            {
+                'arch': 'x86_64',
+                'build_id': 875640,
+                'name': 'mksh',
+                'release': '2.module_bd6e0eb1',
+                'size': 267042,
+                'version': '54'
+            }
+        ]
+
+        builds = [
+            {
+                'build_id': 875640,
+                'name': 'mksh',
+                'release': '2.module_bd6e0eb1',
+                'version': '54',
+                'volume_name': 'prod'
+            },
+            {
+                'build_id': 875636,
+                'name': 'ed',
+                'release': '2.module_bd6e0eb1',
+                'version': '1.14.1',
+                'volume_name': 'prod'
+            },
+            {
+                'build_id': 875991,
+                'name': 'module-build-macros',
+                'release': '1.module_92011fe6',
+                'version': '0.1',
+                'volume_name': 'prod'
+            }
+        ]
+
+        session.listTaggedRPMS.return_value = (rpms, builds)
+        session.opts = {'topurl': 'https://kojipkgs.stg.fedoraproject.org/'}
+        mock_koji_session.return_value = session
+
+        tag = 'module-testmodule-master-20170405123740-build'
+        temp_dir = tempfile.mkdtemp()
+        try:
+            utils.create_local_repo_from_koji_tag(conf, tag, temp_dir)
+        finally:
+            shutil.rmtree(temp_dir)
+
+        url_one = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/module-build-macros/'
+                   '0.1/1.module_92011fe6/noarch/module-build-macros-0.1-1.module_92011fe6.noarch.'
+                   'rpm')
+        url_two = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/ed/1.14.1/'
+                   '2.module_bd6e0eb1/x86_64/ed-1.14.1-2.module_bd6e0eb1.x86_64.rpm')
+        url_three = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/mksh/54/'
+                     '2.module_bd6e0eb1/x86_64/mksh-54-2.module_bd6e0eb1.x86_64.rpm')
+
+        expected_calls = [
+            call(url_one, stream=True, timeout=60),
+            call(url_two, stream=True, timeout=60),
+            call(url_three, stream=True, timeout=60)
+        ]
+        for expected_call in expected_calls:
+            self.assertIn(expected_call, mock_get.call_args_list)
+        self.assertEqual(len(mock_get.call_args_list), len(expected_calls))