modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-02 10:20:31 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove urlgrabber as a dependency

Browse Source
This commit is contained in:
mprahl
2018-01-02 16:16:17 -05:00
parent 56d01be3b0
commit c0ec0f82c1
5 changed files with 179 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@@ -14,7 +14,6 @@ RUN dnf install -y \
python-pip \
python-qpid \
python-futures \
python-urlgrabber \
python2-cffi \
python2-cryptography \
python2-pdc-client \

1
Vagrantfile vendored
View File

@@ -24,7 +24,6 @@ $script = <<SCRIPT
python-m2ext \
python-mock \
python-qpid \
python-urlgrabber \
python-virtualenv \
python-futures \
redhat-rpm-config \

61
module_build_service/builder/utils.py
View File

@@ -1,12 +1,13 @@
import os
import sys
import shutil
import subprocess
import munch
import errno
import logging
import urlgrabber.grabber as grabber
import urlgrabber.progress as progress
from multiprocessing.dummy import Pool as ThreadPool
import requests
from module_build_service import log
@@ -85,8 +86,11 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
# Prepare pathinfo we will use to generate the URL.
pathinfo = koji.PathInfo(topdir=session.opts["topurl"])
# When True, we want to run the createrepo_c.
repo_changed = False
# Prepare the list of URLs to download
urls = []
download_args = []
for rpm in rpms:
build_info = builds[rpm['build_id']]
@@ -96,10 +100,18 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
continue
fname = pathinfo.rpm(rpm)
url = pathinfo.build(build_info) + '/' + fname
urls.append((url, os.path.basename(fname), rpm['size']))
relpath = os.path.basename(fname)
local_fn = os.path.join(repo_dir, relpath)
# Download only when the RPM is not downloaded or the size does not match.
if not os.path.exists(local_fn) or os.path.getsize(local_fn) != rpm['size']:
if os.path.exists(local_fn):
os.remove(local_fn)
repo_changed = True
url = pathinfo.build(build_info) + '/' + fname
download_args.append((url, local_fn))
log.info("Downloading %d packages from Koji tag %s to %s" % (len(urls), tag, repo_dir))
log.info(
"Downloading %d packages from Koji tag %s to %s" % (len(download_args), tag, repo_dir))
# Create the output directory
try:
@@ -108,25 +120,26 @@ def create_local_repo_from_koji_tag(config, tag, repo_dir, archs=None):
if exception.errno != errno.EEXIST:
raise
# When True, we want to run the createrepo_c.
repo_changed = False
def _download_file(url_and_dest):
"""
Download a file in a memory efficient manner
:param url_and_dest: a tuple containing the URL and the destination to download to
:return: None
"""
log.info("Downloading {0}...".format(url_and_dest[0]))
if len(url_and_dest) != 2:
raise ValueError("url_and_dest must have two values")
# Donload the RPMs.
pg = progress.TextMeter(sys.stdout)
multi_pg = progress.TextMultiFileMeter(sys.stdout)
for url, relpath, size in urls:
local_fn = os.path.join(repo_dir, relpath)
rv = requests.get(url_and_dest[0], stream=True, timeout=60)
with open(url_and_dest[1], "wb") as f:
for chunk in rv.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
# Download only when RPM is missing or the size does not match.
if not os.path.exists(local_fn) or os.path.getsize(local_fn) != size:
if os.path.exists(local_fn):
os.remove(local_fn)
repo_changed = True
grabber.urlgrab(url, filename=local_fn, progress_obj=pg,
multi_progress_obj=multi_pg, async=(tag, 5),
text=relpath)
grabber.parallel_wait()
# Download the RPMs four at a time.
pool = ThreadPool(4)
pool.map(_download_file, download_args)
pool.close()
# If we downloaded something, run the createrepo_c.
if repo_changed:

3
requirements.txt
View File

@@ -21,9 +21,8 @@ psutil
pyOpenSSL
python-fedora
qpid-python
requests # Client only
requests
requests_kerberos # Client only
six
sqlalchemy
tabulate
urlgrabber

141
tests/test_builder/test_builder_utils.py Normal file
View File

@@ -0,0 +1,141 @@
# Copyright (c) 2017 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Matt Prahl <mprahl@redhat.com>
import unittest
import tempfile
import shutil
from mock import patch, Mock, call
from module_build_service.builder import utils
from tests import conf
class TestBuilderUtils(unittest.TestCase):
@patch('requests.get')
@patch('koji.ClientSession')
@patch('module_build_service.builder.utils.execute_cmd')
def test_create_local_repo_from_koji_tag(self, mock_exec_cmd, mock_koji_session, mock_get):
session = Mock()
rpms = [
{
'arch': 'src',
'build_id': 875991,
'name': 'module-build-macros',
'release': '1.module_92011fe6',
'size': 6890,
'version': '0.1'
},
{
'arch': 'noarch',
'build_id': 875991,
'name': 'module-build-macros',
'release': '1.module_92011fe6',
'size': 6890,
'version': '0.1'
},
{
'arch': 'x86_64',
'build_id': 875636,
'name': 'ed-debuginfo',
'release': '2.module_bd6e0eb1',
'size': 81438,
'version': '1.14.1'
},
{
'arch': 'x86_64',
'build_id': 875636,
'name': 'ed',
'release': '2.module_bd6e0eb1',
'size': 80438,
'version': '1.14.1'
},
{
'arch': 'x86_64',
'build_id': 875640,
'name': 'mksh-debuginfo',
'release': '2.module_bd6e0eb1',
'size': 578774,
'version': '54'
},
{
'arch': 'x86_64',
'build_id': 875640,
'name': 'mksh',
'release': '2.module_bd6e0eb1',
'size': 267042,
'version': '54'
}
]
builds = [
{
'build_id': 875640,
'name': 'mksh',
'release': '2.module_bd6e0eb1',
'version': '54',
'volume_name': 'prod'
},
{
'build_id': 875636,
'name': 'ed',
'release': '2.module_bd6e0eb1',
'version': '1.14.1',
'volume_name': 'prod'
},
{
'build_id': 875991,
'name': 'module-build-macros',
'release': '1.module_92011fe6',
'version': '0.1',
'volume_name': 'prod'
}
]
session.listTaggedRPMS.return_value = (rpms, builds)
session.opts = {'topurl': 'https://kojipkgs.stg.fedoraproject.org/'}
mock_koji_session.return_value = session
tag = 'module-testmodule-master-20170405123740-build'
temp_dir = tempfile.mkdtemp()
try:
utils.create_local_repo_from_koji_tag(conf, tag, temp_dir)
finally:
shutil.rmtree(temp_dir)
url_one = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/module-build-macros/'
'0.1/1.module_92011fe6/noarch/module-build-macros-0.1-1.module_92011fe6.noarch.'
'rpm')
url_two = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/ed/1.14.1/'
'2.module_bd6e0eb1/x86_64/ed-1.14.1-2.module_bd6e0eb1.x86_64.rpm')
url_three = ('https://kojipkgs.stg.fedoraproject.org//vol/prod/packages/mksh/54/'
'2.module_bd6e0eb1/x86_64/mksh-54-2.module_bd6e0eb1.x86_64.rpm')
expected_calls = [
call(url_one, stream=True, timeout=60),
call(url_two, stream=True, timeout=60),
call(url_three, stream=True, timeout=60)
]
for expected_call in expected_calls:
self.assertIn(expected_call, mock_get.call_args_list)
self.assertEqual(len(mock_get.call_args_list), len(expected_calls))