From 067330c97978c0767be909f161fabb83f99792ce Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Mon, 4 Jul 2016 14:49:14 +0200 Subject: [PATCH 1/9] initial scm version --- rida/scm.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 rida/scm.py diff --git a/rida/scm.py b/rida/scm.py new file mode 100644 index 00000000..3bdcc153 --- /dev/null +++ b/rida/scm.py @@ -0,0 +1,144 @@ +# Code kindly copied and then heavily modified from the koji sources: koji/daemon.py +# +# Copyright (c) 2010-2016 Red Hat, Inc. +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; +# version 2.1 of the License. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this software; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +# Original Authors of the koji source: +# Mike McLean +# Mike Bonnet +# Modified by: +# Karsten Hopp + + +import os +import sys +import time +import traceback +import rida + +class SCM(object): + "SCM abstraction class" + + types = {'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'), + 'GIT+SSH': ('git+ssh://',)} + + def is_scm_url(url): + """ + Return True if the url appears to be a valid, accessible source location, False otherwise + """ + for schemes in SCM.types.values(): + for scheme in schemes: + if url.startswith(scheme): + return True + else: + return False + is_scm_url = staticmethod(is_scm_url) + + def __init__(self, url, allowed_scm): + """ + Initialize the SCM object using the specified url. + If url is not in the list of allowed_scm, an error will be raised. + NOTE: only git URLs in the following formats are supported atm: + git:// + git+http:// + git+https:// + git+rsync:// + git+ssh:// + + The initialized SCM object will have the following attributes: + - url (the unmodified url) + - allowed_scm (the list of allowed scm) + """ + + for allowed in allowed_scm: + if url.startswith(allowed): + break + else: + raise RuntimeError, '%s is not in the list of allowed SCMs' % (url) + + if not SCM.is_scm_url(url): + raise RuntimeError, 'Invalid SCM URL: %s' % url + + self.url = url + self.allowed_scm = allowed_scm + + for scmtype, schemes in SCM.types.items(): + if self.url.startswith(schemes): + self.scmtype = scmtype + break + else: + # should never happen + raise RuntimeError, 'Invalid SCM URL: %s' % url + + def _run(self, cmd, chdir=None, _count=[0]): + append = (_count[0] > 0) + _count[0] += 1 + path = cmd[0] + args = cmd + pid = os.fork() + if not pid: + try: + if chdir: + os.chdir(chdir) + flags = os.O_CREAT | os.O_WRONLY + environ = os.environ.copy() + os.execvpe(path, args, environ) + except: + msg = ''.join(traceback.format_exception(*sys.exc_info())) + print msg + os._exit(1) + else: + while True: + status = os.waitpid(pid, os.WNOHANG) + time.sleep(1) + + if status[0] != 0: + return status[1] + + + def checkout(self, scmdir): + """ + Checkout the module from SCM. Accepts the following parameters: + - scmdir: the working directory + + Returns the directory that the module was checked-out into (a subdirectory of scmdir) + """ + # TODO: sanity check arguments + sourcedir = scmdir + + gitrepo = self.url + commonrepo = os.path.dirname(gitrepo) + '/common' + checkout_path = os.path.basename(gitrepo) + if gitrepo.endswith('/.git'): + # If we're referring to the .git subdirectory of the main module, + # assume we need to do the same for the common module + checkout_path = os.path.basename(gitrepo[:-5]) + commonrepo = os.path.dirname(gitrepo[:-5]) + '/common/.git' + elif gitrepo.endswith('.git'): + # If we're referring to a bare repository for the main module, + # assume we need to do the same for the common module + checkout_path = os.path.basename(gitrepo[:-4]) + commonrepo = os.path.dirname(gitrepo[:-4]) + '/common.git' + + sourcedir = '%s/%s' % (scmdir, checkout_path) + module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] + + # perform checkouts + self._run(module_checkout_cmd, chdir=scmdir) + + return sourcedir + + From e5bd3418d20fbb088fa060adf51bad0ef72ac89f Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Mon, 4 Jul 2016 14:49:14 +0200 Subject: [PATCH 2/9] initial scm version --- rida/scm.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 rida/scm.py diff --git a/rida/scm.py b/rida/scm.py new file mode 100644 index 00000000..3bdcc153 --- /dev/null +++ b/rida/scm.py @@ -0,0 +1,144 @@ +# Code kindly copied and then heavily modified from the koji sources: koji/daemon.py +# +# Copyright (c) 2010-2016 Red Hat, Inc. +# +# This is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; +# version 2.1 of the License. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this software; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +# Original Authors of the koji source: +# Mike McLean +# Mike Bonnet +# Modified by: +# Karsten Hopp + + +import os +import sys +import time +import traceback +import rida + +class SCM(object): + "SCM abstraction class" + + types = {'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'), + 'GIT+SSH': ('git+ssh://',)} + + def is_scm_url(url): + """ + Return True if the url appears to be a valid, accessible source location, False otherwise + """ + for schemes in SCM.types.values(): + for scheme in schemes: + if url.startswith(scheme): + return True + else: + return False + is_scm_url = staticmethod(is_scm_url) + + def __init__(self, url, allowed_scm): + """ + Initialize the SCM object using the specified url. + If url is not in the list of allowed_scm, an error will be raised. + NOTE: only git URLs in the following formats are supported atm: + git:// + git+http:// + git+https:// + git+rsync:// + git+ssh:// + + The initialized SCM object will have the following attributes: + - url (the unmodified url) + - allowed_scm (the list of allowed scm) + """ + + for allowed in allowed_scm: + if url.startswith(allowed): + break + else: + raise RuntimeError, '%s is not in the list of allowed SCMs' % (url) + + if not SCM.is_scm_url(url): + raise RuntimeError, 'Invalid SCM URL: %s' % url + + self.url = url + self.allowed_scm = allowed_scm + + for scmtype, schemes in SCM.types.items(): + if self.url.startswith(schemes): + self.scmtype = scmtype + break + else: + # should never happen + raise RuntimeError, 'Invalid SCM URL: %s' % url + + def _run(self, cmd, chdir=None, _count=[0]): + append = (_count[0] > 0) + _count[0] += 1 + path = cmd[0] + args = cmd + pid = os.fork() + if not pid: + try: + if chdir: + os.chdir(chdir) + flags = os.O_CREAT | os.O_WRONLY + environ = os.environ.copy() + os.execvpe(path, args, environ) + except: + msg = ''.join(traceback.format_exception(*sys.exc_info())) + print msg + os._exit(1) + else: + while True: + status = os.waitpid(pid, os.WNOHANG) + time.sleep(1) + + if status[0] != 0: + return status[1] + + + def checkout(self, scmdir): + """ + Checkout the module from SCM. Accepts the following parameters: + - scmdir: the working directory + + Returns the directory that the module was checked-out into (a subdirectory of scmdir) + """ + # TODO: sanity check arguments + sourcedir = scmdir + + gitrepo = self.url + commonrepo = os.path.dirname(gitrepo) + '/common' + checkout_path = os.path.basename(gitrepo) + if gitrepo.endswith('/.git'): + # If we're referring to the .git subdirectory of the main module, + # assume we need to do the same for the common module + checkout_path = os.path.basename(gitrepo[:-5]) + commonrepo = os.path.dirname(gitrepo[:-5]) + '/common/.git' + elif gitrepo.endswith('.git'): + # If we're referring to a bare repository for the main module, + # assume we need to do the same for the common module + checkout_path = os.path.basename(gitrepo[:-4]) + commonrepo = os.path.dirname(gitrepo[:-4]) + '/common.git' + + sourcedir = '%s/%s' % (scmdir, checkout_path) + module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] + + # perform checkouts + self._run(module_checkout_cmd, chdir=scmdir) + + return sourcedir + + From e37310ab8bf06584699979689a44177c8021ad72 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Mon, 4 Jul 2016 15:22:57 +0200 Subject: [PATCH 3/9] drop -n from git clone command --- rida/scm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scm.py b/rida/scm.py index 3bdcc153..bbc3acc2 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -134,7 +134,7 @@ class SCM(object): commonrepo = os.path.dirname(gitrepo[:-4]) + '/common.git' sourcedir = '%s/%s' % (scmdir, checkout_path) - module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] + module_checkout_cmd = ['git', 'clone', gitrepo, sourcedir] # perform checkouts self._run(module_checkout_cmd, chdir=scmdir) From a9bcef7d29b313e69ba47ef1c9ef8fcbde25baec Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Tue, 5 Jul 2016 12:18:36 +0200 Subject: [PATCH 4/9] add a few functions to return either the git hash or a git URL of master/HEAD of either the git object or a package name --- rida/scm.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/rida/scm.py b/rida/scm.py index 6cb3802c..9ec88c29 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -27,6 +27,7 @@ import os import sys import time import traceback +import subprocess import rida class SCM(object): @@ -67,10 +68,10 @@ class SCM(object): if url.startswith(allowed): break else: - raise RuntimeError, '%s is not in the list of allowed SCMs' % (url) + raise RuntimeError('%s is not in the list of allowed SCMs' % url) if not SCM.is_scm_url(url): - raise RuntimeError, 'Invalid SCM URL: %s' % url + raise RuntimeError('Invalid SCM URL: %s' % url) self.url = url self.allowed_scm = allowed_scm @@ -81,7 +82,7 @@ class SCM(object): break else: # should never happen - raise RuntimeError, 'Invalid SCM URL: %s' % url + raise RuntimeError('Invalid SCM URL: %s' % url) def _run(self, cmd, chdir=None, _count=[0]): append = (_count[0] > 0) @@ -93,12 +94,11 @@ class SCM(object): try: if chdir: os.chdir(chdir) - flags = os.O_CREAT | os.O_WRONLY environ = os.environ.copy() os.execvpe(path, args, environ) except: msg = ''.join(traceback.format_exception(*sys.exc_info())) - print msg + print(msg) os._exit(1) else: while True: @@ -142,4 +142,54 @@ class SCM(object): return sourcedir + def get_git_master_head_giturl(self): + """ + Return the git hash of this git object's master HEAD + """ + # drop git hash if url contains it: + gitrepo = self.url.split('?')[0] + (status , output) = subprocess.getstatusoutput('git ls-remote %s' % gitrepo) + if status != 0: + raise RuntimeError('can\'t get git hash of master HEAD in %s' % self.url) + b = output.split(os.linesep) + ret = '' + for line in b: + if 'refs/heads/master' in line: + ret = gitrepo + '?#' + line.split('\t')[0] + break + return ret + +def get_fedpkg_url_git_master_head_pkgname(pkgname=None): + """ + Return the complete git URL to master HEAD of the given package. + Accepts the following parameters: + - pkgname: the package name + """ + pkghash = get_hash_of_git_master_head_pkgname(pkgname) + if pkghash is not '': + return 'git://pkgs.fedoraproject.org/rpms/' + pkgname + '?#' + pkghash + else: + return '' + +def get_hash_of_git_master_head_pkgname(pkgname=None): + """ + Return the git hash of master HEAD + Accepts the following parameters: + - pkgname: the package name + """ + if not isinstance(pkgname, str): + raise RuntimeError('pkgname needs to be a string') + gitrepo = 'git://pkgs.fedoraproject.org/rpms/' + pkgname + (status , output) = subprocess.getstatusoutput('git ls-remote %s' % gitrepo) + if status != 0: + raise RuntimeError('can\'t get git hash of master HEAD in %s' % gitrepo) + b = output.split(os.linesep) + ret = '' + for line in b: + if 'refs/heads/master' in line: + #ret = gitrepo + '?#' + line.split('\t')[0] + ret = line.split('\t')[0] + break + return ret + From 740a2f576e2d85b619f87259cd6335bd7998eb50 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Tue, 5 Jul 2016 13:36:02 +0200 Subject: [PATCH 5/9] no need to set an environment, use execvp --- rida/scm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rida/scm.py b/rida/scm.py index 9ec88c29..81f73b95 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -94,8 +94,7 @@ class SCM(object): try: if chdir: os.chdir(chdir) - environ = os.environ.copy() - os.execvpe(path, args, environ) + os.execvp(path, args) except: msg = ''.join(traceback.format_exception(*sys.exc_info())) print(msg) From f3c7eac83764076070728a6a79fb8ddffb83b344 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Tue, 5 Jul 2016 14:09:51 +0200 Subject: [PATCH 6/9] add a function that takes a string with a dist-git url as parameter and returns a cgit url --- rida/scm.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/rida/scm.py b/rida/scm.py index 81f73b95..134a094c 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -191,4 +191,21 @@ def get_hash_of_git_master_head_pkgname(pkgname=None): break return ret - +def convert_giturl_to_cgiturl(giturl=None): + """ + dist-pkg giturls are of the form + git://pkgs.fedoraproject.org/rpms/ed?#abc0235d4923930745ef05d873646f361a365457 + cgit urls look like this: + http://pkgs.fedoraproject.org/cgit/rpms/ed.git/commit/?id=abc0235d4923930745ef05d873646f361a365457 + This function takes a string with a dist-git url as parameter and returns a cgit url + """ + if not isinstance(giturl, str): + return '' + try: + url = giturl[giturl.index('://')+3:] + except: + raise RuntimeError('%s is not a dist-git URL' % giturl) + url = url.replace('/rpms/','/cgit/rpms/') + url = url.replace('?#','.git/commit/?id=') + return 'http://' + url + From d033c32113a81c6716faf9dc12dfa3c8068a6dfb Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Tue, 5 Jul 2016 17:23:07 +0200 Subject: [PATCH 7/9] add function to check it a giturl exists in dist-git --- rida/scm.py | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/rida/scm.py b/rida/scm.py index 134a094c..160f8942 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -186,11 +186,26 @@ def get_hash_of_git_master_head_pkgname(pkgname=None): ret = '' for line in b: if 'refs/heads/master' in line: - #ret = gitrepo + '?#' + line.split('\t')[0] ret = line.split('\t')[0] break return ret - + +def check_giturl_syntax(giturl=None): + """ + dist-pkg giturls are of the form + git://pkgs.fedoraproject.org/rpms/ed?#abc0235d4923930745ef05d873646f361a365457 + Returns True if giturl has this format, False otherwise. + """ + if not isinstance(giturl, str): + return False + if giturl[:6] != 'git://': + return False + if giturl[6:34] != 'pkgs.fedoraproject.org/rpms/' and giturl[6:38] != 'pkgs.stg.fedoraproject.org/rpms/': + return False + if not '?#' in giturl.split('/')[-1]: + return False + return True + def convert_giturl_to_cgiturl(giturl=None): """ dist-pkg giturls are of the form @@ -198,9 +213,9 @@ def convert_giturl_to_cgiturl(giturl=None): cgit urls look like this: http://pkgs.fedoraproject.org/cgit/rpms/ed.git/commit/?id=abc0235d4923930745ef05d873646f361a365457 This function takes a string with a dist-git url as parameter and returns a cgit url + Accepts the following parameters: + - giturl - dist-git url ('fedpkg giturl') """ - if not isinstance(giturl, str): - return '' try: url = giturl[giturl.index('://')+3:] except: @@ -209,3 +224,26 @@ def convert_giturl_to_cgiturl(giturl=None): url = url.replace('?#','.git/commit/?id=') return 'http://' + url +def check_if_remote_gitcommit_exists(giturl=None): + """ + Instead of checking out a git repo and then looking through all the + git hashes, this function uses http to connect to cgit and checks + for availability of p.e. + http://pkgs.fedoraproject.org/cgit/rpms/ed.git/commit/?id=abc0235d4923930745ef05d873646f361a365457 + Accepts the following parameters: + - giturl - dist-git url ('fedpkg giturl') + """ + if not check_giturl_syntax(giturl): + return False + import http.client + import os + cgiturl = convert_giturl_to_cgiturl(giturl) + urlpath = cgiturl[cgiturl.index('://')+3:] + urlpath = urlpath[urlpath.index('/'):] + http_obj = http.client.HTTPConnection('pkgs.fedoraproject.org') + http_obj.request('HEAD',urlpath) + res = http_obj.getresponse() + if res.status == 200: + return True + else: + return False From 8494e696bc1740df9cb97554423a5eefc9477f68 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Wed, 6 Jul 2016 14:23:20 +0200 Subject: [PATCH 8/9] add simple retry mechanism --- rida/scm.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/rida/scm.py b/rida/scm.py index 160f8942..6d45b081 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -84,21 +84,25 @@ class SCM(object): # should never happen raise RuntimeError('Invalid SCM URL: %s' % url) - def _run(self, cmd, chdir=None, _count=[0]): - append = (_count[0] > 0) - _count[0] += 1 + def _run(self, cmd, chdir=None): + numretry = 0 path = cmd[0] args = cmd pid = os.fork() if not pid: - try: - if chdir: - os.chdir(chdir) - os.execvp(path, args) - except: - msg = ''.join(traceback.format_exception(*sys.exc_info())) - print(msg) - os._exit(1) + while numretry <= 3: + numretry += 1 + try: + if chdir: + os.chdir(chdir) + os.execvp(path, args) + except: # XXX maybe switch to subprocess (python-3.5) where + # we can check for return codes and timeouts + msg = ''.join(traceback.format_exception(*sys.exc_info())) + print(msg) + if numretry == 3: + os._exit(1) + time.sleep(10) else: while True: status = os.waitpid(pid, os.WNOHANG) @@ -107,7 +111,6 @@ class SCM(object): if status[0] != 0: return status[1] - def checkout(self, scmdir): """ Checkout the module from SCM. Accepts the following parameters: @@ -134,7 +137,6 @@ class SCM(object): sourcedir = '%s/%s' % (scmdir, checkout_path) module_checkout_cmd = ['git', 'clone', gitrepo, sourcedir] - module_checkout_cmd = ['git', 'clone', '-n', gitrepo, sourcedir] # perform checkouts self._run(module_checkout_cmd, chdir=scmdir) From d30d50165798e9c2fca14a253e4da2db3ce67611 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 7 Jul 2016 14:01:44 +0200 Subject: [PATCH 9/9] only a few routines remain from the original koji sources. change comment --- rida/scm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rida/scm.py b/rida/scm.py index 6d45b081..b9992e8f 100644 --- a/rida/scm.py +++ b/rida/scm.py @@ -1,4 +1,4 @@ -# Code kindly copied and then heavily modified from the koji sources: koji/daemon.py +# some functions kindly copied and then heavily modified from the koji sources: koji/daemon.py # # Copyright (c) 2010-2016 Red Hat, Inc. #