# -*- coding: utf-8 -*- # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # # Some functions kindly copied and then heavily modified from the koji sources: koji/daemon.py # # Copyright (c) 2010-2016 Red Hat, Inc. # # This is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; # version 2.1 of the License. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this software; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # Original Authors of the koji source: # Mike McLean # Mike Bonnet # Modified by: # Karsten Hopp # Petr Ĺ abata """SCM handler functions.""" import os import sys import time import traceback import subprocess import re class SCM(object): "SCM abstraction class" types = {'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'), 'GIT+SSH': ('git+ssh://',)} @staticmethod def is_scm_url(url): """ Return True if the url appears to be a valid, accessible source location, False otherwise """ for schemes in SCM.types.values(): for scheme in schemes: if url.startswith(scheme): return True else: return False def __init__(self, url, allowed_scm=None): """ Initialize the SCM object using the specified url. If url is not in the list of allowed_scm, an error will be raised. NOTE: only git URLs in the following formats are supported atm: git:// git+http:// git+https:// git+rsync:// git+ssh:// The initialized SCM object will have the following attributes: - url (the unmodified url) - allowed_scm (the list of allowed scm, optional) """ if allowed_scm: for allowed in allowed_scm: if url.startswith(allowed): break else: raise RuntimeError('%s is not in the list of allowed SCMs' % url) if not SCM.is_scm_url(url): raise RuntimeError('Invalid SCM URL: %s' % url) self.url = url for scmtype, schemes in SCM.types.items(): if self.url.startswith(schemes): self.scheme = scmtype break else: # should never happen raise RuntimeError('Invalid SCM URL: %s' % url) if self.scheme.startswith("GIT"): match = re.search(r"^(?P.*/(?P[^?]*))(\?#(?P.*))?", url) self.repository = match.group("repository") self.name = match.group("name") if self.name.endswith(".git"): self.name = self.name[:-4] self.commit = match.group("commit") else: raise RuntimeError("Unhandled SCM scheme: %s" % self.scheme) def _run(self, cmd, chdir=None): numretry = 0 path = cmd[0] args = cmd pid = os.fork() if not pid: while numretry <= 3: numretry += 1 try: if chdir: os.chdir(chdir) os.execvp(path, args) except: # XXX maybe switch to subprocess (python-3.5) where # we can check for return codes and timeouts msg = ''.join(traceback.format_exception(*sys.exc_info())) print(msg) if numretry == 3: os._exit(1) time.sleep(10) else: while True: status = os.waitpid(pid, os.WNOHANG) time.sleep(1) if status[0] != 0: return status[1] def checkout(self, scmdir): """ Checkout the module from SCM. Accepts the following parameters: - scmdir: the working directory Returns the directory that the module was checked-out into (a subdirectory of scmdir) """ # TODO: sanity check arguments if self.scheme.startswith("GIT"): sourcedir = '%s/%s' % (scmdir, self.name) module_clone_cmd = ['git', 'clone', '-q'] if self.commit: module_checkout_cmd = ['git', 'checkout', '-q', self.commit] else: module_clone_cmd.extend(['--depth', '1']) module_clone_cmd.extend([self.repository, sourcedir]) # perform checkouts self._run(module_clone_cmd, chdir=scmdir) if self.commit: self._run(module_checkout_cmd, chdir=sourcedir) else: raise RuntimeError("checkout: Unhandled SCM scheme.") return sourcedir def get_latest(self): """Returns the latest commit ID, for example the git master HEAD.""" if self.scheme.startswith("GIT"): (status , output) = subprocess.getstatusoutput("git ls-remote %s" % self.repository) if status != 0: raise RuntimeError("Cannot get git hash of master HEAD in %s" % self.repository) for line in output.split(os.linesep): # FIXME: Be more precise here, we don't want # refs/heads/masterfoo, for example... if 'refs/heads/master' in line: return line.split("\t")[0] raise RuntimeError("Couldn't determine the git master HEAD hash in %s" % self.repository) else: raise RuntimeError("get_latest: Unhandled SCM scheme.") @property def url(self): """The original scmurl.""" return self._url @url.setter def url(self, s): self._url = str(s) @property def scheme(self): """The SCM scheme.""" return self._scheme @scheme.setter def scheme(self, s): self._scheme = str(s) @property def repository(self): """The repository part of the scmurl.""" return self._repository @repository.setter def repository(self, s): self._repository = str(s) @property def commit(self): """The commit ID, for example the git hash, or None.""" return self._commit @commit.setter def commit(self, s): self._commit = str(s) if s else None @property def name(self): """The module name.""" return self._name @name.setter def name(self, s): self._name = str(s) def get_fedpkg_url_git_master_head_pkgname(pkgname=None): """ Return the complete git URL to master HEAD of the given package. Accepts the following parameters: - pkgname: the package name """ pkghash = get_hash_of_git_master_head_pkgname(pkgname) if pkghash is not '': return 'git://pkgs.fedoraproject.org/rpms/' + pkgname + '?#' + pkghash else: return '' def get_hash_of_git_master_head_pkgname(pkgname=None): """ Return the git hash of master HEAD Accepts the following parameters: - pkgname: the package name """ if not isinstance(pkgname, str): raise RuntimeError('pkgname needs to be a string') gitrepo = 'git://pkgs.fedoraproject.org/rpms/' + pkgname (status , output) = subprocess.getstatusoutput('git ls-remote %s' % gitrepo) if status != 0: raise RuntimeError('can\'t get git hash of master HEAD in %s' % gitrepo) b = output.split(os.linesep) ret = '' for line in b: if 'refs/heads/master' in line: ret = line.split('\t')[0] break return ret def check_giturl_syntax(giturl=None): """ dist-pkg giturls are of the form git://pkgs.fedoraproject.org/rpms/ed?#abc0235d4923930745ef05d873646f361a365457 Returns True if giturl has this format, False otherwise. """ if not isinstance(giturl, str): return False if giturl[:6] != 'git://': return False if giturl[6:34] != 'pkgs.fedoraproject.org/rpms/' and giturl[6:38] != 'pkgs.stg.fedoraproject.org/rpms/': return False if not '?#' in giturl.split('/')[-1]: return False return True def convert_giturl_to_cgiturl(giturl=None): """ dist-pkg giturls are of the form git://pkgs.fedoraproject.org/rpms/ed?#abc0235d4923930745ef05d873646f361a365457 cgit urls look like this: http://pkgs.fedoraproject.org/cgit/rpms/ed.git/commit/?id=abc0235d4923930745ef05d873646f361a365457 This function takes a string with a dist-git url as parameter and returns a cgit url Accepts the following parameters: - giturl - dist-git url ('fedpkg giturl') """ try: url = giturl[giturl.index('://')+3:] except: raise RuntimeError('%s is not a dist-git URL' % giturl) url = url.replace('/rpms/','/cgit/rpms/') url = url.replace('?#','.git/commit/?id=') return 'http://' + url def check_if_remote_gitcommit_exists(giturl=None): """ Instead of checking out a git repo and then looking through all the git hashes, this function uses http to connect to cgit and checks for availability of p.e. http://pkgs.fedoraproject.org/cgit/rpms/ed.git/commit/?id=abc0235d4923930745ef05d873646f361a365457 Accepts the following parameters: - giturl - dist-git url ('fedpkg giturl') """ if not check_giturl_syntax(giturl): return False import http.client import os cgiturl = convert_giturl_to_cgiturl(giturl) urlpath = cgiturl[cgiturl.index('://')+3:] urlpath = urlpath[urlpath.index('/'):] http_obj = http.client.HTTPConnection('pkgs.fedoraproject.org') http_obj.request('HEAD',urlpath) res = http_obj.getresponse() if res.status == 200: return True else: return False