mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-13 16:59:52 +08:00
Merge #270 Replace submit-build.py by brand new 'mbs-build' command
This commit is contained in:
Jan Kaluža
321
contrib/mbs-build
Executable file
321
contrib/mbs-build
Executable file
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python
|
||||
from __future__ import print_function
|
||||
import os
|
||||
import sys
|
||||
import openidc_client
|
||||
import argparse
|
||||
import logging
|
||||
import subprocess
|
||||
import requests
|
||||
import koji
|
||||
import time
|
||||
from copy import copy
|
||||
|
||||
DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/"
|
||||
DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org"
|
||||
|
||||
openidc_client.WEB_PORTS = [13747]
|
||||
|
||||
def watch_build(server, build_id):
|
||||
"""
|
||||
Watches the MBS build in a loop, updates every 30 seconds.
|
||||
Returns when build state is 'failed' or 'done' or 'ready' or when
|
||||
user hits ctrl+c.
|
||||
"""
|
||||
if not server:
|
||||
server = DEFAULT_MBS_SERVER
|
||||
|
||||
done = False
|
||||
while not done:
|
||||
# Clear the screen
|
||||
print(chr(27) + "[2J")
|
||||
|
||||
state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()])
|
||||
state_names[None] = "undefined"
|
||||
|
||||
idx = int(build_id)
|
||||
|
||||
response = requests.get(server + '/module-build-service/1/module-builds/%i?verbose=true' % idx)
|
||||
data = response.json()
|
||||
|
||||
tasks = data['tasks']['rpms']
|
||||
states = list(set([task['state'] for task in tasks.values()]))
|
||||
inverted = dict()
|
||||
for name, task in tasks.items():
|
||||
state = task['state']
|
||||
inverted[state] = inverted.get(state, [])
|
||||
inverted[state].append(name)
|
||||
|
||||
if 0 in inverted:
|
||||
print("Still building:")
|
||||
for name in inverted[0]:
|
||||
task = tasks[name]
|
||||
print(" ", name, "https://koji.fedoraproject.org/koji/taskinfo?taskID=%s" % task['task_id'])
|
||||
|
||||
if 3 in inverted:
|
||||
print("Failed:")
|
||||
for name in inverted[3]:
|
||||
task = tasks[name]
|
||||
print(" ", name, "https://koji.fedoraproject.org/koji/taskinfo?taskID=%s" % task['task_id'])
|
||||
|
||||
print()
|
||||
print("Summary:")
|
||||
for state in states:
|
||||
print(" ", len(inverted[state]), "components in the", state_names[state], "state")
|
||||
|
||||
done = data["state_name"] in ["failed", "done", "ready"]
|
||||
|
||||
print('Module {name} is in state {state_name} (reason {state_reason})'.format(**data))
|
||||
time.sleep(30)
|
||||
|
||||
# Ideally we would use oidc.send_request here, but it doesn't support
|
||||
# custom HTTP verbs/methods like "PATCH". It sends just "POST"...
|
||||
# TODO: Remove this method once python-openidc-client with verb support
|
||||
# is released and updated in Fedora.
|
||||
def _send_oidc_request(oidc, verb, *args, **kwargs):
|
||||
ckwargs = copy(kwargs)
|
||||
|
||||
scopes = ckwargs.pop('scopes')
|
||||
new_token = ckwargs.pop('new_token', True)
|
||||
auto_refresh = ckwargs.pop('auto_refresh', True)
|
||||
|
||||
is_retry = False
|
||||
if oidc.token_to_try:
|
||||
is_retry = True
|
||||
token = oidc.token_to_try
|
||||
oidc.token_to_try = None
|
||||
else:
|
||||
token = oidc.get_token(scopes, new_token=new_token)
|
||||
if not token:
|
||||
return None
|
||||
|
||||
if oidc.use_post:
|
||||
if 'json' in ckwargs:
|
||||
raise ValueError('Cannot provide json in a post call')
|
||||
|
||||
if 'data' not in ckwargs:
|
||||
ckwargs['data'] = {}
|
||||
ckwargs['data']['access_token'] = token
|
||||
else:
|
||||
if 'headers' not in ckwargs:
|
||||
ckwargs['headers'] = {}
|
||||
ckwargs['headers']['Authorization'] = 'Bearer %s' % token
|
||||
|
||||
resp = requests.request(verb, *args, **ckwargs)
|
||||
if resp.status_code == 401 and not is_retry:
|
||||
if not auto_refresh:
|
||||
return resp
|
||||
|
||||
oidc.token_to_try = oidc.report_token_issue()
|
||||
if not oidc.token_to_try:
|
||||
return resp
|
||||
return _send_oidc_request(oidc, verb, *args, **kwargs)
|
||||
elif resp.status_code == 401:
|
||||
# We got a 401 and this is a retry. Report error
|
||||
oidc.report_token_issue()
|
||||
return resp
|
||||
else:
|
||||
return resp
|
||||
|
||||
def send_authorized_request(verb, server, id_provider, url, body, **kwargs):
|
||||
"""
|
||||
Sends authorized request to server.
|
||||
"""
|
||||
if not server:
|
||||
server = DEFAULT_MBS_SERVER
|
||||
if not id_provider:
|
||||
id_provider = DEFAULT_ID_PROVIDER
|
||||
|
||||
logging.info("Trying to get the token from %s", id_provider)
|
||||
|
||||
# Get the auth token using the OpenID client.
|
||||
oidc = openidc_client.OpenIDCClient(
|
||||
"mbs_build", id_provider,
|
||||
{'Token': 'Token', 'Authorization': 'Authorization'},
|
||||
'mbs-authorizer', "notsecret")
|
||||
|
||||
scopes = ['openid', 'https://id.fedoraproject.org/scope/groups',
|
||||
'https://mbs.fedoraproject.org/oidc/submit-build']
|
||||
|
||||
logging.debug("Sending body: %s", body)
|
||||
resp = _send_oidc_request(oidc, verb, "%s/%s" % (server, url), json=body,
|
||||
scopes=scopes, **kwargs)
|
||||
return resp
|
||||
|
||||
def get_scm_url(scm_url, pyrpkg, local=False):
|
||||
"""
|
||||
If `scm_url` it not set, returns the scm_url based on git repository
|
||||
in the `os.getcwd()`. When local is True, file:// scheme is used,
|
||||
otherwise `pyrpkg` is used to determine public URL to git repository.
|
||||
"""
|
||||
if scm_url:
|
||||
return scm_url
|
||||
|
||||
logging.info("You have not provided SCM URL or branch. Trying to get "
|
||||
"it from current working directory")
|
||||
|
||||
if local:
|
||||
# Just get the local URL from the current working directory.
|
||||
scm_url = "file://%s" % os.getcwdu()
|
||||
return scm_url
|
||||
else:
|
||||
# Get the url using pyrpkg implementation.
|
||||
process = subprocess.Popen([pyrpkg, 'giturl'], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, err = process.communicate()
|
||||
if process.returncode != 0 and len(err) != 0:
|
||||
logging.error("Cannot get the giturl from current "
|
||||
"working directory using the %s", pyrpkg)
|
||||
logging.error(err)
|
||||
return None
|
||||
scm_url = out[:-1] # remove new-line
|
||||
return scm_url
|
||||
|
||||
def get_scm_branch(branch):
|
||||
"""
|
||||
If `branch` it not set, returns the branch name based on git repository
|
||||
in the `os.getcwd()`.
|
||||
"""
|
||||
if branch:
|
||||
return branch
|
||||
|
||||
process = subprocess.Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
out, err = process.communicate()
|
||||
if process.returncode != 0 and len(err) != 0:
|
||||
logging.error("Cannot get the branch name from current "
|
||||
"working directory.")
|
||||
logging.error(err)
|
||||
return None
|
||||
branch = out[:-1] # remove new-line
|
||||
return branch
|
||||
|
||||
def submit_module_build(scm_url, branch, server, id_provider, pyrpkg):
|
||||
"""
|
||||
Submits the module defined by `scm_url` to MBS instance defined
|
||||
by `server`. Returns build_id or negative error code.
|
||||
"""
|
||||
scm_url = get_scm_url(scm_url, pyrpkg)
|
||||
branch = get_scm_branch(branch)
|
||||
if not scm_url or not branch:
|
||||
return -2
|
||||
|
||||
logging.info("Submitting module build %s", scm_url)
|
||||
body = {'scmurl': scm_url, 'branch': branch}
|
||||
resp = send_authorized_request(
|
||||
"POST", server, id_provider, "/module-build-service/1/module-builds/",
|
||||
body)
|
||||
logging.info(resp.text)
|
||||
|
||||
data = resp.json()
|
||||
if 'id' in data:
|
||||
return data['id']
|
||||
return -3
|
||||
|
||||
def do_local_build(scm_url, branch):
|
||||
"""
|
||||
Starts the local build using the 'mbs-manager build_module_locally'
|
||||
command. Returns exit code of that command or None when scm_url or
|
||||
branch are not set and cannot be obtained from the CWD.
|
||||
"""
|
||||
scm_url = get_scm_url(scm_url, None, local=True)
|
||||
branch = get_scm_branch(branch)
|
||||
if not scm_url or not branch:
|
||||
return None
|
||||
|
||||
logging.info("Starting local build of %s, branch %s", scm_url, branch)
|
||||
process = subprocess.Popen(['mbs-manager', 'build_module_locally',
|
||||
scm_url, branch])
|
||||
process.communicate()
|
||||
return process.returncode
|
||||
|
||||
def cancel_module_build(server, id_provider, build_id):
|
||||
"""
|
||||
Cancels the module build.
|
||||
"""
|
||||
logging.info("Cancelling module build %s", build_id)
|
||||
resp = send_authorized_request(
|
||||
"PATCH", server, id_provider,
|
||||
"/module-build-service/1/module-builds/" + str(build_id),
|
||||
{'state': 'failed'})
|
||||
logging.info(resp.text)
|
||||
|
||||
def main():
|
||||
# Parse command line arguments
|
||||
parser = argparse.ArgumentParser(description="Submits and manages module builds.")
|
||||
subparsers = parser.add_subparsers(dest="cmd_name")
|
||||
parser.add_argument('-v', dest='verbose', action='store_true',
|
||||
help="shows verbose output")
|
||||
parser.add_argument('-q', dest='quiet', action='store_true',
|
||||
help="shows only warnings and errors")
|
||||
parser.add_argument('-s', dest='server', action='store',
|
||||
help="defines the hostname[:port] of the Module Build Service")
|
||||
parser.add_argument('-i', dest='idprovider', action='store',
|
||||
help="defines the OpenID Connect identity provider")
|
||||
parser.add_argument('-p', dest='pyrpkg_client', action='store',
|
||||
help="defines the name of pyrpkg client executable",
|
||||
default="fedpkg")
|
||||
|
||||
parser_submit = subparsers.add_parser(
|
||||
'submit', help="submit module build",
|
||||
description="Submits the module build. When 'scm_url' or 'branch' "
|
||||
"is not set, it presumes you are executing this command in "
|
||||
"the directory with the cloned git repository with a module.")
|
||||
parser_submit.add_argument("scm_url", nargs='?')
|
||||
parser_submit.add_argument("branch", nargs='?')
|
||||
parser_submit.add_argument('-w', dest="watch", action='store_true',
|
||||
help="watch the build progress")
|
||||
|
||||
parser_watch = subparsers.add_parser(
|
||||
'watch', help="watch module build",
|
||||
description="Watches the build progress of a build submitted by "
|
||||
"the 'submit' subcommand.")
|
||||
parser_watch.add_argument("build_id")
|
||||
|
||||
parser_cancel = subparsers.add_parser(
|
||||
'cancel', help="cancel module build",
|
||||
description="Cancels the build submitted by 'submit' subcommand.")
|
||||
parser_cancel.add_argument("build_id")
|
||||
parser_local = subparsers.add_parser(
|
||||
'local', help="do local build of module",
|
||||
description="Starts local build of a module using the Mock backend. "
|
||||
"When 'scm_url' or 'branch' is not set, it presumes you are "
|
||||
"executing this command in the directory with the cloned git "
|
||||
"repository with a module.")
|
||||
parser_local.add_argument("scm_url", nargs='?')
|
||||
parser_local.add_argument("branch", nargs='?')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Initialize the logging.
|
||||
if args.verbose:
|
||||
loglevel = logging.DEBUG
|
||||
elif args.quiet:
|
||||
loglevel = logging.WARNING
|
||||
else:
|
||||
loglevel = logging.INFO
|
||||
logging.basicConfig(level=loglevel, format="%(levelname)s: %(message)s")
|
||||
|
||||
if args.cmd_name == "submit":
|
||||
# Submit the module build.
|
||||
build_id = submit_module_build(args.scm_url, args.branch, args.server,
|
||||
args.idprovider, args.pyrpkg_client)
|
||||
if build_id < 0:
|
||||
sys.exit(build_id)
|
||||
|
||||
if args.watch:
|
||||
watch_build(args.server, build_id)
|
||||
elif args.cmd_name == "local":
|
||||
sys.exit(do_local_build(args.scm_url, args.branch))
|
||||
elif args.cmd_name == "watch":
|
||||
# Watch the module build.
|
||||
try:
|
||||
watch_build(args.server, args.build_id)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
elif args.cmd_name == "cancel":
|
||||
# Cancel the module build
|
||||
cancel_module_build(args.server, args.idprovider, args.build_id)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"scmurl": "git://pkgs.stg.fedoraproject.org/modules/testmodule.git?#789dc7b",
|
||||
"branch": "master"
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
import socket
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
|
||||
try:
|
||||
from urllib.parse import urlencode # py3
|
||||
except ImportError:
|
||||
from urllib import urlencode # py2
|
||||
|
||||
def listen_for_token():
|
||||
"""
|
||||
Listens on port 13747 on localhost for a redirect request by OIDC
|
||||
server, parses the response and returns the "access_token" value.
|
||||
"""
|
||||
TCP_IP = '0.0.0.0'
|
||||
TCP_PORT = 13747
|
||||
BUFFER_SIZE = 1024
|
||||
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
s.bind((TCP_IP, TCP_PORT))
|
||||
s.listen(1)
|
||||
|
||||
conn, addr = s.accept()
|
||||
print 'Connection address:', addr
|
||||
data = ""
|
||||
sent_resp = False
|
||||
while 1:
|
||||
try:
|
||||
r = conn.recv(BUFFER_SIZE)
|
||||
except:
|
||||
conn.close()
|
||||
break
|
||||
if not r: break
|
||||
data += r
|
||||
|
||||
if not sent_resp:
|
||||
response = "Token has been handled."
|
||||
conn.send("""HTTP/1.1 200 OK
|
||||
Content-Length: %s
|
||||
Content-Type: text/plain
|
||||
Connection: Closed
|
||||
|
||||
%s""" % (len(response), response))
|
||||
conn.close()
|
||||
sent_resp = True
|
||||
|
||||
s.close()
|
||||
|
||||
data = data.split("\n")
|
||||
for line in data:
|
||||
variables = line.split("&")
|
||||
for var in variables:
|
||||
kv = var.split("=")
|
||||
if not len(kv) == 2:
|
||||
continue
|
||||
if kv[0] == "access_token":
|
||||
return kv[1]
|
||||
return None
|
||||
|
||||
mbs_host = "localhost:5000"
|
||||
token = None
|
||||
if len(sys.argv) > 2:
|
||||
token = sys.argv[2]
|
||||
if len(sys.argv) > 1:
|
||||
mbs_host = sys.argv[1]
|
||||
|
||||
print "Usage: submit_build.py [mbs_host] [oidc_token]"
|
||||
print ""
|
||||
if not token:
|
||||
print "Provide token as command line argument or visit following URL to obtain the token:"
|
||||
|
||||
query = urlencode({
|
||||
'response_type': 'token',
|
||||
'response_mode': 'form_post',
|
||||
'nonce': random.randint(100, 10000),
|
||||
'scope': ' '.join([
|
||||
'openid',
|
||||
'https://id.fedoraproject.org/scope/groups',
|
||||
'https://mbs.fedoraproject.org/oidc/submit-build',
|
||||
]),
|
||||
'client_id': 'mbs-authorizer',
|
||||
}) + "&redirect_uri=http://localhost:13747/"
|
||||
print "https://id.fedoraproject.org/openidc/Authorization?" + query
|
||||
print "We are waiting for you to finish the token generation..."
|
||||
|
||||
if not token:
|
||||
token = listen_for_token()
|
||||
if not token:
|
||||
print "Failed to get a token from response"
|
||||
os._exit(1)
|
||||
|
||||
print "Submitting build of ..."
|
||||
with open("submit-build.json", "r") as build:
|
||||
print build.read()
|
||||
print "Using https://%s/module_build_service/module-builds/" % mbs_host
|
||||
print "NOTE: You need to be a Fedora packager for this to work"
|
||||
print
|
||||
|
||||
os.system("curl -k -H 'Authorization: Bearer %s' -H 'Content-Type: text/json' --data @submit-build.json https://%s/module-build-service/1/module-builds/ -v" % (token, mbs_host))
|
||||
@@ -122,13 +122,13 @@ def cleardb():
|
||||
|
||||
|
||||
@manager.command
|
||||
def build_module_locally(url):
|
||||
def build_module_locally(url, branch):
|
||||
""" Performs local module build using Mock
|
||||
"""
|
||||
conf.set_item("system", "mock")
|
||||
|
||||
# Use our own local SQLite3 database.
|
||||
confdir = os.path.abspath(os.path.dirname(__file__))
|
||||
confdir = os.path.abspath(os.getcwd())
|
||||
dbdir = os.path.abspath(os.path.join(confdir, '..')) if confdir.endswith('conf') \
|
||||
else confdir
|
||||
dbpath = '/{0}'.format(os.path.join(dbdir, '.mbs_local_build.db'))
|
||||
@@ -149,7 +149,7 @@ def build_module_locally(url):
|
||||
db.create_all()
|
||||
|
||||
username = getpass.getuser()
|
||||
submit_module_build_from_scm(username, url, "master", allow_local_url=True)
|
||||
submit_module_build_from_scm(username, url, branch, allow_local_url=True)
|
||||
|
||||
stop = module_build_service.scheduler.make_simple_stop_condition(db.session)
|
||||
initial_messages = [MBSModule("local module build", 1, 1)]
|
||||
|
||||
1
setup.py
1
setup.py
@@ -31,6 +31,7 @@ setup(name='module-build-service',
|
||||
'moksha.consumer': 'mbsconsumer = module_build_service.scheduler.consumer:MBSConsumer',
|
||||
'moksha.producer': 'mbspoller = module_build_service.scheduler.producer:MBSProducer',
|
||||
},
|
||||
scripts=["contrib/mbs-build"],
|
||||
data_files=[('/etc/module-build-service/', ['conf/cacert.pem',
|
||||
'conf/config.py',
|
||||
'conf/copr.conf',
|
||||
|
||||
Reference in New Issue
Block a user