From b104850fd783974180a077b782b8afbb2f425686 Mon Sep 17 00:00:00 2001 From: mprahl Date: Tue, 19 Sep 2017 14:07:12 -0400 Subject: [PATCH 1/4] Fix Vagrant SSHFS issue --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 6fbeefb2..8d0b68b5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -60,7 +60,7 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: $script config.vm.provision "shell", inline: $script_services, run: "always" config.vm.provider "libvirt" do |v, override| - override.vm.synced_folder "./", "/tmp/module_build_service", type: "sshfs" + override.vm.synced_folder "./", "/tmp/module_build_service", type: "sshfs", sshfs_opts_append: "-o nonempty" v.memory = 1024 #v.cpus = 2 end From 6165fd50886164eb402e1b89d0a653ed219a4ace Mon Sep 17 00:00:00 2001 From: mprahl Date: Tue, 19 Sep 2017 14:34:06 -0400 Subject: [PATCH 2/4] Add an API that exposes information about the MBS server --- README.rst | 20 ++++++++++++++++++++ module_build_service/__init__.py | 5 +++++ module_build_service/views.py | 32 +++++++++++++++++++++++++++++--- tests/test_views/test_views.py | 11 ++++++++++- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index f7cf55be..b51eb2aa 100644 --- a/README.rst +++ b/README.rst @@ -493,6 +493,26 @@ parameters:: } } + +Listing about +------------- + +This API shows information about the MBS server:: + + GET /module-build-service/1/about/ + +:: + + HTTP 200 OK + +:: + + { + "auth_method": "oidc", + "version": "1.3.26" + } + + HTTP Response Codes ------------------- diff --git a/module_build_service/__init__.py b/module_build_service/__init__.py index 24fbbd77..66921b2f 100644 --- a/module_build_service/__init__.py +++ b/module_build_service/__init__.py @@ -40,6 +40,7 @@ for a number of tasks: infrastructure services can pick up the work. """ +import pkg_resources from flask import Flask, has_app_context, url_for from flask_sqlalchemy import SQLAlchemy from logging import getLogger @@ -53,6 +54,10 @@ from module_build_service.errors import ( from module_build_service.config import init_config from module_build_service.proxy import ReverseProxy +try: + version = pkg_resources.get_distribution('module-build-service').version +except pkg_resources.DistributionNotFound: + version = 'unknown' app = Flask(__name__) app.wsgi_app = ReverseProxy(app.wsgi_app) diff --git a/module_build_service/views.py b/module_build_service/views.py index f557820f..123d2743 100644 --- a/module_build_service/views.py +++ b/module_build_service/views.py @@ -31,14 +31,13 @@ import module_build_service.auth from flask import request, jsonify from flask.views import MethodView -from module_build_service import app, conf, log -from module_build_service import models, db +from module_build_service import app, conf, log, models, db, version from module_build_service.utils import ( pagination_metadata, filter_module_builds, filter_component_builds, submit_module_build_from_scm, submit_module_build_from_yaml, get_scm_url_re) from module_build_service.errors import ( - ValidationError, Forbidden, NotFound) + ValidationError, Forbidden, NotFound, ProgrammingError) api_v1 = { 'module_builds': { @@ -73,6 +72,12 @@ api_v1 = { 'methods': ['GET'], } }, + 'about': { + 'url': '/module-build-service/1/about/', + 'options': { + 'methods': ['GET'] + } + } } @@ -203,6 +208,20 @@ class ModuleBuildAPI(MethodView): return jsonify(module.json()), 200 +class AboutAPI(MethodView): + def get(self): + json = {'version': version} + config_items = ['auth_method'] + for item in config_items: + config_item = getattr(conf, item) + # All config items have a default, so if doesn't exist it is a programming error + if not config_item: + raise ProgrammingError( + 'An invalid config item of "{0}" was specified'.format(item)) + json[item] = config_item + return jsonify(json), 200 + + class BaseHandler(object): def __init__(self, request): self.username, self.groups = module_build_service.auth.get_user(request) @@ -291,6 +310,7 @@ def register_api_v1(): """ Registers version 1 of MBS API. """ module_view = ModuleBuildAPI.as_view('module_builds') component_view = ComponentBuildAPI.as_view('component_builds') + about_view = AboutAPI.as_view('about') for key, val in api_v1.items(): if key.startswith('component_build'): app.add_url_rule(val['url'], @@ -302,7 +322,13 @@ def register_api_v1(): endpoint=key, view_func=module_view, **val['options']) + elif key.startswith('about'): + app.add_url_rule(val['url'], + endpoint=key, + view_func=about_view, + **val['options']) else: raise NotImplementedError("Unhandled api key.") + register_api_v1() diff --git a/tests/test_views/test_views.py b/tests/test_views/test_views.py index 3e74ac6d..cfb4207f 100644 --- a/tests/test_views/test_views.py +++ b/tests/test_views/test_views.py @@ -37,7 +37,8 @@ import hashlib from tests import app, init_data from module_build_service.errors import UnprocessableEntity from module_build_service.models import ComponentBuild, ModuleBuild -from module_build_service import conf, db +from module_build_service import conf, db, version +import module_build_service.config as mbs_config import module_build_service.scheduler.handlers.modules @@ -928,3 +929,11 @@ class TestViews(unittest.TestCase): allow_custom_scmurls.return_value = True res2 = submit('git://some.custom.url.org/modules/testmodule.git?#68931c9') self.assertEquals(res2.status_code, 201) + + def test_about(self): + with patch.object(mbs_config.Config, 'auth_method', new_callable=PropertyMock) as auth: + auth.return_value = 'kerberos' + rv = self.client.get('/module-build-service/1/about/') + data = json.loads(rv.data) + self.assertEqual(rv.status_code, 200) + self.assertEquals(data, {'auth_method': 'kerberos', 'version': version}) From 5d9a16e8f034a6ca25b36a9ce1cc68c5e5e6862c Mon Sep 17 00:00:00 2001 From: mprahl Date: Tue, 19 Sep 2017 16:14:39 -0400 Subject: [PATCH 3/4] Support insecure mode on "mbs-build overview" and supress warnings from doing so --- contrib/mbs-build | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/mbs-build b/contrib/mbs-build index d4d05934..fee706a4 100755 --- a/contrib/mbs-build +++ b/contrib/mbs-build @@ -13,6 +13,7 @@ import operator from tabulate import tabulate from multiprocessing.dummy import Pool as ThreadPool from copy import copy +import urllib3 try: from urllib.parse import urljoin @@ -325,7 +326,7 @@ def cancel_module_build(server, id_provider, build_id, verify=True): logging.info(resp.text) -def show_overview(server, finished, limit=30): +def show_overview(server, finished, limit=30, verify=True): if not server: server = DEFAULT_MBS_SERVER @@ -340,7 +341,7 @@ def show_overview(server, finished, limit=30): """ Yields modules with state `state`. """ - response = requests.get(baseurl, params=dict(page=page, state=state)) + response = requests.get(baseurl, params=dict(page=page, state=state), verify=verify) data = response.json() for item in data['items']: yield item @@ -488,6 +489,9 @@ def main(): loglevel = logging.WARNING logging.basicConfig(level=loglevel, format="%(levelname)s: %(message)s") + if args.verify is False: + urllib3.disable_warnings() + if args.cmd_name == "submit": # Submit the module build. build_id, errmsg = submit_module_build(args.scm_url, args.branch, args.server, @@ -514,7 +518,7 @@ def main(): # Cancel the module build cancel_module_build(args.server, args.idprovider, args.build_id, args.verify) elif args.cmd_name == "overview": - show_overview(args.server, finished=args.finished, limit=args.limit) + show_overview(args.server, finished=args.finished, limit=args.limit, verify=args.verify) elif args.cmd_name == "info": show_module_info(args.server, args.build_id) From 97ac3de347e34510c6fea4d280aaaad3def8c3cd Mon Sep 17 00:00:00 2001 From: mprahl Date: Tue, 19 Sep 2017 16:30:07 -0400 Subject: [PATCH 4/4] Support Kerberos authentication in mbs-build --- contrib/mbs-build | 73 ++++++++++++++++++++++++++++++++++------------- requirements.txt | 2 ++ 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/contrib/mbs-build b/contrib/mbs-build index fee706a4..b11dba16 100755 --- a/contrib/mbs-build +++ b/contrib/mbs-build @@ -14,6 +14,8 @@ from tabulate import tabulate from multiprocessing.dummy import Pool as ThreadPool from copy import copy import urllib3 +import json +import requests_kerberos try: from urllib.parse import urljoin @@ -22,7 +24,8 @@ except ImportError: DEFAULT_ID_PROVIDER = "https://id.fedoraproject.org/openidc/" DEFAULT_MBS_SERVER = "https://mbs.fedoraproject.org" -DEFAULT_MBS_REST_API = "/module-build-service/1/module-builds/" +DEFAULT_MBS_REST_PREFIX = "/module-build-service/1/" +DEFAULT_MBS_REST_API = "{0}module-builds/".format(DEFAULT_MBS_REST_PREFIX) DEFAULT_KOJI_TASK_URL = "https://koji.fedoraproject.org/koji/taskinfo" openidc_client.WEB_PORTS = [13747] @@ -39,6 +42,17 @@ BUILD_STATES = { INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()} +def get_auth_method(server, verify=True): + config_url = '{0}{1}about/'.format(server.rstrip('/'), DEFAULT_MBS_REST_PREFIX) + rv = requests.get(config_url, timeout=30, verify=verify) + # Assume that if the connection fails, it's because the config API doesn't + # exist on the server yet + if not rv.ok: + return 'oidc' + rv_json = rv.json() + return rv_json['auth_method'] + + def fetch_module_info(server, build_id): if not server: server = DEFAULT_MBS_SERVER @@ -176,29 +190,50 @@ def _send_oidc_request(oidc, verb, *args, **kwargs): return resp -def send_authorized_request(verb, server, id_provider, url, body, **kwargs): +def send_authorized_request(verb, server, url, body, id_provider=None, **kwargs): """ Sends authorized request to server. """ if not server: server = DEFAULT_MBS_SERVER - if not id_provider: - id_provider = DEFAULT_ID_PROVIDER - logging.info("Trying to get the token from %s", id_provider) + full_url = urljoin(server, url) + verify = kwargs.get('verify', True) + auth_method = get_auth_method(server, verify=verify) - # Get the auth token using the OpenID client. - oidc = openidc_client.OpenIDCClient( - "mbs_build", id_provider, - {'Token': 'Token', 'Authorization': 'Authorization'}, - 'mbs-authorizer', "notsecret") + if auth_method == 'oidc': + if not id_provider: + id_provider = DEFAULT_ID_PROVIDER - scopes = ['openid', 'https://id.fedoraproject.org/scope/groups', - 'https://mbs.fedoraproject.org/oidc/submit-build'] + logging.info("Trying to get the token from %s", id_provider) - logging.debug("Sending body: %s", body) - resp = _send_oidc_request(oidc, verb, urljoin(server, url), json=body, - scopes=scopes, **kwargs) + # Get the auth token using the OpenID client. + oidc = openidc_client.OpenIDCClient( + "mbs_build", id_provider, + {'Token': 'Token', 'Authorization': 'Authorization'}, + 'mbs-authorizer', "notsecret") + + scopes = ['openid', 'https://id.fedoraproject.org/scope/groups', + 'https://mbs.fedoraproject.org/oidc/submit-build'] + + logging.debug("Sending body: %s", body) + resp = _send_oidc_request(oidc, verb, full_url, json=body, + scopes=scopes, **kwargs) + elif auth_method == 'kerberos': + if type(body) is dict: + data = json.dumps(body) + else: + data = body + auth = requests_kerberos.HTTPKerberosAuth(mutual_authentication=requests_kerberos.OPTIONAL) + resp = requests.request(verb, full_url, data=data, auth=auth, verify=verify) + if resp.status_code == 401: + logging.error('Authentication using Kerberos failed. Make sure you have a valid ' + 'Kerberos ticket.') + sys.exit(1) + else: + logging.exception('The MBS server requires an unsupported authentication method of ' + '"{0}"'.format(auth_method)) + sys.exit(1) return resp @@ -271,8 +306,7 @@ def submit_module_build(scm_url, branch, server, id_provider, pyrpkg, verify=Tru return -5, "Optional arguments are not in a proper arg=value format." body.update(optional_dict) resp = send_authorized_request( - "POST", server, id_provider, DEFAULT_MBS_REST_API, - body, verify=verify) + "POST", server, DEFAULT_MBS_REST_API, body, id_provider=id_provider, verify=verify) logging.info(resp.text) data = resp.json() @@ -320,9 +354,8 @@ def cancel_module_build(server, id_provider, build_id, verify=True): """ logging.info("Cancelling module build %s", build_id) resp = send_authorized_request( - "PATCH", server, id_provider, - "%s/%s" % (DEFAULT_MBS_REST_API, build_id), - {'state': 'failed'}, verify=verify) + "PATCH", server, "%s/%s" % (DEFAULT_MBS_REST_API, build_id), + {'state': 'failed'}, id_provider=id_provider, verify=verify) logging.info(resp.text) diff --git a/requirements.txt b/requirements.txt index 1d8a34d9..1b27a3f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,8 @@ psutil pyOpenSSL python-fedora qpid-python +requests # Client only +requests_kerberos # Client only six sqlalchemy tabulate