# -*- coding: utf-8 -*- # Copyright (c) 2016 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # # Written by Lubos Kocman """PDC handler functions.""" import modulemd from pdc_client import PDCClient import inspect import pprint import logging log = logging.getLogger() import six def get_pdc_client_session(config): """ :param config: instance of module_build_service.config.Config :return pdc_client.PDCClient instance """ if 'ssl_verify' in inspect.getargspec(PDCClient.__init__).args: # New API return PDCClient( server=config.pdc_url, develop=config.pdc_develop, ssl_verify=not config.pdc_insecure, ) else: # Old API return PDCClient( server=config.pdc_url, develop=config.pdc_develop, insecure=config.pdc_insecure, ) def get_variant_dict(data): """ :param data: one of following pdc variant_dict {'variant_id': value, 'variant_version': value, } module dict {'name': value, 'version': value } modulemd :return final list of module_info which pass repoclosure """ def is_module_dict(data): if not isinstance(data, dict): return False for attr in ('name', 'version'): if attr not in data.keys(): return False return True def is_variant_dict(data): if not isinstance(data, dict): return False for attr in ('variant_id', 'variant_stream'): if attr not in data.keys(): return False return True def is_modulemd(data): return isinstance(data, modulemd.ModuleMetadata) def is_module_str(data): return isinstance(data, six.string_types) result = None if is_module_str(data): result = variant_dict_from_str(data) elif is_modulemd(data): result = {'variant_id': data.name} # Check if this is an old modulemd that doesn't use the new nomenclature if hasattr(data, 'release'): result['variant_release'] = data.release result['variant_version'] = data.version else: result['variant_release'] = data.version result['variant_version'] = data.stream elif is_variant_dict(data): result = data.copy() # This is a transitionary thing until we've ported PDC away from the old nomenclature if 'variant_version' not in result and 'variant_stream' in result: result['variant_version'] = result['variant_stream'] del result['variant_stream'] # ensure that variant_type is in result if 'variant_type' not in result.keys(): result['variant_type'] = 'module' elif is_module_dict(data): result = {'variant_id': data['name'], 'variant_version': data['version']} if 'release' in data: result['variant_release'] = data['release'] if not result: raise ValueError("Couldn't get variant_dict from %s" % data) return result def variant_dict_from_str(module_str): """ :param module_str: a string to match in PDC :return module_info dict Example minimal module_info {'variant_id': module_name, 'variant_version': module_version, 'variant_type': 'module'} """ log.debug("variant_dict_from_str(%r)" % module_str) # best match due several filters not being provided such as variant type ... module_info = {} release_start = module_str.rfind('-') version_start = module_str.rfind('-', 0, release_start) module_info['variant_release'] = module_str[release_start+1:] module_info['variant_version'] = module_str[version_start+1:release_start] module_info['variant_id'] = module_str[:version_start] module_info['variant_type'] = 'module' return module_info def get_module(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: pdc variant_dict, str, mmd or module dict :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. :return final list of module_info which pass repoclosure """ module_info = get_variant_dict(module_info) query = dict( variant_id=module_info['variant_id'], variant_version=module_info['variant_version'], ) if module_info.get('variant_release'): query['variant_release'] = module_info['variant_release'] retval = session['unreleasedvariants'](page_size=-1, **query) # ordering=variant_release... # Error handling if not retval: if strict: raise ValueError("Failed to find module in PDC %r" % query) else: return None module = None # If we specify 'variant_release', we expect only single module to be # returned, but otherwise we have to pick the one with the highest # release ourselves. if 'variant_release' in query: assert len(retval) <= 1, pprint.pformat(retval) module = retval[0] else: module = retval[0] for m in retval: if int(m['variant_release']) > int(module['variant_release']): module = m return module def get_module_tag(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: list of module_info dicts :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. :return: koji tag string """ return get_module(session, module_info, strict=strict)['koji_tag'] def get_module_modulemd(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info: list of module_info dicts :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. :return: ModuleMetadata instance """ yaml = None module = get_module(session, module_info, strict=strict) if module: yaml = module['modulemd'] if not yaml: if strict: raise ValueError("Failed to find modulemd entry in PDC for " "%r" % module_info) else: return None mmd = modulemd.ModuleMetadata() mmd.loads(yaml) return mmd def resolve_profiles(session, mmd, keys, seen=None): """ :param session : PDCClient instance :param mmd: ModuleMetadata instance of module :param keys: list of modulemd installation profiles to include in the result. :return: Dictionary with keys set according to `keys` param and values set to union of all components defined in all installation profiles matching the key recursively using the buildrequires. https://pagure.io/fm-orchestrator/issue/181 """ seen = seen or [] # Initialize to an empty list. results = {} for key in keys: results[key] = set() for module_name, module_info in mmd.xmd['mbs']['buildrequires'].items(): # First, guard against infinite recursion if module_name in seen: continue # Find the dep in the built modules in PDC module_info = { 'variant_id': module_name, 'variant_stream': module_info['stream'], 'variant_release': module_info['version']} dep_mmd = get_module_modulemd(session, module_info, True) # Take note of what rpms are in this dep's profile. for key in keys: if key in dep_mmd.profiles: results[key] |= dep_mmd.profiles[key].rpms # And recurse to all modules that are deps of our dep. rec_results = resolve_profiles(session, dep_mmd, keys, seen + [module_name]) for rec_key, rec_result in rec_results.items(): results[rec_key] |= rec_result # Return the union of all rpms in all profiles of the given keys. return results def module_depsolving_wrapper(session, module_list, strict=True): """ :param session : PDCClient instance :param module_list: list of module_info dicts :return final list of module_info which pass repoclosure """ log.debug("module_depsolving_wrapper(%r, strict=%r)" % (module_list, strict)) # TODO: implement this # Make sure that these are dicts from PDC ... ensures all values module_list = set([get_module_tag(session, x, strict) for x in module_list]) # pdc-updater adds "module-" prefix to koji_tag, but here we expect koji_tag # is just NVR of a module, so remove the "module-" prefix. module_list = set([x[len("module-"):] if x.startswith("module-") else x for x in module_list]) seen = set() # don't query pdc for the same items all over again while True: if seen == module_list: break for module in module_list: if module in seen: continue info = get_module(session, module, strict) assert info, "Module '%s' not found in PDC" % module module_list.update([x['dependency'] for x in info['build_deps']]) seen.add(module) module_list.update(info['build_deps']) return list(module_list) def get_module_runtime_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_infos : a dict containing filters for pdc :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. Example minimal module_info {'variant_id': module_name, 'variant_version': module_version, 'variant_type': 'module'} """ log.debug("get_module_runtime_dependencies(%r, strict=%r)" % (module_info, strict)) # XXX get definitive list of modules deps = [] module_info = get_module(session, module_info, strict=strict) if module_info and module_info.get('runtime_deps', None): deps = [dict(variant_id=x['dependency'], variant_stream=x['stream']) for x in module_info['runtime_deps']] deps = module_depsolving_wrapper(session, deps, strict=strict) return deps def get_module_build_dependencies(session, module_info, strict=False): """ :param session : PDCClient instance :param module_info : a dict containing filters for pdc :param strict: Normally this function returns None if no module can be found. If strict=True, then a ValueError is raised. :return final list of module_infos which pass repoclosure Example minimal module_info {'variant_id': module_name, 'variant_version': module_version, 'variant_type': 'module'} """ log.debug("get_module_build_dependencies(%r, strict=%r)" % (module_info, strict)) # XXX get definitive list of modules deps = [] queried_mmd = get_module_modulemd(session, module_info, strict=strict) if not queried_mmd or not queried_mmd.xmd.get('mbs') or not \ queried_mmd.xmd['mbs'].get('buildrequires'): raise RuntimeError( 'The module "{0!r}" did not contain its modulemd or did not have ' 'its xmd attribute filled out in PDC'.format(module_info)) deps = [dict(name=dep_name, version=dep_info['stream'], release=dep_info['version']) for dep_name, dep_info in queried_mmd.xmd['mbs']['buildrequires'].items()] deps = module_depsolving_wrapper(session, deps, strict=strict) return deps def get_module_commit_hash_and_version(session, module_info): """ Gets the commit hash and version of a module stored in PDC :param module_info: a dict containing filters for PDC :param session: a PDC session instance :return: a tuple containing the string of the commit hash and the version of the module stored in PDC. If a value is not found, None is returned for the values that aren't found. """ commit_hash = None version = None module = get_module(session, module_info) if module: if module.get('modulemd'): mmd = modulemd.ModuleMetadata() mmd.loads(module['modulemd']) if mmd.xmd.get('mbs') and mmd.xmd['mbs'].get('commit'): commit_hash = mmd.xmd['mbs']['commit'] if module.get('variant_release'): version = module['variant_release'] if not commit_hash: # TODO: Should this eventually be an exception? log.warn( 'The commit hash for {0!r} was not part of the modulemd in PDC' .format(module_info)) if not version: # TODO: Should this eventually be an exception? log.warn( 'The version for {0!r} was not in PDC'.format(module_info)) return commit_hash, version