mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-02-03 05:03:43 +08:00
7976b1f084
This will enable building modules from the new libmodulemd v3 format. This format fixes major issue with modularity which enables clear upgrade paths for multicontext modules. Signed-off-by: Martin Curlej <mcurlej@redhat.com>
259 lines
8.6 KiB
Python
259 lines
8.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
from __future__ import absolute_import
|
|
from datetime import datetime
|
|
from functools import partial
|
|
import os
|
|
import time
|
|
|
|
from gi.repository.GLib import Error as ModuleMDError
|
|
from six import string_types, text_type
|
|
|
|
from module_build_service.common import conf, log
|
|
from module_build_service.common.errors import UnprocessableEntity
|
|
from module_build_service.common.modulemd import Modulemd
|
|
|
|
|
|
def to_text_type(s):
|
|
"""
|
|
Converts `s` to `text_type`. In case it fails, returns `s`.
|
|
"""
|
|
try:
|
|
return text_type(s, "utf-8")
|
|
except TypeError:
|
|
return s
|
|
|
|
|
|
def load_mmd(yaml, is_file=False):
|
|
if not yaml:
|
|
raise UnprocessableEntity('The input modulemd was empty')
|
|
|
|
try:
|
|
if is_file:
|
|
mmd = Modulemd.read_packager_file(yaml)
|
|
else:
|
|
mmd = Modulemd.read_packager_string(to_text_type(yaml))
|
|
|
|
mmd_version = mmd.get_mdversion()
|
|
|
|
# both legacy packager v1 and stream v1 are directly upgraded to v2
|
|
if mmd_version == 1:
|
|
mmd = mmd.upgrade(2)
|
|
|
|
except ModuleMDError as e:
|
|
error = None
|
|
if is_file:
|
|
if os.path.exists(yaml):
|
|
with open(yaml, "rt") as yaml_hdl:
|
|
log.debug("The modulemd file '%s' failed to load:\n%s",
|
|
yaml,
|
|
yaml_hdl.read())
|
|
else:
|
|
error = "The modulemd file {} was not found.".format(yaml)
|
|
log.error("The modulemd file %s was not found.", yaml)
|
|
|
|
if not error:
|
|
if is_file:
|
|
error = ("The modulemd {} is invalid. The error was:\n'{}'\nPlease verify the"
|
|
" syntax is correct.").format(os.path.basename(yaml), str(e))
|
|
else:
|
|
error = ("The modulemd is invalid. The error was:\n'{}'\nPlease verify the"
|
|
" syntax is correct.").format(str(e))
|
|
|
|
log.exception(error)
|
|
raise UnprocessableEntity(error)
|
|
|
|
return mmd
|
|
|
|
|
|
load_mmd_file = partial(load_mmd, is_file=True)
|
|
|
|
|
|
def import_mmd(db_session, mmd, check_buildrequires=True):
|
|
"""
|
|
Imports new module build defined by `mmd` to MBS database using `session`.
|
|
If it already exists, it is updated.
|
|
|
|
The ModuleBuild.koji_tag is set according to xmd['mbs]['koji_tag'].
|
|
The ModuleBuild.state is set to "ready".
|
|
The ModuleBuild.rebuild_strategy is set to "all".
|
|
The ModuleBuild.owner is set to "mbs_import".
|
|
|
|
:param db_session: SQLAlchemy session object.
|
|
:param mmd: module metadata being imported into database.
|
|
:type mmd: Modulemd.ModuleStream
|
|
:param bool check_buildrequires: When True, checks that the buildrequires defined in the MMD
|
|
have matching records in the `mmd["xmd"]["mbs"]["buildrequires"]` and also fills in
|
|
the `ModuleBuild.buildrequires` according to this data.
|
|
:return: module build (ModuleBuild),
|
|
log messages collected during import (list)
|
|
:rtype: tuple
|
|
"""
|
|
from module_build_service.common import models
|
|
|
|
xmd = mmd.get_xmd()
|
|
# Set some defaults in xmd["mbs"] if they're not provided by the user
|
|
if "mbs" not in xmd:
|
|
xmd["mbs"] = {"mse": True}
|
|
|
|
if not mmd.get_context():
|
|
mmd.set_context(models.DEFAULT_MODULE_CONTEXT)
|
|
|
|
# NSVC is used for logging purpose later.
|
|
nsvc = mmd.get_nsvc()
|
|
if nsvc is None:
|
|
msg = "Both the name and stream must be set for the modulemd being imported."
|
|
log.error(msg)
|
|
raise UnprocessableEntity(msg)
|
|
|
|
name = mmd.get_module_name()
|
|
stream = mmd.get_stream_name()
|
|
version = str(mmd.get_version())
|
|
context = mmd.get_context()
|
|
|
|
xmd_mbs = xmd["mbs"]
|
|
|
|
disttag_marking = xmd_mbs.get("disttag_marking")
|
|
|
|
# If it is a base module, then make sure the value that will be used in the RPM disttags
|
|
# doesn't contain a dash since a dash isn't allowed in the release field of the NVR
|
|
if name in conf.base_module_names:
|
|
if disttag_marking and "-" in disttag_marking:
|
|
msg = "The disttag_marking cannot contain a dash"
|
|
log.error(msg)
|
|
raise UnprocessableEntity(msg)
|
|
if not disttag_marking and "-" in stream:
|
|
msg = "The stream cannot contain a dash unless disttag_marking is set"
|
|
log.error(msg)
|
|
raise UnprocessableEntity(msg)
|
|
|
|
virtual_streams = xmd_mbs.get("virtual_streams", [])
|
|
|
|
# Verify that the virtual streams are the correct type
|
|
if virtual_streams and (
|
|
not isinstance(virtual_streams, list)
|
|
or any(not isinstance(vs, string_types) for vs in virtual_streams)
|
|
):
|
|
msg = "The virtual streams must be a list of strings"
|
|
log.error(msg)
|
|
raise UnprocessableEntity(msg)
|
|
|
|
if check_buildrequires:
|
|
deps = mmd.get_dependencies()
|
|
if len(deps) > 1:
|
|
raise UnprocessableEntity(
|
|
"The imported module's dependencies list should contain just one element")
|
|
|
|
if "buildrequires" not in xmd_mbs:
|
|
# Always set buildrequires if it is not there, because
|
|
# get_buildrequired_base_modules requires xmd/mbs/buildrequires exists.
|
|
xmd_mbs["buildrequires"] = {}
|
|
mmd.set_xmd(xmd)
|
|
|
|
if len(deps) > 0:
|
|
brs = set(deps[0].get_buildtime_modules())
|
|
xmd_brs = set(xmd_mbs["buildrequires"].keys())
|
|
if brs - xmd_brs:
|
|
raise UnprocessableEntity(
|
|
"The imported module buildrequires other modules, but the metadata in the "
|
|
'xmd["mbs"]["buildrequires"] dictionary is missing entries'
|
|
)
|
|
|
|
if "koji_tag" not in xmd_mbs:
|
|
log.warning("'koji_tag' is not set in xmd['mbs'] for module {}".format(nsvc))
|
|
log.warning("koji_tag will be set to None for imported module build.")
|
|
|
|
# Log messages collected during import
|
|
msgs = []
|
|
|
|
# Get the ModuleBuild from DB.
|
|
build = models.ModuleBuild.get_build_from_nsvc(db_session, name, stream, version, context)
|
|
if build:
|
|
msg = "Updating existing module build {}.".format(nsvc)
|
|
log.info(msg)
|
|
msgs.append(msg)
|
|
else:
|
|
build = models.ModuleBuild()
|
|
db_session.add(build)
|
|
|
|
build.name = name
|
|
build.stream = stream
|
|
build.version = version
|
|
build.koji_tag = xmd_mbs.get("koji_tag")
|
|
build.state = models.BUILD_STATES["ready"]
|
|
build.modulemd = mmd_to_str(mmd)
|
|
build.context = context
|
|
build.owner = "mbs_import"
|
|
build.rebuild_strategy = "all"
|
|
now = datetime.utcnow()
|
|
build.time_submitted = now
|
|
build.time_modified = now
|
|
build.time_completed = now
|
|
if build.name in conf.base_module_names:
|
|
build.stream_version = models.ModuleBuild.get_stream_version(stream)
|
|
|
|
# Record the base modules this module buildrequires
|
|
if check_buildrequires:
|
|
for base_module in build.get_buildrequired_base_modules(db_session):
|
|
if base_module not in build.buildrequires:
|
|
build.buildrequires.append(base_module)
|
|
|
|
build.update_virtual_streams(db_session, virtual_streams)
|
|
|
|
db_session.commit()
|
|
|
|
msg = "Module {} imported".format(nsvc)
|
|
log.info(msg)
|
|
msgs.append(msg)
|
|
|
|
return build, msgs
|
|
|
|
|
|
def mmd_to_str(mmd):
|
|
"""
|
|
Helper method to convert a Modulemd.ModuleStream object to a YAML string.
|
|
|
|
:param Modulemd.ModuleStream mmd: the modulemd to convert
|
|
:return: the YAML string of the modulemd
|
|
:rtype: str
|
|
"""
|
|
index = Modulemd.ModuleIndex()
|
|
index.add_module_stream(mmd)
|
|
return to_text_type(index.dump_to_string())
|
|
|
|
|
|
def provide_module_stream_version_from_timestamp(timestamp=None):
|
|
"""
|
|
Provides a module stream version from a unix timestamp. If the timestamp is not defined
|
|
it will will generate one..
|
|
|
|
:param timestamp: modulemd object representing module metadata.
|
|
:type mmd: Modulemd.Module
|
|
:return: module stream version
|
|
:rtype: int
|
|
"""
|
|
if timestamp:
|
|
dt = datetime.utcfromtimestamp(int(timestamp))
|
|
else:
|
|
dt = datetime.utcfromtimestamp(int(time.time()))
|
|
|
|
return int(dt.strftime("%Y%m%d%H%M%S"))
|
|
|
|
|
|
def provide_module_stream_version_from_mmd(mmd):
|
|
"""
|
|
Provides a module stream version for a module stream. If a mmd v2 already contains
|
|
a version it will return it else it will generate a new one.
|
|
|
|
:param mmd: modulemd object representing module metadata.
|
|
:type mmd: Modulemd.Module
|
|
:return: module stream version
|
|
:rtype: int
|
|
"""
|
|
|
|
if mmd.get_mdversion() == 2 and mmd.get_version():
|
|
return mmd.get_version()
|
|
|
|
dt = datetime.utcfromtimestamp(int(time.time()))
|
|
return int(dt.strftime("%Y%m%d%H%M%S"))
|