mirror of
https://pagure.io/fm-orchestrator.git
synced 2026-04-13 17:29:45 +08:00
1c8e2a07d8
Logging during a build that occurs before the build directory is created used to be logged to the console, but not retained in the build log file. This made referring to the build log file confusing. Solve this by buffering logs in memory until the log file is created and replaying them. A little bit of hackery is needed to avoid saving dangling references to libsolv objects.
286 lines
9.1 KiB
Python
Executable File
286 lines
9.1 KiB
Python
Executable File
# -*- coding: utf-8 -*-
|
|
# SPDX-License-Identifier: MIT
|
|
from __future__ import absolute_import, print_function
|
|
from collections import defaultdict
|
|
import click
|
|
import getpass
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import flask_migrate
|
|
from flask.cli import FlaskGroup
|
|
from werkzeug.datastructures import FileStorage
|
|
|
|
from module_build_service import app, db
|
|
from module_build_service.builder.MockModuleBuilder import (
|
|
import_builds_from_local_dnf_repos, load_local_builds
|
|
)
|
|
from module_build_service.common import build_logs, conf, models
|
|
from module_build_service.common.errors import StreamAmbigous, StreamNotXyz
|
|
from module_build_service.common.utils import load_mmd_file, import_mmd
|
|
import module_build_service.scheduler.consumer
|
|
from module_build_service.scheduler.db_session import db_session
|
|
import module_build_service.scheduler.local
|
|
from module_build_service.web.submit import submit_module_build_from_yaml
|
|
|
|
migrations_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "migrations")
|
|
migrate = flask_migrate.Migrate(app, db, directory=migrations_dir)
|
|
|
|
|
|
@click.group(cls=FlaskGroup, create_app=lambda *args, **kwargs: app)
|
|
def cli():
|
|
"""MBS manager"""
|
|
|
|
|
|
@cli.command("upgradedb")
|
|
def upgradedb():
|
|
""" Upgrades the database schema to the latest revision
|
|
"""
|
|
app.config["SERVER_NAME"] = "localhost"
|
|
with app.app_context():
|
|
flask_migrate.upgrade(directory=migrations_dir)
|
|
|
|
|
|
def upgradedb_entrypoint():
|
|
"""Entrypoint for command mbs-upgradedb"""
|
|
# Work around issue with FlaskGroup not being initiated
|
|
cli(["upgradedb"])
|
|
|
|
|
|
@cli.command("cleardb")
|
|
def cleardb():
|
|
""" Clears the database
|
|
"""
|
|
models.ModuleBuild.query.delete()
|
|
models.ComponentBuild.query.delete()
|
|
|
|
|
|
@cli.command("import_module")
|
|
@click.argument("mmd_file", type=click.Path(exists=True))
|
|
def import_module(mmd_file):
|
|
""" Imports the module from mmd_file
|
|
"""
|
|
mmd = load_mmd_file(mmd_file)
|
|
import_mmd(db.session, mmd)
|
|
|
|
|
|
def collect_dep_overrides(overrides):
|
|
collected = defaultdict(list)
|
|
for value in overrides:
|
|
parts = value.split(":")
|
|
if len(parts) != 2:
|
|
raise ValueError("dependency overrides must be in the form name:stream")
|
|
name, stream = parts
|
|
collected[name].append(stream)
|
|
|
|
return collected
|
|
|
|
|
|
@cli.command("build_module_locally")
|
|
@click.option("--stream", metavar="STREAM")
|
|
@click.option(
|
|
"--file", "yaml_file",
|
|
metavar="FILE",
|
|
required=True,
|
|
type=click.Path(exists=True),
|
|
)
|
|
@click.option("--srpm", "srpms", metavar="SRPM", multiple=True)
|
|
@click.option("--skiptests", is_flag=True)
|
|
@click.option("--offline", is_flag=True)
|
|
@click.option(
|
|
'--buildrequires', "buildrequires", multiple=True,
|
|
metavar='name:stream', default=[],
|
|
help='Buildrequires to override in the form of "name:stream"'
|
|
)
|
|
@click.option(
|
|
'--requires', "requires", multiple=True,
|
|
metavar='name:stream', default=[],
|
|
help='Requires to override in the form of "name:stream"'
|
|
)
|
|
@click.option("-d", "--debug", "log_debug", is_flag=True)
|
|
@click.option(
|
|
"-l", "--add-local-build", "local_build_nsvs",
|
|
metavar="NSV", multiple=True
|
|
)
|
|
@click.option(
|
|
"-s", "--set-stream", "default_streams",
|
|
metavar="STREAM", multiple=True
|
|
)
|
|
@click.option(
|
|
"-r", "--platform-repo-file", "platform_repofiles",
|
|
metavar="FILE",
|
|
type=click.Path(exists=True),
|
|
multiple=True
|
|
)
|
|
@click.option("-p", "--platform-id", metavar="PLATFORM_ID")
|
|
def build_module_locally(
|
|
stream=None,
|
|
yaml_file=None,
|
|
srpms=None,
|
|
skiptests=False,
|
|
offline=False,
|
|
log_debug=False,
|
|
local_build_nsvs=None,
|
|
default_streams=None,
|
|
platform_repofiles=None,
|
|
platform_id=None,
|
|
requires=None,
|
|
buildrequires=None,
|
|
):
|
|
""" Performs local module build using Mock
|
|
"""
|
|
# accumulate initial log messages in memory - we'll output them to a log in the
|
|
# module build directories when we know what they are
|
|
build_logs.buffer_initially()
|
|
|
|
# if debug is not specified, set log level of console to INFO
|
|
if not log_debug:
|
|
for handler in logging.getLogger().handlers:
|
|
if isinstance(handler, logging.StreamHandler):
|
|
handler.setLevel(logging.INFO)
|
|
|
|
if "SERVER_NAME" not in app.config or not app.config["SERVER_NAME"]:
|
|
app.config["SERVER_NAME"] = "localhost"
|
|
|
|
if conf.resolver == "db":
|
|
raise ValueError(
|
|
"Please set RESOLVER to 'mbs' in your configuration for local builds.")
|
|
|
|
conf.set_item("system", "mock")
|
|
conf.set_item("base_module_repofiles", platform_repofiles)
|
|
|
|
# Use our own local SQLite3 database.
|
|
confdir = os.path.abspath(os.getcwd())
|
|
dbdir = \
|
|
os.path.abspath(os.path.join(confdir, "..")) if confdir.endswith("conf") else confdir
|
|
dbpath = "/{0}".format(os.path.join(dbdir, ".mbs_local_build.db"))
|
|
dburi = "sqlite://" + dbpath
|
|
app.config["SQLALCHEMY_DATABASE_URI"] = dburi
|
|
conf.set_item("sqlalchemy_database_uri", dburi)
|
|
if os.path.exists(dbpath):
|
|
os.remove(dbpath)
|
|
|
|
db.create_all()
|
|
# Reconfigure the backend database session registry to use the new the database location
|
|
db_session.remove()
|
|
db_session.configure(bind=db.session.bind)
|
|
|
|
params = {
|
|
"local_build": True,
|
|
"default_streams": dict(ns.split(":") for ns in default_streams),
|
|
"require_overrides": collect_dep_overrides(requires),
|
|
"buildrequire_overrides": collect_dep_overrides(buildrequires),
|
|
}
|
|
if srpms:
|
|
params["srpms"] = srpms
|
|
|
|
username = getpass.getuser()
|
|
if not yaml_file or not yaml_file.endswith(".yaml"):
|
|
raise IOError("Provided modulemd file is not a yaml file.")
|
|
|
|
yaml_file_path = os.path.abspath(yaml_file)
|
|
|
|
if offline:
|
|
import_builds_from_local_dnf_repos(platform_id)
|
|
load_local_builds(local_build_nsvs)
|
|
|
|
with open(yaml_file_path) as fd:
|
|
filename = os.path.basename(yaml_file)
|
|
handle = FileStorage(fd)
|
|
handle.filename = filename
|
|
try:
|
|
module_builds = submit_module_build_from_yaml(
|
|
db_session, username, handle, params,
|
|
stream=str(stream), skiptests=skiptests
|
|
)
|
|
except StreamAmbigous as e:
|
|
logging.error(str(e))
|
|
logging.error("Use '-s module_name:module_stream' to choose the stream")
|
|
return 1
|
|
except StreamNotXyz as e:
|
|
logging.error(str(e))
|
|
logging.error("Use '--buildrequires name:stream' to override the base module stream")
|
|
return 1
|
|
|
|
module_build_ids = [build.id for build in module_builds]
|
|
|
|
try:
|
|
module_build_service.scheduler.local.main(module_build_ids)
|
|
except module_build_service.scheduler.local.BuildFailed as e:
|
|
logging.error("%s", e)
|
|
sys.exit(1)
|
|
|
|
|
|
@cli.command("retire")
|
|
@click.argument("identifier", metavar="NAME:STREAM[:VERSION[:CONTEXT]]")
|
|
@click.option(
|
|
"--confirm",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Perform retire operation without prompting",
|
|
)
|
|
def retire(identifier, confirm=False):
|
|
""" Retire module build(s) by placing them into 'garbage' state.
|
|
"""
|
|
# Parse identifier and build query
|
|
parts = identifier.split(":")
|
|
if len(parts) < 2:
|
|
raise ValueError("Identifier must contain at least NAME:STREAM")
|
|
if len(parts) >= 5:
|
|
raise ValueError("Too many parts in identifier")
|
|
|
|
filter_by_kwargs = {"state": models.BUILD_STATES["ready"], "name": parts[0], "stream": parts[1]}
|
|
|
|
if len(parts) >= 3:
|
|
filter_by_kwargs["version"] = parts[2]
|
|
if len(parts) >= 4:
|
|
filter_by_kwargs["context"] = parts[3]
|
|
|
|
# Find module builds to retire
|
|
module_builds = db_session.query(models.ModuleBuild).filter_by(**filter_by_kwargs).all()
|
|
|
|
if not module_builds:
|
|
logging.info("No module builds found.")
|
|
return
|
|
|
|
logging.info("Found %d module builds:", len(module_builds))
|
|
for build in module_builds:
|
|
logging.info("\t%s", ":".join((build.name, build.stream, build.version, build.context)))
|
|
|
|
# Prompt for confirmation
|
|
confirm_msg = "Retire {} module builds?".format(len(module_builds))
|
|
is_confirmed = confirm or click.confirm(confirm_msg, abort=False)
|
|
if not is_confirmed:
|
|
logging.info("Module builds were NOT retired.")
|
|
return
|
|
|
|
# Retire module builds
|
|
for build in module_builds:
|
|
build.transition(
|
|
db_session, conf, models.BUILD_STATES["garbage"], "Module build retired")
|
|
|
|
db_session.commit()
|
|
|
|
logging.info("Module builds retired.")
|
|
|
|
|
|
@cli.command("run")
|
|
@click.option("-h", "--host", metavar="HOST", help="Bind to this host.")
|
|
@click.option("-p", "--port", metavar="PORT", help="Bind to this port along with --host.")
|
|
@click.option("-d", "--debug", is_flag=True, default=False, help="Run frontend in debug mode.")
|
|
def run(host=None, port=None, debug=None):
|
|
""" Runs the Flask app, locally. Intended for dev instances, should not be used for production.
|
|
"""
|
|
host = host or conf.host
|
|
port = port or conf.port
|
|
debug = debug or conf.debug
|
|
|
|
logging.info("Starting Module Build Service frontend")
|
|
|
|
app.run(host=host, port=port, debug=debug)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|