modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-04-24 10:43:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Separate use of database sessions

Browse Source 
This patch separates the use of database session in different MBS components
and do not mix them together.

In general, MBS components could be separated as the REST API (implemented
based on Flask) and non-REST API including the backend build workflow
(implemented as a fedmsg consumer on top of fedmsg-hub and running
independently) and library shared by them. As a result, there are two kind of
database session used in MBS, one is created and managed by Flask-SQLAlchemy,
and another one is created from SQLAclhemy Session API directly. The goal of
this patch is to make ensure session object is used properly in the right
place.

All the changes follow these rules:

* REST API related code uses the session object db.session created and
  managed by Flask-SQLAlchemy.
* Non-REST API related code uses the session object created with SQLAlchemy
  Session API. Function make_db_session does that.
* Shared code does not created a new session object as much as possible.
  Instead, it accepts an argument db_session.

The first two rules are applicable to tests as well.

Major changes:

* Switch tests back to run with a file-based SQLite database.
* make_session is renamed to make_db_session and SQLAlchemy connection pool
  options are applied for PostgreSQL backend.
* Frontend Flask related code uses db.session
* Shared code by REST API and backend build workflow accepts SQLAlchemy session
  object as an argument. For example, resolver class is constructed with a
  database session, and some functions accepts an argument for database session.
* Build workflow related code use session object returned from make_db_session
  and ensure db.session is not used.
* Only tests for views use db.session, and other tests use db_session fixture
  to access database.
* All argument name session, that is for database access, are renamed to
  db_session.
* Functions model_tests_init_data, reuse_component_init_data and
  reuse_shared_userspace_init_data, which creates fixture data for
  tests, are converted into pytest fixtures from original function
  called inside setup_method or a test method. The reason of this
  conversion is to use fixture ``db_session`` rather than create a
  new one. That would also benefit the whole test suite to reduce the
  number of SQLAlchemy session objects.

Signed-off-by: Chenxiong Qi <cqi@redhat.com>
This commit is contained in:
Chenxiong Qi
2019-07-12 23:43:17 +08:00
parent 64698fbde8
commit 3878affa41
54 changed files with 2692 additions and 2454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

443
module_build_service/resolver/DBResolver.py
View File

@@ -39,21 +39,22 @@ class DBResolver(GenericResolver):
backend = "db"
def __init__(self, config):
def __init__(self, db_session, config):
self.db_session = db_session
self.config = config
def get_module(
self, name, stream, version, context, state=models.BUILD_STATES["ready"], strict=False
self, name, stream, version, context,
state=models.BUILD_STATES["ready"], strict=False
):
with models.make_session(self.config) as session:
mb = models.ModuleBuild.get_build_from_nsvc(
session, name, stream, version, context, state=state)
if mb:
return mb.extended_json()
mb = models.ModuleBuild.get_build_from_nsvc(
self.db_session, name, stream, version, context, state=state)
if mb:
return mb.extended_json(self.db_session)
if strict:
raise UnprocessableEntity(
"Cannot find any module builds for %s:%s" % (name, stream))
if strict:
raise UnprocessableEntity(
"Cannot find any module builds for %s:%s" % (name, stream))
def get_module_count(self, **kwargs):
"""
@@ -62,8 +63,7 @@ class DBResolver(GenericResolver):
:return: the number of modules that match the provided filter
:rtype: int
"""
with models.make_session(self.config) as session:
return models.ModuleBuild.get_module_count(session, **kwargs)
return models.ModuleBuild.get_module_count(self.db_session, **kwargs)
def get_latest_with_virtual_stream(self, name, virtual_stream):
"""
@@ -74,17 +74,17 @@ class DBResolver(GenericResolver):
:return: the module's modulemd or None
:rtype: Modulemd.ModuleStream or None
"""
with models.make_session(self.config) as session:
query = session.query(models.ModuleBuild).filter_by(name=name)
query = models.ModuleBuild._add_virtual_streams_filter(session, query, [virtual_stream])
# Cast the version as an integer so that we get proper ordering
module = query.order_by(
models.ModuleBuild.stream_version.desc(),
sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc(),
).first()
query = self.db_session.query(models.ModuleBuild).filter_by(name=name)
query = models.ModuleBuild._add_virtual_streams_filter(
self.db_session, query, [virtual_stream])
# Cast the version as an integer so that we get proper ordering
module = query.order_by(
models.ModuleBuild.stream_version.desc(),
sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc(),
).first()
if module:
return load_mmd(module.modulemd)
if module:
return load_mmd(module.modulemd)
def get_module_modulemds(self, name, stream, version=None, context=None, strict=False):
"""
@@ -105,17 +105,16 @@ class DBResolver(GenericResolver):
return
return [load_mmd(mmd["modulemd"])]
with models.make_session(self.config) as session:
if not version and not context:
builds = models.ModuleBuild.get_last_builds_in_stream(session, name, stream)
else:
raise NotImplementedError(
"This combination of name/stream/version/context is not implemented")
if not version and not context:
builds = models.ModuleBuild.get_last_builds_in_stream(self.db_session, name, stream)
else:
raise NotImplementedError(
"This combination of name/stream/version/context is not implemented")
if not builds and strict:
raise UnprocessableEntity(
"Cannot find any module builds for %s:%s" % (name, stream))
return [build.mmd() for build in builds]
if not builds and strict:
raise UnprocessableEntity(
"Cannot find any module builds for %s:%s" % (name, stream))
return [build.mmd() for build in builds]
def get_compatible_base_module_modulemds(
self, name, stream, stream_version_lte, virtual_streams, states
@@ -141,21 +140,20 @@ class DBResolver(GenericResolver):
be in.
"""
builds = []
with models.make_session(self.config) as session:
stream_version = None
if stream_version_lte:
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
stream, right_pad=False))) >= 5
if stream_in_xyz_format:
stream_version = models.ModuleBuild.get_stream_version(stream)
else:
log.warning(
"Cannot get compatible base modules, because stream_version_lte is used, "
"but stream %r is not in x.y.z format." % stream)
builds = models.ModuleBuild.get_last_builds_in_stream_version_lte(
session, name, stream_version, virtual_streams, states)
stream_version = None
if stream_version_lte:
stream_in_xyz_format = len(str(models.ModuleBuild.get_stream_version(
stream, right_pad=False))) >= 5
if stream_in_xyz_format:
stream_version = models.ModuleBuild.get_stream_version(stream)
else:
log.warning(
"Cannot get compatible base modules, because stream_version_lte is used, "
"but stream %r is not in x.y.z format." % stream)
builds = models.ModuleBuild.get_last_builds_in_stream_version_lte(
self.db_session, name, stream_version, virtual_streams, states)
return [build.mmd() for build in builds]
return [build.mmd() for build in builds]
def get_buildrequired_modulemds(self, name, stream, base_module_nsvc):
"""
@@ -170,51 +168,50 @@ class DBResolver(GenericResolver):
:return: List of modulemd metadata.
"""
log.debug("Looking for %s:%s buildrequiring %s", name, stream, base_module_nsvc)
with models.make_session(self.config) as session:
query = session.query(models.ModuleBuild)
query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"])
query = self.db_session.query(models.ModuleBuild)
query = query.filter_by(name=name, stream=stream, state=models.BUILD_STATES["ready"])
module_br_alias = aliased(models.ModuleBuild, name="module_br")
# Shorten this table name for clarity in the query below
mb_to_br = models.module_builds_to_module_buildrequires
# The following joins get added:
# JOIN module_builds_to_module_buildrequires
# ON module_builds_to_module_buildrequires.module_id = module_builds.id
# JOIN module_builds AS module_br
# ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id
query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id)
module_br_alias = aliased(models.ModuleBuild, name="module_br")
# Shorten this table name for clarity in the query below
mb_to_br = models.module_builds_to_module_buildrequires
# The following joins get added:
# JOIN module_builds_to_module_buildrequires
# ON module_builds_to_module_buildrequires.module_id = module_builds.id
# JOIN module_builds AS module_br
# ON module_builds_to_module_buildrequires.module_buildrequire_id = module_br.id
query = query.join(mb_to_br, mb_to_br.c.module_id == models.ModuleBuild.id).join(
module_br_alias, mb_to_br.c.module_buildrequire_id == module_br_alias.id)
# Get only modules buildrequiring particular base_module_nsvc
n, s, v, c = base_module_nsvc.split(":")
query = query.filter(
module_br_alias.name == n,
module_br_alias.stream == s,
module_br_alias.version == v,
module_br_alias.context == c,
)
query = query.order_by(
sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc())
all_builds = query.all()
# Get only modules buildrequiring particular base_module_nsvc
n, s, v, c = base_module_nsvc.split(":")
query = query.filter(
module_br_alias.name == n,
module_br_alias.stream == s,
module_br_alias.version == v,
module_br_alias.context == c,
)
query = query.order_by(
sqlalchemy.cast(models.ModuleBuild.version, db.BigInteger).desc())
all_builds = query.all()
# The `all_builds` list contains builds sorted by "build.version". We need only
# the builds with latest version, but in all contexts.
builds = []
latest_version = None
for build in all_builds:
if latest_version is None:
latest_version = build.version
if latest_version != build.version:
break
builds.append(build)
# The `all_builds` list contains builds sorted by "build.version". We need only
# the builds with latest version, but in all contexts.
builds = []
latest_version = None
for build in all_builds:
if latest_version is None:
latest_version = build.version
if latest_version != build.version:
break
builds.append(build)
mmds = [build.mmd() for build in builds]
nsvcs = [
mmd.get_nsvc()
for mmd in mmds
]
log.debug("Found: %r", nsvcs)
return mmds
mmds = [build.mmd() for build in builds]
nsvcs = [
mmd.get_nsvc()
for mmd in mmds
]
log.debug("Found: %r", nsvcs)
return mmds
def resolve_profiles(self, mmd, keys):
"""
@@ -230,44 +227,43 @@ class DBResolver(GenericResolver):
results = {}
for key in keys:
results[key] = set()
with models.make_session(self.config) as session:
for module_name, module_info in mmd.get_xmd()["mbs"]["buildrequires"].items():
local_modules = models.ModuleBuild.local_modules(
session, module_name, module_info["stream"])
if local_modules:
local_module = local_modules[0]
log.info("Using local module {0!r} to resolve profiles.".format(local_module))
dep_mmd = local_module.mmd()
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
continue
build = models.ModuleBuild.get_build_from_nsvc(
session,
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise UnprocessableEntity(
"The module {}:{}:{}:{} was not found".format(
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
)
)
dep_mmd = build.mmd()
# Take note of what rpms are in this dep's profile
for module_name, module_info in mmd.get_xmd()["mbs"]["buildrequires"].items():
local_modules = models.ModuleBuild.local_modules(
self.db_session, module_name, module_info["stream"])
if local_modules:
local_module = local_modules[0]
log.info("Using local module {0!r} to resolve profiles.".format(local_module))
dep_mmd = local_module.mmd()
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
continue
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session,
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise UnprocessableEntity(
"The module {}:{}:{}:{} was not found".format(
module_name,
module_info["stream"],
module_info["version"],
module_info["context"],
)
)
dep_mmd = build.mmd()
# Take note of what rpms are in this dep's profile
for key in keys:
profile = dep_mmd.get_profile(key)
if profile:
results[key] |= set(profile.get_rpms())
# Return the union of all rpms in all profiles of the given keys
return results
@@ -304,53 +300,52 @@ class DBResolver(GenericResolver):
)
module_tags = {}
with models.make_session(self.config) as session:
if mmd:
queried_mmd = mmd
nsvc = ":".join([
mmd.get_module_name(),
mmd.get_stream_name(),
str(mmd.get_version()),
mmd.get_context() or models.DEFAULT_MODULE_CONTEXT,
])
else:
build = models.ModuleBuild.get_build_from_nsvc(
session, name, stream, version, context)
if not build:
raise UnprocessableEntity(
"The module {} was not found".format(
":".join([name, stream, version, context]))
)
queried_mmd = build.mmd()
nsvc = ":".join([name, stream, version, context])
if mmd:
queried_mmd = mmd
nsvc = ":".join([
mmd.get_module_name(),
mmd.get_stream_name(),
str(mmd.get_version()),
mmd.get_context() or models.DEFAULT_MODULE_CONTEXT,
])
else:
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session, name, stream, version, context)
if not build:
raise UnprocessableEntity(
"The module {} was not found".format(
":".join([name, stream, version, context]))
)
queried_mmd = build.mmd()
nsvc = ":".join([name, stream, version, context])
xmd_mbs = queried_mmd.get_xmd().get("mbs", {})
if "buildrequires" not in xmd_mbs:
xmd_mbs = queried_mmd.get_xmd().get("mbs", {})
if "buildrequires" not in xmd_mbs:
raise RuntimeError(
"The module {} did not contain its modulemd or did not have "
"its xmd attribute filled out in MBS".format(nsvc)
)
buildrequires = xmd_mbs["buildrequires"]
for br_name, details in buildrequires.items():
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session,
br_name,
details["stream"],
details["version"],
details["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise RuntimeError(
"The module {} did not contain its modulemd or did not have "
"its xmd attribute filled out in MBS".format(nsvc)
)
"Buildrequired module %s %r does not exist in MBS db" % (br_name, details))
buildrequires = xmd_mbs["buildrequires"]
for br_name, details in buildrequires.items():
build = models.ModuleBuild.get_build_from_nsvc(
session,
br_name,
details["stream"],
details["version"],
details["context"],
state=models.BUILD_STATES["ready"],
)
if not build:
raise RuntimeError(
"Buildrequired module %s %r does not exist in MBS db" % (br_name, details))
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if build.koji_tag is None:
continue
module_tags.setdefault(build.koji_tag, [])
module_tags[build.koji_tag].append(build.mmd())
# If the buildrequire is a meta-data only module with no Koji tag set, then just
# skip it
if build.koji_tag is None:
continue
module_tags.setdefault(build.koji_tag, [])
module_tags[build.koji_tag].append(build.mmd())
return module_tags
@@ -366,70 +361,68 @@ class DBResolver(GenericResolver):
:return: a dictionary
"""
new_requires = {}
with models.make_session(self.config) as session:
for nsvc in requires:
nsvc_splitted = nsvc.split(":")
if len(nsvc_splitted) == 2:
module_name, module_stream = nsvc_splitted
module_version = None
module_context = None
elif len(nsvc_splitted) == 4:
module_name, module_stream, module_version, module_context = nsvc_splitted
else:
raise ValueError(
"Only N:S or N:S:V:C is accepted by resolve_requires, got %s" % nsvc)
local_modules = models.ModuleBuild.local_modules(
session, module_name, module_stream)
if local_modules:
local_build = local_modules[0]
new_requires[module_name] = {
"ref": None,
"stream": local_build.stream,
"version": local_build.version,
"context": local_build.context,
"koji_tag": local_build.koji_tag,
}
continue
if module_version is None or module_context is None:
build = models.ModuleBuild.get_last_build_in_stream(
session, module_name, module_stream)
else:
build = models.ModuleBuild.get_build_from_nsvc(
session, module_name, module_stream, module_version, module_context)
if not build:
raise UnprocessableEntity("The module {} was not found".format(nsvc))
commit_hash = None
mmd = build.mmd()
mbs_xmd = mmd.get_xmd().get("mbs", {})
if mbs_xmd.get("commit"):
commit_hash = mbs_xmd["commit"]
else:
raise RuntimeError(
'The module "{0}" didn\'t contain a commit hash in its xmd'.format(
module_name)
)
if not mbs_xmd.get("mse"):
raise RuntimeError(
'The module "{}" is not built using Module Stream Expansion. '
"Please rebuild this module first".format(nsvc)
)
for nsvc in requires:
nsvc_splitted = nsvc.split(":")
if len(nsvc_splitted) == 2:
module_name, module_stream = nsvc_splitted
module_version = None
module_context = None
elif len(nsvc_splitted) == 4:
module_name, module_stream, module_version, module_context = nsvc_splitted
else:
raise ValueError(
"Only N:S or N:S:V:C is accepted by resolve_requires, got %s" % nsvc)
local_modules = models.ModuleBuild.local_modules(
self.db_session, module_name, module_stream)
if local_modules:
local_build = local_modules[0]
new_requires[module_name] = {
"ref": commit_hash,
"stream": module_stream,
"version": build.version,
"context": build.context,
"koji_tag": build.koji_tag,
"ref": None,
"stream": local_build.stream,
"version": local_build.version,
"context": local_build.context,
"koji_tag": local_build.koji_tag,
}
continue
if module_version is None or module_context is None:
build = models.ModuleBuild.get_last_build_in_stream(
self.db_session, module_name, module_stream)
else:
build = models.ModuleBuild.get_build_from_nsvc(
self.db_session, module_name, module_stream, module_version, module_context)
if not build:
raise UnprocessableEntity("The module {} was not found".format(nsvc))
commit_hash = None
mmd = build.mmd()
mbs_xmd = mmd.get_xmd().get("mbs", {})
if mbs_xmd.get("commit"):
commit_hash = mbs_xmd["commit"]
else:
raise RuntimeError(
'The module "{0}" didn\'t contain a commit hash in its xmd'.format(
module_name)
)
if not mbs_xmd.get("mse"):
raise RuntimeError(
'The module "{}" is not built using Module Stream Expansion. '
"Please rebuild this module first".format(nsvc)
)
new_requires[module_name] = {
"ref": commit_hash,
"stream": module_stream,
"version": build.version,
"context": build.context,
"koji_tag": build.koji_tag,
}
return new_requires
def get_modulemd_by_koji_tag(self, tag):
with models.make_session(self.config) as session:
module = models.ModuleBuild.get_build_by_koji_tag(session, tag)
return module.mmd() if module else None
module = models.ModuleBuild.get_build_by_koji_tag(self.db_session, tag)
return module.mmd() if module else None

17
module_build_service/resolver/MBSResolver.py
View File

@@ -28,7 +28,7 @@
import logging
import kobo.rpmlib
from module_build_service import db, conf
from module_build_service import conf
from module_build_service import models
from module_build_service.errors import UnprocessableEntity
from module_build_service.resolver.base import GenericResolver
@@ -42,7 +42,8 @@ class MBSResolver(GenericResolver):
backend = "mbs"
def __init__(self, config):
def __init__(self, db_session, config):
self.db_session = db_session
self.mbs_prod_url = config.mbs_url
self._generic_error = "Failed to query MBS with query %r returned HTTP status %s"
@@ -195,7 +196,7 @@ class MBSResolver(GenericResolver):
"""
yaml = None
local_modules = models.ModuleBuild.local_modules(db.session, name, stream)
local_modules = models.ModuleBuild.local_modules(self.db_session, name, stream)
if local_modules:
return [m.mmd() for m in local_modules]
@@ -291,7 +292,7 @@ class MBSResolver(GenericResolver):
results[key] = set()
for module_name, module_info in mmd.get_xmd()["mbs"]["buildrequires"].items():
local_modules = models.ModuleBuild.local_modules(
db.session, module_name, module_info["stream"])
self.db_session, module_name, module_info["stream"])
if local_modules:
local_module = local_modules[0]
log.info("Using local module %r to resolve profiles.", local_module)
@@ -375,7 +376,8 @@ class MBSResolver(GenericResolver):
buildrequires = queried_mmd.get_xmd()["mbs"]["buildrequires"]
# Queue up the next tier of deps that we should look at..
for name, details in buildrequires.items():
local_modules = models.ModuleBuild.local_modules(db.session, name, details["stream"])
local_modules = models.ModuleBuild.local_modules(
self.db_session, name, details["stream"])
if local_modules:
for m in local_modules:
# If the buildrequire is a meta-data only module with no Koji tag set, then just
@@ -426,7 +428,8 @@ class MBSResolver(GenericResolver):
"Only N:S or N:S:V:C is accepted by resolve_requires, got %s" % nsvc)
# Try to find out module dependency in the local module builds
# added by utils.load_local_builds(...).
local_modules = models.ModuleBuild.local_modules(db.session, module_name, module_stream)
local_modules = models.ModuleBuild.local_modules(
self.db_session, module_name, module_stream)
if local_modules:
local_build = local_modules[0]
new_requires[module_name] = {
@@ -488,7 +491,7 @@ class MBSResolver(GenericResolver):
# If the module is a base module, then import it in the database so that entries in
# the module_builds_to_module_buildrequires table can be created later on
if module_name in conf.base_module_names:
import_mmd(db.session, mmd)
import_mmd(self.db_session, mmd)
return new_requires

6
module_build_service/resolver/__init__.py
View File

@@ -23,7 +23,6 @@
import pkg_resources
from module_build_service import conf
from module_build_service.resolver.base import GenericResolver
# NOTE: if you are adding a new resolver to MBS please note that you also have to add
@@ -33,8 +32,3 @@ for entrypoint in pkg_resources.iter_entry_points("mbs.resolver_backends"):
if not GenericResolver.backends:
raise ValueError("No resolver plugins are installed or available.")
# Config has the option of which resolver should be used for current MBS run.
# Hence, create a singleton system wide resolver for use. However, resolver
# could be created with other required arguments in concrete cases.
system_resolver = GenericResolver.create(conf)

4
module_build_service/resolver/base.py
View File

@@ -52,7 +52,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
GenericResolver.backends[backend_class.backend] = backend_class
@classmethod
def create(cls, config, backend=None, **extra):
def create(cls, db_session, config, backend=None, **extra):
"""Factory method to create a resolver object
:param config: MBS config object.
@@ -71,7 +71,7 @@ class GenericResolver(six.with_metaclass(ABCMeta)):
backend = conf.resolver
if backend in GenericResolver.backends:
return GenericResolver.backends[backend](config, **extra)
return GenericResolver.backends[backend](db_session, config, **extra)
else:
raise ValueError("Resolver backend='%s' not recognized" % backend)