modules/fm-orchestrator
1
0
Fork 0
mirror of https://pagure.io/fm-orchestrator.git synced 2026-02-13 18:15:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity
Files
efb0ab307e1dc49f8454fe177d40e8d14bcbea3f
fm-orchestrator/tests/__init__.py
Chenxiong Qi 3878affa41 Separate use of database sessions 
This patch separates the use of database session in different MBS components
and do not mix them together.

In general, MBS components could be separated as the REST API (implemented
based on Flask) and non-REST API including the backend build workflow
(implemented as a fedmsg consumer on top of fedmsg-hub and running
independently) and library shared by them. As a result, there are two kind of
database session used in MBS, one is created and managed by Flask-SQLAlchemy,
and another one is created from SQLAclhemy Session API directly. The goal of
this patch is to make ensure session object is used properly in the right
place.

All the changes follow these rules:

* REST API related code uses the session object db.session created and
  managed by Flask-SQLAlchemy.
* Non-REST API related code uses the session object created with SQLAlchemy
  Session API. Function make_db_session does that.
* Shared code does not created a new session object as much as possible.
  Instead, it accepts an argument db_session.

The first two rules are applicable to tests as well.

Major changes:

* Switch tests back to run with a file-based SQLite database.
* make_session is renamed to make_db_session and SQLAlchemy connection pool
  options are applied for PostgreSQL backend.
* Frontend Flask related code uses db.session
* Shared code by REST API and backend build workflow accepts SQLAlchemy session
  object as an argument. For example, resolver class is constructed with a
  database session, and some functions accepts an argument for database session.
* Build workflow related code use session object returned from make_db_session
  and ensure db.session is not used.
* Only tests for views use db.session, and other tests use db_session fixture
  to access database.
* All argument name session, that is for database access, are renamed to
  db_session.
* Functions model_tests_init_data, reuse_component_init_data and
  reuse_shared_userspace_init_data, which creates fixture data for
  tests, are converted into pytest fixtures from original function
  called inside setup_method or a test method. The reason of this
  conversion is to use fixture ``db_session`` rather than create a
  new one. That would also benefit the whole test suite to reduce the
  number of SQLAlchemy session objects.

Signed-off-by: Chenxiong Qi <cqi@redhat.com>
2019-07-18 21:26:50 +08:00

617 lines
24 KiB
Python
Raw Blame History

# Copyright (c) 2016 Red Hat, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Written by Matt Prahl <mprahl@redhat.com
import os
from datetime import datetime, timedelta
from mock import patch
import time
import hashlib
from traceback import extract_stack
from module_build_service.utils import to_text_type, load_mmd
import koji
import module_build_service
from module_build_service import db
from module_build_service.utils import get_rpm_release, import_mmd, mmd_to_str
from module_build_service.config import init_config
from module_build_service.models import (
ModuleBuild, ComponentBuild, VirtualStream, make_db_session, BUILD_STATES,
)
from module_build_service import Modulemd
base_dir = os.path.dirname(__file__)
app = module_build_service.app
conf = init_config(app)
def staged_data_filename(filename):
return os.path.join(base_dir, "staged_data", filename)
def read_staged_data(yaml_name):
"""Read module YAML content from staged_data directory
:param str yaml_name: name of YAML file which could be with or without
extension ``.yaml``. ``.yaml`` will be added if extension is omitted.
:return: module YAML file's content.
:rtype: str
:raises ValueError: if specified module YAML file does not exist in
staged_data directory.
"""
filename = staged_data_filename(
yaml_name if '.' in yaml_name else "{}.yaml".format(yaml_name))
if not os.path.exists(filename):
raise ValueError("Staged data {}.yaml does not exist.".format(yaml_name))
with open(filename, "r") as mmd:
return to_text_type(mmd.read())
def patch_config():
# add test builders for all resolvers
with_test_builders = dict()
for k, v in module_build_service.config.SUPPORTED_RESOLVERS.items():
v["builders"].extend(["test", "testlocal"])
with_test_builders[k] = v
patch("module_build_service.config.SUPPORTED_RESOLVERS", with_test_builders)
patch_config()
def patch_zeromq_time_sleep():
"""
We use moksha.hub in some tests. We used dummy zerombq backend which
connects to /dev/null, but zeromq.py contains time.sleep(1) to ensure
that sockets are listening properly. This is not needed for our dummy
use-case and it slows down tests.
This method patches time.sleep called from "zeromq.py" file to be noop,
but calls the real time.sleep otherwise.
"""
global _orig_time_sleep
_orig_time_sleep = time.sleep
def mocked_time_sleep(n):
global _orig_time_sleep
if n == 1:
tb = extract_stack()
try:
if tb[-4][0].endswith("zeromq.py"):
return
except IndexError:
pass
_orig_time_sleep(n)
ts = patch("time.sleep").start()
ts.side_effect = mocked_time_sleep
patch_zeromq_time_sleep()
def clean_database(add_platform_module=True, add_default_arches=True):
"""Initialize the test database
This function is responsible for dropping all the data in the database and
recreating all the tables from scratch.
Please note that, this function relies on database objects managed by
Flask-SQLAlchemy.
"""
# Ensure all pending transactions are committed and do not block subsequent
# DML on tables.
# TODO: Should the code be fixed that forget to commit?
db.session.commit()
db.drop_all()
db.create_all()
if add_default_arches:
arch_obj = module_build_service.models.ModuleArch(name="x86_64")
db.session.add(arch_obj)
db.session.commit()
if add_platform_module:
mmd = load_mmd(read_staged_data("platform"))
import_mmd(db.session, mmd)
def init_data(data_size=10, contexts=False, multiple_stream_versions=None, scratch=False):
"""
Creates data_size * 3 modules in database in different states and
with different component builds. See _populate_data for more info.
:param bool contexts: If True, multiple streams and contexts in each stream
are generated for 'nginx' module.
:param list/bool multiple_stream_versions: If true, multiple base modules with
difference stream versions are generated. If set to list, the list defines
the generated base module streams.
"""
clean_database()
if multiple_stream_versions:
if multiple_stream_versions is True:
multiple_stream_versions = ["f28.0.0", "f29.0.0", "f29.1.0", "f29.2.0"]
mmd = load_mmd(read_staged_data("platform"))
for stream in multiple_stream_versions:
mmd = mmd.copy("platform", stream)
# Set the virtual_streams based on "fXY" to mark the platform streams
# with the same major stream_version compatible.
xmd = mmd.get_xmd()
xmd["mbs"]["virtual_streams"] = [stream[:3]]
mmd.set_xmd(xmd)
import_mmd(db.session, mmd)
# Just to possibly confuse tests by adding another base module.
mmd = mmd.copy("bootstrap", stream)
import_mmd(db.session, mmd)
with make_db_session(conf) as db_session:
_populate_data(db_session, data_size, contexts=contexts, scratch=scratch)
def _populate_data(db_session, data_size=10, contexts=False, scratch=False):
# Query arch from passed database session, otherwise there will be an error
# like "Object '<ModuleBuild at 0x7f4ccc805c50>' is already attached to
# session '275' (this is '276')" when add new module build object to passed
# session.
arch = db_session.query(module_build_service.models.ModuleArch).get(1)
num_contexts = 2 if contexts else 1
for index in range(data_size):
for context in range(num_contexts):
build_one = ModuleBuild(
name="nginx",
stream="1",
version=2 + index,
state=BUILD_STATES["ready"],
scratch=scratch,
modulemd=read_staged_data("nginx_mmd"),
koji_tag="scrmod-nginx-1.2" if scratch else "module-nginx-1.2",
scmurl="git://pkgs.domain.local/modules/nginx"
"?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
# https://www.youtube.com/watch?v=iQGwrK_yDEg,
owner="Moe Szyslak",
time_submitted=datetime(2016, 9, 3, 11, 23, 20) + timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 25, 32) + timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_one.arches.append(arch)
if contexts:
build_one.stream = str(index)
nsvc = "{}:{}:{}:{}".format(
build_one.name,
build_one.stream,
build_one.version,
context
)
unique_hash = hashlib.sha1(nsvc.encode('utf-8')).hexdigest()
build_one.build_context = unique_hash
build_one.runtime_context = unique_hash
build_one.ref_build_context = unique_hash
combined_hashes = "{0}:{1}".format(unique_hash, unique_hash)
build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8]
db_session.add(build_one)
db_session.commit()
build_one_component_release = get_rpm_release(db_session, build_one)
db_session.add_all([
ComponentBuild(
package="nginx",
scmurl="git://pkgs.domain.local/rpms/nginx?"
"#ga95886c8a443b36a9ce31abda1f9bed22f2f8c3",
format="rpms",
task_id=12312345 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="nginx-1.10.1-2.{0}".format(build_one_component_release),
batch=1,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_nginx_1_2.src.rpm",
format="rpms",
task_id=12312321 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(build_one_component_release),
batch=2,
module_id=2 + index * 3,
tagged=True,
tagged_in_final=True)
])
db_session.commit()
build_two = ModuleBuild(
name="postgressql",
stream="1",
version=2 + index,
state=BUILD_STATES["done"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag="scrmod-postgressql-1.2" if scratch else "module-postgressql-1.2",
scmurl="git://pkgs.domain.local/modules/postgressql"
"?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=2,
owner="some_user",
time_submitted=datetime(2016, 9, 3, 12, 25, 33) + timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 27, 19) + timedelta(minutes=(index * 10)),
time_completed=datetime(2016, 9, 3, 11, 27, 19) + timedelta(minutes=(index * 10)),
rebuild_strategy="changed-and-after",
)
build_two.arches.append(arch)
db_session.add(build_two)
db_session.commit()
build_two_component_release = get_rpm_release(db_session, build_two)
db_session.add_all([
ComponentBuild(
package="postgresql",
scmurl="git://pkgs.domain.local/rpms/postgresql"
"?#dc95586c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="postgresql-9.5.3-4.{0}".format(build_two_component_release),
batch=2,
module_id=3 + index * 3,
tagged=True,
tagged_in_final=True),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_postgresql_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(build_two_component_release),
batch=1,
module_id=3 + index * 3)
])
db_session.commit()
build_three = ModuleBuild(
name="testmodule",
stream="4.3.43",
version=6 + index,
state=BUILD_STATES["wait"],
scratch=scratch,
modulemd=read_staged_data("testmodule"),
koji_tag=None,
scmurl="git://pkgs.domain.local/modules/testmodule"
"?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9",
batch=0,
owner="some_other_user",
time_submitted=datetime(2016, 9, 3, 12, 28, 33) + timedelta(minutes=(index * 10)),
time_modified=datetime(2016, 9, 3, 12, 28, 40) + timedelta(minutes=(index * 10)),
time_completed=None,
rebuild_strategy="changed-and-after",
)
db_session.add(build_three)
db_session.commit()
build_three_component_release = get_rpm_release(db_session, build_three)
db_session.add_all([
ComponentBuild(
package="rubygem-rails",
scmurl="git://pkgs.domain.local/rpms/rubygem-rails"
"?#dd55886c4a443b26a9ce38abda1f9bed22f2f8c3",
format="rpms",
task_id=2433433 + index,
state=koji.BUILD_STATES["FAILED"],
nvr="postgresql-9.5.3-4.{0}".format(build_three_component_release),
batch=2,
module_id=4 + index * 3),
ComponentBuild(
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosWZUPeK/SRPMS/"
"module-build-macros-0.1-1.module_testmodule_1_2.src.rpm",
format="rpms",
task_id=47383993 + index,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-01-1.{0}".format(build_three_component_release),
batch=1,
module_id=4 + index * 3,
tagged=True,
build_time_only=True)
])
db_session.commit()
def scheduler_init_data(db_session, tangerine_state=None, scratch=False):
""" Creates a testmodule in the building state with all the components in the same batch
"""
clean_database()
mmd = load_mmd(read_staged_data("formatted_testmodule"))
mmd.get_rpm_component("tangerine").set_buildorder(0)
module_build = module_build_service.models.ModuleBuild(
name="testmodule",
stream="master",
version='20170109091357',
state=BUILD_STATES["build"],
scratch=scratch,
build_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
runtime_context="ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0",
context="7c29193d",
koji_tag="scrmod-testmodule-master-20170109091357-7c29193d"
if scratch
else "module-testmodule-master-20170109091357-7c29193d",
scmurl="https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79",
batch=3 if tangerine_state else 2,
# https://www.youtube.com/watch?v=iOKymYVSaJE
owner="Buzz Lightyear",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
modulemd=mmd_to_str(mmd),
)
db_session.add(module_build)
db_session.commit()
platform_br = module_build_service.models.ModuleBuild.get_by_id(db_session, 1)
module_build.buildrequires.append(platform_br)
arch = db_session.query(module_build_service.models.ModuleArch).get(1)
module_build.arches.append(arch)
build_one_component_release = get_rpm_release(db_session, module_build)
db_session.add_all([
module_build_service.models.ComponentBuild(
module_id=module_build.id,
package="perl-Tangerine",
scmurl="https://src.fedoraproject.org/rpms/perl-Tangerine"
"?#4ceea43add2366d8b8c5a622a2fb563b625b9abf",
format="rpms",
task_id=90276227,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-Tangerine-0.23-1.{0}".format(build_one_component_release),
batch=2,
ref="4ceea43add2366d8b8c5a622a2fb563b625b9abf",
tagged=True,
tagged_in_final=True,
),
module_build_service.models.ComponentBuild(
module_id=module_build.id,
package="perl-List-Compare",
scmurl="https://src.fedoraproject.org/rpms/perl-List-Compare"
"?#76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
format="rpms",
task_id=90276228,
state=koji.BUILD_STATES["COMPLETE"],
nvr="perl-List-Compare-0.53-5.{0}".format(build_one_component_release),
batch=2,
ref="76f9d8c8e87eed0aab91034b01d3d5ff6bd5b4cb",
tagged=True,
tagged_in_final=True,
),
module_build_service.models.ComponentBuild(
module_id=module_build.id,
package="tangerine",
scmurl="https://src.fedoraproject.org/rpms/tangerine"
"?#fbed359411a1baa08d4a88e0d12d426fbf8f602c",
format="rpms",
batch=3,
ref="fbed359411a1baa08d4a88e0d12d426fbf8f602c",
state=tangerine_state,
task_id=90276315 if tangerine_state else None,
nvr="tangerine-0.22-3.{}".format(build_one_component_release)
if tangerine_state
else None,
tagged=tangerine_state == koji.BUILD_STATES["COMPLETE"],
tagged_in_final=tangerine_state == koji.BUILD_STATES["COMPLETE"],
),
module_build_service.models.ComponentBuild(
module_id=module_build.id,
package="module-build-macros",
scmurl="/tmp/module_build_service-build-macrosqr4AWH/SRPMS/module-build-"
"macros-0.1-1.module_testmodule_master_20170109091357.src.rpm",
format="rpms",
task_id=90276181,
state=koji.BUILD_STATES["COMPLETE"],
nvr="module-build-macros-0.1-1.{}".format(build_one_component_release),
batch=1,
tagged=True,
build_time_only=True,
),
])
db_session.commit()
def make_module(
db_session,
nsvc,
requires_list=None,
build_requires_list=None,
base_module=None,
filtered_rpms=None,
xmd=None,
store_to_db=True,
virtual_streams=None,
arches=None,
):
"""
Creates new models.ModuleBuild defined by `nsvc` string with requires
and buildrequires set according to ``requires_list`` and ``build_requires_list``.
:param db_session: SQLAlchemy database session.
:param str nsvc: name:stream:version:context of a module.
:param list_of_dicts requires_list: List of dictionaries defining the
requires in the mmd requires field format.
:param list_of_dicts build_requires_list: List of dictionaries defining the
build_requires_list in the mmd build_requires_list field format.
:param filtered_rpms: list of filtered RPMs which are added to filter
section in module metadata.
:type filtered_rpms: list[str]
:param dict xmd: a mapping representing XMD section in module metadata. A
custom xmd could be passed for testing a particular scenario and some
default key/value pairs are added if not present.
:param bool store_to_db: whether to store created module metadata to the
database.
:param list virtual_streams: List of virtual streams provided by this module.
:param list arches: List of architectures this module is built against.
If set to None, ["x86_64"] is used as a default.
:return: New Module Build if set to store module metadata to database,
otherwise the module metadata is returned.
:rtype: ModuleBuild or Modulemd.Module
"""
name, stream, version, context = nsvc.split(":")
mmd = Modulemd.ModuleStreamV2.new(name, stream)
mmd.set_version(int(version))
mmd.set_context(context)
mmd.set_summary("foo")
# Test unicode in mmd.
mmd.set_description(u"foo \u2019s")
mmd.add_module_license("GPL")
if filtered_rpms:
for rpm in filtered_rpms:
mmd.add_rpm_filter(rpm)
if requires_list is not None and build_requires_list is not None:
if not isinstance(requires_list, list):
requires_list = [requires_list]
if not isinstance(build_requires_list, list):
build_requires_list = [build_requires_list]
for requires, build_requires in zip(requires_list, build_requires_list):
deps = Modulemd.Dependencies()
for req_name, req_streams in requires.items():
if req_streams == []:
deps.set_empty_runtime_dependencies_for_module(req_name)
else:
for req_stream in req_streams:
deps.add_runtime_stream(req_name, req_stream)
for req_name, req_streams in build_requires.items():
if req_streams == []:
deps.set_empty_buildtime_dependencies_for_module(req_name)
else:
for req_stream in req_streams:
deps.add_buildtime_stream(req_name, req_stream)
mmd.add_dependencies(deps)
# Caller could pass whole xmd including mbs, but if something is missing,
# default values are given here.
xmd = xmd or {"mbs": {}}
xmd_mbs = xmd["mbs"]
if "buildrequires" not in xmd_mbs:
xmd_mbs["buildrequires"] = {}
if "requires" not in xmd_mbs:
xmd_mbs["requires"] = {}
if "commit" not in xmd_mbs:
xmd_mbs["commit"] = "ref_%s" % context
if "mse" not in xmd_mbs:
xmd_mbs["mse"] = "true"
if virtual_streams:
xmd_mbs["virtual_streams"] = virtual_streams
mmd.set_xmd(xmd)
if not store_to_db:
return mmd
module_build = ModuleBuild(
name=name,
stream=stream,
stream_version=ModuleBuild.get_stream_version(stream),
version=version,
context=context,
state=BUILD_STATES["ready"],
scmurl="https://src.stg.fedoraproject.org/modules/unused.git?#ff1ea79",
batch=1,
owner="Tom Brady",
time_submitted=datetime(2017, 2, 15, 16, 8, 18),
time_modified=datetime(2017, 2, 15, 16, 19, 35),
rebuild_strategy="changed-and-after",
build_context=context,
runtime_context=context,
modulemd=mmd_to_str(mmd),
koji_tag=xmd["mbs"]["koji_tag"] if "koji_tag" in xmd["mbs"] else None,
)
if base_module:
module_build.buildrequires.append(base_module)
db_session.add(module_build)
db_session.commit()
if virtual_streams:
for virtual_stream in virtual_streams:
vs_obj = db_session.query(VirtualStream).filter_by(name=virtual_stream).first()
if not vs_obj:
vs_obj = VirtualStream(name=virtual_stream)
db_session.add(vs_obj)
db_session.commit()
if vs_obj not in module_build.virtual_streams:
module_build.virtual_streams.append(vs_obj)
db_session.commit()
if arches is None:
arches = ["x86_64"]
for arch in arches:
arch_obj = db_session.query(module_build_service.models.ModuleArch).filter_by(
name=arch).first()
if not arch_obj:
arch_obj = module_build_service.models.ModuleArch(name=arch)
db_session.add(arch_obj)
db_session.commit()
if arch_obj not in module_build.arches:
module_build.arches.append(arch_obj)
db_session.commit()
return module_build
def module_build_from_modulemd(yaml):
"""
Create a ModuleBuild object and return. It is not written into database.
Please commit by yourself if necessary.
"""
mmd = load_mmd(yaml)
build = ModuleBuild()
build.name = mmd.get_module_name()
build.stream = mmd.get_stream_name()
build.version = mmd.get_version()
build.state = BUILD_STATES["ready"]
build.modulemd = yaml
build.koji_tag = None
build.batch = 0
build.owner = "some_other_user"
build.time_submitted = datetime(2016, 9, 3, 12, 28, 33)
build.time_modified = datetime(2016, 9, 3, 12, 28, 40)
build.time_completed = None
build.rebuild_strategy = "changed-and-after"
return build
Reference in New Issue View Git Blame Copy Permalink