From a0cab4f3cc10f5c68112e0130e138037761fdbb7 Mon Sep 17 00:00:00 2001 From: James Antill Date: Tue, 10 Feb 2026 17:19:28 -0500 Subject: [PATCH] mirror_from_forge: Add mirror_from_forge role, based on mirror_from_pagure. Signed-off-by: James Antill --- roles/mirror_forge_ansible/tasks/main.yml | 125 ++++++++++++++++ .../templates/mirror_forge_ansible.cfg | 87 ++++++++++++ .../templates/mirror_forge_ansible.service | 15 ++ .../templates/mirror_from_forge_bus.py | 134 ++++++++++++++++++ 4 files changed, 361 insertions(+) create mode 100644 roles/mirror_forge_ansible/tasks/main.yml create mode 100644 roles/mirror_forge_ansible/templates/mirror_forge_ansible.cfg create mode 100644 roles/mirror_forge_ansible/templates/mirror_forge_ansible.service create mode 100644 roles/mirror_forge_ansible/templates/mirror_from_forge_bus.py diff --git a/roles/mirror_forge_ansible/tasks/main.yml b/roles/mirror_forge_ansible/tasks/main.yml new file mode 100644 index 0000000000..5b16bd10d2 --- /dev/null +++ b/roles/mirror_forge_ansible/tasks/main.yml @@ -0,0 +1,125 @@ +--- +- name: Install packages + ansible.builtin.package: state=present name={{ item }} + with_items: + - fedora-messaging + - git + tags: + - packages + - mirror_forge_ansible + + +# Create the user the service will run under + +- name: Setup forge user + user: + name: mirror_forge_ansible + shell: /sbin/nologin + comment: "mirror_forge_ansible User" + tags: + - mirror_forge_ansible + - mirror_forge_ansible/user + + +# Ensure the user can write to where we want to store the mirror +- name: Give access to mirror_forge_ansible to /srv + ansible.builtin.command: setfacl -m d:u:mirror_forge_ansible:rwx /srv -m u:mirror_forge_ansible:rwx /srv/ + tags: + - config + - mirror_forge_ansible + + +# configure all the fedora-messaging files + +- name: Create /etc/pki/fedora-messaging + ansible.builtin.file: + dest: /etc/pki/fedora-messaging + mode: "0775" + owner: root + group: root + state: directory + tags: + - config + - mirror_forge_ansible + +- name: Deploy forge/rabbitmq certificate + ansible.builtin.copy: src={{ item.src }} + dest=/etc/pki/fedora-messaging/{{ item.dest }} + owner={{ item.owner }} group={{ item.group}} mode={{ item.mode }} + with_items: + - src: "{{private}}/files/rabbitmq/production/pki/issued/mirror_forge_ansible{{env_suffix}}.crt" + dest: mirror_forge_ansible.crt + owner: mirror_forge_ansible + group: mirror_forge_ansible + mode: "0644" + - src: "{{private}}/files/rabbitmq/production/pki/private/mirror_forge_ansible{{env_suffix}}.key" + dest: mirror_forge_ansible.key + owner: mirror_forge_ansible + group: mirror_forge_ansible + mode: "0600" + - src: "{{private}}/files/rabbitmq/production/ca-combined.crt" + dest: cacert.pem + owner: mirror_forge_ansible + group: mirror_forge_ansible + mode: "0644" + tags: + - forge + - fedora-messaging + +- name: Setup mirror_forge_ansible fedora-messaging config + ansible.builtin.template: + src: mirror_forge_ansible.cfg + dest: /etc/fedora-messaging/mirror_forge_ansible.toml + owner: mirror_forge_ansible + group: mirror_forge_ansible + mode: "0640" + tags: + - config + - mirror_forge_ansible + + +# Install the script + +- name: Create /usr/local/libexec/mirror_forge_ansible + ansible.builtin.file: + dest: /usr/local/libexec/mirror_forge_ansible + mode: "0775" + owner: root + group: root + state: directory + tags: + - config + - mirror_forge_ansible + +- name: Install the consumer + ansible.builtin.template: + src: mirror_from_forge_bus.py + dest: /usr/local/libexec/mirror_forge_ansible/mirror_from_forge_bus.py + tags: + - packages + - mirror_forge_ansible + + +# Install and start the service + +- name: Install the dedicated service file for mirror_forge_ansible + ansible.builtin.template: + src: mirror_forge_ansible.service + dest: /etc/systemd/system/mirror_forge_ansible.service + owner: root + group: root + mode: "0755" + notify: + - Reload systemd + tags: + - config + - mirror_forge_ansible + +- name: Enable and started the service + service: + name: mirror_forge_ansible.service + enabled: yes + state: started + tags: + - config + - mirror_forge_ansible diff --git a/roles/mirror_forge_ansible/templates/mirror_forge_ansible.cfg b/roles/mirror_forge_ansible/templates/mirror_forge_ansible.cfg new file mode 100644 index 0000000000..c2fd17b085 --- /dev/null +++ b/roles/mirror_forge_ansible/templates/mirror_forge_ansible.cfg @@ -0,0 +1,87 @@ +amqp_url = "amqps://mirror_forge_ansible{{ env_suffix }}:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" + +publish_exchange = "amq.topic" +passive_declares = true + +callback = "mirror_from_forge_bus:MirrorFromForge" + +# Don't use topic_prefix, since outgoing message topics are derived from incoming messages. +# topic_prefix = "" + +[[bindings]] +{% if inventory_hostname.startswith('batcave01') %} +queue = "mirror_forge_ansible{{ env_suffix }}" +{% endif %} +exchange = "amq.topic" +# FIXME: This key is probably wrong. +routing_keys = [ + "org.fedoraproject.prod.forgejo.git.receive", +] + +[tls] +ca_cert = "/etc/pki/fedora-messaging/cacert.pem" +keyfile = "/etc/pki/fedora-messaging/mirror_forge_ansible.key" +certfile = "/etc/pki/fedora-messaging/mirror_forge_ansible.crt" + +[client_properties] +app = "mirror_from_forge" +app_url = "https://forge.fedoraproject.org/infra/mirror_from_forge" +app_contacts_email = ["pingou@fedoraproject.org"] + +{% if inventory_hostname.startswith('batcave01') %} +[queues."mirror_forge_ansible{{ env_suffix }}"] +{% endif %} +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[consumer_config] +mirror_folder = "/srv/git/mirrors/" +trigger_names = ["infra/ansible"] +urls = [ + "https://forge.fedoraproject.org/infra/ansible.git", +] + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true + +[log_config.formatters.simple] +format = "[%(levelname)s %(name)s] %(message)s" + +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" + +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.twisted] +level = "INFO" +propagate = false +handlers = ["console"] + +[log_config.loggers.pika] +level = "WARNING" +propagate = false +handlers = ["console"] + +# If your consumer sets up a logger, you must add a configuration for it +# here in order for the messages to show up. e.g. if it set up a logger +# called 'example_printer', you could do: +[log_config.loggers.mirror_from_forge_bus] +level = "DEBUG" +propagate = false +handlers = ["console"] + +[log_config.root] +level = "ERROR" +handlers = ["console"] diff --git a/roles/mirror_forge_ansible/templates/mirror_forge_ansible.service b/roles/mirror_forge_ansible/templates/mirror_forge_ansible.service new file mode 100644 index 0000000000..85b8ad5d6c --- /dev/null +++ b/roles/mirror_forge_ansible/templates/mirror_forge_ansible.service @@ -0,0 +1,15 @@ +[Unit] +Description=Fedora Messaging consumer +Documentation=http://fedora-messaging.readthedocs.io/ + +[Service] +Type=simple +Environment="PYTHONPATH=/usr/local/libexec/mirror_forge_ansible" +ExecStart=/usr/bin/fedora-messaging --conf /etc/fedora-messaging/mirror_forge_ansible.toml consume +Restart=on-failure +User=mirror_forge_ansible +Group=mirror_forge_ansible + +[Install] +WantedBy=multi-user.target + diff --git a/roles/mirror_forge_ansible/templates/mirror_from_forge_bus.py b/roles/mirror_forge_ansible/templates/mirror_from_forge_bus.py new file mode 100644 index 0000000000..73313d84d7 --- /dev/null +++ b/roles/mirror_forge_ansible/templates/mirror_from_forge_bus.py @@ -0,0 +1,134 @@ +""" +This script runs in a loop and clone or update the clone of the ansible repo +hosted in forge.fp.o +""" +from __future__ import print_function + +import logging +import os +import subprocess +import time + +from fedora_messaging import config, message + +# FIXME: This key is probably wrong +_msg_topic = "org.fedoraproject.prod.forgejo.git.receive" + +_log = logging.getLogger("mirror_from_forge_bus") + + +def run_command(command, cwd=None): + """ Run the specified command in a specific working directory if one + is specified. + + :arg command: the command to run + :type command: list + :kwarg cwd: the working directory in which to run this command + :type cwd: str or None + """ + output = None + try: + output = subprocess.check_output(command, cwd=cwd, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + _log.error("Command `%s` return code: `%s`", " ".join(command), e.returncode) + _log.error("Output:\n------\n%s", e.output) + # To enable when we move to python3 + # _log.error("stdout:\n-------\n%s", e.stdout) + # _log.error("stderr:\n-------\n%s", e.stderr) + raise + + return output + + +class MirrorFromForge(object): + """ + A fedora-messaging consumer update a local mirror of a repo hosted on + forge.fp.o + + Three configuration key is used from fedora-messaging's + "consumer_config" key: + - "mirror_folder", which indicates where mirrors should be store + - "urls", which is a list of mirrors to keep up to date + - "triggers_name", the fullname of the project (ie: name or namespace/name) + that we want to trigger a refresh of our clone on + + :: + + [consumer_config] + mirror_folder = "mirrors" + trigger_names = ["infra/ansible"] + urls = ["https://forge.fp.o/infra/ansible.git"] + """ + + def __init__(self): + """Perform some one-time initialization for the consumer.""" + self.path = config.conf["consumer_config"]["mirror_folder"] + self.urls = config.conf["consumer_config"]["urls"] + self.trigger_names = config.conf["consumer_config"]["trigger_names"] + + if not os.path.exists(self.path): + raise OSError("No folder %s found on disk" % self.path) + + _log.info("Ready to consume and trigger on %s", self.trigger_names) + + msg = message.Message + msg.topic = _msg_topic + msg.body = {"repo": {"fullname": self.trigger_names[0]}} + self.__call__(message=msg) + + def __call__(self, message, cnt=0): + """ + Invoked when a message is received by the consumer. + + Args: + message (fedora_messaging.api.Message): The message from AMQP. + """ + _log.info("Received topic: %s", message.topic) + if message.topic == _msg_topic: + repo_name = message.body.get("repo", {}).get("fullname") + if repo_name not in self.trigger_names: + _log.info("%s is not a forge repo of interest, bailing", repo_name) + return + else: + _log.info("Unexpected topic received: %s", message.topic) + return + + try: + for url in self.urls: + _log.info("Syncing %s", url) + name = url.rsplit("/", 1)[-1] + + dest_folder = os.path.join(self.path, name) + if not os.path.exists(dest_folder): + _log.info(" Cloning as new %s", url) + cmd = ["git", "clone", "--mirror", url] + run_command(cmd, cwd=self.path) + + _log.info( + " Running `git -c transfer.fsckObjects=1 fetch` in %s", + dest_folder, + ) + cmd = ["git", "-c", "transfer.fsckObjects=1", "fetch"] + run_command(cmd, cwd=dest_folder) + + cmd = ["git", "remote"] + output = run_command(cmd, cwd=dest_folder).decode("utf-8").strip() + if output: + for remote in output.split("\n"): + if remote == "origin": + continue + _log.info( + " Running git push --mirror %s in %s", + remote, dest_folder) + cmd = ["git", "push", "--mirror", remote] + run_command(cmd, cwd=dest_folder) + else: + _log.info(" No remotes found") + + except Exception: + _log.exception("Something happened while calling git") + if cnt >= 3: + raise + _log.info(" Re-running in 10 seconds") + time.sleep(10) + self.__call__(message, cnt=cnt + 1)