From ad4927fd019ed3278effacab38f03000761afbb8 Mon Sep 17 00:00:00 2001
From: "Owen W. Taylor" <otaylor@fishsoup.net>
Date: Mon, 1 Mar 2021 14:43:28 -0500
Subject: [PATCH] Add a script to check the dist tarball, and use it to fix the
 dist tarball

* Add a script make-dist-tarball.py that runs 'setup.py sdist' and compares
  what gets distributed with git and with a set of patterns of extra files
  added during distribution, and with git files that we don't want to
  distribute.

* Extend MANIFEST.in or add __init__.py files to distribute:

   run-unittests.sh
   tox.in
   conf/client_secrets.json
   tests/test_builder
   tests/integration

* Don't include all of module_build_service - this is too prone to backup
  files and development trash files; instead add just the files we need
  that aren't *.py

* Move global-excludes to the end - they modify the existing list of files;
  and add *.db - for some reason, .mbs_local_build.db files were
  getting disted.
---
 MANIFEST.in                    |  12 ++--
 make-dist-tarball.py           | 104 +++++++++++++++++++++++++++++++++
 tests/integration/__init__.py  |   0
 tests/test_builder/__init__.py |   0
 4 files changed, 112 insertions(+), 4 deletions(-)
 create mode 100755 make-dist-tarball.py
 create mode 100644 tests/integration/__init__.py
 create mode 100644 tests/test_builder/__init__.py

diff --git a/MANIFEST.in b/MANIFEST.in
index 01530cca..61629187 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,12 +1,16 @@
-global-exclude *.pyc *.pyo
 include LICENSE
 include README.rst
 include requirements.txt
+include run-unittests.sh
 include test-requirements.txt
-recursive-include conf config.py cacert.pem *.conf *.cfg *.service
+include tox.ini
+recursive-include conf config.py cacert.pem *.conf *.cfg *.json *.service
 recursive-include docs *.txt *.rst *.md
-recursive-include module_build_service *
+include module_build_service/migrations/README
+include module_build_service/migrations/alembic.ini
+include module_build_service/migrations/script.py.mako
 recursive-include fedmsg.d *
-recursive-include tests *.yaml *.json
+recursive-include tests *.yaml *.json *.rst
 recursive-include tests/scm_data *
 recursive-include client *
+global-exclude *.pyc *.pyo *.db
diff --git a/make-dist-tarball.py b/make-dist-tarball.py
new file mode 100755
index 00000000..ab547a50
--- /dev/null
+++ b/make-dist-tarball.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+if sys.stderr.isatty:
+    def info(*args):
+        print("\033[1m", file=sys.stderr, end="")
+        print(*args, file=sys.stderr, end="")
+        print("\033[0m", file=sys.stderr)
+
+else:
+    def info(*args):
+        print(*args, file=sys.stderr)
+
+
+def ignore_re(patterns):
+    return re.compile('^('
+                      + '|'.join(patterns.strip().split())
+                      + ')$')
+
+
+# These are files that are generated as part of the distribution process
+DIST_IGNORE_RE = ignore_re(r"""
+module_build_service.egg-info/.*
+setup.cfg
+PKG-INFO
+""")
+
+
+# These files are in git, but we don't actually want to distribute them
+GIT_IGNORE_RE = ignore_re(r"""
+\.cico-pr.pipeline
+\.dockerignore
+\.gitignore
+docker/.*
+openshift/.*
+make-dist-tarball.py
+Vagrantfile
+""")
+
+
+# Find the version in setup.py, this determines the generated tarball name
+
+version = None
+with open("setup.py") as f:
+    for line in f:
+        m = re.match(r'\s*version\s*=\s*"([^"]+)', line)
+        if m:
+            version = m.group(1)
+
+assert version, "Version not found in setup.py"
+
+# Make the tarball
+
+subprocess.check_call(['python3', 'setup.py', 'sdist'])
+
+info("Checking distributed files")
+
+# Read what files were included in the generated tarball, ignoring files
+# based on DIST_IGNORE_RE
+
+tarball = Path('dist') / f"module-build-service-{version}.tar.gz"
+disted_files = set()
+with subprocess.Popen(["tar", "tfz", tarball],
+                      stdout=subprocess.PIPE, encoding="UTF-8") as p:
+    for line in p.stdout:
+        filename = re.sub(r'^module-build-service-[\d.]+/', '', line.strip())
+        if filename and not filename.endswith("/") and not DIST_IGNORE_RE.match(filename):
+            disted_files.add(filename)
+
+# And what files are checked into git, ignoring files
+# based on GIT_IGNORE_RE
+
+git_files = set()
+with subprocess.Popen(["git", "ls-tree", "-r", "--name-only", "HEAD"],
+                      stdout=subprocess.PIPE, encoding="UTF-8") as p:
+    for line in p.stdout:
+        filename = line.strip()
+        if not GIT_IGNORE_RE.match(filename):
+            git_files.add(filename)
+
+# Compare and tell the user about differences
+
+disted_extra = disted_files - git_files
+git_extra = git_files - disted_files
+
+if not disted_extra and not git_extra:
+    info(tarball, "- OK")
+else:
+    if disted_extra:
+        info("Extra distributed files")
+        for f in sorted(disted_extra):
+            print('  ', f, file=sys.stderr)
+
+    if git_extra:
+        info("Extra files in git, not distributed")
+        for f in sorted(git_extra):
+            print('  ', f, file=sys.stderr)
+
+    sys.exit(1)
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/test_builder/__init__.py b/tests/test_builder/__init__.py
new file mode 100644
index 00000000..e69de29b