Merge #1684 Add a script to check the dist tarball, and use it to fix the dist tarball

MANIFEST.in

@@ -1,12 +1,16 @@
-global-exclude *.pyc *.pyo
 include LICENSE
 include README.rst
 include requirements.txt
+include run-unittests.sh
 include test-requirements.txt
-recursive-include conf config.py cacert.pem *.conf *.cfg *.service
+include tox.ini
+recursive-include conf config.py cacert.pem *.conf *.cfg *.json *.service
 recursive-include docs *.txt *.rst *.md
-recursive-include module_build_service *
+include module_build_service/migrations/README
+include module_build_service/migrations/alembic.ini
+include module_build_service/migrations/script.py.mako
 recursive-include fedmsg.d *
-recursive-include tests *.yaml *.json
+recursive-include tests *.yaml *.json *.rst
 recursive-include tests/scm_data *
 recursive-include client *
+global-exclude *.pyc *.pyo *.db

make-dist-tarball.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+
+if sys.stderr.isatty:
+    def info(*args):
+        print("\033[1m", file=sys.stderr, end="")
+        print(*args, file=sys.stderr, end="")
+        print("\033[0m", file=sys.stderr)
+
+else:
+    def info(*args):
+        print(*args, file=sys.stderr)
+
+
+def ignore_re(patterns):
+    return re.compile('^('
+                      + '|'.join(patterns.strip().split())
+                      + ')$')
+
+
+# These are files that are generated as part of the distribution process
+DIST_IGNORE_RE = ignore_re(r"""
+module_build_service.egg-info/.*
+setup.cfg
+PKG-INFO
+""")
+
+
+# These files are in git, but we don't actually want to distribute them
+GIT_IGNORE_RE = ignore_re(r"""
+\.cico-pr.pipeline
+\.dockerignore
+\.gitignore
+docker/.*
+openshift/.*
+make-dist-tarball.py
+Vagrantfile
+""")
+
+
+# Find the version in setup.py, this determines the generated tarball name
+
+version = None
+with open("setup.py") as f:
+    for line in f:
+        m = re.match(r'\s*version\s*=\s*"([^"]+)', line)
+        if m:
+            version = m.group(1)
+
+assert version, "Version not found in setup.py"
+
+# Make the tarball
+
+subprocess.check_call(['python3', 'setup.py', 'sdist'])
+
+info("Checking distributed files")
+
+# Read what files were included in the generated tarball, ignoring files
+# based on DIST_IGNORE_RE
+
+tarball = Path('dist') / f"module-build-service-{version}.tar.gz"
+disted_files = set()
+with subprocess.Popen(["tar", "tfz", tarball],
+                      stdout=subprocess.PIPE, encoding="UTF-8") as p:
+    for line in p.stdout:
+        filename = re.sub(r'^module-build-service-[\d.]+/', '', line.strip())
+        if filename and not filename.endswith("/") and not DIST_IGNORE_RE.match(filename):
+            disted_files.add(filename)
+
+# And what files are checked into git, ignoring files
+# based on GIT_IGNORE_RE
+
+git_files = set()
+with subprocess.Popen(["git", "ls-tree", "-r", "--name-only", "HEAD"],
+                      stdout=subprocess.PIPE, encoding="UTF-8") as p:
+    for line in p.stdout:
+        filename = line.strip()
+        if not GIT_IGNORE_RE.match(filename):
+            git_files.add(filename)
+
+# Compare and tell the user about differences
+
+disted_extra = disted_files - git_files
+git_extra = git_files - disted_files
+
+if not disted_extra and not git_extra:
+    info(tarball, "- OK")
+else:
+    if disted_extra:
+        info("Extra distributed files")
+        for f in sorted(disted_extra):
+            print('  ', f, file=sys.stderr)
+
+    if git_extra:
+        info("Extra files in git, not distributed")
+        for f in sorted(git_extra):
+            print('  ', f, file=sys.stderr)
+
+    sys.exit(1)