fix: dind image readiness, exit codes and shutdown order (#1265)

Fixes https://gitea.com/gitea/runner/issues/1263

- Wait for dockerd without a check limit. s6 readiness gave up after 7 checks, so the runner never started on slow hosts, also seen in https://gitea.com/gitea/runner/issues/712, https://gitea.com/gitea/runner/issues/652 and https://gitea.com/gitea/runner/issues/758.
- Let the runner run the docker service's readiness check itself instead of reading s6 status, which can be missing at startup or stale after an unclean stop.
- Exit the container with the first failing service's exit code instead of always 0, so `restart: on-failure` works. A requested stop still exits 0.
- Stop the runner before dockerd, so running jobs can finish within `shutdown_timeout`. The stop grace period has to cover that drain.
- Check the supervision exit codes in `make test-dind`.

---------

Co-authored-by: bircni <bircni@icloud.com>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1265
Reviewed-by: bircni <bircni@icloud.com>

scripts/s6/.s6-svscan/SIGINT

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec /etc/s6/.s6-svscan/SIGTERM

scripts/s6/.s6-svscan/SIGTERM

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+(set -C; echo 0 > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
+s6-svc -wD -d /etc/s6/gitea-runner
+exec s6-svscanctl -t /etc/s6

scripts/s6/.s6-svscan/exitcode-path

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "/etc/s6/.s6-svscan/exitcode.$(cat /proc/sys/kernel/random/boot_id).$(cut -d ' ' -f 22 /proc/1/stat)" # unique per container start, so a killed run's record is ignored

scripts/s6/.s6-svscan/finish

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+exitcode=$(cat "$(/etc/s6/.s6-svscan/exitcode-path)" 2> /dev/null)
+rm -f /etc/s6/.s6-svscan/exitcode.*
+exit "${exitcode:-0}"

scripts/s6/docker/finish

@@ -1,4 +1,8 @@
 #!/usr/bin/env bash
 
-exec s6-svscanctl -t /etc/s6
+if [ "$1" -eq 256 ]; then
+  set -- "$((128 + $2))"
+fi
+(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
 
+s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown

scripts/s6/docker/run

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-exec s6-notifyoncheck dockerd-entrypoint.sh
+exec s6-notifyoncheck -n 0 dockerd-entrypoint.sh

scripts/s6/gitea-runner/finish

@@ -1,3 +1,8 @@
 #!/usr/bin/env bash
 
-exec s6-svscanctl -t /etc/s6
+if [ "$1" -eq 256 ]; then
+  set -- "$((128 + $2))"
+fi
+(set -C; echo "$1" > "$(/etc/s6/.s6-svscan/exitcode-path)") 2> /dev/null
+
+s6-svscanctl -t /etc/s6 && exit 125 # 125 keeps s6 from restarting the service during teardown

scripts/s6/gitea-runner/run

@@ -1,9 +1,7 @@
 #!/usr/bin/env bash
 
-while [ ! -d /etc/s6/docker/supervise ]; do
-    sleep 0.1
+until /etc/s6/docker/data/check &> /dev/null; do # s6-svwait can read missing or stale readiness status
+  sleep 1
 done
 
-s6-svwait -U /etc/s6/docker
-
 exec run.sh

scripts/test-dind.sh

@@ -10,8 +10,8 @@
 # Usage: scripts/test-dind.sh [target] [-- go-test-args...]
 #   target:        dind (default), dind-rootless, or podman to run PODMAN_TEST_IMAGE's API service instead
 #   go-test-args:  passed verbatim to `go test`. Defaults cover image env extraction,
-#                  symlink copying and a mounted Docker job using cached images, or the
-#                  Docker proxy probe for podman.
+#                  symlink copying, a mounted Docker job using cached images and the
+#                  image's s6 supervision, or the Docker proxy probe for podman.
 #
 # Env:
 #   DIND_TEST_PORT     host port for the daemon (default 32375)
@@ -33,7 +33,7 @@ host_docker="${DOCKER_HOST:-$(docker context inspect --format '{{.Endpoints.dock
 test_dir=""
 
 cleanup() {
-  docker -H "$host_docker" rm -fv "$name" >/dev/null 2>&1 || true
+  docker -H "$host_docker" rm -fv "$name" "$name-s6" >/dev/null 2>&1 || true
   if [ -n "$test_dir" ]; then
     rm -rf "$test_dir"
   fi
@@ -147,4 +147,30 @@ if [ "$default_tests" = true ]; then
   echo "==> Running mounted Docker job in a container given the ${target} socket, expecting proxy access"
   docker -H "$host_docker" exec -e DOCKER_HOST="$socket" "$name" docker run --rm -v "${socket#unix://}:/var/run/docker.sock" -v /tmp/gitea-runner-proxy-test:/data -w /data \
     -e ACT_TEST_DOCKER_PROXY=proxy -e ACT_TEST_IMAGE="$job_image" "$job_image" ./runner.test -test.v -test.run '^TestDockerProxyMountedJob$' -test.timeout 3m
+
+  supervised="$name-s6"
+  supervised_exit_is() {
+    if [ "$(docker -H "$host_docker" inspect -f '{{.State.ExitCode}}' "$supervised")" != "$1" ] ||
+      ! docker -H "$host_docker" logs "$supervised" 2>&1 | grep -F "$2" >/dev/null; then
+      docker -H "$host_docker" logs "$supervised" >&2
+      echo "${target} supervision: expected exit $1 after \"$2\"" >&2
+      exit 1
+    fi
+  }
+  echo "==> Checking that a runner failure sets the ${target} container exit code"
+  docker -H "$host_docker" run -d --privileged --name "$supervised" -e DOCKER_TLS_CERTDIR= -e RUNNER_STATE_FILE=/etc/hostname "$image" >/dev/null
+  for _ in $(seq 1 120); do
+    [ "$(docker -H "$host_docker" inspect -f '{{.State.Running}}' "$supervised")" = false ] && break
+    sleep 1
+  done
+  supervised_exit_is 1 'registration file not found'
+  docker -H "$host_docker" rm -fv "$supervised" >/dev/null
+  echo "==> Checking that stopping a started ${target} runner exits 0"
+  docker -H "$host_docker" run -d --privileged --name "$supervised" -e DOCKER_TLS_CERTDIR= "$image" >/dev/null
+  for _ in $(seq 1 120); do
+    docker -H "$host_docker" logs "$supervised" 2>&1 | grep -F 'is missing or not a regular file' >/dev/null && break
+    sleep 1
+  done
+  docker -H "$host_docker" stop "$supervised" >/dev/null
+  supervised_exit_is 0 'is missing or not a regular file'
 fi