fix: job container, daemon startup, action cache and exec fixes (#1294)

- Keep `GITHUB_ENV`, `GITHUB_OUTPUT`, `GITHUB_STATE` and `GITHUB_PATH` when a job times out or is cancelled, so post steps still get their state
- Retry the startup Declare while Gitea is unreachable instead of exiting, still failing fast on rejected registrations, auth and misconfiguration
- Reuse a local image when a platform variant such as `linux/arm/v7` is requested instead of pulling again
- Read `PATH` from the created job container so cross-architecture jobs get tool paths for the right architecture
- Extract action copies relative to the act volume so they never cross an image's `/var/run` symlink, which rootless Podman rejects
- Keep the job container's entrypoint when container options set `--entrypoint`, warning instead of appending it to the keep-alive command
- Lock action cache entries across runner processes that share a cache directory
- `exec`: omit `GITHUB_SERVER_URL` and `GITHUB_API_URL` without `--gitea-instance` instead of exporting a bare `https://`, and read owner and repository from any origin remote

Reviewed-on: https://gitea.com/gitea/runner/pulls/1294
Reviewed-by: bircni <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>

internal/app/cmd/daemon.go

@@ -5,8 +5,10 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -29,6 +31,8 @@ import (
 	"gitea.com/gitea/runner/internal/pkg/ver"
 
 	"connectrpc.com/connect"
+	runnerv1 "gitea.dev/actionslib/runner/v1"
+	"github.com/avast/retry-go/v5"
 	"github.com/mattn/go-isatty"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -185,8 +189,25 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
 		}()
 
 		// declare the labels of the runner before fetching tasks
-		resp, err := runner.Declare(ctx, ls.Names())
+		resp, err := retry.NewWithData[*connect.Response[runnerv1.DeclareResponse]](
+			retry.Context(ctx),
+			retry.UntilSucceeded(),
+			retry.Delay(declareRetryDelay),
+			retry.DelayType(retry.FixedDelay),
+			retry.RetryIf(shouldRetryDeclare),
+			retry.OnRetry(func(attempt uint, err error) {
+				if attempt == 0 {
+					log.WithError(err).Warn("fail to invoke Declare, retrying until Gitea is reachable")
+				} else {
+					log.WithError(err).Debug("fail to invoke Declare, retrying")
+				}
+			}),
+		).Do(func() (*connect.Response[runnerv1.DeclareResponse], error) {
+			return runner.Declare(ctx, ls.Names())
+		})
 		switch {
+		case err != nil && ctx.Err() != nil && !client.IsRegistrationRejected(err):
+			return nil
 		case err != nil && connect.CodeOf(err) == connect.CodeUnimplemented:
 			log.Errorf("Your Gitea version is too old to support runner declare, please upgrade to v1.21 or later")
 			return err
@@ -352,6 +373,20 @@ func initLogging(cfg *config.Config) {
 	}
 }
 
+var declareRetryDelay = 30 * time.Second
+
+func shouldRetryDeclare(err error) bool {
+	if errors.As(err, new(*tls.CertificateVerificationError)) || errors.As(err, new(tls.AlertError)) || errors.Is(err, http.ErrSchemeMismatch) { // misconfigurations never heal
+		return false
+	}
+	switch connect.CodeOf(err) {
+	case connect.CodeUnauthenticated, connect.CodeUnimplemented, connect.CodePermissionDenied, connect.CodeInvalidArgument:
+		return false
+	default:
+		return !client.IsRegistrationRejected(err)
+	}
+}
+
 var commonSocketPaths = []string{
 	"/var/run/docker.sock",
 	"/run/podman/podman.sock",

internal/app/cmd/daemon_test.go

@@ -5,11 +5,13 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -87,12 +89,19 @@ var errUnregisteredRunner = errors.New("rpc error: code = Unauthenticated desc =
 type daemonTestService struct {
 	runnerv1connect.UnimplementedRunnerServiceHandler
 	declareErr, fetchErr error
+	declareCalls         atomic.Int32
 	beforeTask           bool
 	cancel               context.CancelFunc
 }
 
 func (s *daemonTestService) Declare(context.Context, *connect.Request[runnerv1.DeclareRequest]) (*connect.Response[runnerv1.DeclareResponse], error) {
+	if s.declareCalls.Add(1) == 1 {
+		return nil, connect.NewError(connect.CodeUnavailable, errors.New("bad gateway"))
+	}
 	if s.declareErr != nil {
+		if s.beforeTask {
+			s.cancel()
+		}
 		return nil, s.declareErr
 	}
 	return connect.NewResponse(&runnerv1.DeclareResponse{Runner: &runnerv1.Runner{Name: "test"}}), nil
@@ -117,7 +126,9 @@ func (*daemonTestService) UpdateLog(_ context.Context, req *connect.Request[runn
 	return connect.NewResponse(&runnerv1.UpdateLogResponse{AckIndex: req.Msg.Index + int64(len(req.Msg.Rows))}), nil
 }
 
-func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
+func TestDaemonRetriesDeclareOutagesAndRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
+	defer func(delay time.Duration) { declareRetryDelay = delay }(declareRetryDelay)
+	declareRetryDelay = 0
 	for _, tc := range []struct {
 		name                  string
 		ephemeral, beforeTask bool
@@ -132,11 +143,13 @@ func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
 		{"ephemeral proxy 401 on fetch", true, false, nil, connect.NewError(connect.CodeUnauthenticated, errors.New("HTTP status 401 Unauthorized")), false},
 		{"ephemeral rejected on declare", true, false, errUnregisteredRunner, nil, true},
 		{"persistent rejected on declare", false, false, errUnregisteredRunner, nil, false},
+		{"ephemeral stopped during declare outage", true, true, connect.NewError(connect.CodeUnavailable, errors.New("bad gateway")), nil, false},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
 			defer cancel()
-			_, handler := runnerv1connect.NewRunnerServiceHandler(&daemonTestService{declareErr: tc.declareErr, fetchErr: tc.fetchErr, beforeTask: tc.beforeTask, cancel: cancel})
+			service := &daemonTestService{declareErr: tc.declareErr, fetchErr: tc.fetchErr, beforeTask: tc.beforeTask, cancel: cancel}
+			_, handler := runnerv1connect.NewRunnerServiceHandler(service)
 			server := httptest.NewServer(http.StripPrefix("/api/actions", handler))
 			defer server.Close()
 			dir := t.TempDir()
@@ -145,11 +158,12 @@ func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
 			require.NoError(t, os.WriteFile(configFile, []byte("runner:\n  file: "+regFile+"\n  idle_cleanup_interval: 0s\ncache:\n  enabled: false\n"), 0o600))
 			require.NoError(t, config.SaveRegistration(regFile, &config.Registration{Address: server.URL, UUID: "test", Name: "test", Token: "test", Labels: []string{"host:host"}, Ephemeral: tc.ephemeral}))
 			err := runDaemon(ctx, &daemonArgs{Once: true}, &configFile)(nil, nil)
-			if tc.declareErr == nil && tc.fetchErr == nil {
+			if tc.declareErr == nil && tc.fetchErr == nil || tc.beforeTask {
 				require.NoError(t, err)
 			} else {
 				require.Error(t, err)
 			}
+			require.Equal(t, int32(2), service.declareCalls.Load())
 			if tc.removed {
 				require.NoFileExists(t, regFile)
 			} else {
@@ -158,3 +172,20 @@ func TestDaemonRemovesOnlyConsumedEphemeralRegistration(t *testing.T) {
 		})
 	}
 }
+
+func TestShouldRetryDeclare(t *testing.T) {
+	for _, tc := range []struct {
+		err   error
+		retry bool
+	}{
+		{connect.NewError(connect.CodeUnavailable, errors.New("bad gateway")), true},
+		{connect.NewWireError(connect.CodeInternal, errors.New("update runner")), true},
+		{connect.NewError(connect.CodeUnavailable, &tls.CertificateVerificationError{Err: errors.New("unknown authority")}), false},
+		{connect.NewError(connect.CodeUnavailable, http.ErrSchemeMismatch), false},
+		{connect.NewError(connect.CodeUnavailable, tls.AlertError(42)), false},
+		{connect.NewError(connect.CodePermissionDenied, errors.New("forbidden")), false},
+		{connect.NewWireError(connect.CodeUnknown, errors.New("unregistered runner")), false},
+	} {
+		require.Equal(t, tc.retry, shouldRetryDeclare(tc.err), tc.err.Error())
+	}
+}