mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-10-28 19:06:02 +08:00
fix: keep the shared action cache as extracted, re-fetch damaged trees (#1352)
The toolkit patch rewrote bundles in place in the shared action cache. Since https://gitea.com/gitea/runner/pulls/1316 trees are never refreshed, so a bundle left truncated failed every later job using it. Patched bundles now only overwrite the job container's copy, so the runner no longer writes to an extracted tree. Extraction records each file's size beside the tree, and a tree that no longer matches is fetched again, offline mode included, as a missing one always was. This also catches damage from outside the runner, such as storage losing a write, for a stat per file, a fraction of the copy each stage already makes. Trees from older runners have no record and download once more, except in offline mode, which keeps using them as before. Generated with Claude Code. Reviewed-on: https://gitea.com/gitea/runner/pulls/1352 Reviewed-by: Lunny Xiao <xiaolunwen@gmail.com> Reviewed-by: bircni <bircni@icloud.com> Co-authored-by: Zettat123 <zettat123@gmail.com>
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
@@ -28,6 +29,13 @@ import (
|
||||
|
||||
var actionFetches sync.Map
|
||||
|
||||
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
|
||||
|
||||
type treeFile struct {
|
||||
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
|
||||
Size int64
|
||||
}
|
||||
|
||||
type actionFetch struct {
|
||||
done chan struct{}
|
||||
dir, sha string
|
||||
@@ -86,9 +94,14 @@ func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, s
|
||||
record := filepath.Join(input.Dir, "ref")
|
||||
data, _ := os.ReadFile(record)
|
||||
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||
if !isFullObjectID(sha) || !exists(tree(sha)) {
|
||||
switch {
|
||||
case !isFullObjectID(sha):
|
||||
sha, etag = "", ""
|
||||
} else if input.OfflineMode || isFullObjectID(input.Ref) {
|
||||
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
|
||||
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
|
||||
case !treeIntact(input.Dir, sha):
|
||||
sha, etag = "", ""
|
||||
case input.OfflineMode || isFullObjectID(input.Ref):
|
||||
return tree(sha), sha, nil
|
||||
}
|
||||
|
||||
@@ -173,7 +186,7 @@ func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, toke
|
||||
newEtag := response.Header.Get("ETag")
|
||||
gz, err := gzip.NewReader(response.Body)
|
||||
if err == nil {
|
||||
sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||
@@ -191,18 +204,18 @@ func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, e
|
||||
clone := input
|
||||
clone.Dir = filepath.Join(input.Dir, "git")
|
||||
sha, _, err := resolveClone(ctx, clone)
|
||||
if err != nil || exists(filepath.Join(input.Dir, sha)) {
|
||||
if err != nil || treeIntact(input.Dir, sha) {
|
||||
return sha, err
|
||||
}
|
||||
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
|
||||
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
|
||||
return sha, err
|
||||
}
|
||||
|
||||
func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -215,6 +228,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
defer root.Close()
|
||||
|
||||
sha := ""
|
||||
sizes := map[string]int64{}
|
||||
reader := tar.NewReader(archive)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
@@ -225,7 +239,7 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
return "", err
|
||||
}
|
||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
|
||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
|
||||
return sha, nil
|
||||
}
|
||||
continue
|
||||
@@ -241,10 +255,10 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
case tar.TypeDir:
|
||||
err = root.MkdirAll(name, 0o755)
|
||||
case tar.TypeReg:
|
||||
err = writeTreeFile(root, name, header.Mode, reader)
|
||||
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
|
||||
case tar.TypeSymlink:
|
||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||
err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
@@ -257,20 +271,76 @@ func extractTree(archive io.Reader, dir string, strip bool) (string, error) { //
|
||||
if err := root.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sha, os.Rename(temp, filepath.Join(dir, sha))
|
||||
return sha, installTree(ctx, dir, sha, temp, sizes)
|
||||
}
|
||||
|
||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
|
||||
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
|
||||
|
||||
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
|
||||
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
|
||||
files := make([]treeFile, 0, len(sizes))
|
||||
for name, size := range sizes {
|
||||
files = append(files, treeFile{[]byte(name), size})
|
||||
}
|
||||
data, err := json.Marshal(files)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
|
||||
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
|
||||
return err
|
||||
}
|
||||
tree := filepath.Join(dir, sha)
|
||||
if !exists(tree) {
|
||||
return os.Rename(temp, tree)
|
||||
}
|
||||
stale, err := os.MkdirTemp(dir, "tmp-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
|
||||
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temp, tree)
|
||||
}
|
||||
|
||||
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
|
||||
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
|
||||
func treeIntact(dir, sha string) bool {
|
||||
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
|
||||
if err != nil || !exists(filepath.Join(dir, sha)) {
|
||||
return false
|
||||
}
|
||||
var files []treeFile
|
||||
if json.Unmarshal(data, &files) != nil {
|
||||
return false
|
||||
}
|
||||
for _, file := range files {
|
||||
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
|
||||
perm := os.FileMode(0o644)
|
||||
if mode&0o111 != 0 {
|
||||
perm = 0o755
|
||||
}
|
||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
_, err = io.Copy(file, content)
|
||||
return errors.Join(err, file.Close())
|
||||
n, err := io.Copy(file, content)
|
||||
return n, errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func exists(name string) bool {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -206,10 +207,17 @@ func TestFetchAction(t *testing.T) {
|
||||
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, movedSha, offlineSha)
|
||||
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
|
||||
_, _, err = FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
|
||||
reused, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, moved, reused)
|
||||
|
||||
requests, headers := served()
|
||||
assert.Equal(t, 3, requests)
|
||||
require.Len(t, headers, 3)
|
||||
assert.Equal(t, 4, requests)
|
||||
require.Len(t, headers, 4)
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||
@@ -233,6 +241,68 @@ func TestFetchAction(t *testing.T) {
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
})
|
||||
|
||||
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
|
||||
sha := gitRevParse(t, remote, "main")
|
||||
for _, mode := range []string{"prefixed", "none"} {
|
||||
serve(mode, 0)
|
||||
input := newInput(sha)
|
||||
tree, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
yml := filepath.Join(tree, "action.yml")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
|
||||
_, _, err = FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
for _, damage := range []func() error{
|
||||
func() error { return os.Truncate(yml, 4) },
|
||||
func() error { return os.Remove(tree + treeSizes) },
|
||||
} {
|
||||
require.NoError(t, damage(), mode)
|
||||
_, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err, mode)
|
||||
body, err := os.ReadFile(yml)
|
||||
require.NoError(t, err, mode)
|
||||
assert.Equal(t, "name: action\n", string(body), mode)
|
||||
}
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 3, mode)
|
||||
}
|
||||
|
||||
serve("prefixed", 0)
|
||||
input := newInput(sha)
|
||||
tree, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
yml := filepath.Join(tree, "action.yml")
|
||||
require.NoError(t, os.Truncate(yml, 4))
|
||||
unlock, err := AcquireCloneLock(t.Context(), tree)
|
||||
require.NoError(t, err)
|
||||
release := sync.OnceFunc(unlock)
|
||||
defer release()
|
||||
waiting := make(chan struct{})
|
||||
lockTree = func(ctx context.Context, dir string) (func(), error) {
|
||||
close(waiting)
|
||||
return AcquireCloneLock(ctx, dir)
|
||||
}
|
||||
t.Cleanup(func() { lockTree = AcquireCloneLock })
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := FetchAction(t.Context(), input)
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
|
||||
case <-waiting:
|
||||
}
|
||||
info, err := os.Stat(yml)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
|
||||
release()
|
||||
require.NoError(t, <-done)
|
||||
body, err := os.ReadFile(yml)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "name: action\n", string(body))
|
||||
})
|
||||
|
||||
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||
serve("redirect", 2)
|
||||
input := newInput("main")
|
||||
@@ -306,10 +376,30 @@ func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(&archive, dir, false)
|
||||
_, err := extractTree(t.Context(), &archive, dir, false)
|
||||
require.Error(t, err)
|
||||
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("macOS and Windows store no name that is not UTF-8")
|
||||
}
|
||||
sha := strings.Repeat("a", 40)
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
|
||||
_, err := writer.Write([]byte("x"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err = extractTree(t.Context(), &archive, dir, false)
|
||||
require.NoError(t, err)
|
||||
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
|
||||
assert.True(t, treeIntact(dir, sha))
|
||||
}
|
||||
|
||||
@@ -154,12 +154,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
if !rc.Config.NoActionPatch {
|
||||
// Other jobs share the directory, so patch under the copy's lock.
|
||||
defer patchActions(ctx, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))()
|
||||
if err := rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx); err != nil { // an exported commit has no .git to apply gitignore with
|
||||
return err
|
||||
}
|
||||
|
||||
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx) // an exported commit has no .git to apply gitignore with
|
||||
if rc.Config.NoActionPatch {
|
||||
return nil
|
||||
}
|
||||
patched := patchedScripts(ctx, actionDir, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))
|
||||
if len(patched) == 0 {
|
||||
return nil
|
||||
}
|
||||
return rc.JobContainer.Copy(containerActionDirCopy, patched...)(ctx)
|
||||
}
|
||||
|
||||
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
|
||||
|
||||
@@ -12,12 +12,13 @@ import (
|
||||
"slices"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
)
|
||||
|
||||
// Actions bundle the @actions toolkit into their own JavaScript, and two of its lines keep it
|
||||
// from working against Gitea. Both are edited out of the bundle the runner downloaded.
|
||||
// from working against Gitea. Both are edited out of the copy each job container gets.
|
||||
//
|
||||
// isGhes() takes any host that is not github.com, *.ghe.com or *.localhost for GitHub
|
||||
// Enterprise. @actions/cache then forces the v1 API, and @actions/artifact refuses outright,
|
||||
@@ -86,9 +87,10 @@ func quoted(pattern string) string {
|
||||
return "(?:'" + pattern + "'|\"" + pattern + "\"|`" + pattern + "`)"
|
||||
}
|
||||
|
||||
// actionScriptPaths returns a node action's entrypoints and the chunks bundled beside them. Only
|
||||
// remote actions get here: a local one lives in the user's checkout, which the runner does not
|
||||
// rewrite. Scripts may leave dir for the rest of the checkout, as setup-gradle's ../dist does.
|
||||
// actionScriptPaths returns a node action's entrypoints and the chunks bundled beside them,
|
||||
// relative to checkout. Only remote actions get here: a local one lives in the user's checkout,
|
||||
// which the runner does not rewrite. Scripts may leave dir for the rest of the checkout, as
|
||||
// setup-gradle's ../dist does.
|
||||
func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
||||
if action == nil || !action.Runs.Using.IsNode() {
|
||||
return nil
|
||||
@@ -107,12 +109,12 @@ func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
||||
continue
|
||||
}
|
||||
// `runs` is the action's own yaml, and a key resolving outside its checkout is not ours.
|
||||
bundleDir := filepath.Dir(path)
|
||||
if rel, err := filepath.Rel(root, bundleDir); err != nil || !filepath.IsLocal(rel) {
|
||||
bundleDir, err := filepath.Rel(root, filepath.Dir(path))
|
||||
if err != nil || !filepath.IsLocal(bundleDir) {
|
||||
continue
|
||||
}
|
||||
paths = append(paths, path)
|
||||
entries, _ := os.ReadDir(bundleDir)
|
||||
paths = append(paths, filepath.Join(bundleDir, filepath.Base(path)))
|
||||
entries, _ := os.ReadDir(filepath.Dir(path))
|
||||
for _, entry := range entries {
|
||||
if entry.Type().IsRegular() && slices.Contains([]string{".js", ".cjs", ".mjs"}, filepath.Ext(entry.Name())) {
|
||||
paths = append(paths, filepath.Join(bundleDir, entry.Name()))
|
||||
@@ -123,45 +125,38 @@ func actionScriptPaths(checkout, dir string, action *model.Action) []string {
|
||||
return slices.Compact(paths)
|
||||
}
|
||||
|
||||
// patchActions returns a restore: the tree is shared, so a job with patching off gets no edits.
|
||||
func patchActions(ctx context.Context, scripts []string) func() {
|
||||
restore := map[string][]byte{}
|
||||
// patchedScripts returns the scripts needing the edit, patched, to copy over the job container's copy.
|
||||
func patchedScripts(ctx context.Context, checkout string, scripts []string) []*container.FileEntry {
|
||||
var patched []*container.FileEntry
|
||||
for _, script := range scripts {
|
||||
original, err := patchBundle(script)
|
||||
if original != nil {
|
||||
restore[script] = original // also when the write failed part way through
|
||||
}
|
||||
body, mode, err := patchedScript(filepath.Join(checkout, script))
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("actions toolkit: %s left unpatched: %v", script, err)
|
||||
} else if body != nil {
|
||||
patched = append(patched, &container.FileEntry{Name: filepath.ToSlash(script), Mode: int64(mode), Body: string(body)})
|
||||
}
|
||||
}
|
||||
return func() {
|
||||
for script, original := range restore {
|
||||
if err := os.WriteFile(script, original, 0o644); err != nil { //nolint:gosec // as the checkout wrote it
|
||||
common.Logger(ctx).Warnf("actions toolkit: %s left patched in the shared copy: %v", script, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return patched
|
||||
}
|
||||
|
||||
// patchBundle returns the bytes it replaced, or nil when it left the bundle alone.
|
||||
func patchBundle(script string) ([]byte, error) {
|
||||
// patchedScript returns the bundle patched, or nil when it is left alone.
|
||||
func patchedScript(script string) ([]byte, os.FileMode, error) {
|
||||
info, err := os.Stat(script)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
if info.Size() > maxBundleSize {
|
||||
return nil, nil
|
||||
return nil, 0, nil
|
||||
}
|
||||
data, err := os.ReadFile(script)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
patched, ok := patchedBundle(data)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
return nil, 0, nil
|
||||
}
|
||||
return data, os.WriteFile(script, patched, info.Mode().Perm())
|
||||
return patched, info.Mode().Perm(), nil
|
||||
}
|
||||
|
||||
// patchedBundle opens the GHES gate, and where the cache toolkit is present, points the cache
|
||||
|
||||
@@ -122,9 +122,11 @@ func patchedAction(t *testing.T, repo, ref, entrypoint string) string {
|
||||
|
||||
body, err := os.ReadFile(bundleFromGitHub(t, repo, ref, entrypoint))
|
||||
require.NoError(t, err)
|
||||
if patched, ok := patchedBundle(body); ok {
|
||||
body = patched
|
||||
}
|
||||
script := filepath.Join(tempDirPath(t), filepath.Base(entrypoint))
|
||||
require.NoError(t, os.WriteFile(script, body, 0o600))
|
||||
patchActions(t.Context(), []string{script})
|
||||
return script
|
||||
}
|
||||
|
||||
@@ -327,7 +329,9 @@ func TestCodeSplitActionFindsTheCacheService(t *testing.T) {
|
||||
}
|
||||
t.Setenv("CARGO_HOME", tempDirPath(t))
|
||||
|
||||
defer patchActions(t.Context(), actionScriptPaths(dir, dir, action))()
|
||||
for _, entry := range patchedScripts(t.Context(), dir, actionScriptPaths(dir, dir, action)) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, entry.Name), []byte(entry.Body), 0o600))
|
||||
}
|
||||
out := runActionEntrypoint(t, filepath.Join(dir, action.Runs.Main),
|
||||
newActionJobEnv(t, "split-runtime-token", artifactcache.JobCredential{Repo: "testuser/testrepo"}), inputs)
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
@@ -188,34 +190,24 @@ func TestPatchedBundleSurvivesInsideAStringLiteral(t *testing.T) {
|
||||
require.NoError(t, err, "%s", checked)
|
||||
}
|
||||
|
||||
// An action with a pre step is copied, and so patched, twice.
|
||||
func TestPatchBundleIsIdempotent(t *testing.T) {
|
||||
// A bundle an older runner left patched in the shared copy is copied as it is.
|
||||
func TestPatchedScriptIsIdempotent(t *testing.T) {
|
||||
script := bundleFile(t, gateTSC)
|
||||
|
||||
original, err := patchBundle(script)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, original)
|
||||
patched, err := os.ReadFile(script)
|
||||
patched, _, err := patchedScript(script)
|
||||
require.NoError(t, err)
|
||||
require.True(t, gateOpened(string(patched)))
|
||||
|
||||
original, err = patchBundle(script)
|
||||
require.NoError(t, os.WriteFile(script, patched, 0o600))
|
||||
again, _, err := patchedScript(script)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, original, "a patched bundle is not patched again")
|
||||
again, err := os.ReadFile(script)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, string(patched), string(again))
|
||||
assert.Nil(t, again, "a patched bundle is not patched again")
|
||||
}
|
||||
|
||||
func TestPatchBundleLeavesOtherActionsAlone(t *testing.T) {
|
||||
script := bundleFile(t, `console.log("checkout")`)
|
||||
|
||||
original, err := patchBundle(script)
|
||||
func TestPatchedScriptLeavesOtherActionsAlone(t *testing.T) {
|
||||
patched, _, err := patchedScript(bundleFile(t, `console.log("checkout")`))
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, original)
|
||||
body, err := os.ReadFile(script)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, `console.log("checkout")`, string(body))
|
||||
assert.Nil(t, patched)
|
||||
}
|
||||
|
||||
func bundleFile(t *testing.T, body string) string {
|
||||
@@ -242,10 +234,10 @@ func TestActionScriptPaths(t *testing.T) {
|
||||
|
||||
var chunks []string
|
||||
for _, name := range []string{"295.index.js", "cache-CTfQoCTL.js", "chunk.cjs", "chunk.mjs", "restore.js", "save.js"} {
|
||||
chunks = append(chunks, filepath.Join(dist, name))
|
||||
chunks = append(chunks, filepath.Join("dist", name))
|
||||
}
|
||||
node := &model.Action{Runs: model.ActionRuns{Using: "node20", Pre: "pre.js", Main: "dist/restore.js", Post: "dist/save.js"}}
|
||||
assert.Equal(t, append(slices.Clone(chunks), filepath.Join(dir, "pre.js")), actionScriptPaths(dir, dir, node))
|
||||
assert.Equal(t, append(slices.Clone(chunks), "pre.js"), actionScriptPaths(dir, dir, node))
|
||||
|
||||
// Only a node action has a bundle to patch.
|
||||
assert.Nil(t, actionScriptPaths(dir, dir, &model.Action{Runs: model.ActionRuns{Using: "docker", Image: "alpine"}}))
|
||||
@@ -266,7 +258,7 @@ func TestActionScriptPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
copiedBundle := func(t *testing.T, noPatch bool) string {
|
||||
copiedBundles := func(t *testing.T, noPatch bool) ([]string, []*container.FileEntry) {
|
||||
t.Helper()
|
||||
|
||||
cm := &containerMock{}
|
||||
@@ -285,27 +277,39 @@ func TestPatchActionsAtTheContainerCopy(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(script), "index.js"), []byte("require('./cache.js')"), 0o600))
|
||||
require.NoError(t, os.WriteFile(script, []byte(gateTSC), 0o600))
|
||||
|
||||
var copied string
|
||||
cm.On("CopyDir", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(func(context.Context) error {
|
||||
body, err := os.ReadFile(script)
|
||||
require.NoError(t, err)
|
||||
copied = string(body)
|
||||
var calls []string
|
||||
var copied []*container.FileEntry
|
||||
cm.On("CopyDir", "/var/run/act/actions/repo/", sar.actionDir+"/", false, true).Return(func(context.Context) error {
|
||||
calls = append(calls, "CopyDir")
|
||||
return nil
|
||||
})
|
||||
cm.On("Copy", "/var/run/act/actions/repo/", mock.Anything).Run(func(args mock.Arguments) {
|
||||
copied = args.Get(1).([]*container.FileEntry)
|
||||
}).Return(func(context.Context) error {
|
||||
calls = append(calls, "Copy")
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, maybeCopyToActionDir(t.Context(), sar, sar.actionDir, "sub", "/var/run/act/actions/repo/sub"))
|
||||
source, err := os.ReadFile(script)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gateTSC, string(source), "the shared copy is left as it was found")
|
||||
return copied
|
||||
assert.Equal(t, gateTSC, string(source), "the shared copy is never written")
|
||||
return calls, copied
|
||||
}
|
||||
|
||||
t.Run("patched on its way in", func(t *testing.T) {
|
||||
assert.True(t, gateOpened(copiedBundle(t, false)))
|
||||
t.Run("patched over the container's copy", func(t *testing.T) {
|
||||
calls, copied := copiedBundles(t, false)
|
||||
assert.Equal(t, []string{"CopyDir", "Copy"}, calls)
|
||||
require.Len(t, copied, 1)
|
||||
assert.Equal(t, "dist/cache.js", copied[0].Name)
|
||||
assert.Equal(t, int64(0o600), copied[0].Mode)
|
||||
assert.True(t, gateOpened(copied[0].Body))
|
||||
})
|
||||
|
||||
// The escape hatch, for an action the edit breaks: the artifact actions refuse again, and the
|
||||
// cache client keeps to v1.
|
||||
t.Run("as shipped when the runner is told not to patch", func(t *testing.T) {
|
||||
assert.Equal(t, gateTSC, copiedBundle(t, true))
|
||||
calls, copied := copiedBundles(t, true)
|
||||
assert.Equal(t, []string{"CopyDir"}, calls)
|
||||
assert.Empty(t, copied)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -194,7 +194,8 @@ cache:
|
||||
# secret_key_file: "/run/secrets/cache-s3-secret-key"
|
||||
# When true, reuse a cached action instead of fetching from the remote on every job.
|
||||
# A moved tag (e.g. a re-tagged "v6") or an updated branch stays at the cached commit
|
||||
# until its cache entry expires or is manually removed.
|
||||
# until its cache entry expires or is manually removed. A cached action whose files changed size
|
||||
# since it was downloaded is still fetched again, unless an older runner cached it without sizes.
|
||||
#offline_mode: false
|
||||
# Serve the actions cache service v2 API. The actions that use it fall back to v1 on any host
|
||||
# they do not take for GitHub, so reaching it means editing that check out of their own bundle,
|
||||
|
||||
Reference in New Issue
Block a user