Compare commits

..

4 Commits

Author SHA1 Message Date
Lunny Xiao
de9db9c0fa fix: suppress tar leading slash warnings 2026-09-06 20:17:55 -07:00
Lunny Xiao
d1da6ffa52 fix: create guest workspace in real macOS VM integration test 2026-09-06 18:37:15 -07:00
Lunny Xiao
880e0a148a fix: prepare macOS VM workspace and archive without AppleDouble 2026-09-05 21:56:15 -07:00
Lunny Xiao
2db24c7401 enhance: add macOS VM executor backend
Replace the Tart-based prototype with a runner-oriented macOS VM
executor backed by runner-executor-macos and Apple Virtualization.

Assisted-by: Codet:MODEL_VERSION
2026-09-05 13:20:03 -07:00
272 changed files with 13828 additions and 22725 deletions

View File

@@ -45,13 +45,8 @@ cpu.out
/bin
/dist
/.tmp
/.git/hooks
/.git/logs
/.git/rr-cache
/.env
/.runner
/.runner.lock
/config.yaml
/Dockerfile
.DS_Store

View File

@@ -1,29 +0,0 @@
name: go-setup
description: Set up go and restore its caches, which only the cache-seeder workflow saves
runs:
using: composite
steps:
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
id: setup-go
with:
go-version-file: go.mod
check-latest: true
cache: false
- if: ${{ github.workflow == 'cache-seeder' }}
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
- if: ${{ github.workflow != 'cache-seeder' }}
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/.cache/golangci-lint
key: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum', '.golangci.yml', 'Makefile', '.gitea/actions/go-setup/action.yml', '.gitea/workflows/cache-seeder.yml') }}
restore-keys: go-${{ runner.os }}-${{ runner.arch }}-${{ steps.setup-go.outputs.go-version }}-

View File

@@ -1,26 +0,0 @@
name: cache-seeder
on:
workflow_dispatch:
push:
branches:
- main
paths:
- go.mod
- go.sum
- .golangci.yml
- Makefile
- .gitea/actions/go-setup/action.yml
- .gitea/workflows/cache-seeder.yml
jobs:
seed:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- run: make deps-tools lint build
- name: compile the test binaries without running them
run: |
make test GOTEST_FLAGS='-race -exec true'
go test -tags e2e -exec true ./e2e/...

View File

@@ -19,7 +19,7 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- run: make lint-pr-title

View File

@@ -11,6 +11,7 @@ on:
env:
DOCKER_ORG: gitea
DOCKER_LATEST: nightly
jobs:
goreleaser:
@@ -35,7 +36,7 @@ jobs:
go-version-file: "go.mod"
check-latest: true
- name: goreleaser
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
with:
distribution: goreleaser-pro
args: release --nightly
@@ -73,13 +74,13 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -93,7 +94,7 @@ jobs:
echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile

View File

@@ -35,7 +35,7 @@ jobs:
passphrase: ${{ secrets.PASSPHRASE }}
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
- name: goreleaser
uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7
with:
distribution: goreleaser-pro
args: release
@@ -69,6 +69,7 @@ jobs:
image: catthehacker/ubuntu:act-latest
env:
DOCKER_ORG: gitea
DOCKER_LATEST: latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -76,13 +77,13 @@ jobs:
fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
- name: Set up Docker BuildX
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Login to DockerHub
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -102,7 +103,7 @@ jobs:
suffix=${{ matrix.variant.tag_suffix }},onlatest=true
- name: Build and push
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
file: ./Dockerfile

View File

@@ -1,7 +1,9 @@
name: checks
on:
push:
branches:
- main
pull_request:
workflow_dispatch:
env:
# The runner image ships a stale docker.io login; point docker at an empty config so
@@ -17,24 +19,25 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
# Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
# daemon retains them between runs, and their digest pins skip the registry when it has them.
# daemon retains them between runs, so this is usually a fast manifest re-check.
- name: pre-pull test images
env:
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:a3554c0d0b48177b1413c26e01f695334428020d9df01243fbae3e11cfd17417 # renovate: datasource=docker
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:ba849c60be29959425b8734d57b8b4b7d56f98edd9504c9af091d5281095a71e # renovate: datasource=docker
TEST_SERVICE_IMAGE: nginx:alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
run: |
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
if [ "$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null)" != linux/amd64 ]; then
for attempt in 1 2 3; do
docker pull "$image" && break
[ "$attempt" = 3 ] || sleep 5
done
fi
for attempt in 1 2 3; do
docker pull "$image" && break
[ "$attempt" = 3 ] || sleep 5
done
docker tag "$image" "${image%@*}"
done
- name: lint
@@ -50,8 +53,6 @@ jobs:
# after `make test` so the images it needs are already present on the host daemon.
- name: test against dind image
run: make test-dind
- name: test against podman
run: make test-dind TARGET=podman
- name: coverage report
run: |
make coverage-report
@@ -67,7 +68,10 @@ jobs:
- gitea/gitea:main-nightly
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.gitea/actions/go-setup
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
- name: e2e compatibility

View File

@@ -39,13 +39,6 @@ linters:
deny:
- pkg: golang.org/x/exp
desc: it's experimental and unreliable
logging:
files:
- $all
- "!**/act/common/logging/**"
deny:
- pkg: log/slog
desc: log through gitea.com/gitea/runner/act/common/logging
nolintlint:
allow-unused: false
require-explanation: true

View File

@@ -28,18 +28,38 @@ builds:
ignore:
- goos: darwin
goarch: arm
- goos: darwin
goarch: ppc64le
- goos: darwin
goarch: s390x
- goos: windows
goarch: ppc64le
- goos: windows
goarch: s390x
- goos: windows
goarch: arm
goarm: "5"
- goos: windows
goarch: arm
goarm: "6"
- goos: windows
goarch: arm
goarm: "7"
- goos: windows
goarch: arm64
- goos: freebsd
goarch: ppc64le
- goos: freebsd
goarch: s390x
- goos: freebsd
goarch: arm
goarm: "5"
- goos: freebsd
goarch: arm
goarm: "6"
- goos: freebsd
goarch: arm
goarm: "7"
- goos: freebsd
goarch: arm64
flags:

1
CLAUDE.md Normal file
View File

@@ -0,0 +1 @@
@AGENTS.md

View File

@@ -28,30 +28,8 @@ depending on the prefix:
- **No prefix?** Do not escape, and split multi-line values into one row each.
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
encoded forms too.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
handler folds an annotation's location into its decoded message and escapes the result again.
## Logging
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
fields.
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
Going around either leaks them.
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
job logger.
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
package functions.
## Built-in actions
Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions` in
`act/runner/step_action_builtin.go`.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
decodes exactly those two when folding a location into an annotation.
## End-to-end compatibility tests

View File

@@ -1,7 +1,7 @@
### BUILDER STAGE
#
#
FROM golang:1.27-alpine AS builder
FROM golang:1.27-alpine3.23 AS builder
# Do not remove `git` here, it is required for getting runner version when executing `make build`
RUN apk add --no-cache make git
@@ -12,14 +12,12 @@ ENV GOPROXY=${GOPROXY:-}
COPY . /opt/src/runner
WORKDIR /opt/src/runner
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
make clean && make build
RUN make clean && make build
### DIND VARIANT
#
#
FROM docker:29.9.1-dind AS dind
FROM docker:29.7.2-dind AS dind
ARG VERSION=dev
@@ -34,14 +32,12 @@ COPY scripts/s6 /etc/s6
VOLUME /data
ENV TINI_SUBREAPER=1
ENTRYPOINT ["s6-svscan","/etc/s6"]
### DIND-ROOTLESS VARIANT
#
#
FROM docker:29.9.1-dind-rootless AS dind-rootless
FROM docker:29.7.2-dind-rootless AS dind-rootless
ARG VERSION=dev
@@ -60,7 +56,6 @@ VOLUME /data
RUN mkdir -p /data && chown -R rootless:rootless /etc/s6 /data
ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
ENV TINI_SUBREAPER=1
USER rootless
ENTRYPOINT ["s6-svscan","/etc/s6"]

View File

@@ -1,22 +1,25 @@
DIST := dist
EXECUTABLE := gitea-runner
DIST_DIRS := $(DIST)/binaries $(DIST)/release
GO ?= go
SHASUM ?= shasum -a 256
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go
XGO_VERSION := go-1.27.x
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.16 # renovate: datasource=go
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:8856b200466e9a5af9cc0d4de4ca96f73547833bcdddf9a567189ff772c4d372 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
LINUX_ARCHS ?= linux/amd64,linux/arm64
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
WINDOWS_ARCHS ?= windows/amd64
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
DOCKER_IMAGE ?= gitea/runner
DOCKER_TAG ?= nightly
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.9.0 # renovate: datasource=go
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 # renovate: datasource=go
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.7.0 # renovate: datasource=go
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
@@ -29,16 +32,22 @@ endif
ifeq ($(HAS_GO), GO)
GOPATH ?= $(shell $(GO) env GOPATH)
export PATH := $(GOPATH)/bin:$(PATH)
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
endif
ifeq ($(OS), Windows_NT)
GOFLAGS := -v -buildmode=exe
EXECUTABLE ?= $(EXECUTABLE).exe
GO_ENV_WINDOWS := set GOOS=windows&&
else ifeq ($(OS), Windows)
GOFLAGS := -v -buildmode=exe
EXECUTABLE ?= $(EXECUTABLE).exe
GO_ENV_WINDOWS := set GOOS=windows&&
else
GOFLAGS := -v
EXECUTABLE ?= $(EXECUTABLE)
GO_ENV_WINDOWS := GOOS=windows
endif
@@ -87,12 +96,19 @@ go-check:
fi
.PHONY: fmt-check
fmt-check: ## check the Go code formatting
$(GO) run $(GOLANGCI_LINT_PACKAGE) fmt --diff-colored
fmt-check: fmt
@diff=$$(git diff --color=always -- '*.go'); \
if [ -n "$$diff" ]; then \
echo "Please run 'make fmt' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: deps-tools
deps-tools: ## install tool dependencies
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
$(GO) install $(GXZ_PACKAGE) & \
$(GO) install $(XGO_PACKAGE) & \
$(GO) install $(GOVULNCHECK_PACKAGE) & \
wait
@@ -133,8 +149,13 @@ tidy: ## run go mod tidy
fi
.PHONY: tidy-check
tidy-check: ## check that go.mod and go.sum are tidy
$(GO) mod tidy -diff
tidy-check: tidy
@diff=$$(git diff --color=always -- go.mod go.sum); \
if [ -n "$$diff" ]; then \
echo "Please run 'make tidy' and commit the result:"; \
printf "%s" "$${diff}"; \
exit 1; \
fi
.PHONY: test
test: ## test everything (integration tests self-skip without docker/network)
@@ -147,14 +168,17 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
@echo "Wrote .tmp/coverage.md"
.PHONY: test-dind
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless)
@./scripts/test-dind.sh $(TARGET)
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 # renovate: datasource=docker
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
E2E_CONCURRENCY ?= 8
.PHONY: test-e2e
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
.PHONY: install
install: $(GOFILES) ## install the runner binary via `go install`
@@ -170,6 +194,45 @@ $(EXECUTABLE): $(GOFILES)
deps-backend: ## install backend dependencies
$(GO) mod download
.PHONY: release
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
$(DIST_DIRS):
mkdir -p $(DIST_DIRS)
.PHONY: release-windows
release-windows: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-linux
release-linux: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-darwin
release-darwin: | $(DIST_DIRS)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
ifeq ($(CI),true)
cp -r /build/* $(DIST)/binaries/
endif
.PHONY: release-copy
release-copy: | $(DIST_DIRS)
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
.PHONY: release-check
release-check: | $(DIST_DIRS)
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
.PHONY: release-compress
release-compress: | $(DIST_DIRS)
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
.PHONY: docker
docker: ## build the docker image
if ! docker buildx version >/dev/null 2>&1; then \

216
README.md
View File

@@ -4,19 +4,7 @@
### Prerequisites
On the runner:
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
Where steps run, in the job image or on the runner in host mode:
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
- `node` for JavaScript actions and `hashFiles()`.
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
### Download pre-built binary
@@ -91,24 +79,53 @@ If the registry succeed, it will run immediately. Next time, you could run the r
### Run with docker
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
```bash
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> -v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:nightly
```
The volumes keep the runner's registration and the job image cache across container recreation.
Mount a volume on `/data` if you want the registration file and optional config to survive container recreation (see [scripts/run.sh](scripts/run.sh)).
> **`/data` does not hold the image cache.** It is the runner's working directory and contains only the `.runner` registration file and, optionally, your config file. Images pulled for jobs live in the *Docker daemon's* data root, which for the `dind` flavours is inside the container (`/var/lib/docker`, or `/home/rootless/.local/share/docker` for `dind-rootless`). To keep the image cache across restarts, give that path its own volume as well — otherwise every new container re-pulls the job images. With the `basic` flavour the images live on whichever daemon you point the runner at, so there is nothing extra to persist.
### Image flavours
| Tag | Docker daemon for jobs |
| --- | --- |
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
The image is published in three flavours, all built from the single multi-stage [Dockerfile](Dockerfile) in this repository. They differ only in how a Docker daemon is made available to the jobs the runner executes; the `gitea-runner` binary inside them is identical.
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
| Tag | Build target | Base image | Docker daemon | Process supervisor | Runs as |
| --- | --- | --- | --- | --- | --- |
| `latest` (and `<version>`) | `basic` | `alpine` | none — uses an external daemon you provide | [`tini`](https://github.com/krallin/tini) | `root` |
| `latest-dind` | `dind` | `docker:dind` | bundled, started inside the container | [`s6`](https://skarnet.org/software/s6/) | `root` (privileged) |
| `latest-dind-rootless` | `dind-rootless` | `docker:dind-rootless` | bundled, started rootless inside the container | [`s6`](https://skarnet.org/software/s6/) | `rootless` (UID 1000) |
#### `latest` — basic
The default flavour ships only the runner on a minimal Alpine base. It contains **no Docker daemon of its own**: jobs that use `docker://` images need a daemon supplied from outside the container, typically by bind-mounting the host's socket:
```bash
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
-v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:latest
```
`tini` is the entrypoint (it reaps zombie processes), and it just runs [`scripts/run.sh`](scripts/run.sh), which registers the runner on first start and then execs `gitea-runner daemon`. This flavour does not need `--privileged`. The trade-off is that jobs share the host's daemon, so they can see other containers and images on that daemon.
#### `latest-dind` — Docker-in-Docker
This flavour is based on the official `docker:dind` image and bundles its own Docker daemon, so it needs no external socket — only the `--privileged` flag that Docker-in-Docker requires:
```bash
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
--name my_runner gitea/runner:latest-dind
```
Two processes have to run side by side here (the Docker daemon and the runner), so the entrypoint is the [`s6`](https://skarnet.org/software/s6/) supervision tree under [`scripts/s6`](scripts/s6) instead of `tini`. `s6` starts `dockerd`, and the runner service waits for the daemon to come up (`s6-svwait`) before launching [`run.sh`](scripts/run.sh). Each container has a private daemon isolated from the host's, at the cost of running privileged.
#### `latest-dind-rootless` — rootless Docker-in-Docker
Same idea as `dind`, but built on `docker:dind-rootless` so the bundled daemon and the runner run as an unprivileged user (`rootless`, UID 1000) rather than `root`. `DOCKER_HOST` is preset to `unix:///run/user/1000/docker.sock` so the runner talks to the rootless daemon. This reduces the blast radius compared to the privileged `dind` flavour, but rootless Docker carries the usual rootless limitations (networking, cgroups, storage drivers, and some operations that need additional host configuration such as `/etc/subuid` / `/etc/subgid` mappings and unprivileged user-namespace support).
> **The UID is fixed at 1000.** It comes from the `rootless` user baked into the upstream `docker:dind-rootless` base image, and the bundled daemon always listens on `/run/user/1000/docker.sock` inside the container, so running this flavour as a different user (`--user 1001`) does not work. If you need the runner to talk to a *host* rootless daemon that runs under some other UID, use the `basic` flavour instead and bind-mount that daemon's socket (see [examples/vm/rootless-docker.md](examples/vm/rootless-docker.md)); pointing `DOCKER_HOST` at a host socket from inside `dind-rootless` will not work. Changing the UID otherwise means rebuilding the image from a base with a different `rootless` user.
> **Note on Podman:** these images target the Docker daemon. The bundled `dind`/`dind-rootless` daemons are `dockerd`, not Podman, and the `basic` flavour expects a Docker-compatible socket. Running them under rootless Podman is not a supported configuration, though pointing the `basic` flavour at a Podman socket that emulates the Docker API may work for some workloads.
### Configuration
@@ -169,7 +186,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
#### Environment variables
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
### Labels
@@ -184,8 +201,8 @@ A label is written as:
| Part | Meaning |
| --- | --- |
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
| `schema` | Either `docker`, `host`, or `macos-vm`. Defaults to `host` when omitted. |
| `args` | Used by `docker` for the image, and by `macos-vm` for the source VM. |
Three schemas are supported:
@@ -195,29 +212,29 @@ Three schemas are supported:
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
```
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
```text
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
```
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
```text
macos:host
```
- **`macos-vm://<source-vm>`** — the job runs inside an ephemeral macOS VM created by `runner-executor-macos`. The VM image must include the runner guest agent. Service containers are not supported with this schema. See `docs/macos-vm-executor.md` for configuration, integration testing, and cleanup details:
```text
macos-latest:macos-vm://ghcr.io/cirruslabs/macos-sonoma-base:latest
```
So with the labels
```text
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host,macos-latest:macos-vm://ghcr.io/cirruslabs/macos-sonoma-base:latest
```
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, one with `runs-on: macos` is executed directly on the host, and one with `runs-on: macos-latest` is executed in the macOS VM.
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host`, `docker`, and `macos-vm` are treated as schemas.
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs: in `runner.default_image` where docker is available, on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
@@ -242,7 +259,7 @@ A job's `services` are started before its steps run. When a service's image or i
```yaml
services:
postgres:
image: postgres:18
image: postgres:17
options: >-
--health-cmd pg_isready
--health-interval 5s
@@ -255,17 +272,6 @@ A job in a container reaches a service by its id on the job network, on the port
Unlike GitHub, a job whose steps run on the host (a `host` label without `container:`) starts no service containers, so `job.services` and `job.container` stay empty. Give such a job a `container:` when it needs services.
#### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
Containers a job starts through the Docker socket can bind-mount the workspace by the job's own path, as on a host, for example `docker run -v "$PWD:/src"`. Without the Docker proxy, for example with a remote `DOCKER_HOST`, use `GITEA_DOCKER_WORKSPACE`, the path the daemon sees, as the prefix of workspace binds, with `.` as the fallback for local use:
```yaml
volumes:
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
```
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
#### Proxy
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
@@ -280,14 +286,14 @@ The runner uses them for its own requests and gives them to every job, in lower
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
- the built-in cache server, whose address is assigned at startup
- the cache server
- `localhost`, `127.0.0.1` and `::1`
- the job's service containers
- the Docker daemon, when it is reached over `tcp://`
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
For one step, set it in the step's `env:`, for the rest of a job, write it to `$GITHUB_ENV`. Setting it at workflow or job level or in `container.env` has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
To change a value for one job, set it in a step's `env:` or in the job's `container.env`. Setting it at workflow or job level has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
@@ -295,68 +301,63 @@ Dockerfile actions are built with these variables as build arguments, so their `
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
#### Built-in actions (`uses: builtin:<name>`)
Use `uses: builtin:checkout` to check out a repository without downloading `actions/checkout` or needing Node in the job image, see **[docs/builtin-actions.md](docs/builtin-actions.md)**.
#### Caching (`actions/cache`)
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
**Sharing a cache between runners**
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
```yaml
# cache-server.yaml
cache:
dir: /data/actcache
port: 8088
external_secret: "<secret>" # or external_secret_file: /path/to/secret
```
```bash
gitea-runner -c cache-server.yaml cache-server
```
```yaml
# each runner's config
cache:
external_server: "http://<cache-server-host>:8088/"
external_secret: "<secret>"
```
`--dir`, `--host` and `--port` override the matching `cache` keys. Eviction settings take effect on the cache server, not on the runners.
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower cache requests.
**Storing the cache in an S3-compatible bucket**
The cache can also be stored in an S3-compatible bucket such as AWS S3, MinIO or RustFS, with no cache server. Runners with the same settings use the same cache:
```yaml
cache:
s3:
endpoint: "http://minio:9000"
region: "us-east-1"
bucket: "runner-cache"
prefix: "gitea-runner-cache"
path_style: true
access_key_file: "/run/secrets/cache-s3-access-key"
secret_key_file: "/run/secrets/cache-s3-secret-key"
```
A cache saved on one runner can be restored on the others within 30 seconds. The eviction settings below do not apply to the bucket, so add lifecycle rules to it that expire old caches and abort incomplete multipart uploads.
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
**Eviction**
Entries not read or written within `retention` (default `168h`) are removed. A repository over `repo_size_limit` (default `10GB`) loses its least recently used entries, and `size_limit` (off by default) caps the whole cache the same way. Entries in use are never removed. The cache also keeps 1024 MiB free on its volume, or `health_check.min_free_disk_space_mb` when health checks are enabled. See [config.example.yaml](internal/pkg/config/config.example.yaml) for all options.
An entry nothing has read or written for `retention` is removed, and a repository past `repo_size_limit` loses its least recently accessed entries until it fits; `size_limit` caps the whole cache the same way. Age alone never retires an entry still in use, and whatever these allow, the cache keeps free space above `health_check.min_free_disk_space_mb` when health checks are enabled.
These apply where the cache server runs, so on a shared server they belong in *its* config, not the runners'. See `retention`, `repo_size_limit`, `size_limit` and `sweep_interval` in [config.example.yaml](internal/pkg/config/config.example.yaml) for units and defaults.
**Cache service v2**
`actions/cache` v3.4.0, v4.2.0 and later use the cache service v2 API, which the runner serves by default. These actions fall back to v1 on hosts they do not recognize as GitHub, so the runner removes that check from them while a job runs. This also lets the stock `actions/upload-artifact` v4.4.0 and `actions/download-artifact` v4.1.8 and later work without the `gitea-upload-artifact` fork. With `runner.patch_actions: false`, the cache stays on v1 and the artifact actions fail.
`actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with:
With v2, artifact calls also go through the cache server, so jobs must be able to reach it. `cache.v2: false` sends them to Gitea directly, unless the Gitea URL has a path or `runner.insecure` is set with HTTPS.
```yaml
cache:
v2: false
```
Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle on its way into the job, undone whenever the action is downloaded again. A bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork, so it is made whatever `v2` says: that setting only governs the API the runner advertises. Set `runner.patch_actions: false` to leave every bundle exactly as shipped, an escape hatch for an action the edit breaks. The artifact actions then refuse again and the cache client keeps to v1.
**Shared cache across multiple runners**
Run one dedicated `gitea-runner cache-server` that all runners point at.
1. Create a config file for the cache server host:
```yaml
cache:
dir: /data/actcache
port: 8088
external_secret: "replace-with-a-strong-random-secret"
# external_secret_file: /path/to/secret # secret can also be passed via a file
```
2. Start the server:
```bash
gitea-runner -c cache-server-config.yaml cache-server
```
3. On every runner:
```yaml
cache:
external_server: "http://<cache-server-host>:8088/"
external_secret: "replace-with-a-strong-random-secret" # must match the server
# external_secret_file: /path/to/secret # secret can also be passed via a file
```
Jobs reach the cache server at `external_server`, so when a reverse proxy fronts the server, point `external_server` at the proxy. The cache server itself needs no extra configuration.
Alternatively, mount the same NFS/CIFS share on every runner and point `cache.dir` at it — simpler, but with weaker isolation between repositories.
**S3 / MinIO** — mount object storage as a FUSE filesystem (e.g. [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) or [goofys](https://github.com/kahing/goofys)) and set `cache.dir` to the mount point.
Flags `--dir`, `--host`, and `--port` on `cache-server` override the corresponding `cache.*` YAML keys; all other settings, including `external_secret`, require the config file.
#### Official Docker image
@@ -368,8 +369,7 @@ While the runner is idle it cleans up after earlier jobs:
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
#### Post-task script (`runner.post_task_script`)
@@ -392,10 +392,6 @@ Both hooks are synchronous and block the job while they run. Either one exiting
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
#### OpenTelemetry (`OTEL_*`)
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
#### Local job logs (`log.job.dir`)
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -15,6 +15,8 @@ import (
"strconv"
"strings"
"time"
"github.com/julienschmidt/httprouter"
)
// The cache service v2 API. A client on this version talks twirp to
@@ -37,25 +39,24 @@ const (
blobUploadURLTTL = time.Hour
twirpInternal = "internal"
twirpUnavailable = "unavailable"
twirpUnauthenticated = "unauthenticated"
// twirpInternal is the only error code that is not the client's fault.
twirpInternal = "internal"
)
func (h *Handler) registerV2Routes(router *http.ServeMux) {
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
func (h *Handler) registerV2Routes(router *httprouter.Router) {
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
}
// An entry that already exists is reported as not ok, which is how the client learns to skip
// the upload.
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2CreateRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
if req.Key == "" || req.Version == "" {
@@ -82,36 +83,16 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
return
}
// A second live reservation is finalized by whichever job calls last, against the other's upload.
owner := hashedToken(bearerToken(r))
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
if pending.Owner != owner {
h.twirpNotOK(w, r)
return
}
h.touch(db, pending) // still uploading, so it must not go stale under the sweep
h.responseJSON(w, r, http.StatusOK, map[string]any{ // this job retrying its own reservation
"ok": true,
"signed_upload_url": h.signedURL(cred, blobPath, blobUploadPurpose, pending.ID, time.Now().Add(blobUploadURLTTL)),
})
return
}
now := time.Now()
now := time.Now().Unix()
cache := &Cache{
Repo: cred.Repo,
Key: req.Key,
Version: req.Version,
Owner: owner,
Size: -1, // the size is only known at finalize time
CreatedAt: now.Unix(),
CreatedAtNanos: now.Nanosecond(),
UsedAt: now.Unix(),
Repo: cred.Repo,
Key: req.Key,
Version: req.Version,
Size: -1, // the size is only known at finalize time
CreatedAt: now,
UsedAt: now,
}
if err := h.insertCache(db, cache); err != nil {
if err := insertCache(db, cache); err != nil {
h.twirpError(w, r, twirpInternal, err)
return
}
@@ -122,11 +103,11 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
})
}
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -138,13 +119,6 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
defer db.Close()
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
cache = nil // not the reservation this job made, so not this job's to commit
}
if err == nil && cache == nil {
// A retry whose first response was lost finds the entry already committed.
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
}
if err != nil {
h.twirpError(w, r, twirpInternal, err)
return
@@ -153,15 +127,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
h.twirpNotOK(w, r)
return
}
db.Close() // commitCache needs the store closed
if !cache.Complete {
db.Close() // commitCache needs the store closed
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r)
return
}
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
if err := h.commitCache(cache); err != nil {
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
h.twirpNotOK(w, r)
return
}
h.responseJSON(w, r, http.StatusOK, map[string]any{
@@ -171,11 +143,11 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
})
}
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context())
req, err := decodeTwirpRequest[v2DownloadRequest](r)
if err != nil {
h.twirpError(w, r, "malformed", err)
h.twirpError(w, r, "malformed_request", err)
return
}
@@ -206,8 +178,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
// cache is a single PUT, a large one is staged as blocks that a final block list puts
// in order.
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
if err != nil {
h.responseJSON(w, r, http.StatusBadRequest, err)
return
@@ -258,13 +230,8 @@ func (h *Handler) twirpNotOK(w http.ResponseWriter, r *http.Request) {
func (h *Handler) twirpError(w http.ResponseWriter, r *http.Request, code string, err error) {
h.logger.Debugf("%s %s: %v", r.Method, r.URL.Path, err)
status := http.StatusBadRequest
switch code {
case twirpInternal:
if code == twirpInternal {
status = http.StatusInternalServerError
case twirpUnavailable:
status = http.StatusServiceUnavailable
case twirpUnauthenticated:
status = http.StatusUnauthorized
}
h.responseJSON(w, r, status, map[string]any{"code": code, "msg": err.Error()})
}

View File

@@ -10,7 +10,6 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"testing"
@@ -19,6 +18,8 @@ import (
"github.com/stretchr/testify/require"
)
// v2Call posts a twirp request to the cache service and returns the decoded response.
// Field names are the proto ones, which is what the toolkit's client sends.
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
t.Helper()
@@ -35,6 +36,7 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
return got
}
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
func putBlob(t *testing.T, url string, content []byte) int {
t.Helper()
@@ -44,6 +46,8 @@ func putBlob(t *testing.T, url string, content []byte) int {
require.NoError(t, err)
defer resp.Body.Close()
if resp.StatusCode == http.StatusCreated {
// The Azure SDK client dereferences this header without checking, so a blob upload that
// omits it panics the caller rather than failing it.
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
}
return resp.StatusCode
@@ -62,6 +66,8 @@ func getURL(t *testing.T, url string) []byte {
return body
}
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
// with the upload URL it used.
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
t.Helper()
@@ -77,8 +83,10 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
}), uploadURL
}
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
// or to replace a finalized entry.
func TestCacheServiceV2RoundTrip(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
content := []byte("the cached archive")
@@ -89,8 +97,10 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
require.Equal(t, true, finalized["ok"])
assert.NotEmpty(t, finalized["entry_id"])
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
// and it is an upload URL only: nothing reads a blob back through it.
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
resp, err := http.Get(uploadURL)
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
@@ -103,8 +113,9 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
assert.Equal(t, content, getURL(t, downloadURL))
}
// A large archive is staged as blocks and only put in order by the final block list, so
// blocks that arrive out of order must still be assembled the way the client asked.
func TestCacheServiceV2BlockUpload(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "blocks", "version": "v1"})
@@ -118,6 +129,7 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
blocks[blockID] = []byte(part)
order = append(order, blockID)
}
// Upload in an order that is not the block list order.
for _, blockID := range []string{order[2], order[0], order[1]} {
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
}
@@ -129,17 +141,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
}
list.WriteString(`</BlockList>`)
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()))
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()),
"a client that lost the first answer retries the list it already sent")
var reordered bytes.Buffer
reordered.WriteString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
for _, blockID := range []string{order[1], order[0], order[2]} {
fmt.Fprintf(&reordered, "<Latest>%s</Latest>", blockID)
}
reordered.WriteString(`</BlockList>`)
assert.Equal(t, http.StatusInternalServerError, putBlob(t, uploadURL+"&comp=blocklist", reordered.Bytes()),
"the blocks are already assembled, so a different order would silently disagree with them")
finalized := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "blocks", "version": "v1", "size_bytes": len("hello world!"),
@@ -152,7 +153,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
}
func TestCacheServiceV2Lookups(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
saved, _ := saveV2(t, handler, "deps-abc", "v1", []byte("x"))
require.Equal(t, true, saved["ok"])
@@ -162,6 +162,8 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// The toolkit serialises with the proto field names; the camelCase spellings of the same
// proto JSON mapping are accepted alongside them.
for _, field := range []string{"restore_keys", "restoreKeys"} {
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
@@ -177,6 +179,9 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, again["ok"])
})
// A key that is only a prefix of an existing one is a different entry, so the
// reservation check must be exact and not a restore-key prefix match, or the shorter
// key would be reported as existing and silently never saved.
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
require.Equal(t, true, reserved["ok"])
@@ -202,6 +207,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// The size the client declares is what Commit validates the assembled archive against.
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
@@ -212,6 +218,8 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
// Both API versions are served from one store, so an entry written through v2 is a hit for
// a v1 client asking for the same key and version.
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
require.NoError(t, err)
@@ -224,6 +232,7 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.NotEmpty(t, got["archiveLocation"])
})
// The cache of one repository must stay invisible to another, as it does for the v1 API.
t.Run("another repository sees nothing", func(t *testing.T) {
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
@@ -232,95 +241,3 @@ func TestCacheServiceV2Lookups(t *testing.T) {
assert.Equal(t, false, got["ok"])
})
}
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
tests := []struct {
name string
token string
body string
status int
code string
}{
{"no bearer at all", "", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a bearer nobody registered", "not-a-job", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
{"a body that is not the request", testToken, `{`, http.StatusBadRequest, "malformed"},
{"a request missing its key", testToken, `{"version":"v"}`, http.StatusBadRequest, "invalid_argument"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
handler.ExternalURL()+cacheServiceV2Path+"/CreateCacheEntry", strings.NewReader(tt.body))
require.NoError(t, err)
if tt.token != "" {
req.Header.Set("Authorization", "Bearer "+tt.token)
}
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.status, resp.StatusCode)
got := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &got))
assert.Equal(t, tt.code, got["code"])
assert.NotEmpty(t, got["msg"])
})
}
}
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
t.Parallel()
handler := newTestHandler(t, Policy{})
content := []byte("the cached archive")
const otherJob = "another-jobs-token"
defer handler.RegisterJob(otherJob, JobCredential{Repo: testRepo})()
reserve := func(client *http.Client) map[string]any {
return v2Call(t, handler, client, "CreateCacheEntry", map[string]any{"key": "shared", "version": "v1"})
}
first := reserve(testClient)
require.Equal(t, true, first["ok"])
retry := reserve(testClient)
assert.Equal(t, true, retry["ok"], "a job retrying its own reservation is not a conflict")
entryOf := func(reserved map[string]any) string {
parsed, err := url.Parse(reserved["signed_upload_url"].(string))
require.NoError(t, err)
return parsed.Path
}
assert.Equal(t, entryOf(first), entryOf(retry), "the same reservation, however freshly its URL is signed")
assert.Equal(t, false, reserve(&http.Client{Transport: &bearerTransport{token: otherJob}})["ok"],
"another job's reservation would be finalized against the first upload")
otherClient := &http.Client{Transport: &bearerTransport{token: otherJob}}
reserved := v2Call(t, handler, otherClient, "CreateCacheEntry", map[string]any{"key": "owned", "version": "v1"})
require.Equal(t, true, reserved["ok"])
require.Equal(t, http.StatusCreated, putBlob(t, reserved["signed_upload_url"].(string), content))
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "owned", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"another job's upload is not this job's to finalize")
base := handler.ExternalURL() + apiPath
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
require.NoError(t, err)
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
resp, err := testClient.Do(upload)
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
map[string]any{"key": "v1-made", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
"a reservation this job did not make through v2 is not its to commit")
finalized, _ := saveV2(t, handler, "deps", "v1", content)
require.Equal(t, true, finalized["ok"])
again := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
"key": "deps", "version": "v1", "size_bytes": strconv.Itoa(len(content)),
})
assert.Equal(t, true, again["ok"], "a retry whose first answer was lost must not report a failed save")
assert.Equal(t, finalized["entry_id"], again["entry_id"])
}

View File

@@ -4,8 +4,6 @@
package artifactcache
import "time"
type Request struct {
Key string `json:"key" `
Version string `json:"version"`
@@ -13,18 +11,12 @@ type Request struct {
}
type Cache struct {
ID uint64 `json:"id"`
Repo string `json:"repo"`
Key string `json:"key"`
Version string `json:"version"`
Size int64 `json:"cacheSize"`
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
Complete bool `json:"complete"`
UsedAt int64 `json:"usedAt"`
CreatedAt int64 `json:"createdAt"`
CreatedAtNanos int `json:"createdAtNanos,omitzero"`
}
func (cache *Cache) createdAtNano() int64 {
return cache.CreatedAt*int64(time.Second) + int64(cache.CreatedAtNanos)
ID uint64 `json:"id" boltholdKey:"ID"`
Repo string `json:"repo" boltholdIndex:"Repo"`
Key string `json:"key" boltholdIndex:"Key"`
Version string `json:"version" boltholdIndex:"Version"`
Size int64 `json:"cacheSize"`
Complete bool `json:"complete" boltholdIndex:"Complete"`
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
}

View File

@@ -4,10 +4,7 @@
package artifactcache
import (
"cmp"
"crypto/tls"
"errors"
"fmt"
"net/http"
"net/http/httputil"
"net/url"
@@ -18,38 +15,23 @@ import (
// Gitea implements only the artifact half of it. Forwarding that half from here makes this origin
// the whole service, so ACTIONS_RESULTS_URL can point at it truthfully, which is what the clients
// this runner cannot patch need, docker buildx among them.
//
// The instance to forward to travels with the job registration rather than with configuration, so
// a cache server shared between runners serves each of their instances.
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService/"
// forwardOrNotFound is the router's fallback: the artifact service of the instance the job
// registered with, and the 404 the router would have written otherwise.
func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.URL.Path, artifactServicePath) {
cred, ok := h.lookupCredential(bearerToken(r))
if !ok || cred.Results == "" || !strings.HasPrefix(r.URL.Path, artifactServicePath) {
http.NotFound(w, r)
return
}
cred, ok := h.lookupCredential(bearerToken(r))
if !ok {
h.twirpError(w, r, twirpUnauthenticated, errors.New("unknown bearer token"))
return
}
if cred.Results == "" {
h.twirpError(w, r, twirpInternal, errors.New("no instance is registered for this job"))
return
}
target, err := url.Parse(strings.TrimSuffix(cred.Results, "/"))
if err == nil && (target.Hostname() == "" || (target.Scheme != "http" && target.Scheme != "https")) {
err = errors.New("not an absolute http or https URL the job could use")
}
if err != nil {
h.logger.Errorf("artifact service forward to %q: %v", cred.Results, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("artifact service address %q is unusable: %w", cred.Results, err))
return
}
if !h.mustProxy(r, cred, target) {
redirect := target.JoinPath(r.URL.EscapedPath())
redirect.RawQuery = mergeQuery(target.RawQuery, r.URL.RawQuery)
h.logger.Debugf("%s %s: redirecting to %s", r.Method, r.URL.Path, target)
http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect)
w.WriteHeader(http.StatusBadGateway)
return
}
h.logger.Debugf("%s %s: forwarding to %s", r.Method, r.URL.Path, target)
@@ -62,9 +44,9 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
// directly and never through here.
r.Out.Host = target.Host
},
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
h.logger.Warnf("artifact service forward to %s: %v", target, err)
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
w.WriteHeader(http.StatusBadGateway)
},
}
if cred.InsecureTLS {
@@ -73,17 +55,5 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
proxy.ServeHTTP(w, r)
}
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
}
func mergeQuery(target, request string) string {
if target == "" || request == "" {
return target + request
}
return target + "&" + request
}
// insecureTransport is shared, because a transport per request would pool no connections.
var insecureTransport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
var insecureTransport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so

View File

@@ -4,8 +4,6 @@
package artifactcache
import (
"cmp"
"encoding/json/v2"
"io"
"net/http"
"net/http/httptest"
@@ -16,120 +14,43 @@ import (
"github.com/stretchr/testify/require"
)
// The artifact half is forwarded under the Host Gitea knows itself by, so the URLs it hands back
// still point at Gitea, and nothing else is proxied.
func TestFrontResultsService(t *testing.T) {
t.Parallel()
var gotHost, gotPath, gotProto string
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotHost, gotPath, gotProto = r.Host, r.URL.Path, r.Header.Get("X-Forwarded-Proto")
_, _ = io.WriteString(w, `{"ok":true}`)
}))
defer gitea.Close()
selfSigned := httptest.NewTLSServer(gitea.Config.Handler)
defer selfSigned.Close()
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"})
require.NoError(t, err)
defer handler.Close()
const token, rpc = "forward-token", artifactServicePath + "CreateArtifact"
const token = "forward-token"
noRedirect := func(token string) *http.Client {
return &http.Client{
Transport: &bearerTransport{token: token},
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
}
post := func(path string) (int, map[string]string) {
client := &http.Client{Transport: &bearerTransport{token: token}}
post := func(path string) int {
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+path, nil)
require.NoError(t, err)
resp, err := noRedirect(token).Do(req)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
body := map[string]string{}
_ = json.UnmarshalRead(resp.Body, &body)
return resp.StatusCode, body
resp.Body.Close()
return resp.StatusCode
}
status, body := post(rpc)
assert.Equal(t, http.StatusUnauthorized, status, "an unregistered token is told so, not left to parse a 404 page")
assert.Equal(t, twirpUnauthenticated, body["code"])
assert.Contains(t, body["msg"], "bearer")
assert.NotEmpty(t, body["msg"])
assert.Equal(t, http.StatusNotFound, post(artifactServicePath+"CreateArtifact"),
"an unregistered token is forwarded nowhere")
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: selfSigned.URL, InsecureTLS: true})()
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: gitea.URL})()
status, _ = post(rpc)
require.Equal(t, http.StatusOK, status, "an http job reaching an https instance must proxy")
assert.Equal(t, strings.TrimPrefix(selfSigned.URL, "https://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Equal(t, http.StatusOK, post(artifactServicePath+"CreateArtifact"))
assert.Equal(t, strings.TrimPrefix(gitea.URL, "http://"), gotHost, "Gitea must see the host it mints its URLs from")
assert.Empty(t, gotProto, "a forwarded scheme would make an https Gitea mint http URLs")
assert.Equal(t, rpc, gotPath)
assert.Equal(t, artifactServicePath+"CreateArtifact", gotPath)
gotPath = ""
status, _ = post("/twirp/github.actions.results.api.v1.OtherService/Do")
assert.Equal(t, http.StatusNotFound, status)
status, _ = post("/api/v1/repos/owner/repo")
assert.Equal(t, http.StatusNotFound, status)
assert.Equal(t, http.StatusNotFound, post("/twirp/github.actions.results.api.v1.OtherService/Do"))
assert.Equal(t, http.StatusNotFound, post("/api/v1/repos/owner/repo"))
assert.Empty(t, gotPath, "only the artifact service is forwarded")
tests := []struct {
name string
cred JobCredential
forwardedProto string
reqQuery string
want int
location string
wantMsg string
}{
{name: "an instance the job reaches itself", cred: JobCredential{Results: gitea.URL}, want: http.StatusTemporaryRedirect},
{name: "a sub-path instance keeps its prefix", cred: JobCredential{Results: gitea.URL + "/sub"}, want: http.StatusTemporaryRedirect},
{name: "an instance with no certificate to distrust", cred: JobCredential{Results: gitea.URL, InsecureTLS: true}, want: http.StatusTemporaryRedirect},
{
name: "both queries survive", cred: JobCredential{Results: gitea.URL + "?tenant=one"}, reqQuery: "?run=7",
want: http.StatusTemporaryRedirect, location: gitea.URL + rpc + "?tenant=one&run=7",
},
{
name: "a trusted https instance is reached by the job itself",
cred: JobCredential{Results: selfSigned.URL}, forwardedProto: "https", want: http.StatusTemporaryRedirect,
},
{
name: "the job does not share this server's disregard for the certificate",
cred: JobCredential{Results: selfSigned.URL, InsecureTLS: true}, forwardedProto: "https", want: http.StatusOK,
},
{name: "an http job is not redirected to an https instance", cred: JobCredential{Results: selfSigned.URL}, want: http.StatusInternalServerError},
{
name: "a forwarded https scheme onto an http instance is proxied",
cred: JobCredential{Results: gitea.URL}, forwardedProto: "https", want: http.StatusOK,
},
{
name: "a scheme the job could not use is refused before it is sent one",
cred: JobCredential{Results: "ftp://gitea.example"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
{
name: "an address with no host is refused before it is sent one",
cred: JobCredential{Results: "http://:3000"}, want: http.StatusInternalServerError, wantMsg: "unusable",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
defer handler.RegisterJob(tt.name, tt.cred)()
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+rpc+tt.reqQuery, nil)
require.NoError(t, err)
if tt.forwardedProto != "" {
req.Header.Set("X-Forwarded-Proto", tt.forwardedProto)
}
resp, err := noRedirect(tt.name).Do(req)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, tt.want, resp.StatusCode)
switch tt.want {
case http.StatusTemporaryRedirect:
assert.Equal(t, cmp.Or(tt.location, tt.cred.Results+rpc), resp.Header.Get("Location"))
case http.StatusInternalServerError:
body := map[string]string{}
require.NoError(t, json.UnmarshalRead(resp.Body, &body))
assert.Equal(t, twirpInternal, body["code"])
assert.Contains(t, body["msg"], cmp.Or(tt.wantMsg, tt.cred.Results))
}
})
}
}

View File

@@ -1,348 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"bytes"
"cmp"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/json/v2"
"encoding/xml"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net/http"
"net/url"
"os"
"path"
"slices"
"strconv"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"golang.org/x/sync/errgroup"
)
// S3Options configures the bucket that stores completed cache entries.
type S3Options struct {
Endpoint string
Region string
Bucket string
Prefix string
PathStyle *bool // nil uses virtual-hosted requests on AWS and path-style ones elsewhere
AccessKey string
SecretKey string
}
type cacheStorage interface {
Exist(uint64) (bool, error)
Write(uint64, int64, io.Reader) error
WriteBlock(uint64, string, io.Reader) error
OrderBlocks(uint64, []string) error
Commit(uint64, int64) (int64, error)
Serve(http.ResponseWriter, *http.Request, uint64)
Remove(uint64) error
}
const s3PartSize = 64 << 20
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
type s3Storage struct {
*Storage
client *http.Client
bucketURL *url.URL
opts S3Options
}
type s3Part struct {
PartNumber int
ETag string
}
func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
endpoint, err := url.Parse(opts.Endpoint)
if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") {
return nil, fmt.Errorf("invalid S3 endpoint %q", opts.Endpoint)
}
transport, _ := http.DefaultTransport.(*http.Transport)
transport = transport.Clone()
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
if opts.PathStyle != nil {
pathStyle = *opts.PathStyle
}
if pathStyle {
endpoint.Path = "/" + opts.Bucket
} else {
endpoint.Host = opts.Bucket + "." + endpoint.Host
endpoint.Path = "/"
}
opts.Region = cmp.Or(opts.Region, "us-east-1")
return &s3Storage{Storage: staging, client: &http.Client{Transport: transport}, bucketURL: endpoint, opts: opts}, nil
}
func (s *s3Storage) blobKey(id uint64) string {
return path.Join(s.opts.Prefix, "blobs", strconv.FormatUint(id, 10))
}
func (s *s3Storage) metadataDir() string {
return path.Join(s.opts.Prefix, "metadata") + "/"
}
func (s *s3Storage) metadataKey(id uint64) string {
return s.metadataDir() + strconv.FormatUint(id, 10) + ".json"
}
func (s *s3Storage) url(key string, query url.Values) *url.URL {
u := *s.bucketURL
u.Path = path.Join(u.Path, key)
u.RawPath = strings.NewReplacer("+", "%20", "%2F", "/").Replace(url.QueryEscape(u.Path)) // SigV4 encoding
u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20")
return &u
}
// sign authorizes req with AWS Signature Version 4, leaving the payload unsigned so bodies can stream.
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
func (s *s3Storage) sign(req *http.Request, now time.Time) {
date := now.UTC().Format("20060102T150405Z")
req.Header.Set("X-Amz-Date", date)
if req.Header.Get("X-Amz-Content-Sha256") == "" {
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
}
headers := map[string]string{"host": req.URL.Host}
for name, values := range req.Header {
if name = strings.ToLower(name); strings.HasPrefix(name, "x-amz-") {
headers[name] = strings.Join(values, ",")
}
}
names := slices.Sorted(maps.Keys(headers))
canonical := []string{req.Method, req.URL.EscapedPath(), req.URL.RawQuery}
for _, name := range names {
canonical = append(canonical, name+":"+headers[name])
}
canonical = append(canonical, "", strings.Join(names, ";"), req.Header.Get("X-Amz-Content-Sha256"))
scope := date[:8] + "/" + s.opts.Region + "/s3/aws4_request"
signature := []byte("AWS4" + s.opts.SecretKey)
for _, data := range []string{date[:8], s.opts.Region, "s3", "aws4_request", fmt.Sprintf("AWS4-HMAC-SHA256\n%s\n%s\n%x", date, scope, sha256.Sum256([]byte(strings.Join(canonical, "\n"))))} { // key derivation, the last round signs
mac := hmac.New(sha256.New, signature)
_, _ = mac.Write([]byte(data))
signature = mac.Sum(nil)
}
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%x", s.opts.AccessKey, scope, strings.Join(names, ";"), signature))
}
// do returns the response to a signed request when it succeeded, fs.ErrNotExist for a missing key.
func (s *s3Storage) do(ctx context.Context, method, key string, query url.Values, header http.Header, body io.Reader) (*http.Response, error) {
var size int64
if sized, ok := body.(interface{ Size() int64 }); ok {
size = sized.Size()
}
if size == 0 {
body = nil
}
req, err := http.NewRequestWithContext(ctx, method, s.url(key, query).String(), body)
if err != nil {
return nil, err
}
req.ContentLength = size
maps.Copy(req.Header, header)
s.sign(req, time.Now())
resp, err := s.client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode < http.StatusMultipleChoices {
return resp, nil
}
defer drain(resp)
var failure struct{ Code, Message string }
_ = xml.NewDecoder(resp.Body).Decode(&failure)
if resp.StatusCode == http.StatusNotFound && (failure.Code == "NoSuchKey" || failure.Code == "" && method == http.MethodHead) {
return nil, fs.ErrNotExist
}
return nil, fmt.Errorf("S3 %s %s: %s", method, key, cmp.Or(strings.TrimSpace(failure.Code+" "+failure.Message), resp.Status))
}
func (s *s3Storage) call(ctx context.Context, method, key string, query url.Values, body io.Reader, result any) (http.Header, error) {
resp, err := s.do(ctx, method, key, query, nil, body)
if err != nil {
return nil, err
}
defer drain(resp)
if result == nil {
return resp.Header, nil
}
return resp.Header, xml.NewDecoder(resp.Body).Decode(result)
}
// drain reads a response to its end before closing it, so its connection gets reused.
func drain(resp *http.Response) {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
func (s *s3Storage) Exist(id uint64) (bool, error) {
_, err := s.call(context.Background(), http.MethodHead, s.blobKey(id), nil, nil, nil)
if errors.Is(err, fs.ErrNotExist) {
return false, nil
}
return err == nil, err
}
func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
written, err := s.Storage.Commit(id, size)
if err != nil {
return 0, err
}
defer func() { _ = s.Storage.Remove(id) }()
file, err := os.Open(s.filename(id))
if err != nil {
return 0, err
}
defer file.Close()
if err := s.upload(context.Background(), s.blobKey(id), file, written); err != nil {
return 0, fmt.Errorf("upload cache to S3: %w", err)
}
return written, nil
}
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
if size <= s3PartSize {
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
return err
}
var created struct {
UploadID string `xml:"UploadId"`
}
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
return err
}
defer func() {
if uploadErr != nil && created.UploadID != "" {
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
}
}()
var completion struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []s3Part `xml:"Part"`
}
partSize := max(int64(s3PartSize), (size-1)/10000+1)
completion.Parts = make([]s3Part, (size-1)/partSize+1)
workers, workerCtx := errgroup.WithContext(ctx)
workers.SetLimit(4)
for part := range completion.Parts {
workers.Go(func() (err error) {
defer common.RecoverPanic(&err)
offset := int64(part) * partSize
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
return err
})
}
if err := workers.Wait(); err != nil {
return err
}
body, err := xml.Marshal(completion)
if err != nil {
return err
}
var result struct{ Code, Message string } // a completion can fail after its 200 status
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploadId": {created.UploadID}}, bytes.NewReader(body), &result); err != nil {
return err
}
if result.Code != "" {
return fmt.Errorf("S3 complete upload %s: %s %s", key, result.Code, result.Message)
}
return nil
}
func (s *s3Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
header := http.Header{}
if byteRange := r.Header.Get("Range"); byteRange != "" {
header.Set("Range", byteRange)
}
resp, err := s.do(r.Context(), http.MethodGet, s.blobKey(id), nil, header, nil)
if errors.Is(err, fs.ErrNotExist) {
http.NotFound(w, r)
return
} else if err != nil {
http.Error(w, "read cache from S3", http.StatusBadGateway)
return
}
defer resp.Body.Close()
for _, name := range []string{"Content-Length", "Content-Range"} {
if value := resp.Header.Get(name); value != "" {
w.Header().Set(name, value)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = io.Copy(w, resp.Body)
}
// Remove unpublishes an entry before deleting its blob, so no runner imports it without its bytes.
func (s *s3Storage) Remove(id uint64) error {
for _, key := range []string{s.metadataKey(id), s.blobKey(id)} {
if _, err := s.call(context.Background(), http.MethodDelete, key, nil, nil, nil); err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("remove %s from S3: %w", key, err)
}
}
return s.Storage.Remove(id)
}
func (s *s3Storage) publish(cache *Cache) error {
data, err := json.Marshal(cache)
if err != nil {
return err
}
if _, err := s.call(context.Background(), http.MethodPut, s.metadataKey(cache.ID), nil, bytes.NewReader(data), nil); err != nil {
return fmt.Errorf("publish cache to S3: %w", err)
}
return nil
}
func (s *s3Storage) published(ctx context.Context) (map[uint64]bool, error) {
ids := make(map[uint64]bool)
query := url.Values{"list-type": {"2"}, "prefix": {s.metadataDir()}}
for {
var page struct {
Contents []struct{ Key string }
IsTruncated bool
NextContinuationToken string
}
if _, err := s.call(ctx, http.MethodGet, "", query, nil, &page); err != nil {
return nil, fmt.Errorf("list S3 cache metadata: %w", err)
}
for _, object := range page.Contents {
if id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(object.Key, s.metadataDir()), ".json"), 10, 64); err == nil {
ids[id] = true
}
}
if !page.IsTruncated {
return ids, nil
}
query.Set("continuation-token", page.NextContinuationToken)
}
}
func (s *s3Storage) fetch(ctx context.Context, id uint64) (*Cache, error) {
resp, err := s.do(ctx, http.MethodGet, s.metadataKey(id), nil, nil, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
cache := &Cache{}
if err := json.UnmarshalRead(resp.Body, cache); err != nil {
return nil, fmt.Errorf("read S3 cache metadata %d: %w", id, err)
}
return cache, nil
}

View File

@@ -1,141 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"encoding/xml"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
t.Parallel()
var objects sync.Map
var uploads atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
key := strings.TrimPrefix(request.URL.Path, "/cache/")
switch {
case request.Method == http.MethodPut:
if strings.Contains(key, "/blobs/") {
uploads.Add(1)
}
body, err := io.ReadAll(request.Body)
assert.NoError(t, err)
objects.Store(key, body)
case request.URL.Query().Has("list-type"):
_, _ = io.WriteString(writer, "<ListBucketResult>")
objects.Range(func(stored, _ any) bool {
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
return true
})
_, _ = io.WriteString(writer, "</ListBucketResult>")
default:
if body, ok := objects.Load(key); ok {
_, _ = writer.Write(body.([]byte))
} else {
writer.WriteHeader(http.StatusNotFound)
}
}
}))
t.Cleanup(server.Close)
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
entry := map[string]any{"key": "deps", "version": "v1"}
started := time.Now()
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
require.Equal(t, true, finalized["ok"])
assert.EqualValues(t, 1, uploads.Load())
id, err := strconv.ParseUint(finalized["entry_id"].(string), 10, 64)
require.NoError(t, err)
cache, err := saver.s3.fetch(t.Context(), id)
require.NoError(t, err)
assert.WithinRange(t, time.Unix(cache.CreatedAt, int64(cache.CreatedAtNanos)), started, time.Now())
require.NoError(t, restorer.importMetadata(t.Context()))
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
require.Equal(t, true, found["ok"])
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
for _, cache := range []*Cache{
{ID: 2, Repo: testRepo, Key: "deps-old", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt},
{ID: 1, Repo: testRepo, Key: "deps-new", Version: "v1", Complete: true, CreatedAt: cache.CreatedAt, CreatedAtNanos: 1},
} {
require.NoError(t, saver.s3.publish(cache))
objects.Store(saver.s3.blobKey(cache.ID), []byte(cache.Key))
}
require.NoError(t, restorer.importMetadata(t.Context()))
found = v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", map[string]any{"key": "deps-", "version": "v1"})
require.Equal(t, true, found["ok"])
assert.Equal(t, "deps-new", found["matched_key"])
assert.Equal(t, []byte("deps-new"), getURL(t, found["signed_download_url"].(string)))
objects.Clear()
require.NoError(t, restorer.importMetadata(t.Context()))
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
}
type s3TestTransport func(*http.Request) (*http.Response, error)
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return transport(req)
}
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
t.Parallel()
for _, failingPart := range []string{"", "2"} {
var parts, aborts atomic.Int32
var completion struct {
Parts []s3Part `xml:"Part"`
}
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
switch {
case query.Has("uploads"):
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
case req.Method == http.MethodPut:
parts.Add(1)
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
if query.Get("partNumber") == failingPart {
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
}
case req.Method == http.MethodPost:
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
case req.Method == http.MethodDelete:
aborts.Add(1)
}
return response, nil
})}
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
assert.EqualValues(t, 3, parts.Load())
if failingPart == "" {
require.NoError(t, err)
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
assert.Zero(t, aborts.Load())
} else {
require.ErrorContains(t, err, "InvalidPart")
assert.EqualValues(t, 1, aborts.Load())
}
}
}
func TestS3StorageSignsLikeAWS(t *testing.T) {
t.Parallel()
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
storage.sign(req, time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
assert.Equal(t, "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", req.Header.Get("Authorization"))
}

View File

@@ -5,10 +5,8 @@
package artifactcache
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json/v2"
"fmt"
"io"
"net/http"
@@ -32,11 +30,13 @@ func NewStorage(rootDir string) (*Storage, error) {
}
func (s *Storage) Exist(id uint64) (bool, error) {
_, err := os.Stat(s.filename(id))
if os.IsNotExist(err) {
name := s.filename(id)
if _, err := os.Stat(name); os.IsNotExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return err == nil, err
return true, nil
}
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
@@ -66,18 +66,6 @@ func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error
// rename pass is safe because a staged name always carries blockFilePrefix and a target name
// never does, so no rename can collide with a block not yet moved.
func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
wanted, err := json.Marshal(blockIDs) // a block id is client-chosen, so it cannot be a separator
if err != nil {
return err
}
list := filepath.Join(s.tempDir(id), blockFilePrefix+"list")
if recorded, err := os.ReadFile(list); err == nil {
// A retry repeats the list it already sent; another one would reorder what is now staged.
if !bytes.Equal(recorded, wanted) {
return fmt.Errorf("cache %d was already assembled from a different block list", id)
}
return nil
}
for i, blockID := range blockIDs {
if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
if os.IsNotExist(err) {
@@ -86,11 +74,13 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
return err
}
}
return os.WriteFile(list, wanted, 0o600)
return nil
}
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
defer func() {
_ = os.RemoveAll(s.tempDir(id))
}()
name := s.filename(id)
tempNames, err := s.tempNames(id)
@@ -149,7 +139,8 @@ func assemble(name string, tempNames []string) (int64, error) {
}
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
http.ServeFile(w, r, s.filename(id))
name := s.filename(id)
http.ServeFile(w, r, name)
}
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can

View File

@@ -11,15 +11,15 @@ import (
"fmt"
"io"
"io/fs"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/julienschmidt/httprouter"
)
type FileContainerResourceURL struct {
@@ -66,44 +66,39 @@ func writeJSON(w http.ResponseWriter, value any) {
}
}
func uploads(router *http.ServeMux, baseDir string) {
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
func uploads(router *httprouter.Router, baseDir string) {
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId")
writeJSON(w, FileContainerResourceURL{
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
})
})
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
itemPath := req.URL.Query().Get("itemPath")
runID := params.ByName("runId")
if req.Header.Get("Content-Encoding") == "gzip" {
itemPath += gzipExtension
}
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
safeRunPath := safeResolve(baseDir, runID)
safePath := safeResolve(safeRunPath, itemPath)
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
panic(err)
}
var start int64
if contentRange := req.Header.Get("Content-Range"); contentRange != "" {
if _, err := fmt.Sscanf(contentRange, "bytes %d-", &start); err != nil || start < 0 {
http.Error(w, "invalid content range", http.StatusBadRequest)
return
}
}
flags := os.O_CREATE | os.O_WRONLY
if start == 0 {
flags |= os.O_TRUNC
flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC
appendUpload := req.Header.Get("Content-Range")
if appendUpload != "" && !strings.HasPrefix(appendUpload, "bytes 0-") {
flags = os.O_CREATE | os.O_WRONLY | os.O_APPEND
}
file, err := os.OpenFile(safePath, flags, 0o644)
if err != nil {
panic(err)
}
defer file.Close()
if _, err := file.Seek(start, io.SeekStart); err != nil {
panic(err)
}
if req.Body == nil {
panic(errors.New("no body given"))
}
@@ -118,19 +113,21 @@ func uploads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
writeJSON(w, ResponseMessage{
Message: "success",
})
})
}
func downloads(router *http.ServeMux, baseDir string) {
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
runID := req.PathValue("runId")
func downloads(router *httprouter.Router, baseDir string) {
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
runID := params.ByName("runId")
entries, err := os.ReadDir(safeResolve(baseDir, runID))
if err != nil && !os.IsNotExist(err) {
safePath := safeResolve(baseDir, runID)
entries, err := os.ReadDir(safePath)
if err != nil {
panic(err)
}
@@ -148,16 +145,13 @@ func downloads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
container := req.PathValue("container")
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
container := params.ByName("container")
itemPath := req.URL.Query().Get("itemPath")
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
var files []ContainerItem
err := filepath.WalkDir(safePath, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() {
rel, err := filepath.Rel(safePath, path)
if err != nil {
@@ -166,16 +160,20 @@ func downloads(router *http.ServeMux, baseDir string) {
// if it was upload as gzip
rel = strings.TrimSuffix(rel, gzipExtension)
path := filepath.Join(itemPath, rel)
rel = filepath.ToSlash(rel)
path = filepath.ToSlash(path)
files = append(files, ContainerItem{
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
Path: path,
ItemType: "file",
ContentLocation: (&url.URL{Scheme: "http", Host: req.Host, Path: "/artifact/" + container + "/" + itemPath + "/" + filepath.ToSlash(rel)}).String(),
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
})
}
return nil
})
if err != nil && !os.IsNotExist(err) {
if err != nil {
panic(err)
}
@@ -184,8 +182,10 @@ func downloads(router *http.ServeMux, baseDir string) {
})
})
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
safePath := safeResolve(baseDir, req.PathValue("path"))
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
path := params.ByName("path")[1:]
safePath := safeResolve(baseDir, path)
file, err := os.Open(safePath)
if err != nil {
@@ -213,24 +213,23 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
return cancel
}
router := http.NewServeMux()
router := httprouter.New()
logger.Debugf("Artifacts base path '%s'", artifactPath)
uploads(router, artifactPath)
downloads(router, artifactPath)
server := &http.Server{
Addr: net.JoinHostPort(addr, port),
Addr: fmt.Sprintf("%s:%s", addr, port),
ReadHeaderTimeout: 2 * time.Second,
Handler: router,
}
// run server
go func() {
logger.Infof("Start server on http://%s", server.Addr)
logger.Infof("Start server on http://%s:%s", addr, port)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
logger.Error(err)
os.Exit(1)
logger.Fatal(err)
}
}()

View File

@@ -19,13 +19,14 @@ import (
"testing"
"time"
"github.com/julienschmidt/httprouter"
"github.com/stretchr/testify/require"
)
func TestArtifactFlow(t *testing.T) {
artifactPath := t.TempDir()
router := http.NewServeMux()
router := httprouter.New()
uploads(router, artifactPath)
downloads(router, artifactPath)
server := httptest.NewServer(router)
@@ -48,17 +49,8 @@ func TestArtifactFlow(t *testing.T) {
return resp.StatusCode, data
}
t.Run("no-artifacts", func(t *testing.T) {
status, data := request(t, http.MethodGet, baseURL+"/_apis/pipelines/workflows/0/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
var list NamedFileContainerResourceURLResponse
require.NoError(t, json.Unmarshal(data, &list))
require.Zero(t, list.Count)
require.Empty(t, list.Value)
})
t.Run("upload-and-download", func(t *testing.T) {
const runID, item, content = "1", "my-artifact/data#1%25.txt", "hello artifact\n"
const runID, item, content = "1", "my-artifact/data.txt", "hello artifact\n"
status, data := request(t, http.MethodPost, baseURL+"/_apis/pipelines/workflows/"+runID+"/artifacts", nil, nil)
require.Equal(t, http.StatusOK, status, string(data))
@@ -90,13 +82,13 @@ func TestArtifactFlow(t *testing.T) {
require.NoError(t, json.Unmarshal(data, &items))
require.Len(t, items.Value, 1)
require.Equal(t, "file", items.Value[0].ItemType)
require.Equal(t, item, items.Value[0].Path)
require.Equal(t, "my-artifact/data.txt", items.Value[0].Path)
status, data = request(t, http.MethodGet, items.Value[0].ContentLocation, nil, nil)
require.Equal(t, http.StatusOK, status)
require.Equal(t, content, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, filepath.FromSlash(item)))
stored, err := os.ReadFile(filepath.Join(artifactPath, runID, "my-artifact", "data.txt"))
require.NoError(t, err)
require.Equal(t, content, string(stored))
})
@@ -104,25 +96,16 @@ func TestArtifactFlow(t *testing.T) {
t.Run("content-range", func(t *testing.T) {
const rawURL = "/upload/4?itemPath=chunks.txt"
status, data := request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("first"),
http.Header{"Content-Range": []string{"bytes 0-4/12"}})
http.Header{"Content-Range": []string{"bytes 0-4/11"}})
require.Equal(t, http.StatusOK, status, string(data))
for range 2 {
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
http.Header{"Content-Range": []string{"bytes 5-11/12"}})
require.Equal(t, http.StatusOK, status, string(data))
}
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader("-second"),
http.Header{"Content-Range": []string{"bytes 5-11/11"}})
require.Equal(t, http.StatusOK, status, string(data))
stored, err := os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
require.NoError(t, err)
require.Equal(t, "first-second", string(stored))
status, data = request(t, http.MethodPut, baseURL+rawURL, strings.NewReader(""),
http.Header{"Content-Range": []string{"bytes 0--1/0"}})
require.Equal(t, http.StatusOK, status, string(data))
stored, err = os.ReadFile(filepath.Join(artifactPath, "4", "chunks.txt"))
require.NoError(t, err)
require.Empty(t, stored)
})
t.Run("gzip-roundtrip", func(t *testing.T) {

View File

@@ -4,15 +4,15 @@
package common
import (
"context"
"errors"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/sirupsen/logrus"
)
func TestDryrunContext(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
if Dryrun(ctx) {
t.Fatal("plain context should not be dryrun")
}
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
}
func TestJobErrorContainer(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
err := errors.New("job failed")
SetJobError(ctx, err)
@@ -40,27 +40,34 @@ func TestJobErrorContainer(t *testing.T) {
}
}
func TestLoggerAndHandlerContext(t *testing.T) {
ctx := t.Context()
if Logger(ctx) != logging.Default() {
t.Fatal("plain context should use the default logger")
func TestLoggerAndHookContext(t *testing.T) {
ctx := context.Background()
if Logger(ctx) != logrus.StandardLogger() {
t.Fatal("plain context should use standard logger")
}
if LoggerHandler(ctx) != nil {
t.Fatal("plain context should not have a logger handler")
}
if StepIDs(ctx) != nil {
t.Fatal("plain context should not have step IDs")
if LoggerHook(ctx) != nil {
t.Fatal("plain context should not have a logger hook")
}
logger := logging.New(logtest.New())
logger := logrus.New()
ctx = WithLogger(ctx, logger)
if Logger(ctx) != logger {
t.Fatal("WithLogger should set logger")
}
handler := logtest.New()
ctx = WithLoggerHandler(ctx, handler)
if LoggerHandler(ctx) != handler {
t.Fatal("WithLoggerHandler should set handler")
hook := testHook{}
ctx = WithLoggerHook(ctx, hook)
if LoggerHook(ctx) != hook {
t.Fatal("WithLoggerHook should set hook")
}
}
type testHook struct{}
func (testHook) Levels() []logrus.Level {
return logrus.AllLevels
}
func (testHook) Fire(*logrus.Entry) error {
return nil
}

View File

@@ -14,8 +14,13 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
// Dryrun returns true if the current context is dryrun
func Dryrun(ctx context.Context) bool {
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
return dryrun
val := ctx.Value(dryrunContextKeyVal)
if val != nil {
if dryrun, ok := val.(bool); ok {
return dryrun
}
}
return false
}
// WithDryrun adds a value to the context for dryrun

View File

@@ -9,7 +9,7 @@ import (
"fmt"
"runtime/debug"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
)
// Executor define contract for the steps of a workflow
@@ -21,7 +21,8 @@ type Conditional func(ctx context.Context) bool
// NewInfoExecutor is an executor that logs messages
func NewInfoExecutor(format string, args ...any) Executor {
return func(ctx context.Context) error {
Logger(ctx).Infof(format, args...)
logger := Logger(ctx)
logger.Infof(format, args...)
return nil
}
}
@@ -29,7 +30,8 @@ func NewInfoExecutor(format string, args ...any) Executor {
// NewDebugExecutor is an executor that logs messages
func NewDebugExecutor(format string, args ...any) Executor {
return func(ctx context.Context) error {
Logger(ctx).Debugf(format, args...)
logger := Logger(ctx)
logger.Debugf(format, args...)
return nil
}
}
@@ -37,7 +39,9 @@ func NewDebugExecutor(format string, args ...any) Executor {
// NewPipelineExecutor creates a new executor from a series of other executors
func NewPipelineExecutor(executors ...Executor) Executor {
if len(executors) == 0 {
return NewErrorExecutor(nil)
return func(ctx context.Context) error {
return nil
}
}
var rtn Executor
for _, executor := range executors {
@@ -57,13 +61,6 @@ func NewErrorExecutor(err error) Executor {
}
}
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
if recovered := recover(); recovered != nil {
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
*err = fmt.Errorf("panic: %v", recovered)
}
}
// NewParallelExecutor creates a new executor from a parallel of other executors
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
if len(executors) == 0 {
@@ -77,29 +74,34 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
errs := make(chan error, len(executors))
if 1 > parallel {
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
parallel = 1
}
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
for workerID := range parallel {
go func() {
logging.Debugf("Worker %d started", workerID)
for i := 0; i < parallel; i++ {
go func(workerID int, work <-chan Executor, errs chan<- error) {
log.Debugf("Worker %d started", workerID)
taskCount := 0
for executor := range work {
taskCount++
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
log.Debugf("Worker %d executing task %d", workerID, taskCount)
// Recover from panics in executors to avoid crashing the worker
// goroutine which would leave the runner process hung.
// https://gitea.com/gitea/runner/issues/371
errs <- func() (err error) {
defer RecoverPanic(&err)
defer func() {
if r := recover(); r != nil {
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
err = fmt.Errorf("panic: %v", r)
}
}()
return executor(ctx)
}()
}
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
}()
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
}(i, work, errs)
}
for i := range executors {
@@ -148,7 +150,12 @@ func (e Executor) If(conditional Conditional) Executor {
// IfNot only runs this executor if conditional is true
func (e Executor) IfNot(conditional Conditional) Executor {
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
return func(ctx context.Context) error {
if !conditional(ctx) {
return e(ctx)
}
return nil
}
}
// IfBool only runs this executor if conditional is true
@@ -164,10 +171,7 @@ func (e Executor) Finally(finally Executor) Executor {
err := e(ctx)
err2 := finally(ctx)
if err2 != nil {
if err != nil {
return fmt.Errorf("error occurred running finally: %w (original error: %w)", err2, err)
}
return fmt.Errorf("error occurred running finally: %w (original error: %v)", err2, err)
return fmt.Errorf("error occurred running finally: %v (original error: %v)", err2, err)
}
return err
}

View File

@@ -7,7 +7,8 @@ package common
import (
"context"
"errors"
"fmt"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
@@ -17,25 +18,42 @@ import (
)
func TestNewWorkflow(t *testing.T) {
ctx := t.Context()
require.NoError(t, NewPipelineExecutor()(ctx))
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
assert := assert.New(t)
ctx := context.Background()
// empty
emptyWorkflow := NewPipelineExecutor()
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
// error case
errorWorkflow := NewErrorExecutor(errors.New("test error"))
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
// multiple success case
runcount := 0
increment := func(_ctx context.Context) error {
runcount++
return nil
}
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
assert.Equal(t, 2, runcount)
successWorkflow := NewPipelineExecutor(
func(ctx context.Context) error {
runcount++
return nil
},
func(ctx context.Context) error {
runcount++
return nil
})
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(2, runcount)
}
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
// find the gate already open so the last one still finishes with no partner left.
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
var counted, active, peak atomic.Int32
var once sync.Once
reached := make(chan struct{})
return func(_ctx context.Context) error {
return func(ctx context.Context) error {
counted.Add(1)
running := active.Add(1)
for {
@@ -54,72 +72,93 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
}
func TestNewParallelExecutor(t *testing.T) {
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
t.Run(name, func(t *testing.T) {
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
assert.Equal(t, int32(3), count.Load())
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
})
}
ctx := context.Background()
exec, count, maxActive := concurrencyProbe(2)
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
// parallelism below 1 falls back to a single worker
exec, count, maxActive = concurrencyProbe(1)
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
}
func TestNewParallelExecutorEmpty(t *testing.T) {
assert := assert.New(t)
ctx := t.Context()
ctx := context.Background()
require.NoError(t, NewParallelExecutor(2)(ctx))
canceledCtx, cancel := context.WithCancel(t.Context())
canceledCtx, cancel := context.WithCancel(context.Background())
cancel()
err := NewParallelExecutor(2)(canceledCtx)
assert.ErrorIs(err, context.Canceled)
}
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
for name, tc := range map[string]struct {
canceled bool
parallel, tasks int
}{
"failure": {false, 2, 5},
"canceled": {true, 2, 5},
"canceled single task": {true, 1, 1},
} {
t.Run(name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if tc.canceled {
cancel()
}
var calls atomic.Int32
executors := make([]Executor, tc.tasks)
for i := range executors {
executors[i] = func(_ctx context.Context) error {
calls.Add(1)
if i == tc.tasks/2 {
return errors.New("fake error")
}
return nil
}
}
func TestNewParallelExecutorFailed(t *testing.T) {
assert := assert.New(t)
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
if tc.canceled {
require.ErrorIs(t, err, context.Canceled)
} else {
require.EqualError(t, err, "fake error")
ctx, cancel := context.WithCancel(context.Background())
cancel()
count := 0
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count++
return errors.New("fake error")
})
err := NewParallelExecutor(1, errorWorkflow)(ctx)
assert.Equal(1, count)
assert.ErrorIs(context.Canceled, err)
}
func TestNewParallelExecutorCanceled(t *testing.T) {
assert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
cancel()
errExpected := errors.New("fake error")
var count atomic.Int32
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count.Add(1)
return nil
})
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count.Add(1)
return errExpected
})
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
assert.Equal(int32(3), count.Load())
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
}
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
var successCount atomic.Int32
executors := make([]Executor, 5)
for i := range executors {
executors[i] = func(ctx context.Context) error {
if i == 2 {
return errors.New("fake error")
}
assert.Equal(t, int32(tc.tasks), calls.Load())
})
successCount.Add(1)
return nil
}
}
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
}
func TestExecutorConditionalsAndFinally(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
var calls []string
record := func(name string) Executor {
return func(_ctx context.Context) error {
return func(ctx context.Context) error {
calls = append(calls, name)
return nil
}
@@ -131,17 +170,19 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
require.NoError(t, record("if-bool").IfBool(true)(ctx))
require.NoError(t, record("main").Finally(record("finally"))(ctx))
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("calls = %v, want %v", calls, want)
}
}
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
mainErr := errors.New("main failed")
finalErr := errors.New("cleanup failed")
for _, mainErr := range []error{context.Canceled, nil} {
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
require.EqualError(t, err, fmt.Sprintf("error occurred running finally: cleanup failed (original error: %v)", mainErr))
require.ErrorIs(t, err, finalErr)
if mainErr != nil {
require.ErrorIs(t, err, mainErr)
}
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
require.Error(t, err)
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
t.Fatalf("finally error = %q, want both cleanup and original error", err)
}
}

View File

@@ -1,349 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"errors"
"io"
"math/rand/v2"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
"runtime"
"strconv"
"strings"
"sync"
"time"
"gitea.com/gitea/runner/act/common"
)
var actionFetches sync.Map
const treeSizes = ".sizes" // beside the tree rather than in it, which is copied into job containers as it is
type treeFile struct {
Name []byte // not a string, which JSON would turn into UTF-8, as git allows any bytes but NUL
Size int64
}
type actionFetch struct {
done chan struct{}
dir, sha string
err error
retry bool // the leader was cancelled or panicked, so followers fetch themselves
}
var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/runner backs off
if seconds, err := strconv.Atoi(retryAfter); err == nil {
return min(max(time.Duration(seconds)*time.Second, 10*time.Second), 10*time.Minute)
}
return time.Duration(10+rand.IntN(21)) * time.Second
}
func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) { // input.Dir is one clone URL and ref's, holding a directory per commit
if !reflect.ValueOf(input).Comparable() {
return fetchAction(ctx, input)
}
for ctx.Err() == nil {
call := &actionFetch{done: make(chan struct{}), retry: true}
previous, loaded := actionFetches.LoadOrStore(input, call)
if !loaded {
func() {
defer func() { actionFetches.Delete(input); close(call.done) }()
call.dir, call.sha, call.err = fetchAction(ctx, input)
call.retry = ctx.Err() != nil
}()
if call.retry {
continue
}
return call.dir, call.sha, call.err
}
call, _ = previous.(*actionFetch)
select {
case <-ctx.Done():
case <-call.done:
if !call.retry && ctx.Err() == nil {
return call.dir, call.sha, call.err
}
}
}
return "", "", ctx.Err()
}
func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) {
unlock, err := AcquireCloneLock(ctx, input.Dir)
if err != nil {
return "", "", err
}
defer unlock()
if err := os.MkdirAll(input.Dir, 0o755); err != nil {
return "", "", err
}
tree := func(sha string) string { return filepath.Join(input.Dir, sha) }
record := filepath.Join(input.Dir, "ref")
data, _ := os.ReadFile(record)
sha, etag, _ := strings.Cut(string(data), "\n")
switch {
case !isFullObjectID(sha):
sha, etag = "", ""
case input.OfflineMode && exists(tree(sha)) && !exists(tree(sha)+treeSizes):
return tree(sha), sha, nil // from an older runner, with no sizes to check, so offline mode uses it as before
case !treeIntact(input.Dir, sha):
sha, etag = "", ""
case input.OfflineMode || isFullObjectID(input.Ref):
return tree(sha), sha, nil
}
newSha, newEtag, err := downloadArchive(ctx, input, sha, etag)
if err != nil && ctx.Err() == nil {
common.Logger(ctx).Debugf("Cloning %s@%s: %v", input.URL, input.Ref, err)
newEtag = ""
newSha, err = exportClone(ctx, input)
}
if err != nil {
return "", "", err
}
if newSha != sha || newEtag != etag {
if err := os.WriteFile(record, []byte(newSha+"\n"+newEtag), 0o600); err != nil {
return "", "", err
}
}
return tree(newSha), newSha, nil
}
func downloadArchive(ctx context.Context, input NewGitCloneExecutorInput, sha, etag string) (string, string, error) {
rewritten, err := gitOutput(ctx, "", "ls-remote", "--get-url", input.URL)
if err != nil {
return "", "", err
}
source, err := url.Parse(rewritten)
if err != nil || source.Scheme != "http" && source.Scheme != "https" || hexRefRegex.MatchString(input.Ref) && !isFullObjectID(input.Ref) { // only git tells a short commit from a branch
return "", "", errors.New("no archive")
}
archive := source.JoinPath("archive", input.Ref+".tar.gz")
if source.Host == "github.com" {
source.Host = "codeload.github.com"
archive = source.JoinPath("tar.gz", input.Ref)
}
client := &http.Client{Transport: input.Transport, Timeout: 5 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.New("stopped after 10 redirects")
}
if request.URL.Host != via[0].URL.Host || request.URL.Scheme != via[0].URL.Scheme { // the token stays on its origin
request.Header.Del("Authorization")
}
return nil
}}
for attempt := 1; ; attempt++ {
newSha, newEtag, retryAfter, err := downloadArchiveOnce(ctx, client, archive.String(), input.Token, sha, etag, input.Dir)
if retryAfter == 0 || attempt == 3 || ctx.Err() != nil {
return newSha, newEtag, err
}
common.Logger(ctx).Warnf("Failed to download action '%s'. Error: %v. Back off %s before retry.", archive.Redacted(), err, retryAfter)
select {
case <-ctx.Done():
return "", "", ctx.Err()
case <-time.After(retryAfter):
}
}
}
func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, token, sha, etag, dir string) (string, string, time.Duration, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archive, nil)
if err != nil {
return "", "", 0, err
}
if etag != "" {
request.Header.Set("If-None-Match", etag)
}
if token != "" {
request.SetBasicAuth("token", token)
}
response, err := client.Do(request)
if err != nil {
var opErr *net.OpError
if errors.As(err, &opErr) && opErr.Op == "dial" {
return "", "", 0, err
}
return "", "", archiveRetryDelay(""), err
}
defer response.Body.Close()
switch status := response.StatusCode; {
case status == http.StatusNotModified && sha != "":
return sha, etag, 0, nil
case status == http.StatusOK:
newEtag := response.Header.Get("ETag")
gz, err := gzip.NewReader(response.Body)
if err == nil {
sha, err = extractTree(ctx, gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
}
var netErr net.Error
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
return "", "", archiveRetryDelay(""), err
}
return sha, newEtag, 0, err
case status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError:
return "", "", archiveRetryDelay(response.Header.Get("Retry-After")), errors.New(response.Status)
default:
return "", "", 0, errors.New(response.Status)
}
}
func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, error) {
clone := input
clone.Dir = filepath.Join(input.Dir, "git")
sha, _, err := resolveClone(ctx, clone)
if err != nil || treeIntact(input.Dir, sha) {
return sha, err
}
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
if err != nil {
return "", err
}
_, err = extractTree(ctx, bytes.NewReader(archive), input.Dir, false)
return sha, err
}
func extractTree(ctx context.Context, archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
temp, err := os.MkdirTemp(dir, "tmp-")
if err != nil {
return "", err
}
defer os.RemoveAll(temp)
root, err := os.OpenRoot(temp)
if err != nil {
return "", err
}
defer root.Close()
sha := ""
sizes := map[string]int64{}
reader := tar.NewReader(archive)
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return "", err
}
if header.Typeflag == tar.TypeXGlobalHeader {
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && treeIntact(dir, sha) {
return sha, nil
}
continue
}
name := header.Name
if strip {
_, name, _ = strings.Cut(name, "/")
}
if name = strings.TrimSuffix(name, "/"); name == "" {
continue
}
switch header.Typeflag {
case tar.TypeDir:
err = root.MkdirAll(name, 0o755)
case tar.TypeReg:
sizes[name], err = writeTreeFile(root, name, header.Mode, reader)
case tar.TypeSymlink:
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
sizes[name], err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
}
}
if err != nil {
return "", err
}
}
if !isFullObjectID(sha) {
return "", errors.New("the archive names no commit")
}
if err := root.Close(); err != nil {
return "", err
}
return sha, installTree(ctx, dir, sha, temp, sizes)
}
var lockTree = AcquireCloneLock // replaced in tests to see installTree wait for a job holding the tree
// installTree moves temp into place as dir/sha, replacing a tree that failed treeIntact.
func installTree(ctx context.Context, dir, sha, temp string, sizes map[string]int64) error {
files := make([]treeFile, 0, len(sizes))
for name, size := range sizes {
files = append(files, treeFile{[]byte(name), size})
}
data, err := json.Marshal(files)
if err != nil {
return err
}
// Written before the swap: it describes the commit, not a copy of it, so it holds for whichever tree is left.
if err := os.WriteFile(filepath.Join(dir, sha+treeSizes), data, 0o600); err != nil { // a torn record fails treeIntact, so it needs no rename
return err
}
tree := filepath.Join(dir, sha)
if !exists(tree) {
return os.Rename(temp, tree)
}
stale, err := os.MkdirTemp(dir, "tmp-")
if err != nil {
return err
}
defer os.RemoveAll(stale) // after the unlock, as no job reads the old tree once it is moved
unlock, err := lockTree(ctx, tree) // jobs copying the old tree hold it
if err != nil {
return err
}
defer unlock()
if err := os.Rename(tree, filepath.Join(stale, sha)); err != nil {
return err
}
return os.Rename(temp, tree)
}
// treeIntact reports whether dir/sha still holds each file extracted into it at the size it was written.
// Only sizes are compared, not content: a stat per file is cheap enough for every job, hashing is not.
func treeIntact(dir, sha string) bool {
data, err := os.ReadFile(filepath.Join(dir, sha+treeSizes))
if err != nil || !exists(filepath.Join(dir, sha)) {
return false
}
var files []treeFile
if json.Unmarshal(data, &files) != nil {
return false
}
for _, file := range files {
info, err := os.Lstat(filepath.Join(dir, sha, filepath.FromSlash(string(file.Name))))
if err != nil || !info.Mode().IsRegular() || info.Size() != file.Size {
return false
}
}
return true
}
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) (int64, error) {
perm := os.FileMode(0o644)
if mode&0o111 != 0 {
perm = 0o755
}
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
if err != nil {
return 0, err
}
n, err := io.Copy(file, content)
return n, errors.Join(err, file.Close())
}
func exists(name string) bool {
_, err := os.Stat(name)
return err == nil
}

View File

@@ -1,412 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"archive/tar"
"bytes"
"context"
"encoding/base64"
"fmt"
"net/http"
"net/http/cgi"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFetchAction(t *testing.T) {
original := archiveRetryDelay
archiveRetryDelay = func(_retryAfter string) time.Duration { return time.Nanosecond }
t.Cleanup(func() { archiveRetryDelay = original })
root := t.TempDir()
remote := filepath.Join(root, "org", "action")
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
require.NoError(t, gitCmd("-C", remote, "add", "."))
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
expectedSha := gitRevParse(t, remote, "main")
gitPath, err := exec.LookPath("git")
require.NoError(t, err)
backend := &cgi.Handler{Path: gitPath, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
var mu sync.Mutex
var archiveRequests []http.Header
var mirror *httptest.Server
archiveCache := map[string][]byte{}
requests, archives, failures := 0, "", 0
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
mu.Lock()
defer mu.Unlock()
requests++
ref, ok := strings.CutPrefix(request.URL.Path, "/org/action/archive/")
if !ok {
backend.ServeHTTP(writer, request)
return
}
archiveRequests = append(archiveRequests, request.Header.Clone())
switch {
case failures > 0:
failures--
writer.WriteHeader(http.StatusServiceUnavailable)
return
case archives == "none":
http.NotFound(writer, request)
return
case archives == "unreachable":
http.Redirect(writer, request, "http://127.0.0.1:1"+request.URL.Path, http.StatusFound)
return
case archives == "redirect" && request.Host != mirror.Listener.Addr().String():
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
return
}
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
if request.Header.Get("If-None-Match") == etag {
writer.WriteHeader(http.StatusNotModified)
return
}
args := []string{"archive", "--format=tar.gz", sha}
if archives != "unprefixed" {
args = append(args, "--prefix=action/")
}
archiveKey := strings.Join(args, "\x00")
archive, ok := archiveCache[archiveKey]
if !ok {
archive, _ = runGit(request.Context(), remote, nil, args...)
archiveCache[archiveKey] = archive
}
writer.Header().Set("ETag", etag)
_, _ = writer.Write(archive)
})
server := httptest.NewServer(handler)
mirror = httptest.NewServer(handler)
t.Cleanup(server.Close)
t.Cleanup(mirror.Close)
serve := func(mode string, failing int) {
mu.Lock()
defer mu.Unlock()
requests, archives, failures, archiveRequests = 0, mode, failing, nil
}
served := func() (int, []http.Header) {
mu.Lock()
defer mu.Unlock()
return requests, append([]http.Header(nil), archiveRequests...)
}
newInput := func(ref string) NewGitCloneExecutorInput {
return NewGitCloneExecutorInput{URL: server.URL + "/org/action", Ref: ref, Dir: t.TempDir(), Token: "secret"}
}
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
t.Run(scenario, func(t *testing.T) {
serve("prefixed", 0)
input := newInput("main")
started, release := make(chan struct{}), make(chan struct{})
releaseFetches := sync.OnceFunc(func() { close(release) })
t.Cleanup(releaseFetches)
var once sync.Once
var panicked atomic.Bool
transport := actionFetchTransport(func(request *http.Request) (*http.Response, error) {
once.Do(func() { close(started) })
select {
case <-request.Context().Done():
return nil, request.Context().Err()
case <-release:
}
if scenario == "leader panics" && panicked.CompareAndSwap(false, true) {
panic("broken archive")
}
return http.DefaultTransport.RoundTrip(request)
})
input.Transport = &transport
leaderCtx, cancelLeader := context.WithCancel(t.Context())
defer cancelLeader()
waiting := &actionFetchContext{Context: t.Context(), waiting: make(chan struct{})}
followerInput := input
expectedRequests := 1
if scenario == "different token" {
followerInput.Token, expectedRequests = "other", 2
}
leaderDone, followerDone := make(chan error, 1), make(chan error, 1)
fetch := func(ctx context.Context, input NewGitCloneExecutorInput, done chan<- error) {
var err error
defer func() { done <- err }()
defer common.RecoverPanic(&err)
var tree, sha string
if tree, sha, err = FetchAction(ctx, input); err == nil {
assert.Equal(t, expectedSha, sha)
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
}
}
go fetch(leaderCtx, input, leaderDone)
<-started
go fetch(waiting, followerInput, followerDone)
<-waiting.waiting
if scenario == "leader cancelled" {
cancelLeader()
require.ErrorIs(t, <-leaderDone, context.Canceled)
}
releaseFetches()
switch scenario {
case "leader cancelled":
case "leader panics":
require.ErrorContains(t, <-leaderDone, "panic: broken archive")
default:
require.NoError(t, <-leaderDone)
}
require.NoError(t, <-followerDone)
_, headers := served()
assert.Len(t, headers, expectedRequests)
if scenario == "different token" {
assert.NotEqual(t, headers[0].Get("Authorization"), headers[1].Get("Authorization"))
}
})
}
})
t.Run("downloads, revalidates and follows a moved tag", func(t *testing.T) {
serve("prefixed", 0)
input := newInput("v1")
tree, sha, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, expectedSha, sha)
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
assert.NoDirExists(t, filepath.Join(tree, ".git"))
info, err := os.Stat(filepath.Join(tree, "run.sh"))
require.NoError(t, err)
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
link, err := os.Readlink(filepath.Join(tree, "link"))
require.NoError(t, err)
assert.Equal(t, "action.yml", link)
again, _, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, tree, again)
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
expectedSha = gitRevParse(t, remote, "v1")
moved, movedSha, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, expectedSha, movedSha)
assert.NotEqual(t, tree, moved)
assert.FileExists(t, filepath.Join(tree, "action.yml"))
input.OfflineMode = true
_, offlineSha, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, movedSha, offlineSha)
require.NoError(t, os.Truncate(filepath.Join(moved, "action.yml"), 4)) // fetched again, offline or not
_, _, err = FetchAction(t.Context(), input)
require.NoError(t, err)
require.NoError(t, os.Remove(moved+treeSizes)) // from an older runner, used as it is offline
reused, _, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, moved, reused)
requests, headers := served()
assert.Equal(t, 4, requests)
require.Len(t, headers, 4)
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
assert.Empty(t, headers[0].Get("If-None-Match"))
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
assert.Equal(t, headers[1].Get("If-None-Match"), headers[2].Get("If-None-Match"))
})
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
serve("prefixed", 0)
sha := expectedSha
input := newInput(sha)
for range 2 {
tree, got, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, sha, got)
assert.FileExists(t, filepath.Join(tree, "action.yml"))
}
_, _, err := FetchAction(t.Context(), newInput(sha[:7]))
require.ErrorIs(t, err, ErrShortRef)
_, headers := served()
assert.Len(t, headers, 1)
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
})
t.Run("downloads again over a tree whose files changed size or that has no record, not under a job copying it", func(t *testing.T) {
sha := gitRevParse(t, remote, "main")
for _, mode := range []string{"prefixed", "none"} {
serve(mode, 0)
input := newInput(sha)
tree, _, err := FetchAction(t.Context(), input)
require.NoError(t, err, mode)
yml := filepath.Join(tree, "action.yml")
require.NoError(t, os.WriteFile(filepath.Join(tree, "trampoline.js"), nil, 0o644)) // the runner's own, no damage
_, _, err = FetchAction(t.Context(), input)
require.NoError(t, err, mode)
for _, damage := range []func() error{
func() error { return os.Truncate(yml, 4) },
func() error { return os.Remove(tree + treeSizes) },
} {
require.NoError(t, damage(), mode)
_, _, err := FetchAction(t.Context(), input)
require.NoError(t, err, mode)
body, err := os.ReadFile(yml)
require.NoError(t, err, mode)
assert.Equal(t, "name: action\n", string(body), mode)
}
_, headers := served()
assert.Len(t, headers, 3, mode)
}
serve("prefixed", 0)
input := newInput(sha)
tree, _, err := FetchAction(t.Context(), input)
require.NoError(t, err)
yml := filepath.Join(tree, "action.yml")
require.NoError(t, os.Truncate(yml, 4))
unlock, err := AcquireCloneLock(t.Context(), tree)
require.NoError(t, err)
release := sync.OnceFunc(unlock)
defer release()
waiting := make(chan struct{})
lockTree = func(ctx context.Context, dir string) (func(), error) {
close(waiting)
return AcquireCloneLock(ctx, dir)
}
t.Cleanup(func() { lockTree = AcquireCloneLock })
done := make(chan error, 1)
go func() {
_, _, err := FetchAction(t.Context(), input)
done <- err
}()
select {
case err := <-done:
t.Fatalf("FetchAction replaced the tree without waiting for the job holding it: %v", err)
case <-waiting:
}
info, err := os.Stat(yml)
require.NoError(t, err)
assert.Equal(t, int64(4), info.Size(), "the tree a job holds is left alone")
release()
require.NoError(t, <-done)
body, err := os.ReadFile(yml)
require.NoError(t, err)
assert.Equal(t, "name: action\n", string(body))
})
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
serve("redirect", 2)
input := newInput("main")
_, sha, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, expectedSha, sha)
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
_, headers := served()
require.Len(t, headers, 4)
assert.NotEmpty(t, headers[2].Get("Authorization"))
assert.Empty(t, headers[3].Get("Authorization"))
})
t.Run("keeps an unprefixed archive whole, downloads where insteadOf points and exports a clone without an archive or from an unreachable host", func(t *testing.T) {
for _, mode := range []string{"unprefixed", "none", "unreachable"} {
serve(mode, 0)
tree, sha, err := FetchAction(t.Context(), newInput("main"))
require.NoError(t, err, mode)
assert.Equal(t, expectedSha, sha, mode)
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
_, headers := served()
assert.Len(t, headers, 1, mode)
}
serve("prefixed", 0)
config := filepath.Join(t.TempDir(), "gitconfig")
require.NoError(t, os.WriteFile(config, []byte("[url \""+server.URL+"/org/\"]\n\tinsteadOf = https://example.invalid/org/\n"), 0o644))
t.Setenv("GIT_CONFIG_GLOBAL", config)
input := newInput("main")
input.URL = "https://example.invalid/org/action"
_, sha, err := FetchAction(t.Context(), input)
require.NoError(t, err)
assert.Equal(t, expectedSha, sha)
_, headers := served()
assert.Len(t, headers, 1)
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
})
}
type actionFetchTransport func(*http.Request) (*http.Response, error)
func (transport actionFetchTransport) RoundTrip(request *http.Request) (*http.Response, error) {
return transport(request)
}
type actionFetchContext struct {
context.Context
waiting chan struct{}
once sync.Once
}
func (ctx *actionFetchContext) Done() <-chan struct{} {
ctx.once.Do(func() { close(ctx.waiting) })
return ctx.Context.Done()
}
func TestExtractTreeConfinesEntries(t *testing.T) {
outside := t.TempDir()
for name, entries := range map[string][]*tar.Header{
"parent path": {{Name: "../escape", Typeflag: tar.TypeReg}},
"through a symlink": {{Name: "out", Typeflag: tar.TypeSymlink, Linkname: outside}, {Name: "out/escape", Typeflag: tar.TypeReg}},
} {
t.Run(name, func(t *testing.T) {
var archive bytes.Buffer
writer := tar.NewWriter(&archive)
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
for _, entry := range entries {
require.NoError(t, writer.WriteHeader(entry))
}
require.NoError(t, writer.Close())
dir := t.TempDir()
_, err := extractTree(t.Context(), &archive, dir, false)
require.Error(t, err)
assert.NoFileExists(t, filepath.Join(dir, "escape"))
assert.NoFileExists(t, filepath.Join(outside, "escape"))
})
}
}
func TestTreeIntactFindsNamesThatAreNotUTF8(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("macOS and Windows store no name that is not UTF-8")
}
sha := strings.Repeat("a", 40)
var archive bytes.Buffer
writer := tar.NewWriter(&archive)
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": sha}}))
require.NoError(t, writer.WriteHeader(&tar.Header{Name: "caf\xe9.js", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1, Format: tar.FormatGNU}))
_, err := writer.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, writer.Close())
dir := t.TempDir()
_, err = extractTree(t.Context(), &archive, dir, false)
require.NoError(t, err)
require.FileExists(t, filepath.Join(dir, sha, "caf\xe9.js"))
assert.True(t, treeIntact(dir, sha))
}

View File

@@ -6,68 +6,44 @@ package git
import (
"context"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"time"
"sync"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/internal/pkg/lock"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/storer"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/mattn/go-isatty"
log "github.com/sirupsen/logrus"
)
var (
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
}
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
cloneLocks lock.Keyed[string] // key: clone target directory
ErrShortRef = errors.New("short SHA references are not supported")
)
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
dir = filepath.Clean(dir)
if runtime.GOOS == "plan9" {
return cloneLocks.Lock(dir), nil
}
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
return nil, err
}
for {
release, err := lock.TryLock(dir)
if err == nil {
return func() { _ = release() }, nil
}
if !errors.Is(err, lock.ErrLocked) {
return cloneLocks.Lock(dir), nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(50 * time.Millisecond):
}
}
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
func AcquireCloneLock(dir string) func() {
return cloneLocks.Lock(dir)
}
type Error struct {
@@ -75,97 +51,196 @@ type Error struct {
commit string
}
func (e *Error) Error() string { return e.err.Error() }
func (e *Error) Unwrap() error { return e.err }
func (e *Error) Commit() string { return e.commit }
func (e *Error) Error() string {
return e.err.Error()
}
func (e *Error) Unwrap() error {
return e.err
}
func (e *Error) Commit() string {
return e.commit
}
// goGitMu serializes go-git repository access across the process. go-git is not safe for
// concurrent use of the same repository (even read access decodes packfiles into shared
// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded
// operations are fast local reads; gitea runs one job per process, so the lock is effectively
// uncontended in production.
var goGitMu sync.Mutex
// FindGitRevision get the current git revision
func FindGitRevision(ctx context.Context, file string) (string, error) {
func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
goGitMu.Lock()
defer goGitMu.Unlock()
return findGitRevision(ctx, file)
}
func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
logger := common.Logger(ctx)
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
gitDir, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
logger.WithError(err).Error("path", file, "not located inside a git repository")
return "", err
return "", "", err
}
if len(sha) < 7 {
return "", errors.New("HEAD could not be resolved")
head, err := gitDir.Reference(plumbing.HEAD, true)
if err != nil {
return "", "", err
}
logger.Debugf("Found revision: %s", sha)
return sha, nil
if head.Hash().IsZero() {
return "", "", errors.New("HEAD sha1 could not be resolved")
}
hash := head.Hash().String()
logger.Debugf("Found revision: %s", hash)
return hash[:7], strings.TrimSpace(hash), nil
}
// FindGitRef get the current git ref
func FindGitRef(ctx context.Context, file string) (string, error) {
logger := common.Logger(ctx)
logger.Debugf("Loading revision from git directory")
revision, err := FindGitRevision(ctx, file)
if err != nil {
return "", err
}
logger.Debugf("HEAD points to '%s'", revision)
goGitMu.Lock()
defer goGitMu.Unlock()
refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
logger := common.Logger(ctx)
logger.Debugf("Loading revision from git directory")
_, ref, err := findGitRevision(ctx, file)
if err != nil {
return "", err
}
var tag, branch string
for ref := range strings.FieldsSeq(refs) {
switch {
case strings.HasPrefix(ref, "refs/tags/"):
tag = ref
case strings.HasPrefix(ref, "refs/heads/"):
branch = ref
logger.Debugf("HEAD points to '%s'", ref)
// Prefer the git library to iterate over the references and find a matching tag or branch.
refTag := ""
refBranch := ""
repo, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
return "", err
}
iter, err := repo.References()
if err != nil {
return "", err
}
// find the reference that matches the revision's has
err = iter.ForEach(func(r *plumbing.Reference) error {
/* tags and branches will have the same hash
* when a user checks out a tag, it is not mentioned explicitly
* in the go-git package, we must identify the revision
* then check if any tag matches that revision,
* if so then we checked out a tag
* else we look for branches and if matches,
* it means we checked out a branch
*
* If a branches matches first we must continue and check all tags (all references)
* in case we match with a tag later in the interation
*/
if r.Hash().String() == ref {
if r.Name().IsTag() {
refTag = r.Name().String()
}
if r.Name().IsBranch() {
refBranch = r.Name().String()
}
}
// we found what we where looking for
if refTag != "" && refBranch != "" {
return storer.ErrStop
}
return nil
})
if err != nil {
return "", err
}
if tag != "" {
return tag, nil
// order matters here see above comment.
if refTag != "" {
return refTag, nil
}
if branch != "" {
return branch, nil
if refBranch != "" {
return refBranch, nil
}
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)
}
// FindGithubRepo get the repo
func FindGithubRepo(ctx context.Context, file string) (string, error) {
url, err := findGitRemoteURL(ctx, file)
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
goGitMu.Lock()
defer goGitMu.Unlock()
url, err := findGitRemoteURL(ctx, file, "origin")
if err != nil {
return "", err
}
return findGitSlug(url)
_, slug := findGitSlug(url, githubInstance)
return slug, nil
}
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
repo, err := git.PlainOpenWithOptions(
file,
&git.PlainOpenOptions{
DetectDotGit: true,
EnableDotGitCommonDir: true,
},
)
if err != nil {
return "", err
}
if urls == "" {
return "", errors.New("remote 'origin' exists but has no URL")
remote, err := repo.Remote(remoteName)
if err != nil {
return "", err
}
url, _, _ := strings.Cut(urls, "\n")
return url, nil
if len(remote.Config().URLs) < 1 {
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)
}
return remote.Config().URLs[0], nil
}
func findGitSlug(remoteURL string) (string, error) {
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
return matches[2], nil
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
return matches[2], nil
func findGitSlug(url, githubInstance string) (string, string) {
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2]
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
return "CodeCommit", matches[2]
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if githubInstance != "github.com" {
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
}
}
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
}
parsed, err := url.Parse(remoteURL)
if err != nil {
return "", errors.New("cannot parse the origin remote URL")
}
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
owner = path.Base(owner)
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
return "", errors.New("cannot determine owner and repository from the origin remote URL")
}
return owner + "/" + repo, nil
return "", url
}
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
@@ -180,296 +255,298 @@ type NewGitCloneExecutorInput struct {
// 0 for full clone.
Depth int
// Quiet drops the informational clone line to debug level, for callers that log their own
// download summary (the setup section's action report).
Quiet bool
// For Gitea
InsecureSkipTLS bool
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
InsecureSkipTLS bool
}
// CloneIfRequired reports whether an existing local clone was reused.
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
var origin string
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused.
func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) {
r, err := git.PlainOpen(input.Dir)
if err == nil {
origin, err = findGitRemoteURL(ctx, input.Dir)
// Verify the cached clone still points to the resolved URL before reusing it.
remote, err := r.Remote("origin")
if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL {
// Reuse existing clone
return r, true, nil
}
switch {
case err != nil:
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
case len(remote.Config().URLs) == 0:
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
default:
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
}
if err := os.RemoveAll(input.Dir); err != nil {
return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
}
}
if err == nil && origin == input.URL {
return true, nil
var progressWriter io.Writer
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
if entry, ok := logger.(*log.Entry); ok {
progressWriter = entry.WriterLevel(log.DebugLevel)
} else if lgr, ok := logger.(*log.Logger); ok {
progressWriter = lgr.WriterLevel(log.DebugLevel)
} else {
log.Errorf("Unable to get writer from logger (type=%T)", logger)
progressWriter = os.Stdout
}
}
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
return false, ctxErr
cloneOptions := git.CloneOptions{
URL: input.URL,
Progress: progressWriter,
InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea
}
if err == nil {
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
if input.Token != "" {
cloneOptions.Auth = &http.BasicAuth{
Username: "token",
Password: input.Token,
}
}
if err := os.RemoveAll(input.Dir); err != nil {
return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
r, err = cloneAtDepth(ctx, input, cloneOptions, logger)
if err != nil {
logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err)
return nil, false, err
}
if err := clone(ctx, input); err != nil {
return false, err
if err = os.Chmod(input.Dir, 0o755); err != nil {
return nil, false, err
}
if err := os.Chmod(input.Dir, 0o755); err != nil {
return false, err
return r, false, nil
}
func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.PullOptions) {
fetchOptions.RefSpecs = []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}
fetchOptions.Force = true
pullOptions.Force = true
if token != "" {
auth := &http.BasicAuth{
Username: "token",
Password: token,
}
fetchOptions.Auth = auth
pullOptions.Auth = auth
}
return false, nil
return fetchOptions, pullOptions
}
// staleRefreshErr reports why a failed refresh must abort: the resolve and
// checkout that follow are local and succeed on a cancelled context, which
// would hand back the cached revision as if it were fresh.
func staleRefreshErr(ctx context.Context, err error) error {
if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) {
return nil
}
return ctx.Err()
}
// NewGitCloneExecutor creates an executor to clone git repos
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
if input.Quiet {
logger.Debugf("git clone '%s' # ref=%s", input.URL, input.Ref)
} else {
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
}
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
unlock, err := AcquireCloneLock(ctx, input.Dir)
if err != nil {
return err
}
defer unlock()
resolved, reused, err := resolveClone(ctx, input)
defer AcquireCloneLock(input.Dir)()
refName := plumbing.ReferenceName("refs/heads/" + input.Ref)
r, reused, err := CloneIfRequired(ctx, refName, input, logger)
if err != nil {
return err
}
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
return fmt.Errorf("checkout %s: %w", resolved, err)
isOfflineMode := input.OfflineMode
// fetch latest changes
fetchOptions, pullOptions := gitOptions(input.Token)
if input.InsecureSkipTLS { // For Gitea
fetchOptions.InsecureSkipTLS = true
pullOptions.InsecureSkipTLS = true
}
// Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth
// Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone.
shallow := isShallow(r)
if shallow {
fetchOptions.Depth = 1
if spec, ok := shallowFetchRefSpec(r, input.Ref); ok {
fetchOptions.RefSpecs = []config.RefSpec{spec}
}
}
if !isOfflineMode {
err = r.FetchContext(ctx, &fetchOptions)
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
return err
}
}
var hash *plumbing.Hash
rev := plumbing.Revision(input.Ref)
if hash, err = r.ResolveRevision(rev); err != nil {
// ResolveRevision returns a nil hash on error, and a branch ref legitimately fails
// here (no local refs/heads/<ref>); the duck-typing below resolves it.
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
} else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) {
return &Error{
err: ErrShortRef,
commit: hash.String(),
}
}
// At this point we need to know if it's a tag or a branch
// And the easiest way to do it is duck typing
//
// If err is nil, it's a tag so let's proceed with that hash like we would if
// it was a sha
refType := "tag"
rev = plumbing.Revision(path.Join("refs", "tags", input.Ref))
if _, err := r.Tag(input.Ref); errors.Is(err, git.ErrTagNotFound) {
rName := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
if _, err := r.Reference(rName, false); errors.Is(err, plumbing.ErrReferenceNotFound) {
refType = "sha"
rev = plumbing.Revision(input.Ref)
} else {
refType = "branch"
rev = plumbing.Revision(rName)
}
}
if hash, err = r.ResolveRevision(rev); err != nil {
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
return err
}
var w *git.Worktree
if w, err = r.Worktree(); err != nil {
return err
}
// If the hash resolved doesn't match the ref provided in a workflow then we're
// using a branch or tag ref, not a sha
//
// Repos on disk point to commit hashes, and need to checkout input.Ref before
// we try and pull down any changes
if hash.String() != input.Ref && refType == "branch" {
logger.Debugf("Provided ref is not a sha. Checking out branch before pulling changes")
sourceRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
if err = w.Checkout(&git.CheckoutOptions{
Branch: sourceRef,
Force: true,
}); err != nil {
logger.Errorf("Unable to checkout %s: %v", sourceRef, err)
return err
}
}
reusedMsg := ""
if input.OfflineMode && reused {
switch {
case !isOfflineMode && !shallow:
// In shallow mode the depth-limited fetch above already advanced the ref.
if err = w.PullContext(ctx, &pullOptions); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
logger.Debugf("Unable to pull %s: %v", refName, err)
}
if err := staleRefreshErr(ctx, err); err != nil {
return err
}
case isOfflineMode && reused:
reusedMsg = " (reused in offline mode)"
}
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
if hash.String() != input.Ref && refType == "branch" {
logger.Debugf("Provided ref is not a sha. Updating branch ref after pull")
if hash, err = r.ResolveRevision(rev); err != nil {
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
return err
}
}
if err = w.Checkout(&git.CheckoutOptions{
Hash: *hash,
Force: true,
}); err != nil {
logger.Errorf("Unable to checkout %s: %v", *hash, err)
return err
}
if err = w.Reset(&git.ResetOptions{
Mode: git.HardReset,
Commit: *hash,
}); err != nil {
logger.Errorf("Unable to reset to %s: %v", hash.String(), err)
return err
}
logger.Debugf("Checked out %s", input.Ref)
return nil
}
}
func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
logger := common.Logger(ctx)
reused, err := CloneIfRequired(ctx, input, logger)
if err != nil {
return "", false, err
}
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
return "", false, err
}
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
if err := fetch(ctx, input); err != nil {
return "", false, err
}
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
}
if err != nil {
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
}
return resolved, reused, err
}
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
if input.Depth > 0 && isFullObjectID(input.Ref) {
if err := shallowPinnedClone(ctx, input); err == nil {
return nil
} else if err := removePartialClone(input.Dir); err != nil {
return err
}
}
args := []string{"clone", "--no-checkout"}
// cloneAtDepth clones input.URL into input.Dir using opts.
// With input.Depth > 0 it first tries a shallow, single-branch clone of input.Ref, falling back when error.
func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
if input.Depth > 0 {
args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
args = append(args, "--branch", input.Ref)
}
}
args = append(args, input.URL, input.Dir)
if _, err := runGit(ctx, "", &input, args...); err == nil {
return nil
} else if input.Depth == 0 {
return err
}
if err := removePartialClone(input.Dir); err != nil {
return err
}
_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
return err
}
for _, refName := range []plumbing.ReferenceName{
plumbing.NewBranchReferenceName(input.Ref),
plumbing.NewTagReferenceName(input.Ref),
} {
shallowOpts := opts
shallowOpts.Depth = input.Depth
shallowOpts.SingleBranch = true
shallowOpts.ReferenceName = refName
shallowOpts.Tags = git.NoTags
func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
return err
}
if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
return err
}
_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
return err
}
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
args := []string{"fetch", "--force", "--no-prune"}
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
args = append(args, "--depth", "1", "--no-tags")
}
_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
return err
}
func cachedRef(ref string) string {
return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
}
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
if !isFullObjectID(ref) {
revisions := []string{cachedRef(ref) + "^{commit}", "refs/tags/" + ref + "^{commit}", "refs/remotes/origin/" + ref + "^{commit}"}
if strings.ContainsAny(ref, "\x00\n") {
for _, revision := range revisions {
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", revision); err == nil {
return sha, nil
}
r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
if err == nil {
return r, nil
}
} else if output, err := runGitWithStdin(ctx, dir, nil, strings.NewReader(strings.Join(revisions, "\n")+"\n"), "cat-file", "--batch-check=%(objectname)"); err == nil {
for sha := range strings.SplitSeq(string(output), "\n") {
if isFullObjectID(sha) {
return sha, nil
}
logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
if rmErr := os.RemoveAll(input.Dir); rmErr != nil {
return nil, fmt.Errorf("remove partial clone %s: %w", input.Dir, rmErr)
}
}
logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
}
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
return "", &Error{err: ErrShortRef, commit: sha}
}
return sha, err
return git.PlainCloneContext(ctx, input.Dir, false, &opts)
}
func isFullObjectID(ref string) bool {
return ObjectFormat(ref) != ""
// isShallow reports whether the local repository was cloned with a limited depth.
func isShallow(r *git.Repository) bool {
shallows, err := r.Storer.Shallow()
return err == nil && len(shallows) > 0
}
func isFullHash(ref, resolved string) bool {
return isFullObjectID(ref) && len(ref) == len(resolved)
}
// ObjectFormat returns the object format of a full commit hash, empty for anything else.
func ObjectFormat(sha string) string {
switch {
case !hexRefRegex.MatchString(sha):
return ""
case len(sha) == 40:
return "sha1"
case len(sha) == 64:
return "sha256"
}
return ""
}
func removePartialClone(dir string) error {
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("remove partial clone %s: %w", dir, err)
}
return nil
}
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
output, err := runGit(ctx, dir, nil, args...)
return strings.TrimSpace(string(output)), err
}
func RemoteConfig(token string, insecureSkipTLS bool) []string {
var config []string
if insecureSkipTLS {
config = append(config, "http.sslVerify=false")
}
if token != "" {
config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
}
return config
}
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
func ConfigEnv(config ...string) map[string]string {
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
for i, entry := range config {
key, value, _ := strings.Cut(entry, "=")
env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
}
return env
}
func ListFiles(ctx context.Context, dir string) ([]string, error) {
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
if err != nil {
return nil, err
}
var files, repos []string
for entry := range strings.SplitSeq(string(listed), "\x00") {
tag, file, ok := strings.Cut(entry, " ")
if tag != "?" {
var meta string
meta, file, ok = strings.Cut(file, "\t")
if strings.HasPrefix(meta, "160000 ") {
file += "/"
}
}
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
repos = append(repos, repo)
} else if ok {
files = append(files, file)
}
}
slices.Sort(repos)
for _, repo := range slices.Compact(repos) {
files = append(files, repo)
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
continue // git lists an uninitialized submodule as "./" from inside it
}
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
if err != nil {
return nil, err
}
for _, file := range nested {
files = append(files, repo+"/"+file)
}
}
return files, nil
}
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
return runGitWithStdin(ctx, dir, input, nil, args...)
}
func runGitWithStdin(ctx context.Context, dir string, input *NewGitCloneExecutorInput, stdin io.Reader, args ...string) ([]byte, error) {
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
if input != nil {
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
if input.ClientCertFile != "" {
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
}
}
if dir != "" {
args = append([]string{"-C", dir}, args...)
}
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Stdin = stdin
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
name, _, _ := strings.Cut(entry, "=")
return slices.Contains(LocalEnvVars, name)
})
for key, value := range ConfigEnv(config...) {
cmd.Env = append(cmd.Env, key+"="+value)
}
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
if runtime.GOOS != "windows" {
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
}
cmd.WaitDelay = time.Second
var stderr strings.Builder
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
}
return output, nil
// shallowFetchRefSpec returns the single refspec that updates only input.Ref, keeping a shallow clone from re-downloading every branch's history.
// ok is false when the ref is not present locally as a tag or remote-tracking branch, in which case the broad default refspec is used.
func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
tagRef := plumbing.NewTagReferenceName(ref)
if _, err := r.Reference(tagRef, false); err == nil {
return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true
}
remoteRef := plumbing.NewRemoteReferenceName("origin", ref)
if _, err := r.Reference(remoteRef, false); err == nil {
branchRef := plumbing.NewBranchReferenceName(ref)
return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
}
return "", false
}

View File

@@ -6,7 +6,7 @@ package git
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/http/httptest"
@@ -18,70 +18,42 @@ import (
"sync"
"syscall"
"testing"
"testing/synctest"
"time"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"gitea.com/gitea/runner/internal/pkg/lock"
"gitea.com/gitea/runner/act/common"
gogit "github.com/go-git/go-git/v5"
gogitconfig "github.com/go-git/go-git/v5/config"
log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var gitFixtureDir string
func TestMain(m *testing.M) {
var err error
gitFixtureDir, err = os.MkdirTemp("", "runner-git-fixture-")
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
code := 1
if err = gitCmd("init", "--quiet", "--template=", "--bare", "--initial-branch=main", filepath.Join(gitFixtureDir, "remote")); err == nil {
err = gitCmd("init", "--quiet", "--template=", "--initial-branch=main", filepath.Join(gitFixtureDir, "work"))
}
if err == nil {
code = m.Run()
} else {
fmt.Fprintln(os.Stderr, err)
}
if err := os.RemoveAll(gitFixtureDir); err != nil {
fmt.Fprintln(os.Stderr, err)
code = 1
}
os.Exit(code)
}
func TestFindGitSlug(t *testing.T) {
for _, testcase := range []struct {
remoteURL string
slug string
assert := assert.New(t)
slugTests := []struct {
url string // input
provider string // expected result
slug string // expected result
}{
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "my-repo-name"},
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "my-repo"},
{"git@github.com:nektos/act.git", "nektos/act"},
{"git@github.com:nektos/act", "nektos/act"},
{"https://github.com/nektos/act.git", "nektos/act"},
{"http://github.com/nektos/act.git", "nektos/act"},
{"https://github.com/nektos/act", "nektos/act"},
{"http://github.com/nektos/act", "nektos/act"},
{"git+ssh://git@github.com/owner/repo.git", "owner/repo"},
{"https://example.com:3000/gitea/owner/repo.git/", "owner/repo"},
{"ssh://git@example.com:2222/gitea/owner/repo.git", "owner/repo"},
{"example.com:owner/repo.git", "owner/repo"},
{"http://myotherrepo.com/act.git", ""},
{"https://user:secret@example.com/act.git", ""},
{"/tmp/owner/repo.git", ""},
{"./owner/repo:branch", ""},
} {
t.Run(testcase.remoteURL, func(t *testing.T) {
slug, err := findGitSlug(testcase.remoteURL)
assert.Equal(t, testcase.slug == "", err != nil)
assert.NotContains(t, fmt.Sprint(err), "secret")
assert.Equal(t, testcase.slug, slug)
})
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "CodeCommit", "my-repo-name"},
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "CodeCommit", "my-repo"},
{"git@github.com:nektos/act.git", "GitHub", "nektos/act"},
{"git@github.com:nektos/act", "GitHub", "nektos/act"},
{"https://github.com/nektos/act.git", "GitHub", "nektos/act"},
{"http://github.com/nektos/act.git", "GitHub", "nektos/act"},
{"https://github.com/nektos/act", "GitHub", "nektos/act"},
{"http://github.com/nektos/act", "GitHub", "nektos/act"},
{"git+ssh://git@github.com/owner/repo.git", "GitHub", "owner/repo"},
{"http://myotherrepo.com/act.git", "", "http://myotherrepo.com/act.git"},
}
for _, tt := range slugTests {
provider, slug := findGitSlug(tt.url, "github.com")
assert.Equal(tt.provider, provider)
assert.Equal(tt.slug, slug)
}
}
@@ -92,119 +64,144 @@ func TestErrorWrapsCommitAndCause(t *testing.T) {
require.Equal(t, "abc123", err.Commit())
}
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
t.Parallel()
func cleanGitHooks(dir string) error {
hooksDir := filepath.Join(dir, ".git", "hooks")
files, err := os.ReadDir(hooksDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
for _, f := range files {
if f.IsDir() {
continue
}
relName := filepath.Join(hooksDir, f.Name())
if err := os.Remove(relName); err != nil {
return err
}
}
return nil
}
func TestFindGithubRepoUsesOrigin(t *testing.T) {
basedir := t.TempDir()
const remoteURL = "https://github.com/owner/repo.git"
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
require.NoError(t, os.RemoveAll(filepath.Join(basedir, ".git", "hooks")))
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
require.NoError(t, gitCmd("init", basedir))
require.NoError(t, cleanGitHooks(basedir))
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
sha, err := FindGitRevision(t.Context(), basedir)
require.NoError(t, err)
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
require.Len(t, sha, 64)
ref, err := FindGitRef(t.Context(), basedir)
require.NoError(t, err)
require.Equal(t, "refs/heads/main", ref)
url, err := findGitRemoteURL(t.Context(), basedir)
url, err := findGitRemoteURL(context.Background(), basedir, "origin")
require.NoError(t, err)
require.Equal(t, remoteURL, url)
slug, err := FindGithubRepo(t.Context(), basedir)
slug, err := FindGithubRepo(context.Background(), basedir, "github.com")
require.NoError(t, err)
require.Equal(t, "owner/repo", slug)
}
func TestGitFindRef(t *testing.T) {
t.Parallel()
basedir := t.TempDir()
for name, tt := range map[string]struct {
commands [][]string
ref string
Prepare func(t *testing.T, dir string)
Assert func(t *testing.T, ref string, err error)
}{
"new_repo": {},
"new_repo": {
Prepare: func(t *testing.T, dir string) {},
Assert: func(t *testing.T, ref string, err error) {
require.Error(t, err)
},
},
"new_repo_with_commit": {
commands: [][]string{{"commit", "--allow-empty", "-m", "msg"}},
ref: "refs/heads/master",
Prepare: func(t *testing.T, dir string) {
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
},
Assert: func(t *testing.T, ref string, err error) {
require.NoError(t, err)
require.Equal(t, "refs/heads/master", ref)
},
},
"current_head_is_tag": {
commands: [][]string{
{"commit", "--allow-empty", "-m", "commit msg"},
{"tag", "v1.2.3"},
{"checkout", "v1.2.3"},
Prepare: func(t *testing.T, dir string) {
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "commit msg"))
require.NoError(t, gitCmd("-C", dir, "tag", "v1.2.3"))
require.NoError(t, gitCmd("-C", dir, "checkout", "v1.2.3"))
},
Assert: func(t *testing.T, ref string, err error) {
require.NoError(t, err)
require.Equal(t, "refs/tags/v1.2.3", ref)
},
ref: "refs/tags/v1.2.3",
},
"current_head_is_same_as_tag": {
commands: [][]string{
{"commit", "--allow-empty", "-m", "1.4.2 release"},
{"tag", "v1.4.2"},
Prepare: func(t *testing.T, dir string) {
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "1.4.2 release"))
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
},
Assert: func(t *testing.T, ref string, err error) {
require.NoError(t, err)
require.Equal(t, "refs/tags/v1.4.2", ref)
},
ref: "refs/tags/v1.4.2",
},
"current_head_is_not_tag": {
commands: [][]string{
{"commit", "--allow-empty", "-m", "msg"},
{"tag", "v1.4.2"},
{"commit", "--allow-empty", "-m", "msg2"},
Prepare: func(t *testing.T, dir string) {
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg2"))
},
Assert: func(t *testing.T, ref string, err error) {
require.NoError(t, err)
require.Equal(t, "refs/heads/master", ref)
},
ref: "refs/heads/master",
},
"current_head_is_another_branch": {
commands: [][]string{
{"checkout", "-b", "mybranch"},
{"commit", "--allow-empty", "-m", "msg"},
Prepare: func(t *testing.T, dir string) {
require.NoError(t, gitCmd("-C", dir, "checkout", "-b", "mybranch"))
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
},
Assert: func(t *testing.T, ref string, err error) {
require.NoError(t, err)
require.Equal(t, "refs/heads/mybranch", ref)
},
ref: "refs/heads/mybranch",
},
} {
t.Run(name, func(t *testing.T) {
dir := filepath.Join(basedir, name)
require.NoError(t, os.CopyFS(dir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
require.NoError(t, os.WriteFile(filepath.Join(dir, ".git", "HEAD"), []byte("ref: refs/heads/master\n"), 0o644))
for _, command := range tt.commands {
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
}
ref, err := FindGitRef(t.Context(), dir)
if tt.ref == "" {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, tt.ref, ref)
}
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
require.NoError(t, cleanGitHooks(dir))
tt.Prepare(t, dir)
ref, err := FindGitRef(context.Background(), dir)
tt.Assert(t, ref, err)
})
}
}
func TestGitCloneExecutor(t *testing.T) {
remoteDir, workDir := newGitRemote(t)
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
// Build a local bare "remote" so this runs offline and fast. The cases below mirror
// the tag/branch/sha/short-sha ref paths the executor handles, formerly exercised by
// cloning actions/checkout and anchore/scan-action over the network.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
// A branch with a dash in the name (mirrors the historical scan-action@act-fails case).
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
fullSha := gitRevParse(t, workDir, "main")
out, err := exec.Command("git", "-C", workDir, "rev-parse", "main").Output()
require.NoError(t, err)
fullSha := strings.TrimSpace(string(out))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
t.Setenv("GIT_DIR", t.TempDir())
var clones sync.WaitGroup
for name, tt := range map[string]struct {
Err error
Ref string
@@ -225,47 +222,34 @@ func TestGitCloneExecutor(t *testing.T) {
Err: &Error{ErrShortRef, fullSha},
Ref: fullSha[:7],
},
"HEAD": {
Ref: "HEAD",
},
"hex branch named like its own commit": {
Ref: fullSha[:4],
},
} {
clones.Go(func() {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: "https://example.invalid/action",
Ref: tt.Ref,
Dir: dir,
})
err := clone(t.Context())
if tt.Err != nil {
require.Error(t, err)
assert.Equal(t, tt.Err, err)
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
return
}
require.NoError(t, err)
marker := filepath.Join(dir, "marker")
require.NoError(t, os.WriteFile(marker, nil, 0o644))
require.NoError(t, clone(t.Context()))
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
require.NoError(t, err)
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
t.Run(name, func(t *testing.T) {
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: tt.Ref,
Dir: t.TempDir(),
})
err := clone(context.Background())
if tt.Err != nil {
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.Err, err)
} else {
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
}
})
}
clones.Wait()
}
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
t.Parallel()
createRemote := func(message string) string {
remoteDir, workDir := newGitRemote(t, message)
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
return remoteDir
@@ -281,27 +265,16 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
Dir: cacheDir,
})(t.Context()))
nested := filepath.Join(cacheDir, "nested")
require.NoError(t, os.Mkdir(nested, 0o755))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context()))
assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository")
markerPath := filepath.Join(cacheDir, "stale-marker")
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
cancelled, cancel := context.WithCancel(t.Context())
cancel()
_, err := CloneIfRequired(cancelled, NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: cacheDir}, logging.New(logtest.New()))
require.ErrorIs(t, err, context.Canceled)
require.FileExists(t, markerPath)
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: newRemoteDir,
Ref: "main",
Dir: cacheDir,
})(t.Context()))
originURL, err := findGitRemoteURL(t.Context(), cacheDir)
originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin")
require.NoError(t, err)
assert.Equal(t, newRemoteDir, originURL)
@@ -313,58 +286,87 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning")
}
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
t.Parallel()
remoteDir, workDir := newGitRemote(t, "initial", "feature-1")
func TestGitCloneExecutorNonFastForwardRef(t *testing.T) {
// Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes
// non-fast-forward between two fetches. Before the fix, the fetch used Force=false,
// causing go-git to return ErrForceNeeded and short-circuit the checkout.
// Create a bare "remote" repo with an initial commit on main and a feature branch.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
// We need a working clone to push commits from.
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
require.NoError(t, clone(t.Context()))
require.NoError(t, gitCmd("-C", dir, "config", "fetch.prune", "true"))
require.NoError(t, clone(t.Context()))
// Create a feature branch (simulates refs/pull/N/head).
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature"))
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
// First clone via the executor — should succeed and cache the repo.
cloneDir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "main",
Dir: cloneDir,
})
require.NoError(t, clone(context.Background()))
// Now force-push the feature branch to a non-fast-forward commit (simulates
// a PR rebase). This makes refs/heads/feature non-fast-forward.
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature"))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
require.NoError(t, clone(t.Context()), "a non-fast-forward ref must update")
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature"))
tagDir := t.TempDir()
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: tagDir})
require.Error(t, tagClone(t.Context()))
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
require.NoError(t, tagClone(t.Context()), "a tag published after a failed lookup must be fetched")
// Also advance main so we can verify the clone picks up the new commit.
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "tag-moved"))
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v9"))
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "v9"))
for range 2 {
require.NoError(t, tagClone(t.Context()), "a moved tag must update and stay current")
assert.Equal(t, "tag-moved", gitHeadSubject(t, tagDir))
}
// Second clone to the same directory — before the fix this returned ErrForceNeeded
// and left the working tree at the old commit.
err := clone(context.Background())
require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true")
// Verify the working tree was actually updated to the latest main commit.
out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output()
require.NoError(t, err)
assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit")
}
func TestGitCloneExecutorOfflineMode(t *testing.T) {
t.Parallel()
remoteDir, workDir := newGitRemote(t, "initial")
// Build a local "remote" with a single commit on main.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
// Prime the cache with an online clone of main.
cacheDir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "main",
Dir: cacheDir,
})(t.Context()))
})(context.Background()))
t.Run("cached branch resolves without fetching", func(t *testing.T) {
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
// finds no local refs/heads/<ref>.
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "main",
Dir: cacheDir,
OfflineMode: true,
})(t.Context())
})(context.Background())
require.NoError(t, err)
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
@@ -373,147 +375,120 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
})
t.Run("unresolvable cached ref returns error", func(t *testing.T) {
// The ref was never cached; offline mode cannot resolve it and must return an error.
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "never-fetched",
Dir: cacheDir,
OfflineMode: true,
})(t.Context())
})(context.Background())
require.Error(t, err)
})
}
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
// Quiet callers report the download themselves, so the clone line must not reach the job log.
for name, quiet := range map[string]bool{"quiet": true, "not quiet": false} {
t.Run(name, func(t *testing.T) {
logger, hook := logrustest.NewNullLogger()
logger.SetLevel(log.InfoLevel)
ctx := common.WithLogger(context.Background(), logger.WithField("job", "j1"))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir,
Ref: "main",
Dir: t.TempDir(),
Quiet: quiet,
})(ctx))
var cloneLines int
for _, entry := range hook.AllEntries() {
if strings.HasPrefix(entry.Message, "git clone ") {
cloneLines++
}
}
if quiet {
assert.Zero(t, cloneLines)
} else {
assert.Equal(t, 1, cloneLines)
}
})
}
}
func TestGitCloneExecutorShallow(t *testing.T) {
t.Parallel()
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
remoteDir := t.TempDir()
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
workDir := t.TempDir()
require.NoError(t, gitCmd("clone", remoteDir, workDir))
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
for _, m := range []string{"c1", "c2", "c3"} {
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
}
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
sha := gitRevParse(t, workDir, "HEAD~1") // c2, a SHA that go-git cannot shallow-clone
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
})(t.Context()))
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
assert.Equal(t, "c3", gitHeadSubject(t, dir))
})
}
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
remote := filepath.Join(t.TempDir(), "remote.git")
require.NoError(t, gitCmd("clone", "--bare", remoteDir, remote))
require.NoError(t, gitCmd("-C", remote, "config", "uploadpack.allowAnySHA1InWant", strconv.FormatBool(allowUnadvertised)))
return remote
}
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
t.Run("branch is cloned shallowly", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
})(t.Context()))
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
assert.Equal(t, "c3", gitHeadSubject(t, dir))
})
t.Run("commit hash is fetched shallowly, moved to another hash, then reused without reaching the remote", func(t *testing.T) {
remote := unadvertisedRemote(t, true)
t.Run("tag is cloned shallowly", func(t *testing.T) {
dir := t.TempDir()
cloneAt := func(ref string) error {
return NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remote, Ref: ref, Dir: dir, Depth: 1,
})(t.Context())
}
require.NoError(t, cloneAt(unadvertisedSHA))
assert.FileExists(t, shallowMarker(dir))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "v1", Dir: dir, Depth: 1,
})(t.Context()))
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
assert.Equal(t, 1, gitRevCount(t, dir))
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
olderSHA := gitRevParse(t, workDir, "HEAD~2")
require.NoError(t, cloneAt(olderSHA))
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
require.NoError(t, os.RemoveAll(remote))
require.NoError(t, cloneAt(olderSHA))
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
assert.Equal(t, "c3", gitHeadSubject(t, dir))
})
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
t.Run("SHA falls back to a full clone", func(t *testing.T) {
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1,
})
require.NoError(t, clone(t.Context()))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: sha, Dir: dir, Depth: 1,
})(t.Context()))
// go-git cannot shallow-clone a raw SHA, so it falls back to a full clone; the absence of a shallow marker proves the fallback happened.
assert.NoFileExists(t, shallowMarker(dir), "a SHA ref must not produce a shallow clone")
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
})
t.Run("moving branch updates while staying shallow", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
})(t.Context()))
require.Equal(t, "c3", gitHeadSubject(t, dir))
// Advance main on the remote, then reuse the existing shallow clone.
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
require.NoError(t, clone(t.Context()))
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
})(t.Context()))
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
assert.Equal(t, 1, gitRevCount(t, dir))
})
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
remote := filepath.Join(t.TempDir(), "remote.git")
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
work := t.TempDir()
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
sha := gitRevParse(t, work, "HEAD")
dir := t.TempDir()
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
require.NoError(t, clone(t.Context()))
assert.FileExists(t, shallowMarker(dir))
require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef)
require.NoError(t, os.RemoveAll(remote))
require.NoError(t, clone(t.Context()))
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
})
}
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
t.Parallel()
remoteDir, workDir := newGitRemote(t, "pinned")
pinned := gitRevParse(t, workDir, "HEAD")
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
require.NoError(t, gitCmd("-C", workDir, "push", "origin", pinned))
for name, depth := range map[string]int{"shallow": 1, "full clone": 0} {
t.Run(name, func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
URL: remoteDir, Ref: pinned, Dir: dir, Depth: depth,
})(t.Context()))
assert.Equal(t, pinned, gitRevParse(t, dir, "HEAD"))
})
}
}
func newGitRemote(t *testing.T, messages ...string) (string, string) {
t.Helper()
remoteDir, workDir := t.TempDir(), t.TempDir()
require.NoError(t, os.CopyFS(remoteDir, os.DirFS(filepath.Join(gitFixtureDir, "remote"))))
require.NoError(t, os.CopyFS(workDir, os.DirFS(filepath.Join(gitFixtureDir, "work"))))
config, err := os.ReadFile(filepath.Join(workDir, ".git", "config"))
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(workDir, ".git", "config"),
fmt.Appendf(config, "\n[remote \"origin\"]\n\turl = %s\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", strconv.Quote(remoteDir)), 0o644))
for _, message := range messages {
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
}
return remoteDir, workDir
}
func gitRevParse(t *testing.T, dir, rev string) string {
@@ -543,6 +518,9 @@ func gitCmd(args ...string) error {
cmd := exec.Command("git", args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
// Inject a deterministic identity and ignore the host's global/system config so commits
// succeed regardless of the host having no user.name/user.email (e.g. CI, GITHUB_ACTIONS
// unset) or a global commit.gpgsign, and without mutating the developer's ~/.gitconfig.
cmd.Env = append(os.Environ(),
"GIT_AUTHOR_NAME=Unit Test",
"GIT_AUTHOR_EMAIL=test@test.com",
@@ -563,42 +541,14 @@ func gitCmd(args ...string) error {
}
func TestAcquireCloneLock(t *testing.T) {
t.Run("blocks on a lock held by another process despite a trailing separator", func(t *testing.T) {
dir := filepath.Join(t.TempDir(), "cache")
release, err := lock.TryLock(dir)
require.NoError(t, err)
defer func() { require.NoError(t, release()) }()
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
_, err := AcquireCloneLock(ctx, dir+string(filepath.Separator))
require.ErrorIs(t, err, context.DeadlineExceeded)
})
})
t.Run("read-only parent falls back to an in-process lock", func(t *testing.T) {
parent := t.TempDir()
dir := filepath.Join(parent, "cache")
require.NoError(t, os.Mkdir(dir, 0o755))
require.NoError(t, os.Chmod(parent, 0o555))
defer func() { require.NoError(t, os.Chmod(parent, 0o755)) }()
unlock, err := AcquireCloneLock(t.Context(), dir)
require.NoError(t, err)
unlock()
})
t.Run("same directory serializes", func(t *testing.T) {
dir := t.TempDir()
unlock1, err := AcquireCloneLock(t.Context(), dir)
require.NoError(t, err)
unlock1 := AcquireCloneLock(dir)
secondAcquired := make(chan struct{})
go func() {
unlock, err := AcquireCloneLock(t.Context(), dir)
if !assert.NoError(t, err) {
return
}
unlock := AcquireCloneLock(dir)
close(secondAcquired)
unlock()
}()
@@ -622,16 +572,12 @@ func TestAcquireCloneLock(t *testing.T) {
dirA := t.TempDir()
dirB := t.TempDir()
unlockA, err := AcquireCloneLock(t.Context(), dirA)
require.NoError(t, err)
unlockA := AcquireCloneLock(dirA)
defer unlockA()
done := make(chan struct{})
go func() {
unlock, err := AcquireCloneLock(t.Context(), dirB)
if !assert.NoError(t, err) {
return
}
unlock := AcquireCloneLock(dirB)
unlock()
close(done)
}()
@@ -644,17 +590,13 @@ func TestAcquireCloneLock(t *testing.T) {
})
}
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
t.Parallel()
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
block := make(chan struct{})
reached := make(chan struct{})
var once sync.Once
var authorization string
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
once.Do(func() {
authorization = request.Header.Get("Authorization")
close(reached)
})
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
once.Do(func() { close(reached) })
<-block
}))
t.Cleanup(func() {
@@ -663,15 +605,17 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
})
dir := filepath.Join(t.TempDir(), "cached-action")
require.NoError(t, gitCmd("init", dir))
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
repo, err := gogit.PlainInit(dir, false)
require.NoError(t, err)
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
require.NoError(t, err)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx)
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
}()
select {
@@ -679,7 +623,6 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
case <-time.After(10 * time.Second):
t.Fatal("the executor never reached the remote")
}
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
cancel()
select {
@@ -689,3 +632,12 @@ func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
t.Fatal("fetch ignored context cancellation")
}
}
func TestStaleRefreshErr(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
require.NoError(t, staleRefreshErr(ctx, errors.New("remote hung up")))
cancel()
require.ErrorIs(t, staleRefreshErr(ctx, errors.New("remote hung up")), context.Canceled)
require.NoError(t, staleRefreshErr(ctx, gogit.NoErrAlreadyUpToDate))
}

View File

@@ -14,8 +14,13 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
// JobError returns the job error for current context if any
func JobError(ctx context.Context) error {
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
return container["error"]
val := ctx.Value(jobErrorContextKeyVal)
if val != nil {
if container, ok := val.(map[string]error); ok {
return container["error"]
}
}
return nil
}
func SetJobError(ctx context.Context, err error) {
@@ -26,5 +31,6 @@ func SetJobError(ctx context.Context, err error) {
// WithJobErrorContainer adds a value to the context as a container for an error
func WithJobErrorContainer(ctx context.Context) context.Context {
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
container := map[string]error{}
return context.WithValue(ctx, jobErrorContextKeyVal, container)
}

View File

@@ -26,7 +26,9 @@ type lineWriter struct {
// NewLineWriter creates a new instance of a line writer
func NewLineWriter(handlers ...LineHandler) io.Writer {
return &lineWriter{handlers: handlers}
w := new(lineWriter)
w.handlers = handlers
return w
}
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
@@ -37,15 +39,24 @@ func FlushWriter(w io.Writer) {
}
}
func (lw *lineWriter) Write(p []byte) (int, error) {
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
_, _ = lw.buffer.Write(line)
if len(line) > 0 && line[len(line)-1] == '\n' {
lw.handleLine(lw.buffer.String())
lw.buffer.Reset()
func (lw *lineWriter) Write(p []byte) (n int, err error) {
pBuf := bytes.NewBuffer(p)
written := 0
for {
line, err := pBuf.ReadString('\n')
w, _ := lw.buffer.WriteString(line)
written += w
if err != nil {
if err == io.EOF {
break
}
return written, err
}
lw.handleLine(lw.buffer.String())
lw.buffer.Reset()
}
return len(p), nil
return written, nil
}
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
@@ -61,7 +72,8 @@ func (lw *lineWriter) Flush() {
func (lw *lineWriter) handleLine(line string) {
for _, h := range lw.handlers {
if !h(line) {
ok := h(line)
if !ok {
break
}
}

View File

@@ -9,24 +9,24 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
var lines []string
lineWriter := NewLineWriter(func(s string) bool {
func TestLineWriter(t *testing.T) {
lines := make([]string, 0)
lineHandler := func(s string) bool {
lines = append(lines, s)
return true
})
}
lineWriter := NewLineWriter(lineHandler)
assert := assert.New(t)
write := func(s string) {
n, err := lineWriter.Write([]byte(s))
require.NoError(t, err)
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(len(s), n, s)
}
write("")
write("hello")
write(" ")
write("world!!\nextra")
@@ -34,17 +34,39 @@ func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
write(" line\n")
write("no newline here...")
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
assert.Equal(expected, lines)
assert.Len(lines, 4)
assert.Equal("hello world!!\n", lines[0])
assert.Equal("extra line\n", lines[1])
assert.Equal(" and another\n", lines[2])
assert.Equal("last line\n", lines[3])
}
FlushWriter(lineWriter)
expected = append(expected, "no newline here...")
assert.Equal(expected, lines)
func TestLineWriterFlush(t *testing.T) {
lines := make([]string, 0)
lineHandler := func(s string) bool {
lines = append(lines, s)
return true
}
lineWriter := NewLineWriter(lineHandler)
assert := assert.New(t)
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
// Only the newline-terminated line is emitted before flushing.
assert.Equal([]string{"complete line\n"}, lines)
// Flushing emits the buffered, not-yet-terminated trailing line.
FlushWriter(lineWriter)
assert.Equal(expected, lines)
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
// Flushing again is a no-op: nothing is buffered.
FlushWriter(lineWriter)
assert.Len(lines, 2)
}
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
// FlushWriter must be a safe no-op for writers that do not buffer lines.
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
}

View File

@@ -7,51 +7,46 @@ package common
import (
"context"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
)
type loggerContextKey string
const loggerContextKeyVal = loggerContextKey("logging.Logger")
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
// Logger returns the appropriate logger for current context
func Logger(ctx context.Context) *logging.Logger {
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
return logger
func Logger(ctx context.Context) logrus.FieldLogger {
val := ctx.Value(loggerContextKeyVal)
if val != nil {
if logger, ok := val.(logrus.FieldLogger); ok {
return logger
}
}
return logging.Default()
return logrus.StandardLogger()
}
// WithLogger adds a value to the context for the logger
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
return context.WithValue(ctx, loggerContextKeyVal, logger)
}
type loggerHandlerKey string
type loggerHookKey string
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
// It affects the job logger, not the global logger.
func LoggerHandler(ctx context.Context) logging.Handler {
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
return h
// LoggerHook returns the appropriate logger hook for current context
// the hook affects job logger, not global logger
func LoggerHook(ctx context.Context) logrus.Hook {
val := ctx.Value(loggerHookKeyVal)
if val != nil {
if hook, ok := val.(logrus.Hook); ok {
return hook
}
}
return nil
}
// WithLoggerHandler adds the extra handler for the job logger to the context
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
return context.WithValue(ctx, loggerHandlerKeyVal, h)
}
type stepIDsContextKey struct{}
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
func StepIDs(ctx context.Context) []string {
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
return ids
}
// WithStepIDs replaces the step IDs in the context
func WithStepIDs(ctx context.Context, ids []string) context.Context {
return context.WithValue(ctx, stepIDsContextKey{}, ids)
// WithLoggerHook adds a value to the context for the logger hook
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
return context.WithValue(ctx, loggerHookKeyVal, hook)
}

View File

@@ -1,284 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"context"
"fmt"
"io"
"log/slog"
"maps"
"reflect"
"runtime"
"slices"
"strconv"
"strings"
"sync"
"time"
)
const (
ansiReset = "\x1b[0m"
ansiRed = "\x1b[31m"
ansiYellow = "\x1b[33m"
ansiCyan = "\x1b[36m"
ansiDimCyan = "\x1b[2;36m"
ansiDimWhite = "\x1b[2;37m"
)
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
var consoleStart = time.Now()
// console is the state shared by every consoleHandler derived from one another.
type console struct {
mu sync.Mutex
out io.Writer
terminal bool
opts TextOptions
reportCaller bool
level slog.LevelVar
}
func newConsole(out io.Writer) *console {
c := &console{}
c.setOutput(out)
return c
}
func (c *console) setOutput(w io.Writer) {
terminal := checkIfTerminal(w)
c.mu.Lock()
defer c.mu.Unlock()
c.out, c.terminal = w, terminal
}
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
// and a colored prefix with elapsed or full time on a terminal.
type consoleHandler struct {
c *console
attrs []slog.Attr
}
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
return level >= h.c.level.Level()
}
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
}
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
c := h.c
c.mu.Lock()
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
c.mu.Unlock()
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
for _, a := range h.attrs {
data[a.Key] = a.Value.Resolve().Any()
}
r.Attrs(func(a slog.Attr) bool {
data[a.Key] = a.Value.Resolve().Any()
return true
})
var caller *runtime.Frame
if reportCaller && r.PC != 0 {
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
caller = &frame
}
prefixFieldClashes(data, caller != nil)
keys := slices.Sorted(maps.Keys(data))
var b bytes.Buffer
if terminal && !opts.DisableColors {
printColored(&b, opts, r, caller, keys, data)
} else {
printPlain(&b, opts, r, caller, keys, data)
}
line := b.Bytes()
if opts.Filter != nil {
line = opts.Filter(line)
}
c.mu.Lock()
defer c.mu.Unlock()
_, err := c.out.Write(line)
return err
}
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
func prefixFieldClashes(data map[string]any, reportCaller bool) {
for _, key := range []string{"time", "level", "msg"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
delete(data, key)
}
}
if reportCaller {
for _, key := range []string{"func", "file"} {
if v, ok := data[key]; ok {
data["fields."+key] = v
}
}
}
}
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
if !opts.DisableTimestamp {
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
}
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
if r.Message != "" {
appendKeyValue(b, "msg", r.Message)
}
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if funcVal != "" {
appendKeyValue(b, "func", funcVal)
}
if fileVal != "" {
appendKeyValue(b, "file", fileVal)
}
}
for _, k := range keys {
appendKeyValue(b, k, data[k])
}
b.WriteByte('\n')
}
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
if opts.CallerPrettyfier != nil {
return opts.CallerPrettyfier(caller)
}
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
}
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
msg := strings.TrimSuffix(r.Message, "\n")
var callerStr string
if caller != nil {
funcVal, fileVal := callerText(opts, caller)
if opts.CallerPrettyfier == nil && caller.Function != "" {
funcVal += "()"
}
switch {
case fileVal == "":
callerStr = funcVal
case funcVal == "":
callerStr = fileVal
default:
callerStr = fileVal + " " + funcVal
}
}
level := Level(r.Level)
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
switch {
case opts.DisableTimestamp:
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
case !opts.FullTimestamp:
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
default:
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
}
for _, k := range keys {
b.WriteByte(' ')
b.WriteString(colorize(level, k))
b.WriteByte('=')
appendValue(b, data[k])
}
b.WriteByte('\n')
}
// plainLevel spells Warn the way logrus did, as "warning".
func plainLevel(l Level) string {
if s := l.String(); s != "warn" {
return s
}
return "warning"
}
func colorize(l Level, s string) string {
var color string
switch {
case l <= LevelTrace:
color = ansiDimWhite
case l <= LevelDebug:
color = ansiDimCyan
case l <= LevelInfo:
color = ansiCyan
case l <= LevelWarn:
color = ansiYellow
default:
color = ansiRed
}
return color + s + ansiReset
}
func appendKeyValue(b *bytes.Buffer, key string, value any) {
if b.Len() > 0 {
b.WriteByte(' ')
}
b.WriteString(key)
b.WriteByte('=')
appendValue(b, value)
}
func appendValue(b *bytes.Buffer, value any) {
switch v := value.(type) {
case string:
appendString(b, v)
case []byte:
appendString(b, string(v))
case error:
appendFormatted(b, v, v.Error)
case fmt.Stringer:
appendFormatted(b, v, v.String)
default:
appendString(b, fmt.Sprint(value))
}
}
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
defer func() {
if r := recover(); r != nil {
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
appendString(b, "<nil>")
} else {
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
}
}
}()
appendString(b, get())
}
func appendString(b *bytes.Buffer, s string) {
if needsQuoting(s) {
b.WriteString(strconv.Quote(s))
return
}
b.WriteString(s)
}
func needsQuoting(s string) bool {
for i := range len(s) {
c := s[i]
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
default:
return true
}
}
return false
}

View File

@@ -1,78 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"bytes"
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
t.Helper()
var out bytes.Buffer
c := newConsole(&out)
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
c.level.Set(slog.LevelDebug)
build(New(NewSlogHandler(&consoleHandler{c: c})))
return out.String()
}
func TestConsolePlain(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
l.With("msg", "clash", "empty", "").Info("")
})
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
"level=info empty= fields.msg=clash\n", line)
}
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
opts := TextOptions{
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
},
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
}
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
}
func TestConsoleColored(t *testing.T) {
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
l.With("job", "j").Warn("careful")
})
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
assert.Equal(t, "level=info msg=x\n", plain)
}
func TestConsoleLevelAndOutput(t *testing.T) {
var out bytes.Buffer
c := newConsole(&out)
h := &consoleHandler{c: c}
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
c.level.Set(slog.LevelError)
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
rec.AddAttrs(slog.String("c", "3"))
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
assert.Contains(t, out.String(), "a=1 c=3")
}

View File

@@ -1,83 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"io"
"log/slog"
"os"
"runtime"
"sync/atomic"
)
// TextOptions describes how the process-wide Logger writes a record as text.
type TextOptions struct {
DisableColors bool
DisableTimestamp bool
FullTimestamp bool
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
CallerPrettyfier func(*runtime.Frame) (function, file string)
// Filter rewrites each formatted line, for example to mask secrets.
Filter func(line []byte) []byte
}
var (
std atomic.Pointer[Logger]
stdState = newConsole(os.Stderr)
)
func init() {
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
}
// Default returns the process-wide Logger.
func Default() *Logger { return std.Load() }
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
// initial console Logger.
func SetDefault(l *Logger) { std.Store(l) }
// SetLevel sets the minimum level of the initial process-wide Logger.
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
// GetLevel returns the minimum level of the initial process-wide Logger.
func GetLevel() Level { return Level(stdState.level.Level()) }
// SetOutput sets where the initial process-wide Logger writes.
func SetOutput(w io.Writer) { stdState.setOutput(w) }
// Output returns where the initial process-wide Logger writes.
func Output() io.Writer {
stdState.mu.Lock()
defer stdState.mu.Unlock()
return stdState.out
}
// SetTextFormat sets the text format of the initial process-wide Logger.
func SetTextFormat(o TextOptions) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.opts = o
}
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
func SetReportCaller(on bool) {
stdState.mu.Lock()
defer stdState.mu.Unlock()
stdState.reportCaller = on
}
func Enabled(level Level) bool { return Default().Enabled(level) }
func With(args ...any) *Logger { return Default().With(args...) }
func WithError(err error) *Logger { return Default().WithError(err) }
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
func Debug(args ...any) { Default().log(LevelDebug, args) }
func Info(args ...any) { Default().log(LevelInfo, args) }
func Warn(args ...any) { Default().log(LevelWarn, args) }
func Error(args ...any) { Default().log(LevelError, args) }

View File

@@ -1,114 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"errors"
"slices"
"time"
)
// Attr is a structured key/value pair attached to a log record.
type Attr struct {
Key string
Value any
}
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
// Logger.With are accumulated by the Handler.
type Record struct {
Time time.Time
Level Level
Message string
PC uintptr // program counter of the logging call, 0 if unknown
Attrs []Attr
}
// Attr returns the value of the last attribute named key.
func (r Record) Attr(key string) (any, bool) {
for _, a := range slices.Backward(r.Attrs) {
if a.Key == key {
return a.Value, true
}
}
return nil, false
}
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
type Handler interface {
Enabled(ctx context.Context, level Level) bool
Handle(ctx context.Context, r Record) error
// WithAttrs returns a Handler that adds attrs to every record it handles.
WithAttrs(attrs []Attr) Handler
}
type multiHandler []Handler
// NewMultiHandler fans every record out to all handlers that have its level enabled.
func NewMultiHandler(handlers ...Handler) Handler {
return multiHandler(handlers)
}
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
for _, h := range m {
if h.Enabled(ctx, level) {
return true
}
}
return false
}
func (m multiHandler) Handle(ctx context.Context, r Record) error {
var errs []error
for _, h := range m {
if h.Enabled(ctx, r.Level) {
errs = append(errs, h.Handle(ctx, r))
}
}
return errors.Join(errs...)
}
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
out := make(multiHandler, len(m))
for i, h := range m {
out[i] = h.WithAttrs(attrs)
}
return out
}
type discardHandler struct{}
// DiscardHandler drops every record.
var DiscardHandler Handler = discardHandler{}
func (discardHandler) Enabled(context.Context, Level) bool { return false }
func (discardHandler) Handle(context.Context, Record) error { return nil }
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
type levelHandler struct {
h Handler
minLevel Level
}
// NewLevelHandler passes only records at or above minLevel on to h.
func NewLevelHandler(h Handler, minLevel Level) Handler {
return levelHandler{h: h, minLevel: minLevel}
}
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
return level >= l.minLevel && l.h.Enabled(ctx, level)
}
func (l levelHandler) Handle(ctx context.Context, r Record) error {
if r.Level < l.minLevel {
return nil
}
return l.h.Handle(ctx, r)
}
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
}

View File

@@ -1,53 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"fmt"
"strings"
)
// Level is a log severity. The values match log/slog, with Trace added below Debug.
type Level int
const (
LevelTrace Level = -8
LevelDebug Level = -4
LevelInfo Level = 0
LevelWarn Level = 4
LevelError Level = 8
)
func (l Level) String() string {
switch {
case l <= LevelTrace:
return "trace"
case l <= LevelDebug:
return "debug"
case l <= LevelInfo:
return "info"
case l <= LevelWarn:
return "warn"
default:
return "error"
}
}
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
// the Logger has no level above it.
func ParseLevel(s string) (Level, error) {
switch strings.ToLower(s) {
case "trace":
return LevelTrace, nil
case "debug":
return LevelDebug, nil
case "info":
return LevelInfo, nil
case "warn", "warning":
return LevelWarn, nil
case "error", "fatal", "panic":
return LevelError, nil
}
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
}

View File

@@ -1,105 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"fmt"
"runtime"
"time"
)
const badKey = "!BADKEY"
// Logger writes printf-style messages with structured attributes to a Handler.
type Logger struct {
h Handler
ctx context.Context
}
func New(h Handler) *Logger {
return &Logger{h: h, ctx: context.Background()}
}
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
func (l *Logger) With(args ...any) *Logger {
if len(args) == 0 {
return l
}
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
}
func (l *Logger) WithError(err error) *Logger {
return l.With("error", err)
}
// WithContext returns a Logger that passes ctx to its Handler.
func (l *Logger) WithContext(ctx context.Context) *Logger {
return &Logger{h: l.h, ctx: ctx}
}
func (l *Logger) Enabled(level Level) bool {
return l.h.Enabled(l.ctx, level)
}
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
// Trace and the methods below format their arguments like fmt.Sprint.
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
func (l *Logger) logf(level Level, format string, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
}
func (l *Logger) log(level Level, args []any) {
if !l.Enabled(level) {
return
}
l.emit(callerPC(4), level, fmt.Sprint(args...))
}
// emit drops the Handler error, as a logger has nowhere to report it.
func (l *Logger) emit(pc uintptr, level Level, msg string) {
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
}
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
func callerPC(skip int) uintptr {
var pcs [1]uintptr
runtime.Callers(skip, pcs[:])
return pcs[0]
}
func argsToAttrs(args []any) []Attr {
attrs := make([]Attr, 0, len(args)/2+1)
for len(args) > 0 {
switch k := args[0].(type) {
case Attr:
attrs = append(attrs, k)
args = args[1:]
case string:
if len(args) == 1 {
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = nil
} else {
attrs = append(attrs, Attr{Key: k, Value: args[1]})
args = args[2:]
}
default:
attrs = append(attrs, Attr{Key: badKey, Value: k})
args = args[1:]
}
}
return attrs
}

View File

@@ -1,145 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging_test
import (
"errors"
"log/slog"
"runtime"
"strings"
"testing"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseLevel(t *testing.T) {
for in, want := range map[string]logging.Level{
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
} {
got, err := logging.ParseLevel(in)
require.NoError(t, err, in)
assert.Equal(t, want, got, in)
}
_, err := logging.ParseLevel("loud")
require.Error(t, err)
assert.Equal(t, "warn", logging.LevelWarn.String())
}
type countingStringer struct{ n *int }
func (c countingStringer) String() string { *c.n++; return "x" }
func TestLogger(t *testing.T) {
h := logtest.NewWithLevel(logging.LevelTrace)
assert.Nil(t, h.LastRecord())
l := logging.New(h)
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
l.Info("a", 1, 2, "b")
l.With("k").With(7).Debug("x")
recs := h.AllRecords()
require.Len(t, recs, 3)
assert.Equal(t, "step 2", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
assert.False(t, recs[0].Time.IsZero())
job, _ := recs[0].Attr("job")
assert.Equal(t, "j1", job)
errv, _ := recs[0].Attr("error")
require.EqualError(t, errv.(error), "boom")
assert.Equal(t, "a1 2b", recs[1].Message)
bad, _ := recs[2].Attr("!BADKEY")
assert.Equal(t, 7, bad)
}
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
n := 0
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
l.Debugf("%v", countingStringer{&n})
assert.Zero(t, n)
assert.False(t, l.Enabled(logging.LevelInfo))
}
func TestMultiHandler(t *testing.T) {
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
l.Info("i")
l.Error("e")
assert.Len(t, all.AllRecords(), 2)
require.Len(t, warnOnly.AllRecords(), 1)
v, _ := warnOnly.LastRecord().Attr("a")
assert.Equal(t, 1, v)
}
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
t.Helper()
require.NotZero(t, pc)
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
return f
}
func TestCallerPC(t *testing.T) {
l, h := logtest.NewLogger()
l.Infof("method")
l.Info("method")
prev := logging.Default()
t.Cleanup(func() { logging.SetDefault(prev) })
logging.SetDefault(l)
logging.Infof("global")
logging.Error("global")
require.Len(t, h.AllRecords(), 4)
for _, r := range h.AllRecords() {
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
}
}
func TestDefaultLevel(t *testing.T) {
prev := logging.GetLevel()
t.Cleanup(func() { logging.SetLevel(prev) })
logging.SetLevel(logging.LevelWarn)
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
assert.True(t, logging.Enabled(logging.LevelError))
assert.False(t, logging.Enabled(logging.LevelInfo))
}
func TestSlogHandler(t *testing.T) {
var buf strings.Builder
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
assert.False(t, l.Enabled(logging.LevelTrace))
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
out := buf.String()
assert.Contains(t, out, "level=ERROR")
assert.Contains(t, out, `msg="fail x"`)
assert.Contains(t, out, "job=j1")
assert.Contains(t, out, "error=boom")
}
func TestSlogBridge(t *testing.T) {
h := logtest.SetDefault(t)
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
_, _, line, _ := runtime.Caller(0)
slog.Debug("dropped")
recs := h.AllRecords()
require.Len(t, recs, 1)
assert.Equal(t, "from a library", recs[0].Message)
assert.Equal(t, logging.LevelWarn, recs[0].Level)
lib, _ := recs[0].Attr("lib")
assert.Equal(t, "actionslib", lib)
n, _ := recs[0].Attr("n")
assert.EqualValues(t, 1, n)
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
}

View File

@@ -1,83 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// Package logtest provides a Handler that keeps records in memory for assertions.
package logtest
import (
"context"
"slices"
"sync"
"testing"
"gitea.com/gitea/runner/act/common/logging"
)
type store struct {
mu sync.Mutex
records []logging.Record
}
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
type Handler struct {
store *store
attrs []logging.Attr
minLevel logging.Level
}
// New records from LevelInfo up, like the logger of logrus's hooks/test.
func New() *Handler {
return NewWithLevel(logging.LevelInfo)
}
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
func NewLogger() (*logging.Logger, *Handler) {
h := New()
return logging.New(h), h
}
func NewWithLevel(minLevel logging.Level) *Handler {
return &Handler{store: &store{}, minLevel: minLevel}
}
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
h.store.mu.Lock()
defer h.store.mu.Unlock()
h.store.records = append(h.store.records, r)
return nil
}
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
}
func (h *Handler) AllRecords() []logging.Record {
h.store.mu.Lock()
defer h.store.mu.Unlock()
return slices.Clone(h.store.records)
}
// LastRecord returns nil when nothing was logged.
func (h *Handler) LastRecord() *logging.Record {
h.store.mu.Lock()
defer h.store.mu.Unlock()
if len(h.store.records) == 0 {
return nil
}
r := h.store.records[len(h.store.records)-1]
return &r
}
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
func SetDefault(t testing.TB) *Handler {
t.Helper()
prev := logging.Default()
logger, h := NewLogger()
logging.SetDefault(logger)
t.Cleanup(func() { logging.SetDefault(prev) })
return h
}

View File

@@ -1,81 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package logging
import (
"context"
"log/slog"
"slices"
)
func init() {
slog.SetDefault(slog.New(slogBridge{}))
}
// slogBridge hands what libraries log through slog's global logger to the process-wide Logger.
type slogBridge struct {
attrs []Attr
}
func (b slogBridge) Enabled(ctx context.Context, level slog.Level) bool {
return Default().h.Enabled(ctx, Level(level))
}
func (b slogBridge) Handle(ctx context.Context, r slog.Record) error {
h := Default().h
if len(b.attrs) > 0 {
h = h.WithAttrs(b.attrs)
}
attrs := make([]Attr, 0, r.NumAttrs())
r.Attrs(func(a slog.Attr) bool {
attrs = append(attrs, fromSlogAttr(a))
return true
})
return h.Handle(ctx, Record{Time: r.Time, Level: Level(r.Level), Message: r.Message, PC: r.PC, Attrs: attrs})
}
func (b slogBridge) WithAttrs(attrs []slog.Attr) slog.Handler {
out := slices.Clone(b.attrs)
for _, a := range attrs {
out = append(out, fromSlogAttr(a))
}
return slogBridge{attrs: out}
}
func (b slogBridge) WithGroup(string) slog.Handler { return b }
func fromSlogAttr(a slog.Attr) Attr {
return Attr{Key: a.Key, Value: a.Value.Resolve().Any()}
}
type slogHandler struct {
h slog.Handler
}
// NewSlogHandler writes to h. Levels map one to one, as Level uses slog's values.
func NewSlogHandler(h slog.Handler) Handler {
return slogHandler{h: h}
}
func (s slogHandler) Enabled(ctx context.Context, level Level) bool {
return s.h.Enabled(ctx, slog.Level(level))
}
func (s slogHandler) Handle(ctx context.Context, r Record) error {
rec := slog.NewRecord(r.Time, slog.Level(r.Level), r.Message, r.PC)
rec.AddAttrs(toSlogAttrs(r.Attrs)...)
return s.h.Handle(ctx, rec)
}
func (s slogHandler) WithAttrs(attrs []Attr) Handler {
return slogHandler{h: s.h.WithAttrs(toSlogAttrs(attrs))}
}
func toSlogAttrs(attrs []Attr) []slog.Attr {
out := make([]slog.Attr, len(attrs))
for i, a := range attrs {
out[i] = slog.Any(a.Key, a.Value)
}
return out
}

View File

@@ -1,20 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !windows
package logging
import (
"io"
"os"
"golang.org/x/term"
)
func checkIfTerminal(w io.Writer) bool {
if f, ok := w.(*os.File); ok {
return term.IsTerminal(int(f.Fd()))
}
return false
}

View File

@@ -1,27 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build windows
package logging
import (
"io"
"os"
"golang.org/x/sys/windows"
)
// checkIfTerminal also turns on virtual terminal processing, which the console needs to show colors.
func checkIfTerminal(w io.Writer) bool {
f, ok := w.(*os.File)
if !ok {
return false
}
handle := windows.Handle(f.Fd())
var mode uint32
if err := windows.GetConsoleMode(handle, &mode); err != nil {
return false
}
return windows.SetConsoleMode(handle, mode|windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == nil
}

View File

@@ -10,11 +10,6 @@ import (
"strings"
)
type interfaceIP struct {
net.IP
Name string
}
// GetOutboundIP returns an outbound IP address of this machine.
// It tries to access the internet and returns the local IP address of the connection.
// If the machine cannot access the internet, it returns a preferred IP address from network interfaces.
@@ -31,7 +26,11 @@ func GetOutboundIP() net.IP {
// So the machine cannot access the internet. Pick an IP address from network interfaces.
if ifs, err := net.Interfaces(); err == nil {
var ips []interfaceIP
type IP struct {
net.IP
net.Interface
}
var ips []IP
for _, i := range ifs {
if addrs, err := i.Addrs(); err == nil {
for _, addr := range addrs {
@@ -43,41 +42,40 @@ func GetOutboundIP() net.IP {
ip = v.IP
}
if ip.IsGlobalUnicast() {
ips = append(ips, interfaceIP{ip, i.Name})
ips = append(ips, IP{ip, i})
}
}
}
}
return preferredIP(ips)
if len(ips) > 1 {
sort.Slice(ips, func(i, j int) bool {
ifi := ips[i].Interface
ifj := ips[j].Interface
// ethernet is preferred
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
return vi
}
ipi := ips[i].IP
ipj := ips[j].IP
// IPv4 is preferred
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
return vi
}
// en0 is preferred to en1
if ifi.Name != ifj.Name {
return ifi.Name < ifj.Name
}
// fallback
return ipi.String() < ipj.String()
})
return ips[0].IP
}
}
return nil
}
func preferredIP(ips []interfaceIP) net.IP {
if len(ips) == 0 {
return nil
}
sort.Slice(ips, func(i, j int) bool {
first, second := ips[i], ips[j]
// ethernet is preferred
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
return vi
}
// IPv4 is preferred
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
return vi
}
// en0 is preferred to en1
if first.Name != second.Name {
return first.Name < second.Name
}
// fallback
return first.String() < second.String()
})
return ips[0].IP
}

View File

@@ -1,20 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"net"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPreferredIP(t *testing.T) {
assert.Nil(t, preferredIP(nil))
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{{IP: net.ParseIP("2001:db8::1")}}))
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{
{IP: net.ParseIP("192.0.2.1"), Name: "wlan0"},
{IP: net.ParseIP("2001:db8::1"), Name: "eth0"},
}))
}

View File

@@ -9,10 +9,6 @@ import (
"errors"
"fmt"
"io"
"path"
"strings"
"sync"
"sync/atomic"
"gitea.com/gitea/runner/act/common"
@@ -75,31 +71,10 @@ const (
HealthUnhealthy = string(container.Unhealthy)
)
// marks what a runner creates for its jobs, telling its leftovers from another runner's
const runnerUUIDLabel = "com.gitea.runner.uuid"
// ErrContainerNotFound reports a container the daemon no longer knows. Its text is a
// fragment, missingContainerError composes it into the message every operation shares.
var ErrContainerNotFound = errors.New("does not exist")
// DockerProxy is a job's docker socket, fronting the daemon's for the job's lifetime.
type DockerProxy struct {
Socket string
close func(context.Context) error
closeOnce sync.Once
closeErr error
mounts atomic.Value
}
func (p *DockerProxy) SetMounts(mounts map[string]string) {
p.mounts.Store(mounts)
}
func (p *DockerProxy) Close(ctx context.Context) error {
p.closeOnce.Do(func() { p.closeErr = p.close(ctx) })
return p.closeErr
}
// Info is a snapshot of a container, as of one inspect.
type Info struct {
ID string
@@ -109,24 +84,13 @@ type Info struct {
// HealthOutput is the last healthcheck probe's output.
HealthOutput string
Ports map[string]string // container port ("5432") to the host port it is published on
Mounts map[string]string // container path to its source on the daemon
}
// DaemonPath maps target to its daemon path through the nearest mount with a source, empty without one.
func (info *Info) DaemonPath(target string) string {
for dir := target; dir != "/" && dir != "."; dir = path.Dir(dir) {
if source := info.Mounts[dir]; source != "" {
return path.Join(source, strings.TrimPrefix(target, dir))
}
}
return ""
}
// Container for managing docker run containers
type Container interface {
Create(capAdd, capDrop []string) common.Executor
Copy(destPath string, files ...*FileEntry) common.Executor
CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor
CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor
GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
Inspect(ctx context.Context) (*Info, error)
DumpLogs(ctx context.Context) error

View File

@@ -7,33 +7,20 @@
package container
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net"
"net/url"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"strings"
"gitea.com/gitea/runner/act/common"
"github.com/distribution/reference"
"github.com/docker/cli/cli/config"
"github.com/moby/moby/api/types/registry"
)
func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfig, error) {
logger := common.Logger(ctx)
cfg, err := loadDockerConfig(ctx)
// config.LoadDefaultConfigFile panics on nil io.Writer when the config
// file is malformed; use config.Load to route errors through the logger.
cfg, err := config.Load(config.Dir())
if err != nil {
logger.Warnf("Could not load docker config: %v", err)
return registry.AuthConfig{}, err
@@ -47,27 +34,32 @@ func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfi
}
}
authConfig, err := cfg.authConfig(ctx, registryKey)
authConfig, err := cfg.GetAuthConfig(registryKey)
if err != nil {
logger.Warnf("Could not get auth config from docker config: %v", err)
return registry.AuthConfig{}, err
}
return authConfig, nil
return registry.AuthConfig(authConfig), nil
}
func LoadDockerAuthConfigs(ctx context.Context) map[string]registry.AuthConfig {
logger := common.Logger(ctx)
cfg, err := loadDockerConfig(ctx)
cfg, err := config.Load(config.Dir())
if err != nil {
logger.Warnf("Could not load docker config: %v", err)
return nil
}
authConfigs, err := cfg.allAuthConfigs(ctx)
creds, err := cfg.GetAllCredentials()
if err != nil {
logger.Warnf("Could not get docker auth configs: %v", err)
return nil
}
authConfigs := make(map[string]registry.AuthConfig, len(creds))
for k, v := range creds {
authConfigs[k] = registry.AuthConfig(v)
}
return authConfigs
}
@@ -77,226 +69,3 @@ func registryAuthConfigKey(domainName string) string {
}
return domainName
}
type dockerConfig struct {
Auths map[string]registry.AuthConfig `json:"auths"`
CredsStore string `json:"credsStore"`
CredHelpers map[string]string `json:"credHelpers"`
env map[string]registry.AuthConfig // from DOCKER_AUTH_CONFIG, nil when unset or unusable
}
func loadDockerConfig(ctx context.Context) (*dockerConfig, error) {
dir := os.Getenv("DOCKER_CONFIG")
if dir == "" {
home, _ := os.UserHomeDir()
if home == "" && runtime.GOOS != "windows" {
if current, err := user.Current(); err == nil {
home = current.HomeDir
}
}
dir = filepath.Join(home, ".docker")
}
cfg := &dockerConfig{}
filename := filepath.Join(dir, "config.json")
file, err := os.Open(filename)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("loading config file: %w", err)
}
if err == nil {
defer file.Close()
if err := json.NewDecoder(file).Decode(cfg); err != nil && !errors.Is(err, io.EOF) {
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
}
}
for address, auth := range cfg.Auths {
if auth.Auth != "" {
if auth.Username, auth.Password, err = decodeAuth(auth.Auth); err != nil {
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
}
}
auth.Auth, auth.ServerAddress = "", address
cfg.Auths[address] = auth
}
if value := os.Getenv("DOCKER_AUTH_CONFIG"); value != "" {
if cfg.env, err = parseDockerAuthConfigEnv(value); err != nil {
common.Logger(ctx).Warnf("Failed to create credential store from DOCKER_AUTH_CONFIG: %v", err)
}
}
return cfg, nil
}
func decodeAuth(auth string) (string, string, error) {
decoded, err := base64.StdEncoding.DecodeString(auth)
if err != nil {
return "", "", err
}
username, password, ok := strings.Cut(string(decoded), ":")
if !ok || username == "" {
return "", "", errors.New("invalid auth configuration file")
}
return username, strings.Trim(password, "\x00"), nil
}
func parseDockerAuthConfigEnv(value string) (map[string]registry.AuthConfig, error) {
var env struct {
Auths map[string]struct {
Auth string `json:"auth"`
} `json:"auths"`
}
decoder := json.NewDecoder(strings.NewReader(value))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&env); err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if decoder.More() {
return nil, errors.New("DOCKER_AUTH_CONFIG does not support more than one JSON object")
}
auths := make(map[string]registry.AuthConfig, len(env.Auths))
for address, entry := range env.Auths {
if entry.Auth == "" {
return nil, fmt.Errorf("DOCKER_AUTH_CONFIG environment variable is missing key `auth` for %s", address)
}
username, password, err := decodeAuth(entry.Auth)
if err != nil {
return nil, err
}
auths[address] = registry.AuthConfig{Username: username, Password: password, ServerAddress: address}
}
return auths, nil
}
func (c *dockerConfig) authConfig(ctx context.Context, key string) (registry.AuthConfig, error) {
if auth, ok := c.env[key]; ok {
return auth, nil
}
helper := c.CredsStore
if named, ok := c.CredHelpers[key]; ok {
helper = named
}
if helper == "" {
return c.fileAuth(key), nil
}
auth := c.fileAuth(key)
stored, err := helperCredentials(ctx, helper, key)
if err != nil {
return auth, err
}
auth.Username, auth.Password, auth.IdentityToken, auth.ServerAddress = stored.Username, stored.Password, stored.IdentityToken, stored.ServerAddress
return auth, nil
}
func (c *dockerConfig) fileAuth(key string) registry.AuthConfig {
if auth, ok := c.Auths[key]; ok {
return auth
}
for address, auth := range c.Auths {
if registryHostname(address) == key {
return auth
}
}
return registry.AuthConfig{}
}
func (c *dockerConfig) allAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
var auths map[string]registry.AuthConfig
if c.CredsStore == "" {
auths = maps.Clone(c.Auths)
} else {
var err error
if auths, err = c.helperAuthConfigs(ctx); err != nil {
if c.env == nil {
return nil, err
}
common.Logger(ctx).Warnf("Could not list the credentials of docker-credential-%s: %v", c.CredsStore, err)
}
}
if auths == nil {
auths = map[string]registry.AuthConfig{}
}
maps.Copy(auths, c.env)
if c.CredsStore == "" {
c.Auths = maps.Clone(auths) // docker's file store holds the DOCKER_AUTH_CONFIG entries after listing, and credHelpers set to "" read it
}
for address := range c.CredHelpers {
auth, err := c.authConfig(ctx, registryAuthConfigKey(address))
if err != nil {
common.Logger(ctx).Warnf("Failed to get credentials for registry: %s: %v", address, err)
continue
}
auths[address] = auth
}
return auths, nil
}
func (c *dockerConfig) helperAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
out, err := runCredentialHelper(ctx, c.CredsStore, "list", "unused")
if err != nil {
return nil, fmt.Errorf("error listing credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
}
var addresses map[string]string
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&addresses); err != nil {
return nil, err
}
auths := make(map[string]registry.AuthConfig, len(addresses))
for address := range addresses {
stored, err := helperCredentials(ctx, c.CredsStore, address)
if err != nil {
return nil, err
}
auth := c.Auths[address]
auth.Username, auth.Password, auth.IdentityToken = stored.Username, stored.Password, stored.IdentityToken
if auth.ServerAddress == "" {
auth.ServerAddress = stored.ServerAddress
}
auths[address] = auth
}
return auths, nil
}
func helperCredentials(ctx context.Context, helper, address string) (registry.AuthConfig, error) {
out, err := runCredentialHelper(ctx, helper, "get", address)
if err != nil {
if strings.TrimSpace(string(out)) == "credentials not found in native keychain" {
return registry.AuthConfig{}, nil
}
return registry.AuthConfig{}, fmt.Errorf("error getting credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
}
var stored struct{ Username, Secret string }
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&stored); err != nil {
return registry.AuthConfig{}, err
}
if stored.Username == "<token>" {
return registry.AuthConfig{IdentityToken: stored.Secret, ServerAddress: address}, nil
}
return registry.AuthConfig{Username: stored.Username, Password: stored.Secret, ServerAddress: address}, nil
}
func runCredentialHelper(ctx context.Context, helper, action, input string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "docker-credential-"+helper, action)
cmd.Stdin = strings.NewReader(input)
cmd.Stderr = os.Stderr
return cmd.Output()
}
func registryHostname(address string) string {
if _, rest, ok := strings.Cut(address, "://"); ok {
if u, err := url.Parse(address); err == nil && u.Hostname() != "" {
if u.Port() == "" {
return u.Hostname()
}
return net.JoinHostPort(u.Hostname(), u.Port())
}
if host, _, _ := strings.Cut(rest, "/"); host != "" {
if colon := strings.LastIndex(host, ":"); strings.Count(host, ":") > 1 && !strings.HasPrefix(host, "[") && isDigits(host[colon+1:]) {
return net.JoinHostPort(host[:colon], host[colon+1:])
}
return host
}
}
host, _, _ := strings.Cut(address, "/")
return host
}
func isDigits(s string) bool {
return s != "" && strings.Trim(s, "0123456789") == ""
}

View File

@@ -1,46 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
package container
import (
"encoding/base64"
"os"
"path/filepath"
"testing"
"github.com/moby/moby/api/types/registry"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadDockerAuthConfigFromFileHelpersAndEnv(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "docker-credential-fake"), []byte("#!/bin/sh\nread -r server\n[ \"$1\" = get ] && [ \"$server\" = helper.example.com ] && exec echo '{\"Username\":\"helped\",\"Secret\":\"pw\"}'\necho 'credentials not found in native keychain'\nexit 1\n"), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
auth := func(credentials string) string { return base64.StdEncoding.EncodeToString([]byte(credentials)) }
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths": {"https://legacy.example.com/v1/": {"auth": "`+auth("legacy:pw")+`"}}, "credHelpers": {"helper.example.com": "fake", "gone.example.com": "fake", "env-legacy.example.com": ""}}`), 0o600))
t.Setenv("DOCKER_CONFIG", dir)
t.Setenv("DOCKER_AUTH_CONFIG", `{"auths": {"env.example.com": {"auth": "`+auth("env:pw")+`"}, "https://env-legacy.example.com/v1/": {"auth": "`+auth("env:pw")+`"}}}`)
for image, want := range map[string]registry.AuthConfig{
"legacy.example.com/app": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
"helper.example.com/app": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
"env.example.com/app": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
"gone.example.com/app": {},
} {
got, err := LoadDockerAuthConfig(t.Context(), image)
require.NoError(t, err)
assert.Equal(t, want, got, image)
}
assert.Equal(t, map[string]registry.AuthConfig{
"https://legacy.example.com/v1/": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
"helper.example.com": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
"gone.example.com": {},
"env.example.com": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
"https://env-legacy.example.com/v1/": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
"env-legacy.example.com": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
}, LoadDockerAuthConfigs(t.Context()))
}

View File

@@ -7,18 +7,15 @@
package container
import (
"archive/tar"
"context"
"io"
"io/fs"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/moby/go-archive"
"github.com/moby/go-archive/compression"
"github.com/moby/moby/client"
"github.com/moby/patternmatcher"
"github.com/moby/patternmatcher/ignorefile"
@@ -46,8 +43,9 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
logger.Debugf("Building image from '%v'", input.ContextDir)
tags := []string{input.ImageTag}
options := client.ImageBuildOptions{
Tags: []string{input.ImageTag},
Tags: tags,
Remove: true,
AuthConfigs: LoadDockerAuthConfigs(ctx),
Dockerfile: input.Dockerfile,
@@ -74,10 +72,12 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
resp, err := cli.ImageBuild(ctx, buildContext, options)
err = logDockerResponse(logger, resp.Body, err != nil)
if err != nil {
return err
}
return logDockerResponse(logger, resp.Body, false)
return nil
}
}
@@ -115,140 +115,14 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
includes = append(includes, ".dockerignore", relDockerfile)
}
pm, err := patternmatcher.New(excludes)
buildCtx, err := archive.TarWithOptions(contextDir, &archive.TarOptions{
Compression: compression.None,
ExcludePatterns: excludes,
IncludeFiles: includes,
})
if err != nil {
return nil, err
}
reader, writer := io.Pipe()
go func() { writer.CloseWithError(writeBuildContext(ctx, writer, contextDir, includes, pm)) }()
return reader, nil
return buildCtx, nil
}
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
tw := tar.NewWriter(w)
seen := map[string]bool{}
hardlinks := map[uint64]string{}
buffer := make([]byte, 32*1024)
for _, include := range includes {
include = filepath.FromSlash(include)
var parents []parentMatch
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
if err != nil {
return leaveOut(ctx, file, err)
}
rel, err := filepath.Rel(contextDir, file)
if err != nil || rel == "." {
return nil
}
if rel != include {
for len(parents) > 0 && !strings.HasPrefix(rel, parents[len(parents)-1].dir+string(filepath.Separator)) {
parents = parents[:len(parents)-1]
}
var parentInfo patternmatcher.MatchInfo
if len(parents) > 0 {
parentInfo = parents[len(parents)-1].info
}
skip, info, err := pm.MatchesUsingParentResults(rel, parentInfo)
if err != nil {
return err
}
if entry.IsDir() {
parents = append(parents, parentMatch{rel, info})
}
if skip {
if entry.IsDir() && !hasExceptionWithin(pm, rel) {
return filepath.SkipDir
}
return nil
}
}
if seen[rel] {
return nil
}
seen[rel] = true
return addBuildContextEntry(ctx, tw, file, rel, entry, hardlinks, buffer)
})
if err != nil {
return err
}
}
return tw.Close()
}
type parentMatch struct {
dir string
info patternmatcher.MatchInfo
}
func hasExceptionWithin(pm *patternmatcher.PatternMatcher, dir string) bool {
dirPrefix := dir + string(filepath.Separator)
for _, pattern := range pm.Patterns() {
if pattern.Exclusion() && strings.HasPrefix(pattern.String()+string(filepath.Separator), dirPrefix) {
return true
}
}
return false
}
func leaveOut(ctx context.Context, file string, err error) error {
common.Logger(ctx).Warnf("Leaving %s out of the build context: %v", file, err)
return nil
}
func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string, entry fs.DirEntry, hardlinks map[uint64]string, buffer []byte) error {
info, err := entry.Info()
if err != nil {
return leaveOut(ctx, file, err)
}
if info.Mode()&fs.ModeSocket != 0 {
return nil // tar has no socket type
}
var link string
if info.Mode()&fs.ModeSymlink != 0 {
if link, err = os.Readlink(file); err != nil {
return leaveOut(ctx, file, err)
}
}
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
if err != nil {
return err
}
header.Name = filepath.ToSlash(rel)
inode, linked := hardlinkInode(info)
if target, ok := hardlinks[inode]; linked && ok {
header.Typeflag, header.Linkname, header.Size = tar.TypeLink, target, 0
}
var content *os.File
if header.Typeflag == tar.TypeReg && header.Size > 0 {
if content, err = os.Open(file); err != nil {
return leaveOut(ctx, file, err)
}
defer content.Close()
}
if linked && header.Typeflag != tar.TypeLink {
hardlinks[inode] = header.Name
}
if info.IsDir() {
header.Name += "/"
}
header.Format = tar.FormatPAX
header.ModTime = header.ModTime.Truncate(time.Second)
header.AccessTime, header.ChangeTime = time.Time{}, time.Time{}
if runtime.GOOS == "windows" {
header.Mode = header.Mode&0o755 | 0o111 // Windows has no exec bit, so mark everything executable like docker build
}
if err := tw.WriteHeader(header); err != nil {
return err
}
if content != nil {
_, err = io.CopyBuffer(tw, struct{ io.Reader }{content}, buffer) // hides os.File's WriteTo, which would allocate a buffer per file
}
return err
}
type noNameLookups struct{ fs.FileInfo } // leaves Uname and Gname empty like docker build, so packing never waits on NSS
func (noNameLookups) Uname() (string, error) { return "", nil }
func (noNameLookups) Gname() (string, error) { return "", nil }

View File

@@ -1,71 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"archive/tar"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestDockerBuildReportsDaemonError(t *testing.T) {
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("API-Version", "1.47")
if strings.HasSuffix(request.URL.Path, "/build") {
writer.Header().Set("Content-Type", "application/json")
writer.WriteHeader(http.StatusInternalServerError)
_, _ = io.WriteString(writer, `{"message":"build rejected"}`)
}
}))
t.Cleanup(daemon.Close)
t.Setenv("DOCKER_HOST", daemon.URL)
t.Setenv("DOCKER_CONFIG", t.TempDir())
t.Setenv("DOCKER_AUTH_CONFIG", "")
require.ErrorContains(t, NewDockerBuildExecutor(NewDockerBuildExecutorInput{
ImageTag: "action:test", Dockerfile: "Dockerfile", BuildContext: strings.NewReader(""),
})(t.Context()), "build rejected")
}
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
} {
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
}
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
contexts := []string{dir}
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
contexts = append(contexts, link+string(filepath.Separator))
}
for _, contextDir := range contexts {
buildContext, err := createBuildContext(t.Context(), contextDir, "Dockerfile")
require.NoError(t, err)
var names []string
for reader := tar.NewReader(buildContext); ; {
header, err := reader.Next()
if err == io.EOF {
break
}
require.NoError(t, err)
names = append(names, header.Name)
}
require.NoError(t, buildContext.Close())
require.Equal(t, []string{".dockerignore", "a.txt", "build/keep.txt", "empty/", "Dockerfile"}, names, contextDir)
}
}

View File

@@ -1,18 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
package container
import (
"io/fs"
"syscall"
)
func hardlinkInode(info fs.FileInfo) (uint64, bool) {
if stat, ok := info.Sys().(*syscall.Stat_t); ok && !info.IsDir() && stat.Nlink > 1 {
return stat.Ino, true
}
return 0, false
}

View File

@@ -1,35 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
package container
import (
"archive/tar"
"os"
"path/filepath"
"syscall"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateBuildContextKeepsHardlinkedSpecialFilesLinked(t *testing.T) {
dir := t.TempDir()
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "first"), 0o644))
require.NoError(t, os.Link(filepath.Join(dir, "first"), filepath.Join(dir, "second")))
buildContext, err := createBuildContext(t.Context(), dir, "Dockerfile")
require.NoError(t, err)
defer buildContext.Close()
reader := tar.NewReader(buildContext)
first, err := reader.Next()
require.NoError(t, err)
assert.Equal(t, byte(tar.TypeFifo), first.Typeflag)
second, err := reader.Next()
require.NoError(t, err)
assert.Equal(t, byte(tar.TypeLink), second.Typeflag)
assert.Equal(t, first.Name, second.Linkname)
}

View File

@@ -1,10 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER
package container
import "io/fs"
func hardlinkInode(_info fs.FileInfo) (uint64, bool) { return 0, false }

View File

@@ -4,12 +4,10 @@
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts.go with:
// * appended with license information
// * regexp and ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
// * cdiDeviceName in place of the container-device-interface parser's IsQualifiedName
// * jsontext in place of json.Compact for the seccomp profile
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for nat port types
// * regexp and loader.ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
//
// docker/cli is licensed under the Apache License, Version 2.0.
// See DOCKER_LICENSE for the full license text.
@@ -34,12 +32,14 @@ import (
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/docker/cli/cli/compose/loader"
"github.com/docker/cli/opts"
"github.com/docker/go-connections/nat"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
"github.com/spf13/pflag"
cdi "tags.cncf.io/container-device-interface/pkg/parser"
)
const (
@@ -152,8 +152,10 @@ type containerOptions struct {
runtime string
autoRemove bool
init bool
umask opts.UmaskOpt
annotations *opts.MapOpts
Image string
Args []string
}
// addFlags adds all command line flags that will be used by parse to the FlagSet
@@ -324,9 +326,6 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
flags.Var(&copts.shmSize, "shm-size", "Size of /dev/shm")
flags.StringVar(&copts.utsMode, "uts", "", "UTS namespace to use")
flags.StringVar(&copts.runtime, "runtime", "", "Runtime to use for this container")
flags.Var(&copts.umask, "umask", "Set umask for the container")
flags.SetAnnotation("umask", "version", []string{"1.56"})
flags.SetAnnotation("umask", "ostype", []string{"linux"})
flags.BoolVar(&copts.init, "init", false, "Run an init inside the container that forwards signals and reaps processes")
flags.SetAnnotation("init", "version", []string{"1.25"})
@@ -384,7 +383,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
volumes := copts.volumes.GetMap()
// add any bind targets to the list of container volumes
for bind := range volumes {
parsed, err := ParseVolume(bind)
parsed, err := loader.ParseVolume(bind)
if err != nil {
return nil, err
}
@@ -392,7 +391,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
if parsed.Source != "" {
toBind := bind
if parsed.Type == mount.TypeBind {
if parsed.Type == string(mount.TypeBind) {
if hostPart, targetPath, ok := strings.Cut(bind, ":"); ok {
if !filepath.IsAbs(hostPart) && strings.HasPrefix(hostPart, ".") {
if absHostPart, err := filepath.Abs(hostPart); err == nil {
@@ -419,7 +418,12 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
tmpfs[k] = v
}
var entrypoint []string
var runCmd, entrypoint []string
if len(copts.Args) > 0 {
runCmd = copts.Args
}
if copts.entrypoint != "" {
entrypoint = []string{copts.entrypoint}
} else if flags.Changed("entrypoint") {
@@ -440,15 +444,36 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
if err != nil {
return nil, err
}
portBindings, err := convertPortMap(natPortBindings)
if err != nil {
return nil, err
portBindings := network.PortMap{}
for port, bindings := range natPortBindings {
p, err := network.ParsePort(string(port))
if err != nil {
return nil, err
}
portBindings[p] = []network.PortBinding{}
for _, b := range bindings {
var hostIP netip.Addr
if b.HostIP != "" {
hostIP, err = netip.ParseAddr(b.HostIP)
if err != nil {
return nil, err
}
}
portBindings[p] = append(portBindings[p], network.PortBinding{
HostIP: hostIP,
HostPort: b.HostPort,
})
}
}
// Add published ports as exposed ports.
exposedPorts, err := convertPortSet(ports)
if err != nil {
return nil, err
exposedPorts := network.PortSet{}
for port := range ports {
p, err := network.ParsePort(string(port))
if err != nil {
return nil, err
}
exposedPorts[p] = struct{}{}
}
// Merge in exposed ports to the map of published ports
@@ -474,7 +499,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
deviceMappings := make([]container.DeviceMapping, 0, len(devices))
cdiDeviceNames := make([]string, 0, len(devices))
for _, device := range devices {
if cdiDeviceName.MatchString(device) {
if cdi.IsQualifiedName(device) {
cdiDeviceNames = append(cdiDeviceNames, device)
continue
}
@@ -589,10 +614,11 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
deviceRequests := copts.gpus.Value()
if len(cdiDeviceNames) > 0 {
deviceRequests = append(deviceRequests, container.DeviceRequest{
cdiDeviceRequest := container.DeviceRequest{
Driver: "cdi",
DeviceIDs: cdiDeviceNames,
})
}
deviceRequests = append(deviceRequests, cdiDeviceRequest)
}
resources := container.Resources{
@@ -638,6 +664,8 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
AttachStdout: attachStdout,
AttachStderr: attachStderr,
Env: envVariables,
Cmd: runCmd,
Image: copts.Image,
Volumes: volumes,
Entrypoint: entrypoint,
WorkingDir: copts.workingDir,
@@ -693,7 +721,6 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
MaskedPaths: maskedPaths,
ReadonlyPaths: readonlyPaths,
Annotations: copts.annotations.GetAll(),
Umask: copts.umask.Value(),
}
if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() {
@@ -778,8 +805,10 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
// For backward compatibility: if no custom options are provided for the network,
// and only a single network is specified, omit the endpoint-configuration
// on the client (the daemon will still create it when creating the container)
if i == 0 && len(copts.netMode.Value()) == 1 && reflect.ValueOf(*ep).IsZero() {
continue
if i == 0 && len(copts.netMode.Value()) == 1 {
if ep == nil || reflect.ValueOf(*ep).IsZero() {
continue
}
}
endpoints[n.Target] = ep
}
@@ -810,12 +839,14 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
return invalidParameter(errors.New("conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses"))
}
if copts.aliases.Len() > 0 {
n.Aliases = slices.Clone(copts.aliases.GetSlice())
n.Aliases = make([]string, copts.aliases.Len())
copy(n.Aliases, copts.aliases.GetSlice())
}
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
// for the default bridge it defines a legacy-link.
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
n.Links = slices.Clone(copts.links.GetSlice())
n.Links = make([]string, copts.links.Len())
copy(n.Links, copts.links.GetSlice())
}
if copts.ipv4Address != nil {
if ipv4, ok := netip.AddrFromSlice(copts.ipv4Address.To4()); ok {
@@ -854,6 +885,7 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
}
epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
if len(ep.DriverOpts) > 0 {
epConfig.DriverOpts = make(map[string]string)
epConfig.DriverOpts = ep.DriverOpts
}
if len(ep.Links) > 0 {
@@ -897,10 +929,11 @@ func convertToStandardNotation(ports []string) ([]string, error) {
}
func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts)
if loggingDriver == "none" && len(loggingOpts) > 0 {
return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver)
}
return opts.ConvertKVStringsToMap(loggingOpts), nil
return loggingOptsMap, nil
}
// takes a local seccomp daemon, reads the file contents for sending to the daemon
@@ -1158,8 +1191,6 @@ func convertPortSet(ports nat.PortSet) (network.PortSet, error) {
return converted, nil
}
var cdiDeviceName = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?/[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?=[a-zA-Z0-9]([a-zA-Z0-9_.:-]*[a-zA-Z0-9])?$`)
func convertPortMap(bindings nat.PortMap) (network.PortMap, error) {
converted := make(network.PortMap, len(bindings))
for port, portBindings := range bindings {

View File

@@ -2,7 +2,7 @@
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts_test.go with:
// * appended with license information
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
//
@@ -25,11 +25,13 @@ import (
"testing"
"time"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/moby/moby/api/types/container"
networktypes "github.com/moby/moby/api/types/network"
"github.com/spf13/pflag"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
"gotest.tools/v3/skip"
)
func mustParseMAC(s string) networktypes.HardwareAddr {
@@ -69,6 +71,7 @@ func parseRun(args []string) (*container.Config, *container.HostConfig, *network
if err := flags.Parse(args); err != nil {
return nil, nil, nil, err
}
// TODO(dnephin): fix tests to accept ContainerConfig; see https://github.com/moby/moby/pull/31621
containerCfg, err := parse(flags, copts, runtime.GOOS)
if err != nil {
return nil, nil, nil, err
@@ -80,13 +83,14 @@ func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
flags := pflag.NewFlagSet("run", pflag.ContinueOnError)
flags.SetOutput(io.Discard)
flags.Usage = nil
return flags, addFlags(flags)
copts := addFlags(flags)
return flags, copts
}
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
t.Helper()
config, hostConfig, nwConfig, err := parseRun(strings.Split(args, " "))
require.NoError(t, err)
config, hostConfig, nwConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash"))
assert.NilError(t, err)
return config, hostConfig, nwConfig
}
@@ -152,46 +156,65 @@ func TestParseRunLinks(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, hostConfig, netConfig := mustParse(t, tc.input)
assert.Equal(t, tc.expHostConfigLinks, hostConfig.Links)
assert.Len(t, netConfig.EndpointsConfig, len(tc.expNetConfigLinks))
assert.Check(t, is.DeepEqual(hostConfig.Links, tc.expHostConfigLinks))
assert.Check(t, is.Len(netConfig.EndpointsConfig, len(tc.expNetConfigLinks)))
for netName, expLinks := range tc.expNetConfigLinks {
nc, ok := netConfig.EndpointsConfig[netName]
require.True(t, ok)
assert.Equal(t, expLinks, nc.Links)
assert.Assert(t, ok)
assert.Check(t, is.DeepEqual(nc.Links, expLinks))
}
})
}
}
func TestParseRunWithoutUmask(t *testing.T) {
_, hostConfig, _, err := parseRun([]string{"ubuntu", "bash"})
require.NoError(t, err)
require.Nil(t, hostConfig.Umask)
}
func TestParseRunUmask(t *testing.T) {
_, hostConfig, _, err := parseRun([]string{"--umask", "0022", "ubuntu", "bash"})
require.NoError(t, err)
require.NotNil(t, hostConfig.Umask)
require.Equal(t, uint32(0o22), *hostConfig.Umask)
}
func TestParseRunAttach(t *testing.T) {
for _, tc := range []struct {
input string
stdin, stdout, stderr bool
tests := []struct {
input string
expected container.Config
}{
{"", false, true, true},
{"-i", true, true, true},
{"-a stdin", true, false, false},
{"-a stdin -a stdout", true, true, false},
{"-a stdin -a stdout -a stderr", true, true, true},
} {
{
input: "",
expected: container.Config{
AttachStdout: true,
AttachStderr: true,
},
},
{
input: "-i",
expected: container.Config{
AttachStdin: true,
AttachStdout: true,
AttachStderr: true,
},
},
{
input: "-a stdin",
expected: container.Config{
AttachStdin: true,
},
},
{
input: "-a stdin -a stdout",
expected: container.Config{
AttachStdin: true,
AttachStdout: true,
},
},
{
input: "-a stdin -a stdout -a stderr",
expected: container.Config{
AttachStdin: true,
AttachStdout: true,
AttachStderr: true,
},
},
}
for _, tc := range tests {
t.Run(tc.input, func(t *testing.T) {
config, _, _ := mustParse(t, tc.input)
require.Equal(t, tc.stdin, config.AttachStdin)
require.Equal(t, tc.stdout, config.AttachStdout)
require.Equal(t, tc.stderr, config.AttachStderr)
assert.Equal(t, config.AttachStdin, tc.expected.AttachStdin)
assert.Equal(t, config.AttachStdout, tc.expected.AttachStdout)
assert.Equal(t, config.AttachStderr, tc.expected.AttachStderr)
})
}
}
@@ -237,52 +260,109 @@ func TestParseRunWithInvalidArgs(t *testing.T) {
flags, _ := setupRunFlags()
for _, tc := range tests {
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
require.EqualError(t, flags.Parse(tc.args), tc.error)
assert.Error(t, flags.Parse(tc.args), tc.error)
})
}
}
func TestParseWithVolumes(t *testing.T) {
for _, tc := range []struct {
unix, windows []string
binds int
}{
{[]string{`/tmp`}, []string{`c:\tmp`}, 0},
{[]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`}, 0},
{[]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`}, 1},
{[]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`}, 2},
{[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`}, []string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`}, 2},
{[]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, nil, 2},
{[]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, nil, 2},
{[]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`}, 1},
{nil, []string{os.Getenv("SystemDrive") + `\:d:`}, 1},
} {
volumes := tc.unix
if runtime.GOOS == "windows" {
volumes = tc.windows
}
if len(volumes) == 0 {
continue
}
t.Run(strings.Join(volumes, ","), func(t *testing.T) {
var args []string
for _, volume := range volumes {
args = append(args, "-v", volume)
}
config, hostConfig, _, err := parseRun(args)
require.NoError(t, err)
if tc.binds == 0 {
assert.Nil(t, hostConfig.Binds)
} else {
assert.ElementsMatch(t, volumes[:tc.binds], hostConfig.Binds)
}
expectedVolumes := map[string]struct{}{}
for _, volume := range volumes[tc.binds:] {
expectedVolumes[volume] = struct{}{}
}
assert.Equal(t, expectedVolumes, config.Volumes)
})
func TestParseWithVolumes(t *testing.T) { //nolint:gocyclo // verbatim copy from docker/cli tests
// A single volume
arr, tryit := setupPlatformVolume([]string{`/tmp`}, []string{`c:\tmp`})
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
} else if _, exists := config.Volumes[arr[0]]; !exists {
t.Fatalf("Error parsing volume flags, %q is missing from volumes. Received %v", tryit, config.Volumes)
}
// Two volumes
arr, tryit = setupPlatformVolume([]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`})
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
} else if _, exists := config.Volumes[arr[0]]; !exists {
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[0], config.Volumes)
} else if _, exists := config.Volumes[arr[1]]; !exists {
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[1], config.Volumes)
}
// A single bind mount
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`})
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || hostConfig.Binds[0] != arr[0] {
t.Fatalf("Error parsing volume flags, %q should mount-bind the path before the colon into the path after the colon. Received %v %v", arr[0], hostConfig.Binds, config.Volumes)
}
// Two bind mounts.
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`})
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
}
// Two bind mounts, first read-only, second read-write.
// TODO Windows: The Windows version uses read-write as that's the only mode it supports. Can change this post TP4
arr, tryit = setupPlatformVolume(
[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`},
[]string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`})
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
}
// Similar to previous test but with alternate modes which are only supported by Linux
if runtime.GOOS != "windows" {
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, []string{})
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
}
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, []string{})
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
}
}
// One bind mount and one volume
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`})
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] {
t.Fatalf("Error parsing volume flags, %s and %s should only one and only one bind mount %s. Received %s", arr[0], arr[1], arr[0], hostConfig.Binds)
} else if _, exists := config.Volumes[arr[1]]; !exists {
t.Fatalf("Error parsing volume flags %s and %s. %s is missing from volumes. Received %v", arr[0], arr[1], arr[1], config.Volumes)
}
// Root to non-c: drive letter (Windows specific)
if runtime.GOOS == "windows" {
arr, tryit = setupPlatformVolume([]string{}, []string{os.Getenv("SystemDrive") + `\:d:`})
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] || len(config.Volumes) != 0 {
t.Fatalf("Error parsing %s. Should have a single bind mount and no volumes", arr[0])
}
}
}
// setupPlatformVolume takes two arrays of volume specs - a Unix style
// spec and a Windows style spec. Depending on the platform being unit tested,
// it returns one of them, along with a volume string that would be passed
// on the docker CLI (e.g. -v /bar -v /foo).
func setupPlatformVolume(u, w []string) ([]string, string) {
var a []string
if runtime.GOOS == "windows" {
a = w
} else {
a = u
}
s := ""
for _, v := range a {
s = s + "-v " + v + " "
}
return a, s
}
// check if (a == c && b == d) || (a == d && b == c)
// because maps are randomized
func compareRandomizedStrings(a, b, c, d string) error {
if a == c && b == d {
return nil
}
if a == d && b == c {
return nil
}
return errors.New("strings don't match")
}
// Simple parse with MacAddress validation
@@ -303,49 +383,66 @@ func TestRunFlagsParseWithMemory(t *testing.T) {
flags, _ := setupRunFlags()
args := []string{"--memory=invalid", "img", "cmd"}
err := flags.Parse(args)
require.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
assert.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
_, hostconfig, _ := mustParse(t, "--memory=1G")
assert.Equal(t, int64(1073741824), hostconfig.Memory)
assert.Check(t, is.Equal(int64(1073741824), hostconfig.Memory))
}
func TestParseWithMemorySwap(t *testing.T) {
flags, _ := setupRunFlags()
args := []string{"--memory-swap=invalid", "img", "cmd"}
err := flags.Parse(args)
require.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
assert.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
_, hostconfig, _ := mustParse(t, "--memory-swap=1G")
assert.Equal(t, int64(1073741824), hostconfig.MemorySwap)
assert.Check(t, is.Equal(int64(1073741824), hostconfig.MemorySwap))
_, hostconfig, _ = mustParse(t, "--memory-swap=-1")
assert.Equal(t, int64(-1), hostconfig.MemorySwap)
assert.Check(t, is.Equal(int64(-1), hostconfig.MemorySwap))
}
func TestParseHostname(t *testing.T) {
for _, tc := range []struct {
input, hostname, domainname string
}{
{"--hostname=hostname", "hostname", ""},
{"--hostname=host-name", "host-name", ""},
{"--hostname=hostname123", "hostname123", ""},
{"--hostname=123hostname", "123hostname", ""},
{"--hostname=hostname-of-63-bytes-long-should-be-valid-and-without-any-error", "hostname-of-63-bytes-long-should-be-valid-and-without-any-error", ""},
{"--hostname=hostname.domainname", "hostname.domainname", ""},
{"--hostname=hostname.domainname.tld", "hostname.domainname.tld", ""},
{"--domainname=domainname", "", "domainname"},
{"--domainname=domain-name", "", "domain-name"},
{"--domainname=domainname123", "", "domainname123"},
{"--domainname=123domainname", "", "123domainname"},
{"--domainname=domainname-63-bytes-long-should-be-valid-and-without-any-errors", "", "domainname-63-bytes-long-should-be-valid-and-without-any-errors"},
{"--hostname=some.prefix --domainname=domainname", "some.prefix", "domainname"},
{"--hostname=another-prefix --domainname=domainname.tld", "another-prefix", "domainname.tld"},
} {
t.Run(tc.input, func(t *testing.T) {
config, _, _ := mustParse(t, tc.input)
require.Equal(t, tc.hostname, config.Hostname)
require.Equal(t, tc.domainname, config.Domainname)
})
validHostnames := map[string]string{
"hostname": "hostname",
"host-name": "host-name",
"hostname123": "hostname123",
"123hostname": "123hostname",
"hostname-of-63-bytes-long-should-be-valid-and-without-any-error": "hostname-of-63-bytes-long-should-be-valid-and-without-any-error",
}
hostnameWithDomain := "--hostname=hostname.domainname"
hostnameWithDomainTld := "--hostname=hostname.domainname.tld"
for hostname, expectedHostname := range validHostnames {
if config, _, _ := mustParse(t, "--hostname="+hostname); config.Hostname != expectedHostname {
t.Fatalf("Expected the config to have 'hostname' as %q, got %q", expectedHostname, config.Hostname)
}
}
if config, _, _ := mustParse(t, hostnameWithDomain); config.Hostname != "hostname.domainname" || config.Domainname != "" {
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname, got %q", config.Hostname)
}
if config, _, _ := mustParse(t, hostnameWithDomainTld); config.Hostname != "hostname.domainname.tld" || config.Domainname != "" {
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname.tld, got %q", config.Hostname)
}
}
func TestParseHostnameDomainname(t *testing.T) {
validDomainnames := map[string]string{
"domainname": "domainname",
"domain-name": "domain-name",
"domainname123": "domainname123",
"123domainname": "123domainname",
"domainname-63-bytes-long-should-be-valid-and-without-any-errors": "domainname-63-bytes-long-should-be-valid-and-without-any-errors",
}
for domainname, expectedDomainname := range validDomainnames {
if config, _, _ := mustParse(t, "--domainname="+domainname); config.Domainname != expectedDomainname {
t.Fatalf("Expected the config to have 'domainname' as %q, got %q", expectedDomainname, config.Domainname)
}
}
if config, _, _ := mustParse(t, "--hostname=some.prefix --domainname=domainname"); config.Hostname != "some.prefix" || config.Domainname != "domainname" {
t.Fatalf("Expected the config to have 'hostname' as 'some.prefix' and 'domainname' as 'domainname', got %q and %q", config.Hostname, config.Domainname)
}
if config, _, _ := mustParse(t, "--hostname=another-prefix --domainname=domainname.tld"); config.Hostname != "another-prefix" || config.Domainname != "domainname.tld" {
t.Fatalf("Expected the config to have 'hostname' as 'another-prefix' and 'domainname' as 'domainname.tld', got %q and %q", config.Hostname, config.Domainname)
}
}
@@ -364,7 +461,7 @@ func TestParseWithExpose(t *testing.T) {
for expose, expectedError := range tests {
t.Run(expose, func(t *testing.T) {
_, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
require.EqualError(t, err, expectedError)
assert.Error(t, err, expectedError)
})
}
})
@@ -379,38 +476,30 @@ func TestParseWithExpose(t *testing.T) {
for expose, exposedPorts := range tests {
t.Run(expose, func(t *testing.T) {
config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
require.NoError(t, err)
assert.NilError(t, err)
for _, port := range exposedPorts {
_, ok := config.ExposedPorts[port]
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
}
})
}
})
t.Run("merge with published", func(t *testing.T) {
config, hostConfig, _, err := parseRun([]string{"--publish=80", "--publish=127.0.0.1:8080:80", "--publish=[::1]:8081:80", "--expose=80-81/tcp"})
require.NoError(t, err)
assert.Equal(t, networktypes.PortMap{
networktypes.MustParsePort("80/tcp"): {
{},
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "8080"},
{HostIP: netip.MustParseAddr("::1"), HostPort: "8081"},
},
}, hostConfig.PortBindings)
assert.Len(t, config.ExposedPorts, 2)
// Merge with actual published port
config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"})
assert.NilError(t, err)
assert.Check(t, is.Len(config.ExposedPorts, 2))
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")}
for _, port := range ports {
_, ok := config.ExposedPorts[port]
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
}
})
}
func TestParseDevice(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("Windows and macOS validate server-side")
}
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
testCases := []struct {
devices []string
deviceMapping *container.DeviceMapping
@@ -484,17 +573,17 @@ func TestParseDevice(t *testing.T) {
_, hostconfig, _, err := parseRun(args)
require.NoError(t, err)
assert.NilError(t, err)
if tc.deviceMapping != nil {
if assert.Len(t, hostconfig.Devices, 1) {
assert.Equal(t, *tc.deviceMapping, hostconfig.Devices[0])
if assert.Check(t, is.Len(hostconfig.Devices, 1)) {
assert.Check(t, is.DeepEqual(*tc.deviceMapping, hostconfig.Devices[0]))
}
} else {
assert.Empty(t, hostconfig.Devices)
assert.Check(t, is.Len(hostconfig.Devices, 0))
}
assert.Equal(t, tc.deviceRequests, hostconfig.DeviceRequests)
assert.Check(t, is.DeepEqual(tc.deviceRequests, hostconfig.DeviceRequests))
})
}
}
@@ -563,11 +652,11 @@ func TestParseDeviceByServerOS(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
got, err := parseDevice(tc.device, tc.serverOS)
if tc.wantErr != "" {
require.EqualError(t, err, tc.wantErr)
assert.Error(t, err, tc.wantErr)
return
}
require.NoError(t, err)
require.Equal(t, tc.want, got)
assert.NilError(t, err)
assert.Equal(t, got, tc.want)
})
}
}
@@ -746,13 +835,13 @@ func TestParseNetworkConfig(t *testing.T) {
_, hConfig, nwConfig, err := parseRun(tc.flags)
if tc.expectedErr != "" {
require.EqualError(t, err, tc.expectedErr)
assert.Error(t, err, tc.expectedErr)
return
}
require.NoError(t, err)
require.Equal(t, tc.expectedHostCfg.NetworkMode, hConfig.NetworkMode)
require.Equal(t, tc.expected, nwConfig.EndpointsConfig)
assert.NilError(t, err)
assert.DeepEqual(t, hConfig.NetworkMode, tc.expectedHostCfg.NetworkMode)
assert.DeepEqual(t, nwConfig.EndpointsConfig, tc.expected, cmpopts.EquateComparable(netip.Addr{}))
})
}
}
@@ -761,24 +850,24 @@ func TestParseModes(t *testing.T) {
// pid ko
flags, copts := setupRunFlags()
args := []string{"--pid=container:", "img", "cmd"}
require.NoError(t, flags.Parse(args))
assert.NilError(t, flags.Parse(args))
_, err := parse(flags, copts, runtime.GOOS)
require.ErrorContains(t, err, "--pid: invalid PID mode")
assert.ErrorContains(t, err, "--pid: invalid PID mode")
// pid ok
_, hostconfig, _, err := parseRun([]string{"--pid=host", "img", "cmd"})
require.NoError(t, err)
assert.NilError(t, err)
if !hostconfig.PidMode.Valid() {
t.Fatalf("Expected a valid PidMode, got %v", hostconfig.PidMode)
}
// uts ko
_, _, _, err = parseRun([]string{"--uts=container:", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
require.ErrorContains(t, err, "--uts: invalid UTS mode")
assert.ErrorContains(t, err, "--uts: invalid UTS mode")
// uts ok
_, hostconfig, _, err = parseRun([]string{"--uts=host", "img", "cmd"})
require.NoError(t, err)
assert.NilError(t, err)
if !hostconfig.UTSMode.Valid() {
t.Fatalf("Expected a valid UTSMode, got %v", hostconfig.UTSMode)
}
@@ -790,11 +879,11 @@ func TestRunFlagsParseShmSize(t *testing.T) {
args := []string{"--shm-size=a128m", "img", "cmd"}
expectedErr := `invalid argument "a128m" for "--shm-size" flag:`
err := flags.Parse(args)
require.ErrorContains(t, err, expectedErr)
assert.ErrorContains(t, err, expectedErr)
// shm-size ok
_, hostconfig, _, err := parseRun([]string{"--shm-size=128m", "img", "cmd"})
require.NoError(t, err)
assert.NilError(t, err)
if hostconfig.ShmSize != 134217728 {
t.Fatalf("Expected a valid ShmSize, got %d", hostconfig.ShmSize)
}
@@ -865,11 +954,11 @@ func TestParseRestartPolicy(t *testing.T) {
t.Run(tc.input, func(t *testing.T) {
_, hostConfig, _, err := parseRun([]string{"--restart=" + tc.input, "img", "cmd"})
if tc.expectedErr != "" {
assert.Nil(t, hostConfig)
assert.EqualError(t, err, tc.expectedErr)
assert.Check(t, is.Error(err, tc.expectedErr))
assert.Check(t, is.Nil(hostConfig))
} else {
require.NoError(t, err)
assert.Equal(t, tc.expected, hostConfig.RestartPolicy)
assert.NilError(t, err)
assert.Check(t, is.DeepEqual(hostConfig.RestartPolicy, tc.expected))
}
})
}
@@ -878,33 +967,45 @@ func TestParseRestartPolicy(t *testing.T) {
func TestParseRestartPolicyAutoRemove(t *testing.T) {
_, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
const expected = "conflicting options: cannot specify both --restart and --rm"
assert.EqualError(t, err, expected)
assert.Check(t, is.Error(err, expected))
}
func TestParseHealth(t *testing.T) {
for _, tc := range []struct {
args []string
expected *container.HealthConfig
wantErr string
}{
{[]string{"--no-healthcheck", "img", "cmd"}, &container.HealthConfig{Test: []string{"NONE"}}, ""},
{[]string{"--health-cmd=/check.sh -q", "img", "cmd"}, &container.HealthConfig{Test: []string{"CMD-SHELL", "/check.sh -q"}}, ""},
{[]string{"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd"}, nil, "--no-healthcheck conflicts with --health-* options"},
{
[]string{"--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd"},
&container.HealthConfig{Timeout: 2 * time.Second, Retries: 3, Interval: 4500 * time.Millisecond, StartPeriod: 5 * time.Second, StartInterval: time.Second},
"",
},
} {
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
config, _, _, err := parseRun(tc.args)
if tc.wantErr != "" {
require.EqualError(t, err, tc.wantErr)
} else {
require.NoError(t, err)
require.Equal(t, tc.expected, config.Healthcheck)
}
})
checkOk := func(args ...string) *container.HealthConfig {
config, _, _, err := parseRun(args)
if err != nil {
t.Fatalf("%#v: %v", args, err)
}
return config.Healthcheck
}
checkError := func(expected string, args ...string) {
config, _, _, err := parseRun(args)
if err == nil {
t.Fatalf("Expected error, but got %#v", config)
}
if err.Error() != expected {
t.Fatalf("Expected %#v, got %#v", expected, err)
}
}
health := checkOk("--no-healthcheck", "img", "cmd")
if health == nil || len(health.Test) != 1 || health.Test[0] != "NONE" {
t.Fatalf("--no-healthcheck failed: %#v", health)
}
health = checkOk("--health-cmd=/check.sh -q", "img", "cmd")
if len(health.Test) != 2 || health.Test[0] != "CMD-SHELL" || health.Test[1] != "/check.sh -q" {
t.Fatalf("--health-cmd: got %#v", health.Test)
}
if health.Timeout != 0 {
t.Fatalf("--health-cmd: timeout = %s", health.Timeout)
}
checkError("--no-healthcheck conflicts with --health-* options",
"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd")
if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second || health.StartInterval != 1*time.Second {
t.Fatalf("--health-*: got %#v", health)
}
}
@@ -923,66 +1024,93 @@ func TestParseLoggingOpts(t *testing.T) {
}
}
func TestParseFileVariables(t *testing.T) {
for _, tc := range []struct {
flag, file, key string
}{
{"env", "valid.env", "ENV"},
{"label", "valid.label", "LABEL"},
} {
t.Run(tc.flag, func(t *testing.T) {
expectedError := "no such file or directory"
if runtime.GOOS == "windows" {
expectedError = "The system cannot find the file specified."
}
_, _, _, err := parseRun([]string{"--" + tc.flag + "-file=nonexistent"})
require.EqualError(t, err, "--"+tc.flag+"-file: open nonexistent: "+expectedError)
for _, extra := range []bool{false, true} {
args := []string{"--" + tc.flag + "-file=testdata/" + tc.file}
expected := []string{tc.key + "1=value1"}
if extra {
args = append(args, "--"+tc.flag+"="+tc.key+"2=value2")
expected = append(expected, tc.key+"2=value2")
}
config, _, _, err := parseRun(args)
require.NoError(t, err)
if tc.flag == "env" {
assert.Equal(t, expected, config.Env)
} else {
assert.Len(t, config.Labels, len(expected))
for _, label := range expected {
key, value, _ := strings.Cut(label, "=")
assert.Equal(t, value, config.Labels[key])
}
}
}
})
func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
expErr := "--env-file: open nonexistent: no such file or directory"
if runtime.GOOS == "windows" {
expErr = "--env-file: open nonexistent: The system cannot find the file specified."
}
// env ko
if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
}
// env ok
config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"})
if err != nil {
t.Fatal(err)
}
if len(config.Env) != 1 || config.Env[0] != "ENV1=value1" {
t.Fatalf("Expected a config with [ENV1=value1], got %v", config.Env)
}
config, _, _, err = parseRun([]string{"--env-file=testdata/valid.env", "--env=ENV2=value2", "img", "cmd"})
if err != nil {
t.Fatal(err)
}
if len(config.Env) != 2 || config.Env[0] != "ENV1=value1" || config.Env[1] != "ENV2=value2" {
t.Fatalf("Expected a config with [ENV1=value1 ENV2=value2], got %v", config.Env)
}
}
func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env"})
require.NoError(t, err)
assert.Equal(t, []string{"FOO=BAR", "HELLO=您好", "BAR=FOO"}, config.Env)
// UTF8 with BOM
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env", "img", "cmd"})
if err != nil {
t.Fatal(err)
}
env := []string{"FOO=BAR", "HELLO=" + string([]byte{0xe6, 0x82, 0xa8, 0xe5, 0xa5, 0xbd}), "BAR=FOO"}
if len(config.Env) != len(env) {
t.Fatalf("Expected a config with %d env variables, got %v: %v", len(env), len(config.Env), config.Env)
}
for i, v := range env {
if config.Env[i] != v {
t.Fatalf("Expected a config with [%s], got %v", v, []byte(config.Env[i]))
}
}
for _, file := range []string{"utf16.env", "utf16be.env"} {
t.Run(file, func(t *testing.T) {
_, _, _, err := parseRun([]string{"--env-file=testdata/" + file})
require.ErrorContains(t, err, "invalid utf8 bytes at line")
})
// UTF16 with BOM
e := "invalid utf8 bytes at line"
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
t.Fatalf("Expected an error with message '%s', got %v", e, err)
}
// UTF16BE with BOM
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16be.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
t.Fatalf("Expected an error with message '%s', got %v", e, err)
}
}
func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
expErr := "--label-file: open nonexistent: no such file or directory"
if runtime.GOOS == "windows" {
expErr = "--label-file: open nonexistent: The system cannot find the file specified."
}
// label ko
if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
}
// label ok
config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"})
if err != nil {
t.Fatal(err)
}
if len(config.Labels) != 1 || config.Labels["LABEL1"] != "value1" {
t.Fatalf("Expected a config with [LABEL1:value1], got %v", config.Labels)
}
config, _, _, err = parseRun([]string{"--label-file=testdata/valid.label", "--label=LABEL2=value2", "img", "cmd"})
if err != nil {
t.Fatal(err)
}
if len(config.Labels) != 2 || config.Labels["LABEL1"] != "value1" || config.Labels["LABEL2"] != "value2" {
t.Fatalf("Expected a config with [LABEL1:value1 LABEL2:value2], got %v", config.Labels)
}
}
func TestParseEntryPoint(t *testing.T) {
config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"})
require.NoError(t, err)
assert.Equal(t, []string{"anything"}, config.Entrypoint)
assert.NilError(t, err)
assert.Check(t, is.DeepEqual(config.Entrypoint, []string{"anything"}))
}
func TestValidateDevice(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("Windows and macOS validate server-side")
}
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
valid := []string{
"/home",
"/home:/home",
@@ -1081,22 +1209,22 @@ func TestValidateDeviceByServerOS(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
got, err := validateDevice(tc.value, tc.serverOS)
if tc.wantError != "" {
require.EqualError(t, err, tc.wantError)
assert.Error(t, err, tc.wantError)
return
}
require.NoError(t, err)
require.Equal(t, tc.want, got)
assert.NilError(t, err)
assert.Equal(t, got, tc.want)
})
}
}
func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
got, err := validateDeviceCgroupRule("c 1:3 rwm")
require.NoError(t, err)
require.Equal(t, "c 1:3 rwm", got)
assert.NilError(t, err)
assert.Equal(t, got, "c 1:3 rwm")
_, err = validateDeviceCgroupRule("invalid")
require.EqualError(t, err, "invalid device cgroup format 'invalid'")
assert.Error(t, err, "invalid device cgroup format 'invalid'")
if invalidParameter(nil) != nil {
t.Fatal("invalidParameter(nil) should be nil")
@@ -1104,9 +1232,9 @@ func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
cause := errors.New("bad input")
err = invalidParameter(cause)
var invalid interface{ InvalidParameter() }
require.ErrorAs(t, err, &invalid)
require.ErrorIs(t, err, cause)
require.Condition(t, func() bool { return invalidParameter(err) == err })
assert.Assert(t, errors.As(err, &invalid))
assert.Assert(t, errors.Is(err, cause))
assert.Equal(t, invalidParameter(err), err) // already invalid, so not wrapped twice
}
func TestParseSystemPaths(t *testing.T) {
@@ -1157,9 +1285,9 @@ func TestParseSystemPaths(t *testing.T) {
for _, tc := range tests {
securityOpts, maskedPaths, readonlyPaths := parseSystemPaths(tc.in)
require.Equal(t, tc.out, securityOpts)
require.Equal(t, tc.masked, maskedPaths)
require.Equal(t, tc.readonly, readonlyPaths)
assert.DeepEqual(t, securityOpts, tc.out)
assert.DeepEqual(t, maskedPaths, tc.masked)
assert.DeepEqual(t, readonlyPaths, tc.readonly)
}
}
@@ -1181,9 +1309,9 @@ func TestConvertToStandardNotation(t *testing.T) {
for key, ports := range valid {
convertedPorts, err := convertToStandardNotation(ports)
if err != nil {
require.NoError(t, err)
assert.NilError(t, err)
}
require.Equal(t, strings.Split(key, " "), convertedPorts)
assert.DeepEqual(t, strings.Split(key, " "), convertedPorts)
}
for _, ports := range invalid {

View File

@@ -8,10 +8,10 @@ package container
import (
"context"
"fmt"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/client"
specs "github.com/opencontainers/image-spec/specs-go/v1"
)
// ImageExistsLocally returns a boolean indicating if an image with the
@@ -30,23 +30,11 @@ func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool,
return false, err
}
if platform == "" || platform == "any" {
if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform {
return true, nil
}
requested, err := parsePlatform(platform)
if err != nil {
return false, err
}
return platformMatches(requested, &specs.Platform{OS: inspectImage.Os, Architecture: inspectImage.Architecture, Variant: inspectImage.Variant}), nil
}
func platformMatches(requested, image *specs.Platform) bool {
variant := image.Variant
if variant == "" {
variant = map[string]string{"amd64": "v1", "arm": "v7", "arm64": "v8"}[image.Architecture]
}
return image.OS == requested.OS && image.Architecture == requested.Architecture && (requested.Variant == "" || requested.Variant == variant)
return false, nil
}
// RemoveImage removes image from local store, the function is used to run different
@@ -58,14 +46,19 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
}
defer cli.Close()
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
Force: force,
PruneChildren: pruneChildren,
}); cerrdefs.IsNotFound(err) {
inspectImage, err := cli.ImageInspect(ctx, imageName)
if cerrdefs.IsNotFound(err) {
return false, nil
} else if err != nil {
return false, err
}
if _, err = cli.ImageRemove(ctx, inspectImage.ID, client.ImageRemoveOptions{
Force: force,
PruneChildren: pruneChildren,
}); err != nil {
return false, err
}
return true, nil
}

View File

@@ -5,74 +5,65 @@
package container
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
mobyclient "github.com/moby/moby/client"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func init() {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
}
func loadScratchImage(t *testing.T, platform string) string {
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
// network or emulation since there is nothing to run) and returns its tag, removing it after
// the test.
func buildScratchImage(t *testing.T, platform string) string {
t.Helper()
tag := fmt.Sprintf("act-test-exists-%s-%d:latest", strings.TrimPrefix(platform, "linux/"), time.Now().UnixNano())
var archive bytes.Buffer
require.NoError(t, writeFilesTar(t.Context(), &archive, 0, 0,
&FileEntry{Name: "config.json", Mode: 0o600, Body: fmt.Sprintf(`{"architecture":%q,"os":"linux","rootfs":{"type":"layers","diff_ids":[]}}`, strings.TrimPrefix(platform, "linux/"))},
&FileEntry{Name: "manifest.json", Mode: 0o600, Body: fmt.Sprintf(`[{"Config":"config.json","RepoTags":[%q],"Layers":[]}]`, tag)},
))
cli, err := GetDockerClient(t.Context())
require.NoError(t, err)
t.Cleanup(func() { _ = cli.Close() })
response, err := cli.ImageLoad(t.Context(), &archive)
require.NoError(t, err)
defer response.Close()
require.NoError(t, logDockerResponse(common.Logger(t.Context()), response, false))
t.Cleanup(func() {
_, _ = cli.ImageRemove(context.WithoutCancel(t.Context()), tag, mobyclient.ImageRemoveOptions{Force: true})
})
tag := fmt.Sprintf("act-test-exists-%s:latest", strings.TrimPrefix(platform, "linux/"))
cmd := exec.Command("docker", "build", "--platform", platform, "-t", tag, "-")
cmd.Stdin = strings.NewReader("FROM scratch\nLABEL act-test=1\n")
// Force BuildKit: it records the requested architecture in the image config for a
// FROM-scratch build, whereas the classic builder ignores --platform and tags it with the
// host arch, which would break the per-platform existence assertions below.
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, string(out))
t.Cleanup(func() { _ = exec.Command("docker", "rmi", "-f", tag).Run() })
return tag
}
func TestImageExistsLocally(t *testing.T) {
t.Parallel()
requireDocker(t)
ctx := t.Context()
ctx := context.Background()
// a non-existent image is reported absent
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.False(t, missing)
amd64Ref := loadScratchImage(t, "linux/amd64")
arm64Ref := loadScratchImage(t, "linux/arm64")
// Build tiny images for two architectures locally so per-platform existence can be checked
// offline (formerly pulled node:24-bookworm-slim for amd64 and arm64 over the network).
amd64Ref := buildScratchImage(t, "linux/amd64")
arm64Ref := buildScratchImage(t, "linux/arm64")
amd64Exists, err := ImageExistsLocally(ctx, amd64Ref, "linux/amd64")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.True(t, amd64Exists)
// a non-host architecture image is detected for its own architecture
arm64Exists, err := ImageExistsLocally(ctx, arm64Ref, "linux/arm64")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.True(t, arm64Exists)
// a present image is reported absent for a different platform
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.False(t, wrongPlatform)
removed, err := RemoveImage(ctx, amd64Ref, false, false)
require.NoError(t, err)
assert.True(t, removed)
removed, err = RemoveImage(ctx, amd64Ref, false, false)
require.NoError(t, err)
assert.False(t, removed)
}

View File

@@ -12,7 +12,7 @@ import (
"errors"
"io"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
)
type dockerMessage struct {
@@ -26,17 +26,25 @@ type dockerMessage struct {
Progress string `json:"progress"`
}
func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isError bool) error {
func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error {
if dockerResponse == nil {
return nil
}
defer dockerResponse.Close()
scanner := bufio.NewScanner(dockerResponse)
scanner.Buffer(nil, 64*1024*1024) // a build echoes long instructions in one record
msg := dockerMessage{}
for scanner.Scan() {
line := scanner.Bytes()
var msg dockerMessage
msg.ID = ""
msg.Stream = ""
msg.Error = ""
msg.ErrorDetail.Message = ""
msg.Status = ""
msg.Progress = ""
if err := json.Unmarshal(line, &msg); err != nil {
writeLog(logger, false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
continue
@@ -66,10 +74,10 @@ func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isE
}
}
return scanner.Err()
return nil
}
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...any) {
if isError {
logger.Errorf(format, args...)
} else {

View File

@@ -10,20 +10,21 @@ import (
"context"
"errors"
"fmt"
"net/netip"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/client"
)
const (
networkCreateAttempts = 3
networkCreateRetryDelay = time.Second
// marks the networks a runner creates for its jobs, so it can tell its own leftovers from
// those of another runner sharing the daemon
runnerUUIDLabel = "com.gitea.runner.uuid"
)
// RemoveOrphanNetworks removes the networks this runner created for jobs whose teardown did
@@ -31,17 +32,17 @@ const (
// had an endpoint on it at the time. Each one holds a subnet of the daemon's address pool
// until it is removed. Networks created after createdBefore are left alone, so a job starting
// while this runs cannot lose the network it has created but not yet attached a container to.
func RemoveOrphanNetworks(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
}
defer cli.Close()
return removeOrphanNetworks(ctx, cli, runnerUUID, detach, createdBefore)
return removeOrphanNetworks(ctx, cli, runnerUUID, createdBefore)
}
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID, detach string, createdBefore time.Time) error {
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID string, createdBefore time.Time) error {
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID),
})
@@ -50,32 +51,18 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID,
}
var errs []error
networks:
for _, n := range networks.Items {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
if err != nil {
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
continue
}
// the emptiness check, not the label, is what keeps a live job of another process
// sharing this registration safe
if len(result.Network.Containers) > 1 || result.Network.Created.After(createdBefore) {
if len(result.Network.Containers) != 0 || result.Network.Created.After(createdBefore) {
continue
}
for containerID := range result.Network.Containers {
if detach == "" || !strings.HasPrefix(containerID, detach) {
continue networks
}
if err := disconnectNetwork(ctx, cli, n.ID, containerID); err != nil {
errs = append(errs, fmt.Errorf("failed to disconnect container from network %s: %w", n.Name, err))
continue networks
}
}
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
continue
}
@@ -85,60 +72,51 @@ networks:
}
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
return common.Executor(func(ctx context.Context) error {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return createDockerNetwork(ctx, cli, name, opts)
}).IfNot(common.Dryrun)
}
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
// Only create the network if it doesn't exist
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
var options map[string]string
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
common.Logger(ctx).Debugf("Network %v exists", name)
return nil
}
// the daemon's mtu only reaches its default bridge, Docker's default is left out to keep default-network-opts
if mtu := n.Options["com.docker.network.driver.mtu"]; n.Name == network.NetworkBridge && mtu != "" && mtu != "1500" {
options = map[string]string{"com.docker.network.driver.mtu": mtu}
}
}
for i := range networkCreateAttempts {
if i > 0 {
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
}
}
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Scope: "local",
EnableIPv4: opts.EnableIPv4,
EnableIPv6: opts.EnableIPv6,
Options: options,
Labels: runnerLabels(opts.RunnerUUID),
}); err == nil {
return nil
}
if !isAddressPoolExhausted(err) {
// Only create the network if it doesn't exist
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
common.Logger(ctx).Debugf("Network %v exists", name)
return nil
}
}
for i := range networkCreateAttempts {
if i > 0 {
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
}
}
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Scope: "local",
EnableIPv4: opts.EnableIPv4,
EnableIPv6: opts.EnableIPv6,
Labels: runnerLabels(opts.RunnerUUID),
}); err == nil {
return nil
}
if !isAddressPoolExhausted(err) {
return err
}
}
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
}
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
}
func runnerLabels(runnerUUID string) map[string]string {
@@ -156,130 +134,41 @@ func isAddressPoolExhausted(err error) bool {
strings.Contains(msg, "could not find an available, non-overlapping IPv4 address pool among the defaults") // docker 24 and older
}
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
return common.Executor(func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
if err := connectNetwork(ctx, cli, networkName, container); err != nil {
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
}
return nil
}).IfNot(common.Dryrun)
}
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
_, err := cli.NetworkConnect(ctx, networkName, client.NetworkConnectOptions{
Container: container,
EndpointConfig: &network.EndpointSettings{Aliases: []string{container}},
})
if err != nil && (strings.Contains(err.Error(), "already exists in network") || strings.Contains(err.Error(), "already connected")) {
return nil
}
return err
}
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
return common.Executor(func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return disconnectNetwork(ctx, cli, networkName, container)
}).IfNot(common.Dryrun)
}
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
_, err := cli.NetworkDisconnect(ctx, networkName, client.NetworkDisconnectOptions{Container: container, Force: true})
if cerrdefs.IsNotFound(err) || (err != nil && strings.Contains(err.Error(), "is not connected")) {
return nil
}
return err
}
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return common.Executor(func(ctx context.Context) error {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return removeNetwork(ctx, cli, name)
}).IfNot(common.Dryrun)
}
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{}) // NetworkRemove by name refuses duplicates, so remove each by ID
if err != nil {
return err
}
var errs []error
for _, n := range networks.Items {
if n.Name != name {
continue
}
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if cerrdefs.IsNotFound(err) {
continue
} else if err != nil {
// Make sure that all network of the specified name are removed
// cli.NetworkRemove refuses to remove a network if there are duplicates
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
if len(result.Network.Containers) != 0 {
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the orphan cleanup reclaims it once they are gone", name)
continue
}
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
}
}
return errors.Join(errs...)
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
var errs []error
for _, n := range networks.Items {
if n.Name == name {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if err != nil {
return err
}
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
func IsolatedCacheContainer(ctx context.Context, addr netip.Addr, jobNetwork string) (string, error) {
cli, err := GetDockerClient(ctx)
if err != nil {
return "", err
}
defer cli.Close()
return isolatedCacheContainer(ctx, cli, addr, jobNetwork)
}
func isolatedCacheContainer(ctx context.Context, cli client.APIClient, addr netip.Addr, jobNetwork string) (string, error) {
if jobNetwork == "host" || strings.HasPrefix(jobNetwork, "container:") {
return "", nil
}
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{})
if err != nil {
return "", err
}
var holderID, holderContainer string
for _, summary := range containers.Items {
if summary.NetworkSettings == nil {
continue
}
for _, endpoint := range summary.NetworkSettings.Networks {
if endpoint != nil && (endpoint.IPAddress == addr || endpoint.GlobalIPv6Address == addr) {
holderID = endpoint.NetworkID
holderContainer = summary.ID[:12]
// it holds a subnet out of the daemon's pool until something reclaims it
if len(result.Network.Containers) != 0 {
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the idle cleanup reclaims it once they are gone", name)
continue
}
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
}
}
}
return errors.Join(errs...)
}
if holderID == "" {
return "", nil
}
holder, err := cli.NetworkInspect(ctx, holderID, client.NetworkInspectOptions{})
if err != nil || holder.Network.Driver != "bridge" {
return "", err
}
if jobNetwork != "" {
job, err := cli.NetworkInspect(ctx, jobNetwork, client.NetworkInspectOptions{})
if err != nil || job.Network.ID == holderID {
return "", err
}
}
return holderContainer, nil
}

View File

@@ -6,118 +6,27 @@ package container
import (
"context"
"errors"
"net/netip"
"testing"
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIsolatedCacheContainer(t *testing.T) {
ctx := t.Context()
cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
client := &mockDockerClient{}
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
ID: "0123456789abcdef",
NetworkSettings: &container.NetworkSettingsSummary{Networks: map[string]*network.EndpointSettings{
"compose": {NetworkID: "c0ffee", IPAddress: cache},
"lan": {NetworkID: "beef", IPAddress: lan},
}},
}}}, nil)
for ref, inspected := range map[string]network.Network{
"c0ffee": {ID: "c0ffee", Name: "compose", Driver: "bridge"},
"beef": {ID: "beef", Name: "lan", Driver: "macvlan"},
"c0f": {ID: "c0ffee"},
"jobs": {ID: "f00d"},
} {
client.On("NetworkInspect", ctx, ref, mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: inspected}}, nil)
}
check := func(addr netip.Addr, jobNetwork, want string) {
got, err := isolatedCacheContainer(ctx, client, addr, jobNetwork)
require.NoError(t, err)
assert.Equal(t, want, got, jobNetwork)
}
check(cache, "", "0123456789ab")
check(cache, "jobs", "0123456789ab")
check(cache, "c0f", "")
check(cache, "host", "")
check(lan, "", "")
check(netip.MustParseAddr("10.0.0.5"), "", "")
}
func TestIsAddressPoolExhausted(t *testing.T) {
assert.True(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("Error response from daemon: all predefined address pools have been fully subnetted")))
assert.True(t, isAddressPoolExhausted(errors.New("could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network")))
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
}
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
ctx := t.Context()
for _, tc := range []struct {
bridgeOptions, want map[string]string
}{
{
map[string]string{"com.docker.network.bridge.host_binding_ipv4": "127.0.0.1", "com.docker.network.driver.mtu": "1450"},
map[string]string{"com.docker.network.driver.mtu": "1450"},
},
{map[string]string{"com.docker.network.driver.mtu": "1500"}, nil},
} {
client := &mockDockerClient{}
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).
Return(mobyclient.NetworkListResult{Items: []network.Summary{{Name: network.NetworkBridge, Options: tc.bridgeOptions}}}, nil)
client.On("NetworkCreate", ctx, "job-network", mobyclient.NetworkCreateOptions{Driver: "bridge", Scope: "local", Options: tc.want}).
Return(mobyclient.NetworkCreateResult{}, nil)
require.NoError(t, createDockerNetwork(ctx, client, "job-network", NewDockerNetworkCreateExecutorInput{}))
client.AssertExpectations(t)
}
}
func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
}
func (m *mockDockerClient) NetworkDisconnect(ctx context.Context, id string, opts mobyclient.NetworkDisconnectOptions) (mobyclient.NetworkDisconnectResult, error) {
return mobyclient.NetworkDisconnectResult{}, m.Called(ctx, id, opts).Error(0)
}
func TestConnectAndDisconnectNetwork(t *testing.T) {
ctx := t.Context()
connect := func(answer error) error {
client := &mockDockerClient{}
client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
Container: "0123456789ab",
EndpointConfig: &network.EndpointSettings{Aliases: []string{"0123456789ab"}},
}).Return(answer).Once()
return connectNetwork(ctx, client, "jobs", "0123456789ab")
}
require.NoError(t, connect(nil))
require.NoError(t, connect(errors.New("endpoint with name runner already exists in network jobs")))
require.NoError(t, connect(errors.New("container runner already connected to network jobs")))
require.ErrorIs(t, connect(context.Canceled), context.Canceled)
disconnect := func(answer error) error {
client := &mockDockerClient{}
client.On("NetworkDisconnect", ctx, "jobs", mobyclient.NetworkDisconnectOptions{
Container: "0123456789ab", Force: true,
}).Return(answer).Once()
return disconnectNetwork(ctx, client, "jobs", "0123456789ab")
}
require.NoError(t, disconnect(cerrdefs.ErrNotFound))
require.NoError(t, disconnect(errors.New("container runner is not connected to the network jobs")))
require.ErrorIs(t, disconnect(context.Canceled), context.Canceled)
}
// Of this runner's networks, only the ones nothing but its own cache container is attached to
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
func TestRemoveNetworks(t *testing.T) {
ctx := t.Context()
// Of this runner's networks, only the ones nothing is attached to and old enough to predate
// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
// test on its own, since testify has no expectation to match it against.
func TestRemoveOrphanNetworks(t *testing.T) {
ctx := context.Background()
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
client := &mockDockerClient{}
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
@@ -126,46 +35,16 @@ func TestRemoveNetworks(t *testing.T) {
{ID: "orphan"},
{ID: "busy"},
{ID: "starting"},
{ID: "cached"},
{ID: "shared"},
{ID: "gone"},
{ID: "raced"},
}}, nil)
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
Return(mobyclient.NetworkInspectResult{}, nil)
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil).Twice()
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
Network: network.Network{Created: cutoff.Add(time.Second)},
Containers: map[string]network.EndpointResource{"0123456789abcdef": {}},
}}, nil)
client.On("NetworkInspect", ctx, "cached", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
client.On("NetworkInspect", ctx, "gone", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, cerrdefs.ErrNotFound).Twice()
client.On("NetworkInspect", ctx, "raced", mobyclient.NetworkInspectOptions{}).
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
client.On("NetworkRemove", ctx, "raced", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrNotFound).Twice()
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil).Twice()
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
Return(nil).Once()
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
Return(mobyclient.NetworkRemoveResult{}, nil)
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
{ID: "orphan", Name: "job"},
{ID: "busy", Name: "job"},
{ID: "gone", Name: "job"},
{ID: "raced", Name: "job"},
{ID: "shared", Name: "other"},
}}, nil)
require.NoError(t, removeNetwork(ctx, client, "job"))
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
client.AssertExpectations(t)
}

View File

@@ -6,7 +6,6 @@ package container
import (
"testing"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -18,17 +17,32 @@ func TestParsePlatform(t *testing.T) {
assert.Nil(t, got)
})
for input, want := range map[string]specs.Platform{
"linux/amd64": {OS: "linux", Architecture: "amd64"},
"linux/arm/v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
"Linux/AMD64/V8": {OS: "linux", Architecture: "amd64", Variant: "v8"},
} {
t.Run(input, func(t *testing.T) {
got, err := parsePlatform(input)
require.NoError(t, err)
assert.Equal(t, &want, got)
})
}
t.Run("os/arch", func(t *testing.T) {
got, err := parsePlatform("linux/amd64")
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, "linux", got.OS)
assert.Equal(t, "amd64", got.Architecture)
assert.Empty(t, got.Variant)
})
t.Run("os/arch/variant", func(t *testing.T) {
got, err := parsePlatform("linux/arm/v7")
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, "linux", got.OS)
assert.Equal(t, "arm", got.Architecture)
assert.Equal(t, "v7", got.Variant)
})
t.Run("input is lowercased", func(t *testing.T) {
got, err := parsePlatform("Linux/AMD64/V8")
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, "linux", got.OS)
assert.Equal(t, "amd64", got.Architecture)
assert.Equal(t, "v8", got.Variant)
})
for _, bad := range []string{
"amd64",
@@ -47,26 +61,3 @@ func TestParsePlatform(t *testing.T) {
})
}
}
func TestPlatformMatches(t *testing.T) {
for _, testcase := range []struct {
requested, image string
want bool
}{
{"linux/amd64", "linux/amd64", true},
{"linux/amd64/v1", "linux/amd64", true},
{"linux/arm64", "linux/amd64", false},
{"linux/arm64/v8", "linux/arm64", true},
{"linux/arm/v7", "linux/arm/v7", true},
{"linux/arm", "linux/arm/v7", true},
{"linux/arm/v6", "linux/arm/v7", false},
{"linux/arm", "linux/arm/v6", true},
{"windows/arm/v7", "linux/arm/v7", false},
} {
requested, err := parsePlatform(testcase.requested)
require.NoError(t, err)
image, err := parsePlatform(testcase.image)
require.NoError(t, err)
assert.Equal(t, testcase.want, platformMatches(requested, image), "%s on %s", testcase.requested, testcase.image)
}
}

View File

@@ -1,644 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"mime"
"net"
"net/http"
"net/http/httputil"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"slices"
"strings"
"sync"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
)
const (
jobLabel = "com.gitea.runner.job."
maxCreateBody = 8 << 20
dockerProxyProbeTimeout = 5 * time.Second
)
var (
createPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers|networks|volumes)/create$`)
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
)
var dockerProxyDirectories, dockerProxyProbes sync.Map
type dockerProxyDirectory struct {
dir, daemonDir string
socket os.FileInfo
}
type dockerProxyProbe struct {
directory dockerProxyDirectory
done chan struct{}
seen bool
err error
}
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
return nil
}
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
defer cancel()
cli, err := GetDockerClient(probeCtx)
if err != nil {
return nil
}
defer cli.Close()
daemonSocket, ok := strings.CutPrefix(cli.DaemonHost(), "unix://")
if !ok {
return nil
}
socket, err := os.Stat(daemonSocket)
if err != nil || socket.Mode()&os.ModeSocket == 0 {
return nil
}
workdir, _ := os.Getwd()
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
cached, _ := dockerProxyDirectories.Load(key)
directory, seen := cached.(dockerProxyDirectory)
seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
if !seen {
directory, seen, err = probeDockerProxyDirectory(probeCtx, cli, key, socket)
}
if err != nil {
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
return nil
}
if !seen {
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
return nil
}
if ctx.Err() != nil {
return nil
}
proxy, err := StartDockerProxy(daemonSocket, directory.dir, job)
if err != nil {
dockerProxyDirectories.Delete(key)
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
return nil
}
dockerProxyDirectories.Store(key, directory)
proxy.Socket = directory.daemonDir + strings.TrimPrefix(proxy.Socket, directory.dir)
return proxy
}
func probeDockerProxyDirectory(ctx context.Context, cli client.APIClient, key [3]string, socket os.FileInfo) (dockerProxyDirectory, bool, error) {
for {
probe := &dockerProxyProbe{directory: dockerProxyDirectory{socket: socket}, done: make(chan struct{})}
actual, loaded := dockerProxyProbes.LoadOrStore(key, probe)
if loaded {
probe, _ = actual.(*dockerProxyProbe)
if !os.SameFile(probe.directory.socket, socket) {
dockerProxyProbes.CompareAndDelete(key, probe)
continue
}
select {
case <-ctx.Done():
return dockerProxyDirectory{}, false, ctx.Err()
case <-probe.done:
}
if ctx.Err() == nil && (errors.Is(probe.err, context.Canceled) || errors.Is(probe.err, context.DeadlineExceeded)) {
continue
}
} else {
probe.directory.dir, probe.directory.daemonDir = runnerContainerWorkdir(ctx, cli)
probe.seen = probe.directory.daemonDir != ""
if !probe.seen {
if probe.directory.dir, probe.err = filepath.Abs(key[2]); probe.err == nil {
probe.directory.daemonDir = probe.directory.dir
probe.seen, probe.err = daemonSeesDir(ctx, cli, probe.directory.dir, probe.directory.daemonDir)
}
}
dockerProxyProbes.CompareAndDelete(key, probe)
close(probe.done)
}
return probe.directory, probe.seen, probe.err
}
}
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
workdir, err := os.Getwd()
hostname, hostnameErr := os.Hostname()
if err != nil || hostnameErr != nil {
return "", ""
}
self, err := cli.ContainerInspect(ctx, hostname, client.ContainerInspectOptions{})
if err != nil {
return "", ""
}
if daemonDir = containerInfoFromInspect(self.Container).DaemonPath(workdir); daemonDir == "" {
return "", ""
}
if seen, _ := containerSeesMarker(ctx, cli, workdir, self.Container.ID, workdir); !seen { // the hostname may name another container
return "", ""
}
return workdir, daemonDir
}
func containerSeesMarker(ctx context.Context, cli client.APIClient, dir, id, containerDir string) (bool, error) {
marker, err := os.CreateTemp(dir, "gitea-runner-probe-")
if err != nil {
return false, err
}
defer func() {
if err := os.Remove(marker.Name()); err != nil {
common.Logger(ctx).Warnf("removing the docker proxy probe marker failed: %v", err)
}
}()
if err := marker.Close(); err != nil {
return false, err
}
_, err = cli.ContainerStatPath(ctx, id, client.ContainerStatPathOptions{Path: path.Join(containerDir, filepath.Base(marker.Name()))})
if cerrdefs.IsNotFound(err) {
return false, nil
}
return err == nil, err
}
// daemonSeesDir reports whether the daemon opens the files the runner writes in dir by their path in daemonDir,
// which is what a job's proxy socket mounted from there needs.
func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir string) (bool, error) {
images, err := cli.ImageList(ctx, client.ImageListOptions{})
if err != nil {
return false, err
}
if len(images.Items) == 0 {
return false, errors.New("no image available for the docker proxy probe")
}
var created client.ContainerCreateResult
for _, image := range images.Items {
created, err = cli.ContainerCreate(ctx, client.ContainerCreateOptions{
Config: &container.Config{Image: image.ID, Cmd: []string{"true"}},
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
}},
})
if !cerrdefs.IsNotFound(err) {
break
}
}
if cerrdefs.IsInvalidArgument(err) {
return false, nil
}
if err != nil {
return false, err
}
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
defer cancel()
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
return false, fmt.Errorf("removing the docker proxy probe container failed: %w", removeErr)
}
return seen, err
}
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
info, err := os.Stat(daemonSocket)
if err != nil {
return nil, err
}
if info.Mode()&os.ModeSocket == 0 {
return nil, errors.New("docker daemon path is not a Unix socket")
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
instance, err := os.MkdirTemp(dir, "p-")
if err != nil {
return nil, err
}
socket := filepath.Join(instance, "docker.sock")
listener, err := net.Listen("unix", socket)
if err != nil {
return nil, errors.Join(err, os.RemoveAll(instance))
}
if err := copyDockerSocketPermissions(daemonSocket, socket, info); err != nil {
return nil, errors.Join(err, listener.Close(), os.RemoveAll(instance))
}
dial := func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", daemonSocket)
}
transport := &http.Transport{DialContext: dial}
forward := &httputil.ReverseProxy{
Rewrite: func(r *httputil.ProxyRequest) {
r.Out.URL.Scheme = "http"
r.Out.URL.Host = "docker"
},
Transport: transport,
}
proxy := &DockerProxy{Socket: socket}
streams, cancelStreams := context.WithCancel(context.Background())
creates, cancelCreates := context.WithCancel(context.Background())
var admission sync.Mutex
var handlers sync.WaitGroup
server := &http.Server{ReadHeaderTimeout: 30 * time.Second, ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
return context.WithValue(ctx, dockerProxyConnKey{}, conn)
}, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
admission.Lock()
if streams.Err() != nil {
admission.Unlock()
http.Error(w, "docker proxy is closing", http.StatusServiceUnavailable)
return
}
handlers.Add(1)
admission.Unlock()
defer handlers.Done()
creating := r.Method == http.MethodPost && createPath.MatchString(r.URL.Path)
parent, lifetime := r.Context(), streams
if creating {
parent, lifetime = context.WithoutCancel(parent), creates
}
ctx, cancel := context.WithCancel(parent)
defer cancel()
stop := context.AfterFunc(lifetime, func() {
cancel()
if !creating {
if conn, ok := parent.Value(dockerProxyConnKey{}).(net.Conn); ok {
_ = conn.Close()
}
}
})
defer stop()
r = r.WithContext(ctx)
if creating {
r.Body = http.MaxBytesReader(w, r.Body, maxCreateBody)
mounts, _ := proxy.mounts.Load().(map[string]string)
if err := rewriteCreate(r, job, mounts); err != nil {
status := http.StatusBadRequest
if _, ok := errors.AsType[*http.MaxBytesError](err); ok {
status = http.StatusRequestEntityTooLarge
}
http.Error(w, err.Error(), status)
return
}
} else if r.Method == http.MethodPost && rawStreamPath.MatchString(r.URL.Path) {
tunnel(w, r, dial, forward)
return
}
forward.ServeHTTP(w, r)
})}
served := make(chan struct{})
go func() {
defer close(served)
_ = server.Serve(listener)
}()
proxy.close = func(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
admission.Lock()
listenerErr := listener.Close()
cancelStreams()
admission.Unlock()
<-served
shutdownErr := server.Shutdown(ctx)
cancelCreates()
serverErr := server.Close()
handlers.Wait()
transport.CloseIdleConnections()
return errors.Join(ctx.Err(), listenerErr, shutdownErr, serverErr, os.RemoveAll(instance))
}
return proxy, nil
}
func rewriteCreate(r *http.Request, job string, mounts map[string]string) error {
body, err := io.ReadAll(r.Body)
if err != nil {
return err
}
if len(bytes.TrimSpace(body)) == 0 {
body = []byte("{}")
}
var fields map[string]json.RawMessage
var config struct{ Labels map[string]string }
if err := json.Unmarshal(body, &fields); err != nil {
return fmt.Errorf("invalid create request: %w", err)
}
if err := json.Unmarshal(body, &config); err != nil {
return fmt.Errorf("invalid create labels: %w", err)
}
if fields == nil {
fields = make(map[string]json.RawMessage)
}
if len(mounts) > 0 && !hasAmbiguousFields(body) {
translateBinds(fields, createPath.FindStringSubmatch(r.URL.Path)[2], mounts)
}
maps.DeleteFunc(fields, func(name string, _ json.RawMessage) bool {
return strings.EqualFold(name, "Labels")
})
if config.Labels == nil {
config.Labels = make(map[string]string)
}
config.Labels[jobLabel+job] = "" // per-job key, so nested proxies keep each other's
if fields["Labels"], err = json.Marshal(config.Labels); err != nil {
return err
}
if body, err = json.Marshal(fields); err != nil {
return err
}
r.Body = io.NopCloser(bytes.NewReader(body))
r.ContentLength = int64(len(body))
r.TransferEncoding = nil
return nil
}
func translateBinds(fields map[string]json.RawMessage, kind string, mounts map[string]string) {
cleanedSource := func(source string) string {
cleaned := path.Clean(source)
if target := jobMount(cleaned, mounts); mounts[target] != "" {
return mounts[target] + cleaned[len(target):]
}
return source
}
spelledSource := func(source string) string { // dockerd checks these as spelled
target := jobMount(path.Clean(source), mounts)
if rest, spelled := strings.CutPrefix(source, target); mounts[target] != "" && spelled && (rest == "" || rest[0] == '/') && filepath.IsLocal("."+rest) {
return mounts[target] + rest
}
return cleanedSource(source)
}
switch kind {
case "volumes":
var driver string
var options map[string]any
decodeField(fields, "Driver", &driver)
if key := decodeField(fields, "DriverOpts", &options); options != nil {
translateDevice(driver, options, spelledSource)
if encoded, err := json.Marshal(options); err == nil {
fields[key] = encoded
}
}
case "containers":
var hostConfig map[string]any
key := decodeField(fields, "HostConfig", &hostConfig)
binds, _ := field(hostConfig, "Binds").([]any)
for i, bind := range binds {
bind, _ := bind.(string)
source, target, _ := strings.Cut(bind, ":")
if translated := cleanedSource(source); strings.HasPrefix(target, "/") && !strings.Contains(translated, ":") {
binds[i] = translated + ":" + target
}
}
specs, _ := field(hostConfig, "Mounts").([]any)
for _, spec := range specs {
spec, _ := spec.(map[string]any)
switch field(spec, "Type") {
case "bind":
if source, ok := field(spec, "Source").(string); ok {
spec["Source"] = spelledSource(source)
}
case "volume":
volumeOptions, _ := field(spec, "VolumeOptions").(map[string]any)
driverConfig, _ := field(volumeOptions, "DriverConfig").(map[string]any)
translateDevice(field(driverConfig, "Name"), field(driverConfig, "Options"), spelledSource)
}
}
if encoded, err := json.Marshal(hostConfig); err == nil && hostConfig != nil {
fields[key] = encoded
}
}
}
func decodeField(fields map[string]json.RawMessage, name string, value any) string {
for key, raw := range fields {
if strings.EqualFold(key, name) {
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.UseNumber()
_ = decoder.Decode(value) // wrong types stay unset for dockerd to reject
return key
}
}
return ""
}
func translateDevice(driver, options any, translate func(string) string) {
optionMap, _ := options.(map[string]any)
device, ok := optionMap["device"].(string)
flags, _ := optionMap["o"].(string)
tokens := strings.Split(flags, ",")
local := driver == nil || driver == "" || driver == "local"
if ok && local && (slices.Contains(tokens, "bind") || slices.Contains(tokens, "rbind")) && !slices.Contains(tokens, "remount") {
optionMap["device"] = translate(device)
}
}
// field also renames the matched key to name.
func field(object map[string]any, name string) any {
for key, value := range object {
if strings.EqualFold(key, name) {
delete(object, key)
object[name] = value
return value
}
}
return nil
}
var (
requestFields = []string{"hostconfig", "driver", "driveropts"}
asciiFolds = strings.NewReplacer("ſ", "s", "K", "k") // the non-ASCII runes strings.EqualFold matches to ASCII letters
)
// dockerd settles repeated names by order, which re-encoding loses.
func hasAmbiguousFields(body []byte) bool {
type frame struct {
names map[string]bool
key string
expectKey bool
nested bool
}
decoder := json.NewDecoder(bytes.NewReader(body))
decoder.UseNumber()
stack := []*frame{{}}
for {
token, err := decoder.Token()
if err != nil {
return !errors.Is(err, io.EOF)
}
top := stack[len(stack)-1]
if name, ok := token.(string); ok && top.expectKey {
top.key, top.expectKey = strings.ToLower(asciiFolds.Replace(name)), false
if top.names[top.key] && (top.nested || slices.Contains(requestFields, top.key)) {
return true
}
top.names[top.key] = true
continue
}
top.expectKey = top.names != nil
nested := top.nested || len(stack) == 2 && slices.Contains(requestFields, top.key)
switch token {
case json.Delim('{'):
stack = append(stack, &frame{names: map[string]bool{}, expectKey: true, nested: nested})
case json.Delim('['):
stack = append(stack, &frame{nested: nested})
case json.Delim('}'), json.Delim(']'):
stack = stack[:len(stack)-1]
}
}
}
// jobMount returns "" also for a path already naming a daemon source.
func jobMount(source string, mounts map[string]string) string {
target := ""
for destination, daemonSource := range mounts {
if daemonSource != "" && (source == daemonSource || strings.HasPrefix(source, daemonSource+"/")) {
return ""
}
if len(destination) > len(target) && (source == destination || strings.HasPrefix(source, destination+"/")) {
target = destination
}
}
return target
}
type dockerProxyConnKey struct{}
type dockerProxyResponse struct {
response *http.Response
}
func (r dockerProxyResponse) RoundTrip(_ *http.Request) (*http.Response, error) {
return r.response, nil
}
// tunnel splices attach and exec streams, which the daemon hijacks with or without an HTTP upgrade
func tunnel(w http.ResponseWriter, r *http.Request, dial func(context.Context, string, string) (net.Conn, error), forward *httputil.ReverseProxy) {
upstream, err := dial(r.Context(), "", "")
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
defer upstream.Close()
stop := context.AfterFunc(r.Context(), func() { _ = upstream.Close() })
defer stop()
if err := r.Write(upstream); err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
reader := bufio.NewReader(upstream)
var response *http.Response
for {
response, err = http.ReadResponse(reader, r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
if response.StatusCode >= 200 || response.StatusCode == http.StatusSwitchingProtocols {
break
}
maps.Copy(w.Header(), response.Header)
w.WriteHeader(response.StatusCode)
clear(w.Header())
_ = response.Body.Close()
}
defer func() {
_ = upstream.Close()
_ = response.Body.Close()
}()
mediaType, _, _ := mime.ParseMediaType(response.Header.Get("Content-Type"))
if response.StatusCode != http.StatusSwitchingProtocols && (response.StatusCode != http.StatusOK || mediaType != "application/vnd.docker.raw-stream") {
ordinary := *forward
ordinary.Transport = dockerProxyResponse{response: response}
ordinary.ServeHTTP(w, r)
return
}
downstream, buffered, err := http.NewResponseController(w).Hijack()
if err != nil {
return
}
defer downstream.Close()
if _, err := fmt.Fprintf(buffered, "%s %s\r\n", response.Proto, response.Status); err != nil {
return
}
if err := response.Header.Write(buffered); err != nil {
return
}
if _, err := buffered.WriteString("\r\n"); err != nil {
return
}
if err := buffered.Flush(); err != nil {
return
}
done := make(chan struct{})
go func() {
defer close(done)
if _, err := io.Copy(upstream, io.MultiReader(io.LimitReader(buffered, int64(buffered.Reader.Buffered())), downstream)); err != nil { // Bypass net/http after the prefix so stdin EOF preserves output.
_ = upstream.Close()
} else if writer, ok := upstream.(interface{ CloseWrite() error }); ok {
_ = writer.CloseWrite()
} else {
_ = upstream.Close()
}
}()
_, _ = io.Copy(downstream, reader)
_ = downstream.Close()
_ = upstream.Close()
<-done
}
func RemoveDockerJobResources(ctx context.Context, job string) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
return removeLabelled(ctx, cli, job)
}
func removeLabelled(ctx context.Context, cli client.APIClient, job string) error {
logger := common.Logger(ctx)
filters := make(client.Filters).Add("label", jobLabel+job)
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
errs := []error{err}
for _, c := range containers.Items {
logger.Infof("removing container %s the job left behind", strings.TrimPrefix(strings.Join(c.Names, ","), "/"))
errs = append(errs, (&containerReference{cli: cli, id: c.ID}).remove()(ctx))
}
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{Filters: filters})
errs = append(errs, err)
for _, n := range networks.Items {
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); n.Scope == "swarm" && cerrdefs.IsInvalidArgument(err) { // swarm refuses while a service or its tasks use it
logger.Infof("keeping network %s, a swarm service still uses it", n.Name)
} else if err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
}
}
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{Filters: filters})
errs = append(errs, err)
for _, v := range volumes.Items {
if _, err := cli.VolumeRemove(ctx, v.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", v.Name, err))
}
}
return errors.Join(errs...)
}

View File

@@ -1,732 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"archive/tar"
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
"github.com/moby/moby/api/types/volume"
mobyclient "github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func (m *mockDockerClient) VolumeList(ctx context.Context, opts mobyclient.VolumeListOptions) (mobyclient.VolumeListResult, error) {
args := m.Called(ctx, opts)
return args.Get(0).(mobyclient.VolumeListResult), args.Error(1)
}
func (m *mockDockerClient) VolumeRemove(ctx context.Context, id string, opts mobyclient.VolumeRemoveOptions) (mobyclient.VolumeRemoveResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.VolumeRemoveResult), args.Error(1)
}
// unix socket paths are limited to about a hundred bytes, which the default macOS TMPDIR exceeds
func shortTempDir(t testing.TB) string {
t.Helper()
t.Setenv("TMPDIR", "/tmp")
return t.TempDir()
}
func daemonSocketPath(t testing.TB, cli mobyclient.APIClient) string {
t.Helper()
host := cli.DaemonHost()
if !strings.HasPrefix(host, "unix://") {
t.Skipf("skipping: daemon at %s is not a unix socket", host)
}
return strings.TrimPrefix(host, "unix://")
}
func TestDockerProxy(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Unix socket ownership is unavailable on Windows")
}
bodies := make(chan []byte, 8)
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
streamsDone := make(chan struct{}, 2)
var probes atomic.Int32
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
listener, err := net.Listen("unix", daemonSocket)
require.NoError(t, err)
daemon := &http.Server{ReadHeaderTimeout: time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/create"):
body, _ := io.ReadAll(r.Body)
if bytes.Contains(body, []byte(`"Image":"probe-image"`)) {
_, _ = w.Write([]byte(`{"Id":"probe"}`))
return
}
if r.URL.RawQuery == "wait" {
close(createStarted)
<-releaseCreate
}
bodies <- body
w.WriteHeader(http.StatusCreated)
case strings.HasSuffix(r.URL.Path, "/images/json"):
if probes.Add(1) == 1 {
http.Error(w, "probe unavailable", http.StatusServiceUnavailable)
return
}
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
case strings.HasSuffix(r.URL.Path, "/json"):
http.NotFound(w, r)
case strings.HasSuffix(r.URL.Path, "/archive"):
w.Header().Set("X-Docker-Container-Path-Stat", "e30=")
case r.Method == http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
if r.URL.Path == "/exec/error/start" {
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
w.WriteHeader(http.StatusBadRequest)
}
_, _ = w.Write([]byte("ordinary"))
case rawStreamPath.MatchString(r.URL.Path) || r.URL.Path == "/session":
_, _ = io.Copy(io.Discard, r.Body)
conn, buffered, err := w.(http.Hijacker).Hijack()
if err != nil {
return
}
defer conn.Close()
if r.Header.Get("Upgrade") != "" {
_, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\nready\n"))
} else {
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\nready\n"))
}
input, _ := io.ReadAll(buffered)
_, _ = conn.Write(append([]byte("final:"), input...))
streamsDone <- struct{}{}
default:
w.Header().Set("Api-Version", "1.47")
_, _ = w.Write([]byte("OK " + r.Method + " " + r.URL.Path))
}
})}
go func() { _ = daemon.Serve(listener) }()
t.Cleanup(func() { _ = daemon.Close() })
proxy, err := StartDockerProxy(daemonSocket, shortTempDir(t), "job-1")
require.NoError(t, err)
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
proxy.SetMounts(map[string]string{"/workspace/o/r": "/volumes/job/_data", "/workspace/o/r/tmp": "", "/var/run/docker.sock": "/tmp/p/docker.sock", "/volumes": "/daemon/volumes"})
client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", proxy.Socket)
}}}
t.Cleanup(client.CloseIdleConnections)
dialProxy := func(t *testing.T) *net.UnixConn {
t.Helper()
conn, err := net.DialUnix("unix", nil, &net.UnixAddr{Name: proxy.Socket, Net: "unix"})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.Close() })
require.NoError(t, conn.SetDeadline(time.Now().Add(5*time.Second)))
return conn
}
startStream := func(t *testing.T, path, upgrade string) (*net.UnixConn, *bufio.Reader) {
t.Helper()
conn := dialProxy(t)
_, err := fmt.Fprintf(conn, "POST %s HTTP/1.1\r\nHost: docker\r\n%sContent-Length: 2\r\n\r\n{}input", path, upgrade)
require.NoError(t, err)
reader := bufio.NewReader(conn)
resp, err := http.ReadResponse(reader, nil)
require.NoError(t, err)
t.Cleanup(func() {
_ = conn.Close()
_ = resp.Body.Close()
})
prefix, err := reader.ReadString('\n')
require.NoError(t, err)
require.Equal(t, "ready\n", prefix)
return conn, reader
}
t.Run("labels creates", func(t *testing.T) {
for _, testCase := range []struct{ path, body, want string }{
{
"/v1.47/containers/create", `{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":""}}`,
`{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":"","com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Binds":["/workspace/o/r:/src:ro","/workspace/o/r/../r/sub/:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2"}}`,
`{"HostConfig":{"Binds":["/volumes/job/_data:/src:ro","/volumes/job/_data/sub:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2","com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/var/run/docker.sock"},{"type":"bind","source":"/workspace/o/r/sub/","target":"/m"},{"Type":"bind","Source":"/workspace/o/r/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/workspace/o/r/../r","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]}}`,
`{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/tmp/p/docker.sock","Target":"/var/run/docker.sock"},{"Type":"bind","Source":"/volumes/job/_data/sub/","target":"/m"},{"Type":"bind","Source":"/volumes/job/_data/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/volumes/job/_data","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/workspace/o/r/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]}}`,
`{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/volumes/job/_data/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/containers/create", `{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null}`,
`{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null,"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/networks/create", `{"Name":"n"}`,
`{"Name":"n","Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Name":"v","labels":null,"DriverOpts":{"type":"none","o":"rbind,ro","device":"/workspace/o/r/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
`{"Name":"v","Labels":{"com.gitea.runner.job.job-1":""},"DriverOpts":{"type":"none","o":"rbind,ro","device":"/volumes/job/_data/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
},
{
"/volumes/create", `{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"}}`,
`{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"}}`,
`{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", "",
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", "null",
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
},
{
"/volumes/create", `{"Labels":{"discard":"1"},"labels":null,"LABELS":{"own":"1"},"LABELS":{"extra":"2"}}`,
`{"Labels":{"own":"1","extra":"2","com.gitea.runner.job.job-1":""}}`,
},
} {
resp, err := client.Post("http://docker"+testCase.path, "application/json", strings.NewReader(testCase.body))
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, testCase.want, string(<-bodies), testCase.body)
}
})
t.Run("passes other requests through", func(t *testing.T) {
resp, err := client.Get("http://docker/v1.47/_ping")
require.NoError(t, err)
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
assert.Equal(t, "1.47", resp.Header.Get("Api-Version"))
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
})
t.Run("discovery caches only successes", func(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("automatic proxy discovery requires Linux")
}
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
t.Setenv("TMPDIR", shortTempDir(t))
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
for _, job := range []string{"first", "second"} {
proxy := NewDockerProxy(t.Context(), job)
require.NotNil(t, proxy)
require.NoError(t, proxy.Close(context.Background()))
}
assert.EqualValues(t, 2, probes.Load())
dockerProxyDirectories.Range(func(key, value any) bool {
directory, _ := value.(dockerProxyDirectory)
directory.socket, _ = os.Stat(t.TempDir())
dockerProxyDirectories.Store(key, directory)
return true
})
proxy := NewDockerProxy(t.Context(), "replaced daemon")
require.NotNil(t, proxy)
require.NoError(t, proxy.Close(context.Background()))
assert.EqualValues(t, 3, probes.Load())
})
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
for _, stream := range []struct{ path, upgrade string }{
{path: "/v1.47/exec/abc/start"},
{path: "/containers/abc/attach", upgrade: "Connection: Upgrade\r\nUpgrade: tcp\r\n"},
} {
conn, reader := startStream(t, stream.path, stream.upgrade)
require.NoError(t, conn.CloseWrite())
output, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "final:input", string(output))
<-streamsDone
}
})
t.Run("detached and error responses retain keepalive label injection", func(t *testing.T) {
conn := dialProxy(t)
reader := bufio.NewReader(conn)
for endpoint, status := range map[string]int{"detached": http.StatusOK, "error": http.StatusBadRequest} {
_, err := fmt.Fprintf(conn, "POST /exec/%s/start HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}", endpoint)
require.NoError(t, err)
resp, err := http.ReadResponse(reader, nil)
require.NoError(t, err)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
assert.Equal(t, status, resp.StatusCode)
assert.Equal(t, "ordinary", string(body))
_, err = io.WriteString(conn, "POST /volumes/create HTTP/1.1\r\nHost: docker\r\nContent-Length: 0\r\n\r\n")
require.NoError(t, err)
resp, err = http.ReadResponse(reader, nil)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
}
})
t.Run("close joins streams and admitted create", func(t *testing.T) {
t.Cleanup(func() { close(releaseCreate) })
_, raw := startStream(t, "/exec/live/start", "")
_, upgraded := startStream(t, "/session", "Connection: Upgrade\r\nUpgrade: tcp\r\n")
creator := dialProxy(t)
_, err := io.WriteString(creator, "POST /volumes/create?wait HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}")
require.NoError(t, err)
<-createStarted
require.NoError(t, creator.CloseWrite())
closed := make(chan error, 1)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
go func() { closed <- proxy.Close(ctx) }()
for _, reader := range []*bufio.Reader{raw, upgraded} {
_, err := reader.ReadByte()
require.ErrorIs(t, err, io.EOF)
<-streamsDone
}
select {
case err := <-closed:
t.Fatalf("Close returned before admitted create settled: %v", err)
default:
}
releaseCreate <- struct{}{}
resp, err := http.ReadResponse(bufio.NewReader(creator), nil)
require.NoError(t, err)
require.NoError(t, resp.Body.Close())
require.Equal(t, http.StatusCreated, resp.StatusCode)
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
require.NoError(t, <-closed)
})
}
func TestRemoveLabelledRemovesContainersNetworksAndVolumes(t *testing.T) {
containerFailure := errors.New("container removal failed")
listFailure := errors.New("listing failed")
volumeFailure := errors.New("volume removal failed")
ctx := t.Context()
filters := make(mobyclient.Filters).Add("label", jobLabel+"job-1")
cli := &mockDockerClient{}
cli.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true, Filters: filters}).
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil).Once()
cli.On("ContainerKill", ctx, "c1", mock.Anything).Return(mobyclient.ContainerKillResult{}, nil).Once()
cli.On("ContainerRemove", ctx, "c1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
Return(mobyclient.ContainerRemoveResult{}, containerFailure).Once()
cli.On("NetworkList", ctx, mobyclient.NetworkListOptions{Filters: filters}).
Return(mobyclient.NetworkListResult{Items: []network.Summary{{ID: "n1", Name: "stack_default", Scope: "swarm"}}}, listFailure).Once()
cli.On("NetworkRemove", ctx, "n1", mobyclient.NetworkRemoveOptions{}).Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrInvalidArgument).Once()
cli.On("VolumeList", ctx, mobyclient.VolumeListOptions{Filters: filters}).
Return(mobyclient.VolumeListResult{Items: []volume.Volume{{Name: "app_data"}}}, nil).Once()
cli.On("VolumeRemove", ctx, "app_data", mobyclient.VolumeRemoveOptions{}).Return(mobyclient.VolumeRemoveResult{}, volumeFailure).Once()
err := removeLabelled(ctx, cli, "job-1")
require.ErrorIs(t, err, containerFailure)
require.ErrorIs(t, err, listFailure)
require.ErrorIs(t, err, volumeFailure)
require.NotErrorIs(t, err, cerrdefs.ErrInvalidArgument)
require.ErrorContains(t, err, "failed to remove container c1")
require.ErrorContains(t, err, "failed to remove volume app_data")
cli.AssertExpectations(t)
}
func TestDockerProxyWithDaemon(t *testing.T) {
requireDocker(t)
ctx := t.Context()
cleanupCtx := context.WithoutCancel(ctx)
require.NoError(t, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
direct, err := GetDockerClient(ctx)
require.NoError(t, err)
defer direct.Close()
dir := shortTempDir(t)
seen, err := daemonSeesDir(ctx, direct, dir, dir)
require.NoError(t, err)
t.Logf("daemon sees the runner's filesystem: %v", seen)
seen, err = daemonSeesDir(ctx, direct, dir, shortTempDir(t))
require.NoError(t, err)
assert.False(t, seen)
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
if os.Geteuid() != 0 && errors.Is(err, os.ErrPermission) {
t.Skipf("non-root runners that cannot take the daemon socket's ownership get it directly: %v", err)
}
require.NoError(t, err)
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
require.NoError(t, err)
defer viaProxy.Close()
created, err := viaProxy.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{
Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "300"}, Labels: map[string]string{"own": "1"}},
})
require.NoError(t, err)
t.Cleanup(func() {
_, _ = direct.ContainerRemove(cleanupCtx, created.ID, mobyclient.ContainerRemoveOptions{Force: true})
})
_, err = viaProxy.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
require.NoError(t, err)
net, err := viaProxy.NetworkCreate(ctx, job, mobyclient.NetworkCreateOptions{})
require.NoError(t, err)
t.Cleanup(func() { _, _ = direct.NetworkRemove(cleanupCtx, net.ID, mobyclient.NetworkRemoveOptions{}) })
_, err = viaProxy.VolumeCreate(ctx, mobyclient.VolumeCreateOptions{Name: job})
require.NoError(t, err)
t.Cleanup(func() { _, _ = direct.VolumeRemove(cleanupCtx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
inspected, err := direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
require.NoError(t, err)
assert.Subset(t, inspected.Container.Config.Labels, map[string]string{"own": "1", jobLabel + job: ""})
vol, err := direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
require.NoError(t, err)
assert.Contains(t, vol.Volume.Labels, jobLabel+job)
exec, err := viaProxy.ExecCreate(ctx, created.ID, mobyclient.ExecCreateOptions{Cmd: []string{"cat"}, AttachStdin: true, AttachStdout: true, TTY: true})
require.NoError(t, err)
attached, err := viaProxy.ExecAttach(ctx, exec.ID, mobyclient.ExecAttachOptions{TTY: true})
require.NoError(t, err)
_, err = attached.Conn.Write([]byte("hello\n"))
require.NoError(t, err)
echoed, err := attached.Reader.ReadString('\n')
require.NoError(t, err)
assert.Equal(t, "hello\r\n", echoed)
attached.Close()
require.NoError(t, proxy.Close(ctx))
require.NoError(t, RemoveDockerJobResources(ctx, job))
_, err = direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
_, err = direct.NetworkInspect(ctx, net.ID, mobyclient.NetworkInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
_, err = direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
assert.True(t, cerrdefs.IsNotFound(err))
}
func BenchmarkDockerProxy(b *testing.B) {
ctx := context.Background()
direct, err := GetDockerClient(ctx)
require.NoError(b, err)
defer direct.Close()
if _, err := direct.Ping(ctx, mobyclient.PingOptions{}); err != nil {
b.Skipf("docker daemon unreachable: %v", err)
}
proxy, err := StartDockerProxy(daemonSocketPath(b, direct), shortTempDir(b), "bench")
require.NoError(b, err)
defer func() { _ = proxy.Close(ctx) }()
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
require.NoError(b, err)
defer viaProxy.Close()
require.NoError(b, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
created, err := direct.ContainerCreate(ctx, mobyclient.ContainerCreateOptions{Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "600"}}})
require.NoError(b, err)
defer func() { _, _ = direct.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true}) }()
_, err = direct.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
require.NoError(b, err)
var archive bytes.Buffer
writer := tar.NewWriter(&archive)
payload := make([]byte, 64<<20)
require.NoError(b, writer.WriteHeader(&tar.Header{Name: "blob", Mode: 0o600, Size: int64(len(payload))}))
_, _ = writer.Write(payload)
require.NoError(b, writer.Close())
for name, cli := range map[string]mobyclient.APIClient{"direct": direct, "proxy": viaProxy} {
b.Run("ping/"+name, func(b *testing.B) {
for b.Loop() {
if _, err := cli.Ping(ctx, mobyclient.PingOptions{}); err != nil {
b.Fatal(err)
}
}
})
b.Run("copy64MiB/"+name, func(b *testing.B) {
b.SetBytes(int64(archive.Len()))
for b.Loop() {
_, err := cli.CopyToContainer(ctx, created.ID, mobyclient.CopyToContainerOptions{DestinationPath: "/tmp", Content: bytes.NewReader(archive.Bytes())})
if err != nil {
b.Fatal(err)
}
}
})
}
}
type probeClient struct {
mobyclient.APIClient
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
stat func(string) error
mounts []container.MountPoint
}
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
return mobyclient.ContainerInspectResult{Container: container.InspectResponse{Mounts: c.mounts}}, nil
}
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "removed-image"}, {ID: "probe-image"}}}, nil
}
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
return c.create(opts)
}
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
return c.remove(ctx, id, opts)
}
func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobyclient.ContainerStatPathOptions) (mobyclient.ContainerStatPathResult, error) {
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
}
func TestDockerProxyDirectoryProbes(t *testing.T) {
type result struct {
seen bool
err error
}
for name, createErr := range map[string]error{"unavailable directory": nil, "daemon error": errors.New("probe failed")} {
t.Run(name, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
dir := t.TempDir()
socket, err := os.Stat(t.TempDir())
require.NoError(t, err)
key := [3]string{"daemon", dir, dir}
release := make(chan struct{})
var calls, removals atomic.Int32
visible := false
cli := &probeClient{
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
calls.Add(1)
<-release
return mobyclient.ContainerCreateResult{ID: "probe"}, createErr
},
stat: func(_path string) error {
if visible {
return nil
} else {
return cerrdefs.ErrNotFound
}
},
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
removals.Add(1)
return mobyclient.ContainerRemoveResult{}, nil
},
}
results := make(chan result, 8)
for range cap(results) {
go func() {
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
results <- result{seen, err}
}()
}
synctest.Wait()
assert.EqualValues(t, 1, calls.Load())
ctx, cancel := context.WithCancel(t.Context())
cancel()
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
require.ErrorIs(t, err, context.Canceled)
assert.False(t, seen)
replacement, err := os.Stat(t.TempDir())
require.NoError(t, err)
replaced := make(chan result, 1)
go func() {
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, replacement)
replaced <- result{seen, err}
}()
synctest.Wait()
assert.EqualValues(t, 2, calls.Load())
close(release)
for range cap(results) {
result := <-results
require.ErrorIs(t, result.err, createErr)
assert.False(t, result.seen)
}
replacementResult := <-replaced
require.ErrorIs(t, replacementResult.err, createErr)
assert.False(t, replacementResult.seen)
assert.EqualValues(t, 2, calls.Load())
if createErr == nil {
assert.EqualValues(t, 2, removals.Load())
} else {
assert.Zero(t, removals.Load())
}
createErr, visible = nil, true
directory, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
require.NoError(t, err)
assert.True(t, seen)
assert.Equal(t, dir, directory.dir)
assert.Equal(t, dir, directory.daemonDir)
assert.EqualValues(t, 3, calls.Load())
entries, err := os.ReadDir(dir)
require.NoError(t, err)
assert.Empty(t, entries)
})
})
}
t.Run("cancelled probe does not cancel another caller", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
dir := t.TempDir()
socket, err := os.Stat(t.TempDir())
require.NoError(t, err)
key := [3]string{"daemon", dir, dir}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
release := make(chan struct{})
calls := 0
cli := &probeClient{
create: func(_opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
calls++
if calls == 1 {
<-release
return mobyclient.ContainerCreateResult{}, ctx.Err()
} else {
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
}
},
stat: func(_path string) error { return nil },
remove: func(_ctx context.Context, _id string, _opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
return mobyclient.ContainerRemoveResult{}, nil
},
}
leader, follower := make(chan result, 1), make(chan result, 1)
go func() {
_, seen, err := probeDockerProxyDirectory(ctx, cli, key, socket)
leader <- result{seen, err}
}()
synctest.Wait()
go func() {
_, seen, err := probeDockerProxyDirectory(t.Context(), cli, key, socket)
follower <- result{seen, err}
}()
synctest.Wait()
assert.Equal(t, 1, calls)
cancel()
close(release)
leaderResult, followerResult := <-leader, <-follower
require.ErrorIs(t, leaderResult.err, context.Canceled)
assert.False(t, leaderResult.seen)
require.NoError(t, followerResult.err)
assert.True(t, followerResult.seen)
assert.Equal(t, 2, calls)
})
})
}
func TestDaemonSeesDir(t *testing.T) {
dir := t.TempDir()
markers := make(map[string]bool)
for _, testCase := range []struct {
name string
daemonDir string
removeErr error
seen bool
removed bool
}{
{name: "cleanup after cancellation", daemonDir: dir, seen: true, removed: true},
{name: "daemon without the directory", daemonDir: filepath.Join(dir, "missing")},
{name: "daemon with a different directory at that path", daemonDir: t.TempDir(), removed: true},
{name: "cleanup error after cancellation", daemonDir: dir, removeErr: errors.New("cleanup failed"), removed: true},
} {
t.Run(testCase.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
removed := false
var attemptedImages []string
cli := &probeClient{
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
attemptedImages = append(attemptedImages, opts.Config.Image)
if opts.Config.Image == "removed-image" {
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrNotFound
}
cancel()
if _, err := os.Stat(testCase.daemonDir); err != nil {
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
}
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
},
stat: func(path string) error {
marker := strings.TrimPrefix(path, "/gitea-runner-probe/")
assert.False(t, markers[marker])
markers[marker] = true
if _, err := os.Stat(filepath.Join(testCase.daemonDir, marker)); err != nil {
return cerrdefs.ErrNotFound
}
return nil
},
remove: func(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
removed = true
assert.Equal(t, "probe", id)
assert.Equal(t, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true}, opts)
require.NoError(t, ctx.Err())
_, bounded := ctx.Deadline()
assert.True(t, bounded)
return mobyclient.ContainerRemoveResult{}, testCase.removeErr
},
}
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
require.ErrorIs(t, err, testCase.removeErr)
assert.Equal(t, []string{"removed-image", "probe-image"}, attemptedImages)
assert.Equal(t, testCase.seen, seen)
assert.Equal(t, testCase.removed, removed)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
assert.Empty(t, entries)
})
}
}
func TestRunnerContainerWorkdir(t *testing.T) {
workdir := t.TempDir()
t.Chdir(workdir)
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
for _, testCase := range []struct {
name string
mount container.MountPoint
foreign bool
daemonDir string
}{
{name: "volume above the working directory", mount: volume, daemonDir: filepath.Join(volume.Source, filepath.Base(workdir))},
{name: "tmpfs", mount: container.MountPoint{Type: mount.TypeTmpfs, Destination: volume.Destination}},
{name: "hostname of another container", mount: volume, foreign: true},
} {
cli := &probeClient{
mounts: []container.MountPoint{testCase.mount},
stat: func(path string) error {
if _, err := os.Stat(path); err != nil || testCase.foreign {
return cerrdefs.ErrNotFound
}
return nil
},
}
dir, daemonDir := runnerContainerWorkdir(t.Context(), cli)
assert.Equal(t, testCase.daemonDir, daemonDir, testCase.name)
assert.Equal(t, testCase.daemonDir != "", dir == workdir, testCase.name)
}
entries, err := os.ReadDir(workdir)
require.NoError(t, err)
assert.Empty(t, entries)
}

View File

@@ -1,48 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
package container
import (
"encoding/binary"
"errors"
"io/fs"
"math"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
)
func copyDockerSocketPermissions(daemonSocket, socket string, info os.FileInfo) error {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return errors.New("docker socket ownership is unavailable")
}
groupErr := os.Chown(socket, int(stat.Uid), int(stat.Gid))
if groupErr != nil && (!errors.Is(groupErr, fs.ErrPermission) || int(stat.Uid) != os.Geteuid()) {
return groupErr
}
if err := os.Chown(filepath.Dir(socket), int(stat.Uid), -1); err != nil {
return err
}
mode := uint16(info.Mode().Perm())
if err := os.Chmod(socket, fs.FileMode(mode)); err != nil || groupErr == nil {
return err
}
if size, err := unix.Getxattr(daemonSocket, "system.posix_acl_access", nil); err == nil && size > 0 {
return groupErr
}
owner, group, other := mode>>6, mode>>3&7, mode&7
acl := binary.LittleEndian.AppendUint32(nil, 2)
for _, entry := range []struct {
tag, perm uint16
id uint32
}{{1, owner, math.MaxUint32}, {4, other, math.MaxUint32}, {8, group, stat.Gid}, {16, group | other, math.MaxUint32}, {32, other, math.MaxUint32}} {
acl = binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint16(binary.LittleEndian.AppendUint16(acl, entry.tag), entry.perm), entry.id)
}
return unix.Setxattr(socket, "system.posix_acl_access", acl, 0) // names the group a rootless runner cannot chown to, its own group keeps what others had
}

View File

@@ -1,15 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER
package container
import (
"errors"
"os"
)
func copyDockerSocketPermissions(_, _ string, _ os.FileInfo) error {
return errors.New("docker socket ownership cannot be preserved on Windows")
}

View File

@@ -114,7 +114,7 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
if err != nil {
return imagePullOptions, err
}
if authConfig.Username == "" && authConfig.Password == "" && authConfig.IdentityToken == "" && authConfig.RegistryToken == "" {
if authConfig.Username == "" && authConfig.Password == "" {
return imagePullOptions, nil
}
logger.Info("using DockerAuthConfig authentication for docker pull")

View File

@@ -5,24 +5,20 @@
package container
import (
"context"
"io"
"os"
"path/filepath"
"strings"
"testing"
"testing/iotest"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
"github.com/moby/moby/api/pkg/authconfig"
"github.com/moby/moby/api/types/registry"
"github.com/docker/cli/cli/config"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
assert "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func init() {
logging.SetLevel(logging.LevelDebug)
log.SetLevel(log.DebugLevel)
}
func TestCleanImage(t *testing.T) {
@@ -40,15 +36,18 @@ func TestCleanImage(t *testing.T) {
}
for _, table := range tables {
imageOut := cleanImage(t.Context(), table.imageIn)
imageOut := cleanImage(context.Background(), table.imageIn)
assert.Equal(t, table.imageOut, imageOut)
}
}
func TestGetImagePullOptions(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
t.Setenv("DOCKER_CONFIG", "/non-existent/docker")
orig := config.Dir()
t.Cleanup(func() { config.SetDir(orig) })
config.SetDir("/non-existent/docker")
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{})
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
@@ -62,28 +61,13 @@ func TestGetImagePullOptions(t *testing.T) {
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZCJ9", options.RegistryAuth, "Username and Password should be provided")
t.Setenv("DOCKER_CONFIG", "testdata/docker-pull-options")
config.SetDir("testdata/docker-pull-options")
options, err = getImagePullOptions(ctx, NewDockerPullExecutorInput{
Image: "nektos/act",
})
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths":{"identity.example.com":{"identitytoken":"identity-token"},"registry.example.com":{"registrytoken":"registry-token"}}}`), 0o600))
t.Setenv("DOCKER_CONFIG", dir)
for host, want := range map[string]registry.AuthConfig{
"identity.example.com": {IdentityToken: "identity-token", ServerAddress: "identity.example.com"},
"registry.example.com": {RegistryToken: "registry-token", ServerAddress: "registry.example.com"},
} {
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{Image: host + "/app"})
require.NoError(t, err)
require.NotEmpty(t, options.RegistryAuth, host)
got, err := authconfig.Decode(options.RegistryAuth)
require.NoError(t, err)
assert.Equal(t, want, *got)
}
}
// A digest-pinned image is immutable, so its local copy is always current.
@@ -95,15 +79,11 @@ func TestIsPinnedImage(t *testing.T) {
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
// whichever of the two shapes the daemon used.
func TestLogDockerResponseError(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
streamErr := func(line string) error {
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
}
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
require.NoError(t, streamErr(`{"status":"Downloading"}`))
require.NoError(t, streamErr(`{"stream":"`+strings.Repeat("x", 70000)+`"}`))
require.ErrorIs(t, logDockerResponse(logger, io.NopCloser(io.MultiReader(
strings.NewReader(`{"status":"Downloading"}`+"\n"), iotest.ErrReader(io.ErrUnexpectedEOF),
)), false), io.ErrUnexpectedEOF)
}

File diff suppressed because it is too large Load Diff

View File

@@ -5,7 +5,6 @@
package container
import (
"archive/tar"
"bufio"
"bytes"
"context"
@@ -13,18 +12,12 @@ import (
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"gitea.com/gitea/runner/act/common/logging/logtest"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/pkg/stdcopy"
@@ -32,33 +25,50 @@ import (
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
client := &mockDockerClient{}
client.On("ContainerInspect", mock.Anything, "amd64-job", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Env: []string{
"PATH=/opt/node/x64/bin:/usr/bin:/bin", "CONFLICT_VAR=container-value", "SOME_RANDOM_VAR=",
"ANOTHER_ONE=old-value", "ANOTHER_ONE=BUT_I_HAVE_VALUE", "LITERAL='quoted'=$HOME\nsecond line",
}}}}, nil)
func TestDocker(t *testing.T) {
requireDocker(t)
ctx := context.Background()
client, err := GetDockerClient(ctx)
require.NoError(t, err)
defer client.Close()
dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{
ContextDir: "testdata",
ImageTag: "envmergetest",
})
err = dockerBuild(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
cr := &containerReference{
cli: client,
input: &NewContainerInput{
Image: "envmergetest",
},
}
env := map[string]string{
"PATH": "/custom/bin",
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin",
"RANDOM_VAR": "WITH_VALUE",
"ANOTHER_VAR": "",
"CONFLICT_VAR": "step-value",
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
}
require.NoError(t, (&containerReference{cli: client, id: "amd64-job"}).UpdateFromImageEnv(&env)(t.Context()))
envExecutor := cr.extractFromImageEnv(&env)
err = envExecutor(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, map[string]string{
"PATH": "/custom/bin:/opt/node/x64/bin:/usr/bin:/bin",
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin:/this/path/does/not/exists/anywhere:/this/either",
"RANDOM_VAR": "WITH_VALUE",
"ANOTHER_VAR": "",
"SOME_RANDOM_VAR": "",
"ANOTHER_ONE": "BUT_I_HAVE_VALUE",
"CONFLICT_VAR": "step-value",
"LITERAL": "'quoted'=$HOME\nsecond line",
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
}, env)
}
@@ -92,16 +102,6 @@ func (m *mockDockerClient) ContainerAttach(ctx context.Context, containerID stri
return args.Get(0).(mobyclient.ContainerAttachResult), args.Error(1)
}
func (m *mockDockerClient) ContainerLogs(ctx context.Context, containerID string, opts mobyclient.ContainerLogsOptions) (mobyclient.ContainerLogsResult, error) {
args := m.Called(ctx, containerID, opts)
return args.Get(0).(mobyclient.ContainerLogsResult), args.Error(1)
}
func (m *mockDockerClient) ContainerStart(ctx context.Context, containerID string, opts mobyclient.ContainerStartOptions) (mobyclient.ContainerStartResult, error) {
args := m.Called(ctx, containerID, opts)
return args.Get(0).(mobyclient.ContainerStartResult), args.Error(1)
}
func (m *mockDockerClient) ContainerWait(ctx context.Context, containerID string, opts mobyclient.ContainerWaitOptions) mobyclient.ContainerWaitResult {
args := m.Called(ctx, containerID, opts)
return args.Get(0).(mobyclient.ContainerWaitResult)
@@ -137,11 +137,6 @@ func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.Netw
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
}
func (m *mockDockerClient) NetworkCreate(ctx context.Context, name string, opts mobyclient.NetworkCreateOptions) (mobyclient.NetworkCreateResult, error) {
args := m.Called(ctx, name, opts)
return args.Get(0).(mobyclient.NetworkCreateResult), args.Error(1)
}
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
@@ -168,7 +163,6 @@ func (r *interruptReader) Read(_ []byte) (int, error) {
type mockConn struct {
net.Conn
mock.Mock
closed atomic.Bool
}
func (m *mockConn) Write(b []byte) (n int, err error) {
@@ -177,12 +171,11 @@ func (m *mockConn) Write(b []byte) (n int, err error) {
}
func (m *mockConn) Close() (err error) {
m.closed.Store(true)
return nil
}
func TestDockerExecAbort(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
ctx, cancel := context.WithCancel(context.Background())
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
conn := &mockConn{}
@@ -198,9 +191,11 @@ func TestDockerExecAbort(t *testing.T) {
}, nil)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{},
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
channel := make(chan error)
@@ -221,12 +216,14 @@ func TestDockerExecAbort(t *testing.T) {
}
func TestDockerExecFailure(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
conn := &mockConn{}
client := &mockDockerClient{}
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
Conn: &mockConn{},
Conn: conn,
Reader: bufio.NewReader(strings.NewReader("output")),
}, nil)
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
@@ -234,9 +231,11 @@ func TestDockerExecFailure(t *testing.T) {
}, nil)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{},
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
err := cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
@@ -245,21 +244,7 @@ func TestDockerExecFailure(t *testing.T) {
assert.Equal(t, ExitCodeError(1), exitErr)
assert.Equal(t, "Process completed with exit code 1.", err.Error())
client.AssertExpectations(t)
}
func TestDockerExecStreamFailure(t *testing.T) {
ctx := t.Context()
client := &mockDockerClient{}
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).
Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).
Return(mobyclient.ExecAttachResult{
Conn: &mockConn{},
Reader: bufio.NewReader(bytes.NewReader(stdcopyFrame(stdcopy.Systemerr, "unable to start exec"))),
}, nil)
require.ErrorContains(t, (&containerReference{id: "123", cli: client, input: &NewContainerInput{}}).
exec([]string{"sh"}, nil, "", "")(ctx), "unable to start exec")
conn.AssertExpectations(t)
client.AssertExpectations(t)
}
@@ -274,138 +259,128 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
return b
}
// TestDockerAttachFlushesTrailingLine verifies that wait() blocks until the
// attach() streaming goroutine has drained and flushed the container's output,
// so a final line without a trailing newline is not lost.
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
for _, testcase := range []struct {
name string
tty bool
}{
{name: "multiplexed"},
{name: "tty from container options", tty: true},
} {
t.Run(testcase.name, func(t *testing.T) {
ctx := t.Context()
output := []byte("line one\nlast line without newline")
if !testcase.tty {
output = stdcopyFrame(stdcopy.Stdout, string(output))
}
var lines []string
logWriter := common.NewLineWriter(func(line string) bool {
lines = append(lines, line)
return true
})
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "123", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Tty: testcase.tty}}}, nil)
conn := &mockConn{}
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
Return(mobyclient.ContainerAttachResult{
Conn: conn,
Reader: bufio.NewReader(bytes.NewReader(output)),
}, nil)
client.On("ContainerLogs", ctx, "123", mobyclient.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}).
Return(io.NopCloser(bytes.NewReader(output)), nil)
statusCh := make(chan container.WaitResponse, 1)
statusCh <- container.WaitResponse{StatusCode: 0}
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
Return(mobyclient.ContainerWaitResult{Result: statusCh})
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{Stdout: logWriter, Stderr: logWriter}}
ctx := context.Background()
require.NoError(t, cr.attach()(ctx))
require.NoError(t, cr.wait()(ctx))
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
assert.True(t, conn.closed.Load())
require.NoError(t, cr.DumpLogs(ctx))
assert.Equal(t, []string{"line one\n", "last line without newline", "line one\n", "last line without newline"}, lines)
client.AssertExpectations(t)
})
}
}
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
var lines []string
logWriter := common.NewLineWriter(func(s string) bool {
lines = append(lines, s)
return true
})
func TestDockerAttachStreamClosesWithItsContext(t *testing.T) {
conn, peer := net.Pipe()
t.Cleanup(func() { _ = conn.Close() })
t.Cleanup(func() { _ = peer.Close() })
client := &mockDockerClient{}
client.On("ContainerInspect", mock.Anything, "123", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{}}}, nil)
client.On("ContainerAttach", mock.Anything, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
Return(mobyclient.ContainerAttachResult{HijackedResponse: mobyclient.NewHijackedResponse(conn, "")}, nil)
client.On("ContainerStart", mock.Anything, "123", mobyclient.ContainerStartOptions{}).
Return(mobyclient.ContainerStartResult{}, errors.New("container start failed"))
cr := &containerReference{id: "123", cli: client, input: &NewContainerInput{}}
client.On("ContainerAttach", ctx, "123", mock.AnythingOfType("client.ContainerAttachOptions")).
Return(mobyclient.ContainerAttachResult{
Conn: &mockConn{},
Reader: bufio.NewReader(framed),
}, nil)
statusCh := make(chan container.WaitResponse, 1)
statusCh <- container.WaitResponse{StatusCode: 0}
errCh := make(chan error, 1)
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
Return(mobyclient.ContainerWaitResult{
Result: (<-chan container.WaitResponse)(statusCh),
Error: (<-chan error)(errCh),
})
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
Stdout: logWriter,
Stderr: logWriter,
},
}
require.NoError(t, cr.attach()(ctx))
require.NoError(t, cr.wait()(ctx))
// wait() must have blocked until the goroutine drained AND flushed; the
// trailing, non-newline-terminated line must therefore be present. Reading
// lines here is race-free because wait() synchronizes on attachDone, which
// the goroutine closes after the final append.
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
ctx, cancel := context.WithCancel(t.Context())
require.ErrorContains(t, cr.Start(true)(ctx), "container start failed")
cancel()
<-cr.attachDone
client.AssertExpectations(t)
}
func TestDockerWaitFailure(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
statusCh := make(chan container.WaitResponse, 1)
statusCh <- container.WaitResponse{StatusCode: 2}
errCh := make(chan error, 1)
client := &mockDockerClient{}
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
Return(mobyclient.ContainerWaitResult{Result: statusCh})
cr := &containerReference{id: "123", cli: client}
Return(mobyclient.ContainerWaitResult{
Result: (<-chan container.WaitResponse)(statusCh),
Error: (<-chan error)(errCh),
})
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
err := cr.wait()(ctx)
var exitErr ExitCodeError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, ExitCodeError(2), exitErr)
assert.Equal(t, "Process completed with exit code 2.", err.Error())
statusCh <- container.WaitResponse{Error: &container.WaitExitError{Message: "container wait failed"}}
require.EqualError(t, cr.wait()(ctx), "failed to wait for container: container wait failed")
client.AssertExpectations(t)
}
// A remove that raced the daemon's AutoRemove teardown is not a failure and must not
// be logged as one.
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
removeFailure := errors.New("driver failed to remove root filesystem")
removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}
killOpts := mobyclient.ContainerKillOptions{Signal: "SIGKILL"}
for _, tc := range []struct {
name string
err error
wantWait bool
waitErr error
wantErr error
name string
err error
wantWait bool
wantFailure bool
}{
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress"), wantWait: true},
{name: "wait canceled", err: cerrdefs.ErrConflict, wantWait: true, waitErr: context.Canceled, wantErr: context.Canceled},
{name: "removed during wait", err: cerrdefs.ErrConflict, wantWait: true, waitErr: cerrdefs.ErrNotFound},
{name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")},
{name: "removed cleanly", err: nil},
{name: "real failure", err: removeFailure, wantErr: removeFailure},
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantFailure: true},
} {
t.Run(tc.name, func(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
client := &mockDockerClient{}
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
if tc.wantWait {
removed := make(chan container.WaitResponse, 1)
waitErrors := make(chan error, 1)
if tc.waitErr == nil {
removed <- container.WaitResponse{}
} else {
waitErrors <- tc.waitErr
}
removed <- container.WaitResponse{}
client.On("ContainerWait", mock.Anything, "abc", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionRemoved}).
Return(mobyclient.ContainerWaitResult{Result: removed, Error: waitErrors})
Return(mobyclient.ContainerWaitResult{Result: removed})
}
cr := &containerReference{id: "abc", cli: client}
require.ErrorIs(t, cr.remove()(ctx), tc.wantErr)
if tc.wantErr != nil {
require.NoError(t, cr.remove()(ctx))
// a failure keeps the id, so a later Remove() can retry it
if tc.wantFailure {
assert.Equal(t, "abc", cr.id)
assert.Len(t, hook.AllEntries(), 1)
} else {
assert.Empty(t, cr.id)
assert.Empty(t, hook.AllEntries())
}
assert.Empty(t, hook.AllRecords())
client.AssertExpectations(t)
})
}
@@ -415,7 +390,7 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
// create() lost its reply, must still be removed rather than leaking with its network. It
// was never started here, so it is not worth a kill of its own.
func TestRemoveWithoutIDUsesName(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
Return(mobyclient.ContainerRemoveResult{}, nil)
@@ -425,88 +400,70 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
client.AssertExpectations(t)
}
func TestRemovePreservesClientCreationError(t *testing.T) {
t.Setenv("DOCKER_HOST", "://")
require.ErrorContains(t, (&containerReference{input: &NewContainerInput{Name: "job-1"}}).
Remove()(t.Context()), "failed to connect to docker daemon")
}
// find() must drop a stale cached id so later Copy/Exec don't hit the
// daemon with a torn-down container.
func TestFindRevalidatesStaleID(t *testing.T) {
ctx := t.Context()
inspectOpts := mobyclient.ContainerInspectOptions{}
ctx := context.Background()
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
boom := errors.New("daemon unreachable")
for _, tc := range []struct {
name, cachedID string
cachedErr error
lookup container.InspectResponse
lookupErr error
listed []container.Summary
wantID string
retryList bool
}{
{name: "stale id cleared, name lookup empty", cachedID: "stale", cachedErr: notFound, lookupErr: notFound},
{name: "stale id cleared, name lookup repopulates", cachedID: "stale", cachedErr: notFound, lookup: container.InspectResponse{ID: "fresh", Name: "/job-1"}, wantID: "fresh"},
{name: "id match falls back to listing", lookup: container.InspectResponse{ID: "job-1abc", Name: "/somebody-else"}, listed: []container.Summary{{ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh"},
{name: "live id kept", cachedID: "live", wantID: "live"},
{name: "transient inspect error trusts cache", cachedID: "live", cachedErr: boom, wantID: "live"},
{name: "name lookup error falls back to listing", lookupErr: boom, listed: []container.Summary{{ID: "other", Names: []string{"/somebody-else"}}, {ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh", retryList: true},
} {
t.Run(tc.name, func(t *testing.T) {
client := &mockDockerClient{}
cr := &containerReference{id: tc.cachedID, cli: client, input: &NewContainerInput{Name: "job-1"}}
if tc.cachedID != "" {
client.On("ContainerInspect", ctx, tc.cachedID, inspectOpts).Return(mobyclient.ContainerInspectResult{}, tc.cachedErr)
}
if tc.cachedID == "" || cerrdefs.IsNotFound(tc.cachedErr) {
client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{Container: tc.lookup}, tc.lookupErr)
}
if tc.listed != nil {
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: tc.listed}, nil).Once()
}
require.NoError(t, cr.find()(ctx))
assert.Equal(t, tc.wantID, cr.id)
if tc.retryList {
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, boom)
cr.id = ""
require.ErrorIs(t, cr.find()(ctx), boom)
}
client.AssertExpectations(t)
})
newCR := func(id string) (*containerReference, *mockDockerClient) {
client := &mockDockerClient{}
return &containerReference{id: id, cli: client, input: &NewContainerInput{Name: "job-1"}}, client
}
}
listOpts := mobyclient.ContainerListOptions{All: true}
inspectOpts := mobyclient.ContainerInspectOptions{}
func TestRunnerArchCachedPerRun(t *testing.T) {
var infos atomic.Int32
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "application/json")
writer.Header().Set("API-Version", "1.47")
if strings.HasSuffix(request.URL.Path, "/info") {
infos.Add(1)
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
}
}))
t.Cleanup(daemon.Close)
t.Setenv("DOCKER_HOST", daemon.URL)
extensions := &LinuxContainerEnvironmentExtensions{}
run := WithRunnerArch(t.Context())
var lookups sync.WaitGroup
for range 4 {
lookups.Go(func() { assert.Equal(t, "ARM64", extensions.GetRunnerContext(run)["arch"]) })
}
lookups.Wait()
assert.Equal(t, "ARM64", extensions.GetRunnerContext(t.Context())["arch"])
assert.EqualValues(t, 2, infos.Load())
t.Run("stale id cleared, name lookup empty", func(t *testing.T) {
cr, client := newCR("stale")
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, nil)
require.NoError(t, cr.find()(ctx))
assert.Empty(t, cr.id)
client.AssertExpectations(t)
})
t.Run("stale id cleared, name lookup repopulates", func(t *testing.T) {
cr, client := newCR("stale")
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{Items: []container.Summary{
{ID: "other", Names: []string{"/somebody-else"}},
{ID: "fresh", Names: []string{"/job-1"}},
}}, nil)
require.NoError(t, cr.find()(ctx))
assert.Equal(t, "fresh", cr.id)
client.AssertExpectations(t)
})
t.Run("live id kept", func(t *testing.T) {
cr, client := newCR("live")
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, nil)
require.NoError(t, cr.find()(ctx))
assert.Equal(t, "live", cr.id)
client.AssertExpectations(t)
})
t.Run("transient inspect error trusts cache", func(t *testing.T) {
cr, client := newCR("live")
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, boom)
require.NoError(t, cr.find()(ctx))
assert.Equal(t, "live", cr.id)
client.AssertExpectations(t)
})
t.Run("list error propagates", func(t *testing.T) {
cr, client := newCR("")
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, boom)
require.ErrorIs(t, cr.find()(ctx), boom)
client.AssertExpectations(t)
})
}
// Every daemon entry point fails fast with a clear, container-named
// error when no live cr.id is known.
func TestRejectsMissingContainer(t *testing.T) {
ctx := t.Context()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
check := func(op string, err error) {
t.Helper()
@@ -514,7 +471,7 @@ func TestRejectsMissingContainer(t *testing.T) {
assert.Contains(t, err.Error(), `container "job-1" does not exist`, op)
}
check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
check("copyDir", cr.copyDir("/var/run/act", "/src", false, false)(ctx))
check("copyDir", cr.copyDir("/var/run/act", "/src", false)(ctx))
check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
_, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
check("GetContainerArchive", err)
@@ -529,55 +486,36 @@ func TestRejectsMissingContainer(t *testing.T) {
check("Inspect after removal", err)
}
// End-to-end: a stale cr.id is cleared, repopulated from name lookup,
// and the Copy completes against the fresh id.
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
for _, testcase := range []struct {
name, destination, extractionRoot, entry string
directory bool
}{
{name: "files", destination: "/var/run/act", extractionRoot: "/var/run/act", entry: "x"},
{name: "action below act volume", directory: true, destination: "/var/run/act/actions/checkout/", extractionRoot: "/var/run/act", entry: "actions/checkout/x"},
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
} {
t.Run(testcase.name, func(t *testing.T) {
hook := logtest.NewWithLevel(logging.LevelError)
ctx := common.WithLogger(t.Context(), logging.New(hook))
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
client.On("CopyToContainer", ctx, "fresh", mock.Anything).
Run(func(args mock.Arguments) {
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
assert.Equal(t, testcase.extractionRoot, opts.DestinationPath)
header, err := tar.NewReader(opts.Content).Next()
require.NoError(t, err)
assert.Equal(t, testcase.entry, header.Name)
}).Return(mobyclient.CopyToContainerResult{}, nil).Once()
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
Return(mobyclient.ContainerListResult{Items: []container.Summary{
{ID: "fresh", Names: []string{"/job-1"}},
}}, nil)
client.On("CopyToContainer", ctx, "fresh", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act"
})).Return(mobyclient.CopyToContainerResult{}, nil)
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
if testcase.directory {
source := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
} else {
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
}
assert.Equal(t, "fresh", cr.id)
assert.Empty(t, hook.AllRecords())
client.AssertExpectations(t)
})
}
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
require.NoError(t, cr.Copy("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
assert.Equal(t, "fresh", cr.id)
client.AssertExpectations(t)
}
// Type assert containerReference implements ExecutionsEnvironment
var _ ExecutionsEnvironment = &containerReference{}
func TestCheckVolumes(t *testing.T) {
testCases := []struct {
desc string
validVolumes []string
binds []string
expectedBinds []string
mounts []mount.Mount
expectedMounts []mount.Mount
desc string
validVolumes []string
binds []string
expectedBinds []string
}{
{
desc: "match all volumes",
@@ -615,14 +553,11 @@ func TestCheckVolumes(t *testing.T) {
"shared_volume",
"/home/test/data",
"/etc/conf.d/*.json",
"/opt/**",
},
binds: []string{
"shared_volume:/shared_volume",
"/home/test/data:/test_data",
"/etc/conf.d/base.json:/config/base.json",
"/etc/conf.d/sub/nested.json:/config/nested.json",
"/opt/app/cache:/cache",
"sql_data:/sql_data",
"/secrets/keys:/keys",
},
@@ -630,45 +565,21 @@ func TestCheckVolumes(t *testing.T) {
"shared_volume:/shared_volume",
"/home/test/data:/test_data",
"/etc/conf.d/base.json:/config/base.json",
"/opt/app/cache:/cache",
},
},
{
desc: "read-only rules allow only read-only mounts and copy nothing into volumes",
validVolumes: []string{"/etc/ssl:ro", "data:ro", "shared"},
binds: []string{"/etc/ssl:/ssl:ro", "/etc/ssl:/ssl", "data:/data:ro", "data:/data", "shared:/shared:ro"},
expectedBinds: []string{"/etc/ssl:/ssl:ro", "data:/data:ro,nocopy", "shared:/shared:ro"},
mounts: []mount.Mount{
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true, BindOptions: &mount.BindOptions{ReadOnlyNonRecursive: true}},
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true},
},
expectedMounts: []mount.Mount{
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true, VolumeOptions: &mount.VolumeOptions{NoCopy: true}},
},
},
{
desc: "read-only wildcard keeps writable rules writable",
validVolumes: []string{"**:ro", "shared"},
binds: []string{"/etc/ssl:/ssl:ro", "data:/data", "shared:/shared"},
expectedBinds: []string{"/etc/ssl:/ssl:ro", "shared:/shared"},
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
logger, hook := logtest.NewLogger()
ctx := common.WithLogger(t.Context(), logger)
logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
ValidVolumes: tc.validVolumes,
},
}
hostConf := &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts}
cr.sanitizeConfig(ctx, hostConf)
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds})
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts))
assert.Len(t, hook.AllEntries(), len(tc.binds)-len(tc.expectedBinds)) // every drop is warned about
})
}
}
@@ -688,35 +599,41 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
}
// Both of these are read here, on the runner, so a workflow could read the runner's files
// and environment with them.
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
hostFile := filepath.Join(t.TempDir(), "host.env")
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
t.Setenv("RUNNER_SECRET", "s3cr3t")
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
require.ErrorContains(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}), "not allowed in a workflow")
_, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.ErrorContains(t, err, "not allowed in a workflow")
// the runner reading its own files is what those options are for
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
_, _, err = cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
}
logger := logging.New(logtest.New())
// a bare name is no longer resolved from the runner's environment, for either source
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
NetworkMode: "bridge",
RunnerOptions: "--env RUNNER_SECRET",
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
}}
config := &container.Config{}
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), config, &container.HostConfig{}))
config, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
}
func TestSanitizeOptionsHostConfig(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
// every field the sanitizer resets, so a reset dropped in a refactor fails here
hostConfig := &container.HostConfig{
@@ -738,9 +655,6 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
DeviceCgroupRules: []string{"a *:* rwm"},
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}},
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
LogConfig: container.LogConfig{Type: "syslog", Config: map[string]string{"syslog-address": "tcp://169.254.169.254:514"}},
OomScoreAdj: -1000,
OomKillDisable: new(true),
}
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{})
@@ -748,23 +662,23 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
assert.Equal(t, &container.HostConfig{}, hostConfig)
}
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *logtest.Handler) {
// mergeOptions merges both option sources into a bare container, returning the result and its log.
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *test.Hook) {
t.Helper()
logger, hook := logtest.NewLogger()
logger, hook := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
RunnerOptions: runnerOptions,
WorkflowOptions: workflowOptions,
NetworkMode: "bridge",
UsernsMode: "private",
ValidVolumes: []string{"**"},
}}
hostConfig := &container.HostConfig{
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{
Privileged: privileged,
UsernsMode: container.UsernsMode("private"),
NetworkMode: container.NetworkMode("bridge"),
}
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, hostConfig))
})
require.NoError(t, err)
return hostConfig, hook
}
@@ -773,9 +687,7 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
"--security-opt apparmor=unconfined --volumes-from other --isolation process " +
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1 " +
"--log-driver=syslog --log-opt syslog-address=tcp://169.254.169.254:514 " +
"--oom-score-adj=-1000 --oom-kill-disable"
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's
// own options alone describe it, field for field
@@ -802,8 +714,8 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
}
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
logger := logging.New(logtest.New())
ctx := common.WithLogger(t.Context(), logger)
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
base := t.TempDir()
allowed := filepath.Join(base, "allowed")
@@ -818,46 +730,26 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
}
escapingPath := allowed + string(filepath.Separator) + ".." + string(filepath.Separator) + "denied"
hostConf := &container.HostConfig{
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
Binds: []string{escapingPath + ":/mnt"},
}
cr.sanitizeConfig(ctx, hostConf)
})
assert.Empty(t, hostConf.Binds)
linkPath := filepath.Join(allowed, "link")
if err := os.Symlink(denied, linkPath); err != nil {
t.Skipf("cannot create symlink: %v", err)
}
hostConf = &container.HostConfig{
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
Binds: []string{linkPath + ":/mnt"},
}
cr.sanitizeConfig(ctx, hostConf)
})
assert.Empty(t, hostConf.Binds)
hostConf = &container.HostConfig{
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
Binds: []string{filepath.Join(linkPath, "missing") + ":/mnt"},
}
cr.sanitizeConfig(ctx, hostConf)
})
assert.Empty(t, hostConf.Binds)
}
func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
hostConfig, _ := mergeOptions(t, "", "--mount type=tmpfs,dst=/var/lib/mysql", false)
require.Len(t, hostConfig.Mounts, 1)
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
logger, hook := logtest.NewLogger()
sanitized := &container.HostConfig{Mounts: []mount.Mount{
hostConfig.Mounts[0],
{Type: mount.TypeVolume, Target: "/anonymous"},
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
}}
(&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), sanitized)
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
assert.Len(t, hook.AllRecords(), 2)
}
}
func TestContainerInfoFromInspect(t *testing.T) {
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
@@ -909,50 +801,23 @@ func TestContainerInfoFromInspect(t *testing.T) {
assert.Equal(t, "abc123", info.ID)
assert.Equal(t, HealthNone, info.Health)
})
t.Run("reports the mount sources by container path", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
Mounts: []container.MountPoint{
{Type: mount.TypeVolume, Name: "job", Source: "/var/lib/docker/volumes/job/_data", Destination: "/workspace/owner/repo"},
{Type: mount.TypeBind, Source: "/var/run/docker.sock", Destination: "/var/run/docker.sock"},
{Type: mount.TypeVolume, Name: "cache", Source: "/var/lib/docker/volumes/cache/_data", Destination: "/cache"},
{Type: mount.TypeBind, Source: "/custom/resolv.conf", Destination: "/etc/resolv.conf"},
},
HostConfig: &container.HostConfig{
Tmpfs: map[string]string{"/workspace/owner/repo//tmp/": "size=1m"},
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "cache", Target: "/cache/", VolumeOptions: &mount.VolumeOptions{Subpath: "project"}}},
},
})
assert.Equal(t, map[string]string{
"/workspace/owner/repo": "/var/lib/docker/volumes/job/_data",
"/workspace/owner/repo/tmp": "",
"/var/run/docker.sock": "/var/run/docker.sock",
"/cache": "/var/lib/docker/volumes/cache/_data/project",
"/etc/resolv.conf": "/custom/resolv.conf",
"/etc/hosts": "",
"/etc/hostname": "",
}, info.Mounts)
})
}
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
logger := logging.New(logtest.New())
ctx := common.WithLogger(t.Context(), logger)
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
NetworkMode: "bridge",
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
WorkflowOptions: "--volume /unapproved:/var/run/docker.sock",
ValidVolumes: []string{"/host/tools"},
NetworkMode: "bridge",
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
},
}
hostConf := &container.HostConfig{
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
}
require.NoError(t, cr.mergeContainerConfigs(ctx, &container.Config{}, hostConf))
})
require.NoError(t, err)
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
assert.Empty(t, hostConf.Mounts)
}
@@ -960,51 +825,22 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
warnings := func(runnerOptions, workflowOptions string) int {
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
return len(hook.AllRecords())
return len(hook.AllEntries())
}
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
assert.Zero(t, warnings("", "--shm-size 1g"))
assert.Equal(t, 1, warnings("--network host", ""))
assert.Equal(t, 1, warnings("", "--privileged"))
}
func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
for _, entrypoint := range [][]string{nil, {"sleep", "3600"}} {
for _, optionEntrypoint := range []string{"/sbin/dumb-init", ""} {
logger, hook := logtest.NewLogger()
cr := &containerReference{
input: &NewContainerInput{
NetworkMode: "bridge",
Entrypoint: entrypoint,
RunnerOptions: "--entrypoint=runner-entrypoint",
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
},
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
if len(entrypoint) > 0 {
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
} else {
assert.Equal(t, []string{optionEntrypoint}, opts.Config.Entrypoint)
}
require.NotNil(t, opts.HostConfig.Init)
assert.True(t, *opts.HostConfig.Init)
return mobyclient.ContainerCreateResult{ID: "created"}, nil
}},
}
require.NoError(t, cr.create(nil, nil)(common.WithLogger(t.Context(), logger)))
assert.Equal(t, "created", cr.id)
assert.Equal(t, len(entrypoint) > 0, len(hook.AllRecords()) == 1)
}
}
}
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
logger := logging.New(logtest.New())
logger, _ := test.NewNullLogger()
cr := &containerReference{input: &NewContainerInput{
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
}}
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
require.NoError(t, err)
assert.Equal(t, []string{"redis", "redis-primary"}, cr.input.NetworkAliases)
}

View File

@@ -1,111 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"os/exec"
"strings"
"sync"
"syscall"
"time"
"github.com/docker/cli/cli/connhelper/ssh"
)
func sshArgs(daemonURL string) ([]string, error) {
spec, err := ssh.ParseURL(daemonURL)
if err != nil {
return nil, err
}
remote := []string{"docker", "system", "dial-stdio"}
if strings.Trim(spec.Path, "/") != "" {
remote = []string{"docker", "--host=unix://" + spec.Path, "system", "dial-stdio"}
}
return spec.Command([]string{"-o", "ConnectTimeout=30", "-T"}, remote...)
}
func dialCommand(name string, args ...string) (net.Conn, error) {
conn := &commandConn{cmd: exec.Command(name, args...)}
conn.cmd.SysProcAttr = sshSysProcAttr() // a session of its own like docker's, so a ProxyCommand never reads the runner's terminal: https://github.com/docker/cli/issues/1707
conn.cmd.Stderr = &conn.stderr
conn.cmd.WaitDelay = time.Second // a ProxyCommand child keeping stderr open would otherwise block Wait after ssh exits
conn.wait = sync.OnceValue(func() error {
if err := conn.cmd.Wait(); err != nil && !errors.Is(err, exec.ErrWaitDelay) {
return fmt.Errorf("%s: %w: %s", strings.Join(conn.cmd.Args, " "), err, strings.TrimSpace(conn.stderr.String()))
}
return nil
})
var err error
if conn.stdin, err = conn.cmd.StdinPipe(); err != nil {
return nil, err
}
if conn.stdout, err = conn.cmd.StdoutPipe(); err != nil {
return nil, err
}
return conn, conn.cmd.Start()
}
type commandConn struct {
cmd *exec.Cmd
stdin io.WriteCloser
stdout io.ReadCloser
stderr cappedBuffer
wait func() error
}
func (c *commandConn) waitWithin(timeout time.Duration) error {
timer := time.AfterFunc(timeout, func() { _ = c.cmd.Process.Kill() })
defer timer.Stop()
return c.wait()
}
func (c *commandConn) Read(p []byte) (int, error) {
n, err := c.stdout.Read(p)
if errors.Is(err, io.EOF) {
if waitErr := c.waitWithin(10 * time.Second); waitErr != nil {
return n, waitErr
}
}
return n, err
}
func (c *commandConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
// CloseWrite lets the daemon see the end of a hijacked stream, like a TCP half-close.
func (c *commandConn) CloseWrite() error { return c.stdin.Close() }
func (c *commandConn) Close() error {
_ = c.stdin.Close()
if c.cmd.Process.Signal(syscall.SIGTERM) != nil { // SIGTERM lets ssh take its ProxyCommand down, Windows only kills
_ = c.cmd.Process.Kill()
}
_ = c.waitWithin(3 * time.Second)
return nil
}
func (c *commandConn) LocalAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
func (c *commandConn) RemoteAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
func (c *commandConn) SetDeadline(_deadline time.Time) error { return nil }
func (c *commandConn) SetReadDeadline(_deadline time.Time) error { return nil }
func (c *commandConn) SetWriteDeadline(_deadline time.Time) error { return nil }
type cappedBuffer struct{ bytes.Buffer }
func (b *cappedBuffer) Write(p []byte) (int, error) {
if b.Len() > 4096 {
b.Reset()
}
return b.Buffer.Write(p)
}

View File

@@ -1,12 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER
package container
import "syscall"
func sshSysProcAttr() *syscall.SysProcAttr {
return &syscall.SysProcAttr{Setsid: true, Pdeathsig: syscall.SIGKILL}
}

View File

@@ -1,52 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"io"
"os/exec"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSSHArgs(t *testing.T) {
args, err := sshArgs("ssh://me@host:2222/run/user/1000/my docker.sock")
require.NoError(t, err)
assert.Equal(t, []string{"-l", "me", "-p", "2222", "-o", "ConnectTimeout=30", "-T", "--", "host", "docker '--host=unix:///run/user/1000/my docker.sock' system dial-stdio"}, args)
args, err = sshArgs("ssh://[::1]")
require.NoError(t, err)
assert.Equal(t, []string{"-o", "ConnectTimeout=30", "-T", "--", "::1", "docker system dial-stdio"}, args)
for _, invalid := range []string{"ssh://me:secret@host", "ssh://host?x=1", "ssh://host#x", "ssh://"} {
_, err := sshArgs(invalid)
assert.Error(t, err, invalid)
}
}
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("sh"); err != nil {
t.Skip("needs a POSIX shell")
}
conn, err := dialCommand("sh", "-c", "cat")
require.NoError(t, err)
_, err = conn.Write([]byte("ping"))
require.NoError(t, err)
require.NoError(t, conn.(interface{ CloseWrite() error }).CloseWrite())
echoed, err := io.ReadAll(conn)
require.NoError(t, err)
assert.Equal(t, "ping", string(echoed))
require.NoError(t, conn.Close())
conn, err = dialCommand("sh", "-c", "exec 3<&0; cat <&3 >/dev/null & echo 'no route to host' >&2; exit 255")
require.NoError(t, err)
_, err = io.ReadAll(conn)
require.ErrorContains(t, err, "no route to host")
require.NoError(t, conn.Close())
}

View File

@@ -1,12 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
package container
import "syscall"
func sshSysProcAttr() *syscall.SysProcAttr {
return &syscall.SysProcAttr{Setsid: true}
}

View File

@@ -1,10 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !WITHOUT_DOCKER
package container
import "syscall"
func sshSysProcAttr() *syscall.SysProcAttr { return nil }

View File

@@ -9,7 +9,6 @@ package container
import (
"context"
"errors"
"net/netip"
"runtime"
"time"
@@ -18,10 +17,6 @@ import (
"github.com/moby/moby/api/types/system"
)
func IsolatedCacheContainer(context.Context, netip.Addr, string) (string, error) {
return "", nil
}
// ImageExistsLocally returns a boolean indicating if an image with the
// requested name, tag and architecture exists in the local docker image store
func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool, error) {
@@ -34,18 +29,6 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
return false, errors.New("Unsupported Operation")
}
func RemoveDockerJobResources(_ context.Context, _ string) error {
return nil
}
func NewDockerProxy(_ context.Context, _ string) *DockerProxy {
return nil
}
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
return nil, errors.New("Unsupported Operation")
}
// NewDockerBuildExecutor function to create a run executor for the container
func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
return func(ctx context.Context) error {
@@ -91,30 +74,6 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
}
}
func NewDockerNetworkConnectExecutor(_, _ string) common.Executor {
return func(_ context.Context) error {
return nil
}
}
func NewDockerNetworkDisconnectExecutor(_, _ string) common.Executor {
return func(_ context.Context) error {
return nil
}
}
func RemoveOrphanNetworks(_ context.Context, _, _ string, _ time.Time) error {
return nil
}
func SanitizeBinds(_ context.Context, _, binds []string) []string {
return binds
}
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
return nil
}
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
return nil
}

View File

@@ -8,67 +8,40 @@ package container
import (
"context"
"errors"
"fmt"
"time"
"gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/client"
)
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
for _, volumeName := range volumeNames {
if _, err := cli.VolumeCreate(ctx, client.VolumeCreateOptions{
Name: volumeName,
Labels: runnerLabels(runnerUUID),
}); err != nil {
return err
}
}
return nil
}
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
}
defer cli.Close()
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID).Add("dangling", "true"),
})
if err != nil {
return err
}
var errs []error
for _, item := range volumes.Items {
created, err := time.Parse(time.RFC3339, item.CreatedAt)
// an unreadable or recent timestamp may belong to a job that is still starting up
if err != nil || created.After(createdBefore) || item.Labels[runnerUUIDLabel] != runnerUUID {
continue
}
if _, err := cli.VolumeRemove(ctx, item.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", item.Name, err))
continue
}
common.Logger(ctx).Infof("removed docker volume %s left behind by an earlier job", item.Name)
}
return errors.Join(errs...)
}
func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executor {
return func(ctx context.Context) error {
common.Logger(ctx).Debugf("docker volume rm %s", volumeName)
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
list, err := cli.VolumeList(ctx, client.VolumeListOptions{})
if err != nil {
return err
}
for _, vol := range list.Items {
if vol.Name == volumeName {
return removeExecutor(volumeName, force)(ctx)
}
}
// Volume not found - do nothing
return nil
}
}
func removeExecutor(volume string, force bool) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
logger.Debugf("docker volume rm %s", volume)
if common.Dryrun(ctx) {
return nil
@@ -80,10 +53,7 @@ func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executo
}
defer cli.Close()
_, err = cli.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: force})
if cerrdefs.IsNotFound(err) { // already gone is the outcome we wanted
return nil
}
_, err = cli.VolumeRemove(ctx, volume, client.VolumeRemoveOptions{Force: force})
return err
}
}

View File

@@ -4,6 +4,7 @@
package container
import (
"context"
"testing"
mobyclient "github.com/moby/moby/client"
@@ -14,7 +15,7 @@ import (
// so the daemon has to be pinged explicitly here to decide whether to skip.
func requireDocker(t *testing.T) {
t.Helper()
ctx := t.Context()
ctx := context.Background()
cli, err := GetDockerClient(ctx)
if err != nil {
t.Skipf("skipping: docker client unavailable: %v", err)

View File

@@ -7,7 +7,6 @@ package container
import (
"archive/tar"
"bytes"
"cmp"
"context"
"errors"
"fmt"
@@ -28,6 +27,9 @@ import (
"gitea.com/gitea/runner/internal/pkg/process"
"github.com/creack/pty"
"github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"golang.org/x/term"
)
@@ -65,11 +67,15 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
}
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
return common.NewPipelineExecutor()
return func(ctx context.Context) error {
return nil
}
}
func (e *HostEnvironment) Close() common.Executor {
return common.NewPipelineExecutor()
return func(ctx context.Context) error {
return nil
}
}
func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
@@ -86,7 +92,7 @@ func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Exec
}
}
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error {
logger := common.Logger(ctx)
srcPrefix := filepath.Dir(srcPath)
@@ -94,16 +100,24 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGi
srcPrefix += string(filepath.Separator)
}
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
var ignorer gitignore.Matcher
if useGitIgnore {
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
if err != nil {
logger.Debugf("Error loading .gitignore: %v", err)
}
ignorer = gitignore.NewMatcher(ps)
}
fc := &filecollector.FileCollector{
SrcPath: srcPath,
SrcPrefix: srcPrefix,
SkipGitDir: skipGitDir,
UseGitIgnore: useGitIgnore,
Ignorer: ignorer,
SrcPath: srcPath,
SrcPrefix: srcPrefix,
Handler: &filecollector.CopyCollector{
DstDir: destPath,
},
}
return filepath.Walk(srcPath, fc.CollectFiles(ctx))
return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
}
}
@@ -137,7 +151,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
SrcPrefix: srcPrefix,
Handler: tc,
}
err = filepath.Walk(srcPath, fc.CollectFiles(ctx))
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
if err != nil {
return nil, err
}
@@ -165,11 +179,15 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
}
func (e *HostEnvironment) Pull(_ bool) common.Executor {
return common.NewPipelineExecutor()
return func(ctx context.Context) error {
return nil
}
}
func (e *HostEnvironment) Start(_ bool) common.Executor {
return common.NewPipelineExecutor()
return func(ctx context.Context) error {
return nil
}
}
type ptyWriter struct {
@@ -186,6 +204,7 @@ func (w *ptyWriter) Write(buf []byte) (int, error) {
}
if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
_, _ = w.Out.Write([]byte("\n"))
return n, io.EOF
}
return n, io.EOF
}
@@ -225,38 +244,49 @@ func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) {
return ppty, tty, nil
}
func writeKeepAlive(ctx context.Context, ppty io.Writer) {
for ctx.Err() == nil {
if count, err := ppty.Write([]byte{4}); count != 1 || err != nil {
return
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
func writeKeepAlive(ppty io.Writer) {
c := 1
var err error
for c == 1 && err == nil {
c, err = ppty.Write([]byte{4})
<-time.After(time.Second)
}
}
func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) {
defer func() {
finishLog()
}()
if _, err := io.Copy(writer, ppty); err != nil {
return
}
}
func (e *HostEnvironment) UpdateFromImageEnv(_ *map[string]string) common.Executor {
return common.NewPipelineExecutor()
return func(ctx context.Context) error {
return nil
}
}
func getEnvListFromMap(env map[string]string) []string {
envList := make([]string, 0, len(env))
for key, value := range env {
envList = append(envList, key+"="+value)
envList := make([]string, 0)
for k, v := range env {
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
}
return envList
}
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, workdir string) error {
wd := e.Path
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error {
envList := getEnvListFromMap(env)
var wd string
if workdir != "" {
wd = workdir
if !filepath.IsAbs(workdir) {
if filepath.IsAbs(workdir) {
wd = workdir
} else {
wd = filepath.Join(e.Path, workdir)
}
} else {
wd = e.Path
}
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
// does in waitForCommand, so the final line of a command's output is not lost.
@@ -269,8 +299,9 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
cmd := exec.CommandContext(ctx, f)
cmd.Path = f
cmd.Args = command
cmd.Stdin = nil
cmd.Stdout = e.StdOut
cmd.Env = getEnvListFromMap(env)
cmd.Env = envList
cmd.Stderr = e.StdOut
cmd.Dir = wd
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
@@ -302,11 +333,8 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
writer = &ptyWriter{Out: e.StdOut}
var finishLog context.CancelFunc
logctx, finishLog = context.WithCancel(context.Background())
go func() {
defer finishLog()
_, _ = io.Copy(writer, ppty)
}()
go writeKeepAlive(logctx, ppty)
go copyPtyOutput(writer, ppty, finishLog)
go writeKeepAlive(ppty)
}
if err := cmd.Start(); err != nil {
return err
@@ -321,32 +349,32 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
} else {
defer k.Close()
}
err = treeKill.Wait(ctx, cmd)
err = cmd.Wait()
if err != nil {
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return ExitCodeError(exitErr.ExitCode())
}
return err
}
if tty != nil {
writer.AutoStop.Store(true)
if _, err := tty.WriteString("\x04"); err != nil {
common.Logger(ctx).Debug("Failed to write EOT")
}
select {
case <-logctx.Done():
case <-ctx.Done(): // a cancelled step does not wait on output nobody consumes
}
<-logctx.Done()
ppty.Close()
ppty = nil
}
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return ExitCodeError(exitErr.ExitCode())
}
return err
}
func (e *HostEnvironment) Exec(command []string, env map[string]string, _user, workdir string) common.Executor {
return e.ExecWithCmdLine(command, "", env, workdir)
func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor {
return e.ExecWithCmdLine(command, "", env, user, workdir)
}
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, workdir string) common.Executor {
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor {
return func(ctx context.Context) error {
if err := e.exec(ctx, command, cmdline, env, workdir); err != nil {
if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil {
select {
case <-ctx.Done():
return fmt.Errorf("this step has been cancelled: %w", err)
@@ -359,7 +387,7 @@ func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env
}
func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(e, srcPath, env, runtime.GOOS == "windows")
return parseEnvFile(e, srcPath, env)
}
// removeAll is a var so tests can substitute a blocking stub.
@@ -588,7 +616,8 @@ func (*HostEnvironment) GetPathVariableName() string {
}
func (e *HostEnvironment) DefaultPathVariable() string {
return os.Getenv(e.GetPathVariableName())
v, _ := os.LookupEnv(e.GetPathVariableName())
return v
}
func (*HostEnvironment) JoinPathVariable(paths ...string) string {
@@ -598,22 +627,30 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string {
// Reference for Arch values for runner.arch
// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
func goArchToActionArch(arch string) string {
return cmp.Or(map[string]string{
archMapper := map[string]string{
"x86_64": "X64",
"amd64": "X64",
"386": "X86",
"arm": "ARM",
"aarch64": "ARM64",
"arm64": "ARM64",
}[arch], arch)
}
if arch, ok := archMapper[arch]; ok {
return arch
}
return arch
}
func goOsToActionOs(os string) string {
return cmp.Or(map[string]string{
osMapper := map[string]string{
"linux": "Linux",
"darwin": "macOS",
"windows": "Windows",
}[os], os)
}
if os, ok := osMapper[os]; ok {
return os
}
return os
}
func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any {

View File

@@ -10,20 +10,21 @@ import (
"context"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"testing"
"testing/synctest"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/logging"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Type assert HostEnvironment implements ExecutionsEnvironment
var _ ExecutionsEnvironment = &HostEnvironment{}
func TestActionPlatformNames(t *testing.T) {
@@ -32,96 +33,82 @@ func TestActionPlatformNames(t *testing.T) {
}
func TestCopyDir(t *testing.T) {
for _, test := range []struct {
name string
useGitIgnore, skipGitDir bool
}{
{name: "git dir kept for a workspace copy"},
{name: "git dir skipped for an action copy", skipGitDir: true},
{name: "unfiltered outside a git repository", useGitIgnore: true},
} {
t.Run(test.name, func(t *testing.T) {
src := filepath.Join(t.TempDir(), "action")
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
dest := filepath.Join(t.TempDir(), "dest")
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
assert.FileExists(t, filepath.Join(dest, "action.yml"))
if test.skipGitDir {
assert.NoDirExists(t, filepath.Join(dest, ".git"))
} else {
assert.FileExists(t, filepath.Join(dest, ".git", "objects", "pack"))
}
})
dir := t.TempDir()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
ToolCache: filepath.Join(dir, "tool_cache"),
ActPath: filepath.Join(dir, "act_path"),
StdOut: os.Stdout,
Workdir: path.Join("testdata", "scratch"),
}
_ = os.MkdirAll(e.Path, 0o700)
_ = os.MkdirAll(e.TmpDir, 0o700)
_ = os.MkdirAll(e.ToolCache, 0o700)
_ = os.MkdirAll(e.ActPath, 0o700)
err := e.CopyDir(e.Workdir, e.Path, true)(ctx)
assert.NoError(t, err)
}
func TestGetContainerArchive(t *testing.T) {
ctx := t.Context()
e := &HostEnvironment{Path: t.TempDir()}
dir := t.TempDir()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
ToolCache: filepath.Join(dir, "tool_cache"),
ActPath: filepath.Join(dir, "act_path"),
StdOut: os.Stdout,
Workdir: path.Join("testdata", "scratch"),
}
_ = os.MkdirAll(e.Path, 0o700)
_ = os.MkdirAll(e.TmpDir, 0o700)
_ = os.MkdirAll(e.ToolCache, 0o700)
_ = os.MkdirAll(e.ActPath, 0o700)
expectedContent := []byte("sdde/7sh")
err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600)
require.NoError(t, err)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
archive, err := e.GetContainerArchive(ctx, e.Path)
require.NoError(t, err)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
defer archive.Close()
reader := tar.NewReader(archive)
h, err := reader.Next()
require.NoError(t, err)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, "action.yml", h.Name)
content, err := io.ReadAll(reader)
require.NoError(t, err)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, expectedContent, content)
_, err = reader.Next()
assert.ErrorIs(t, err, io.EOF)
}
func TestHostEnvironmentExecExitCode(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("uses POSIX shell")
}
for name, allocatePTY := range map[string]bool{"without PTY": false, "with PTY": true} {
t.Run(name, func(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
reader, writer := io.Pipe()
defer reader.Close()
e := &HostEnvironment{Path: t.TempDir(), StdOut: writer, AllocatePTY: allocatePTY}
t.Cleanup(func() { assert.NoError(t, e.Remove()(t.Context())) })
result := make(chan error, 1)
go func() {
defer writer.Close()
result <- e.Exec([]string{"sh", "-c", "printf 'failed\\n'; exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(t.Context())
}()
first := make([]byte, 1)
_, err := io.ReadFull(reader, first)
require.NoError(t, err)
synctest.Wait()
select {
case err := <-result:
t.Fatalf("Exec returned before its output was consumed: %v", err)
default:
}
remaining, err := io.ReadAll(reader)
require.NoError(t, err)
err = <-result
var exitErr ExitCodeError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, ExitCodeError(3), exitErr)
assert.Equal(t, "Process completed with exit code 3.", err.Error())
assert.Equal(t, "failed\n", string(first)+string(remaining))
})
})
dir := t.TempDir()
ctx := context.Background()
e := &HostEnvironment{
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
ToolCache: filepath.Join(dir, "tool_cache"),
ActPath: filepath.Join(dir, "act_path"),
StdOut: io.Discard,
Workdir: filepath.Join(dir, "path"),
}
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
assert.NoError(t, os.MkdirAll(p, 0o700)) //nolint:testifylint // test setup
}
err := e.Exec([]string{"sh", "-c", "exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(ctx)
var exitErr ExitCodeError
require.ErrorAs(t, err, &exitErr)
assert.Equal(t, ExitCodeError(3), exitErr)
assert.Equal(t, "Process completed with exit code 3.", err.Error())
}
func TestHostEnvironmentAllocatePTY(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("uses POSIX shell")
}
@@ -134,18 +121,25 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
{name: "on", allocPTY: true, expect: "TTY"},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dir := t.TempDir()
buf := &bytes.Buffer{}
e := &HostEnvironment{
Path: t.TempDir(),
Path: filepath.Join(dir, "path"),
TmpDir: filepath.Join(dir, "tmp"),
ToolCache: filepath.Join(dir, "tool_cache"),
ActPath: filepath.Join(dir, "act_path"),
StdOut: buf,
Workdir: filepath.Join(dir, "path"),
AllocatePTY: tc.allocPTY,
}
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
require.NoError(t, os.MkdirAll(p, 0o700))
}
err := e.Exec(
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
map[string]string{"PATH": os.Getenv("PATH")}, "", "",
)(t.Context())
)(context.Background())
require.NoError(t, err)
got := strings.TrimSpace(strings.ReplaceAll(buf.String(), "\r", ""))
assert.Equal(t, tc.expect, got)
@@ -153,91 +147,162 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
}
}
func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
for name, owned := range map[string]bool{"preserves caller workdir": false, "cleans owned workdir": true} {
t.Run(name, func(t *testing.T) {
miscRoot, workdir := t.TempDir(), t.TempDir()
e := &HostEnvironment{
Path: filepath.Join(miscRoot, "hostexecutor"),
Workdir: workdir,
CleanWorkdir: owned,
CleanUp: func() { _ = os.RemoveAll(miscRoot) },
}
require.NoError(t, os.MkdirAll(e.Path, 0o700))
require.NoError(t, e.Remove()(t.Context()))
_, err := os.Stat(workdir)
if owned {
assert.ErrorIs(t, err, os.ErrNotExist)
} else {
require.NoError(t, err)
}
})
func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
miscRoot := filepath.Join(base, "misc")
path := filepath.Join(miscRoot, "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
workdir := filepath.Join(base, "workspace", "owner", "repo")
require.NoError(t, os.MkdirAll(workdir, 0o700))
e := &HostEnvironment{
Path: path,
Workdir: workdir,
CleanUp: func() {
_ = os.RemoveAll(miscRoot)
},
StdOut: os.Stdout,
}
require.NoError(t, e.Remove()(ctx))
_, err := os.Stat(workdir)
require.NoError(t, err)
}
func TestHostEnvironmentRemoveCleansWorkdirWhenOwned(t *testing.T) {
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
miscRoot := filepath.Join(base, "misc")
path := filepath.Join(miscRoot, "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
workdir := filepath.Join(base, "workspace", "123", "owner", "repo")
require.NoError(t, os.MkdirAll(workdir, 0o700))
e := &HostEnvironment{
Path: path,
Workdir: workdir,
CleanWorkdir: true,
CleanUp: func() {
_ = os.RemoveAll(miscRoot)
},
StdOut: os.Stdout,
}
require.NoError(t, e.Remove()(ctx))
_, err := os.Stat(workdir)
assert.ErrorIs(t, err, os.ErrNotExist)
}
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
release := make(chan struct{})
stubDone := make(chan struct{})
release := make(chan struct{})
stubDone := make(chan struct{})
orig := removeAll
removeAll = func(string) error {
defer close(stubDone)
<-release
return nil
}
t.Cleanup(func() {
close(release)
<-stubDone
removeAll = orig
})
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
defer cancel()
err := removeAllWithContext(ctx, t.TempDir())
require.ErrorIs(t, err, context.DeadlineExceeded)
orig := removeAll
removeAll = func(string) error {
defer close(stubDone)
<-release
return nil
}
// removeAllWithContext intentionally leaks the delete goroutine on timeout,
// and that goroutine still references removeAll. Unblock it and wait for it
// to return before restoring the var, so the restore can't race the read.
t.Cleanup(func() {
close(release)
<-stubDone
removeAll = orig
})
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
err := removeAllWithContext(ctx, t.TempDir())
require.ErrorIs(t, err, context.DeadlineExceeded)
}
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
for _, phase := range []string{"CleanUp callback", "path removal"} {
t.Run(phase, func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
origTimeout := hostCleanupTimeout
hostCleanupTimeout = 100 * time.Millisecond
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
release, stubDone := make(chan struct{}), make(chan struct{})
block := func() {
defer close(stubDone)
<-release
}
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
if phase == "CleanUp callback" {
e.CleanUp = block
} else {
origRemoveAll := removeAll
removeAll = func(string) error {
block()
return nil
}
t.Cleanup(func() { removeAll = origRemoveAll })
}
t.Cleanup(func() {
close(release)
<-stubDone
})
require.NoError(t, os.MkdirAll(e.Path, 0o700))
done := make(chan error, 1)
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(10 * time.Second):
t.Fatalf("Remove() hung on a stuck %s", phase)
}
})
})
// TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp guards against a stalled
// CleanUp callback (e.g. an os.RemoveAll blocked by an AV/EDR filter driver or
// an unresponsive mount) wedging the runner slot forever at "Cleaning up
// container". Remove must time out the callback and complete job teardown.
func TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp(t *testing.T) {
// Keep the suite fast: shrink the per-phase teardown timeout for this test.
orig := hostCleanupTimeout
hostCleanupTimeout = 100 * time.Millisecond
t.Cleanup(func() { hostCleanupTimeout = orig })
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
path := filepath.Join(base, "misc", "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
release := make(chan struct{})
t.Cleanup(func() { close(release) }) // unblock the leaked goroutine at test end
e := &HostEnvironment{
Path: path,
CleanUp: func() {
<-release // simulate a delete syscall stuck indefinitely
},
StdOut: os.Stdout,
}
done := make(chan error, 1)
go func() { done <- e.Remove()(ctx) }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(10 * time.Second):
t.Fatal("Remove() hung on a stuck CleanUp callback")
}
}
// TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval guards against a
// stalled os.RemoveAll on the misc/workspace paths (same AV/EDR wedge as
// #1023) wedging job completion after the CleanUp callback has already timed
// out or finished.
func TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval(t *testing.T) {
origTimeout := hostCleanupTimeout
hostCleanupTimeout = 100 * time.Millisecond
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
release := make(chan struct{})
stubDone := make(chan struct{})
origRemoveAll := removeAll
removeAll = func(string) error {
defer close(stubDone)
<-release
return nil
}
// The stuck delete goroutine outlives the timed-out Remove and still reads
// removeAll; unblock it and wait before restoring to avoid a restore/read race.
t.Cleanup(func() {
close(release)
<-stubDone
removeAll = origRemoveAll
})
logger := logrus.New()
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
base := t.TempDir()
path := filepath.Join(base, "misc", "hostexecutor")
require.NoError(t, os.MkdirAll(path, 0o700))
e := &HostEnvironment{
Path: path,
StdOut: os.Stdout,
}
done := make(chan error, 1)
go func() { done <- e.Remove()(ctx) }()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(10 * time.Second):
t.Fatal("Remove() hung on a stuck path removal")
}
}
@@ -245,11 +310,17 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
t.Run("single dir", func(t *testing.T) {
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
// Self-PID guard is essential — without it the script could taskkill
// the PowerShell process running it.
assert.Contains(t, s, "$selfPid = $PID")
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
// Must match both ExecutablePath (binaries from the workspace) and
// CommandLine (system binaries invoked with workspace paths in args),
// both bounded by dir+separator so a name-prefix sibling is spared.
assert.Contains(t, s, `$prefix = $p + '\'`)
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
// Each matched PID must be tree-killed, not just stopped.
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
})
@@ -262,16 +333,25 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
assert.Contains(t, s, `'C:\work\path'`)
assert.Contains(t, s, `'C:\work\workdir'`)
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
// Commas between entries — no trailing comma, no leading comma.
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
})
t.Run("path with single quote is escaped", func(t *testing.T) {
// In PowerShell single-quoted strings the only special char is the
// quote itself, escaped by doubling. A workspace path that ever
// contained `'` would inject a command into the script otherwise.
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
assert.Contains(t, s, `'C:\work\it''s\path'`)
// And it must NOT appear unescaped — otherwise the quote would
// terminate the literal early.
assert.NotContains(t, s, `'C:\work\it's\path'`)
})
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
// A path containing [ ] ? * must be embedded verbatim and matched with
// ordinal String methods, not -like, otherwise the metacharacters would
// be interpreted as wildcards and the leftover process could escape.
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
assert.Contains(t, s, `'C:\work\[job]?1'`)
assert.NotContains(t, s, "-like")
@@ -281,6 +361,7 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
s := buildWindowsWorkspaceKillScript(nil)
// Empty array literal — script runs, matches nothing, is a no-op.
assert.Contains(t, s, "$paths = @()")
assert.Contains(t, s, "Get-CimInstance Win32_Process")
})

View File

@@ -1,614 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"bufio"
"bytes"
"cmp"
"context"
"crypto/rand"
"errors"
"fmt"
"io"
"maps"
"net/http"
"net/url"
"os"
"path"
"regexp"
"slices"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"github.com/kballard/go-shellquote"
"github.com/spf13/pflag"
)
// KubernetesOptions configures the pod a job runs in.
type KubernetesOptions struct {
Kubeconfig string
Namespace string
PodTemplates []map[string]any // merged in order, a later one winning
MaxLifetime time.Duration
ForcePull bool
RunnerUUID string
}
// KubernetesPod runs a job container and its service containers as one pod.
type KubernetesPod struct {
options KubernetesOptions
client *kubeClient
name, secretName string
job *kubernetesContainer
services []*kubernetesContainer
arch, path string
}
type kubernetesContainer struct {
LinuxContainerEnvironmentExtensions
pod *KubernetesPod
id string // the workflow's service id, name is its DNS label form
name string
input *NewContainerInput
probe map[string]any
}
type containerStatus struct {
Name string
Ready bool
State struct {
Waiting *struct{ Reason, Message string }
Running *struct{}
Terminated *struct {
ExitCode int
Message string
}
}
}
type podStatus struct {
Spec struct{ Containers []struct{} }
Status struct {
Phase, Reason, Message string
Conditions []struct{ Type, Status, Message string }
InitContainerStatuses, ContainerStatuses []containerStatus
}
}
func NewKubernetesPod(options KubernetesOptions) (*KubernetesPod, error) {
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
if err != nil {
return nil, fmt.Errorf("connect to Kubernetes: %w", err)
}
return &KubernetesPod{options: options, client: client}, nil
}
// JobContainer returns the container of the pod the job's steps run in.
func (p *KubernetesPod) JobContainer(input *NewContainerInput) ExecutionsEnvironment {
p.job = &kubernetesContainer{pod: p, name: "job", input: input}
return p.job
}
var (
invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
dnsSubdomain = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
)
// kubernetesServiceName returns the container name of service id, made a DNS label as it must be.
func kubernetesServiceName(id string) string {
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
return strings.TrimRight(name[:min(63, len(name))], "-")
}
// KubernetesServiceHostnames returns the hostnames of service id, its container name and, where hostAliases accept it as a DNS subdomain, id lowercased.
func KubernetesServiceHostnames(id string) []string {
name, host := kubernetesServiceName(id), strings.ToLower(id)
if host != name && len(host) <= 253 && dnsSubdomain.MatchString(host) { // https://github.com/kubernetes/apimachinery/blob/v0.35.1/pkg/util/validation/validation.go#L198-L207
return []string{name, host}
}
return []string{name}
}
// ServiceContainer returns a container of the pod the job reaches at KubernetesServiceHostnames(id).
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
service := &kubernetesContainer{pod: p, id: id, name: kubernetesServiceName(id), input: input}
p.services = append(p.services, service)
return service
}
func (p *KubernetesPod) create(ctx context.Context) error {
prefix := strings.Trim(strings.ToLower(p.job.input.Name), "-")
prefix = cmp.Or(strings.TrimRight(prefix[:min(40, len(prefix))], "-"), "gitea-runner") + "-"
name := prefix + strings.ToLower(rand.Text()[:8]) // chosen here, so cleanup knows it even when a create response is lost
labels := map[string]any{"app.kubernetes.io/managed-by": "gitea-runner"}
if p.options.RunnerUUID != "" {
labels[runnerUUIDLabel] = p.options.RunnerUUID
}
metadata := map[string]any{"name": name, "labels": labels}
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
var runner struct{ Metadata struct{ Name, UID string } }
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
} else {
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
}
}
secretData := map[string]string{}
manifest, err := p.manifest(name, secretData, metadata)
if err != nil {
return err
}
for _, service := range p.services {
if !slices.ContainsFunc(KubernetesServiceHostnames(service.id), func(hostname string) bool { return strings.EqualFold(hostname, service.id) }) { // glibc and Go match /etc/hosts case-insensitively
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
}
}
if len(secretData) > 0 {
p.secretName = name // named like the pod, RemoveOrphanKubernetesResources finds it by that
if err := p.client.do(ctx, http.MethodPost, p.client.path("secrets"), map[string]any{"metadata": metadata, "stringData": secretData}, nil); err != nil {
return fmt.Errorf("create secret: %w", err)
}
}
p.name = name
if err := p.client.do(ctx, http.MethodPost, p.client.path("pods"), manifest, nil); err != nil {
return fmt.Errorf("create pod: %w", err)
}
return nil
}
func (p *KubernetesPod) manifest(secret string, secretData map[string]string, metadata map[string]any) (any, error) {
template := map[string]any{"spec": map[string]any{
"automountServiceAccountToken": false,
"enableServiceLinks": false,
"terminationGracePeriodSeconds": 0, // the job container's tail ignores SIGTERM
}}
for _, layer := range p.options.PodTemplates {
template, _ = mergeManifest(template, layer, "").(map[string]any)
}
templateSpec, _ := template["spec"].(map[string]any)
templateContainers, _ := templateSpec["containers"].([]any)
templateVolumes, _ := templateSpec["volumes"].([]any)
var serviceTemplate any
if index := slices.IndexFunc(templateContainers, func(container any) bool { return manifestField(container, "name") == "$services" }); index >= 0 {
serviceTemplate = templateContainers[index]
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
}
var containers, volumes []any
var hostnames []string
names := map[string]bool{}
for _, container := range templateContainers {
names[manifestField(container, "name")] = true
}
for _, c := range append([]*kubernetesContainer{p.job}, p.services...) {
if c != p.job && (c.name == "" || names[c.name]) {
return nil, fmt.Errorf("service %s maps to container %q, which is empty or named like another container of the pod", c.id, c.name)
}
names[c.name] = true
spec, err := c.spec(secret, secretData)
if err != nil {
return nil, err
}
if c == p.job {
containers = append(containers, spec)
continue
}
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
hostnames = append(hostnames, KubernetesServiceHostnames(c.id)...)
}
for _, name := range []string{"workspace", "act"} {
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
volumes = append(volumes, map[string]any{"name": name, "emptyDir": map[string]any{}})
}
}
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
if len(hostnames) > 0 {
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
}
if p.options.MaxLifetime > 0 {
spec["activeDeadlineSeconds"] = max(1, int64(p.options.MaxLifetime.Seconds()))
}
return mergeManifest(template, map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}, ""), nil
}
// mergeManifest merges over onto base like a strategic merge patch: maps key by key, map list items by name or volumeMounts by mountPath, env entries, volumes and other values replaced.
func mergeManifest(base, over any, field string) any {
switch over := over.(type) {
case map[string]any:
merged := map[string]any{}
if base, ok := base.(map[string]any); ok {
maps.Copy(merged, base)
}
for key, value := range over {
merged[key] = mergeManifest(merged[key], value, key)
}
return merged
case []any:
key := cmp.Or(map[string]string{"volumeMounts": "mountPath"}[field], "name")
baseList, _ := base.([]any)
merged := slices.Clone(baseList)
for _, item := range over {
if _, ok := item.(map[string]any); !ok {
return over
}
name := manifestField(item, key)
index := slices.IndexFunc(merged, func(existing any) bool { return name != "" && manifestField(existing, key) == name })
switch {
case index < 0:
merged = append(merged, item)
case field == "env" || field == "volumes": // each holds one source, merging two makes a pod the API rejects
merged[index] = item
default:
merged[index] = mergeManifest(merged[index], item, "")
}
}
return merged
}
return over
}
func manifestField(item any, key string) string {
fields, _ := item.(map[string]any)
value, _ := fields[key].(string)
return value
}
func (c *kubernetesContainer) spec(secret string, secretData map[string]string) (map[string]any, error) {
env := []any{}
for index, entry := range c.input.Env {
key, value, _ := strings.Cut(entry, "=")
if c == c.pod.job {
env = append(env, map[string]any{"name": key, "value": value})
continue
}
secretKey := fmt.Sprintf("%s.%d", c.name, index)
secretData[secretKey] = value
env = append(env, map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": secret, "key": secretKey}}})
}
spec := map[string]any{
"name": c.name, "image": c.input.Image, "command": escapeExpansion(c.input.Entrypoint), "args": escapeExpansion(c.input.Cmd),
"workingDir": c.input.WorkingDir, "env": env,
}
if c.pod.options.ForcePull {
spec["imagePullPolicy"] = "Always"
}
if c == c.pod.job {
spec["volumeMounts"] = []any{
map[string]any{"name": "workspace", "mountPath": c.input.WorkingDir},
map[string]any{"name": "act", "mountPath": c.GetActPath()},
}
}
if err := c.parseOptions(); err != nil {
return nil, fmt.Errorf("service %s: %w", c.name, err)
}
if c.probe != nil {
spec["startupProbe"] = c.probe
}
return spec, nil
}
// escapeExpansion keeps kubelet from expanding $(VAR) and $$ in command and args, which docker passes as is.
func escapeExpansion(args []string) []string {
args = slices.Clone(args)
for index := range args {
args[index] = strings.ReplaceAll(args[index], "$", "$$")
}
return args
}
// parseOptions turns docker health options, the only ones a pod supports, into a startup probe, which like docker gives up after the retries.
func (c *kubernetesContainer) parseOptions() error {
args, err := shellquote.Split(c.input.WorkflowOptions)
if err != nil {
return err
}
flags := pflag.NewFlagSet("options", pflag.ContinueOnError)
flags.SetOutput(io.Discard)
command := flags.String("health-cmd", "", "")
interval := flags.Duration("health-interval", 0, "")
timeout := flags.Duration("health-timeout", 0, "")
retries := flags.Int("health-retries", 0, "")
startPeriod := flags.Duration("health-start-period", 0, "")
if err := flags.Parse(args); err != nil {
return fmt.Errorf("options %q: %w, only --health-* options are supported on Kubernetes", c.input.WorkflowOptions, err)
}
if *command == "" {
return nil
}
period := max(1, int(cmp.Or(*interval, 30*time.Second).Seconds()))
c.probe = map[string]any{
"exec": map[string]any{"command": []string{"/bin/sh", "-c", *command}},
"periodSeconds": period,
"timeoutSeconds": max(1, int(cmp.Or(*timeout, 30*time.Second).Seconds())),
"failureThreshold": max(1, cmp.Or(*retries, 3)) + int(startPeriod.Seconds())/period, // docker does not count failures in the start period
}
return nil
}
func (p *KubernetesPod) get(ctx context.Context) (*podStatus, error) {
var pod podStatus
return &pod, p.client.do(ctx, http.MethodGet, p.client.path("pods/"+p.name), nil, &pod)
}
func (p *KubernetesPod) waitStarted(ctx context.Context) error {
var unscheduled string
for delay := 100 * time.Millisecond; ; delay = min(2*delay, time.Second) {
pod, err := p.get(ctx)
if err != nil {
return err
}
if pod.Status.Phase == "Failed" {
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
}
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
if waiting := status.State.Waiting; waiting != nil {
switch waiting.Reason {
case "ErrImagePull", "ImagePullBackOff", "InvalidImageName", "ErrImageNeverPull", "CreateContainerConfigError", "CreateContainerError", "RunContainerError":
return fmt.Errorf("container %s of pod %s: %s: %s", status.Name, p.name, waiting.Reason, waiting.Message)
}
started = false
}
}
if started {
return nil
}
for _, condition := range pod.Status.Conditions {
if condition.Type == "PodScheduled" && condition.Status == "False" && condition.Message != unscheduled {
unscheduled = condition.Message
common.Logger(ctx).Infof("Waiting for pod %s to be scheduled: %s", p.name, unscheduled)
}
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(delay):
}
}
}
func (c *kubernetesContainer) Create(_, _ []string) common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
return c.pod.create(ctx)
}
}
func (c *kubernetesContainer) Start(_ bool) common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
if err := c.pod.waitStarted(ctx); err != nil {
return err
}
output, err := c.pod.probe(ctx, c.name)
if err != nil {
return err
}
machine, path, _ := strings.Cut(output, "\n")
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
return nil
}
}
// probe retries exec while it fails short of the container, as a new node runs pods before its kubelet has a serving certificate.
func (p *KubernetesPod) probe(ctx context.Context, container string) (string, error) {
var failure string
deadline := time.Now().Add(time.Minute)
for delay := 100 * time.Millisecond; ; delay = min(2*delay, 2*time.Second) {
var stdout bytes.Buffer
err := p.client.exec(ctx, p.name, container, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard)
var exitErr ExitCodeError
var apiErr *apiError
if err == nil || ctx.Err() != nil || errors.As(err, &exitErr) || time.Now().After(deadline) ||
errors.As(err, &apiErr) && apiErr.code < http.StatusInternalServerError {
return stdout.String(), err
}
if err.Error() != failure {
failure = err.Error()
common.Logger(ctx).Infof("Waiting for exec into pod %s: %v", p.name, err)
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(delay):
}
}
}
func (*kubernetesContainer) Pull(bool) common.Executor {
return func(context.Context) error { return nil }
}
var (
shellName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
bashReadonly = []string{"BASHOPTS", "BASH_VERSINFO", "EUID", "PPID", "SHELLOPTS", "UID"} // bash as sh cannot export these
)
// Exec feeds the command to sh on stdin, which exports the env so it stays out of the request URL and, where sh can export a name, out of process arguments.
func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
return func(ctx context.Context) error {
script, args := "", []string{"env", "--"}
for name, value := range env {
if shellName.MatchString(name) && !slices.Contains(bashReadonly, name) {
script += "command export " + shellquote.Join(name+"="+value) + "\n"
} else {
args = append(args, name+"="+value)
}
}
script += "exec " + shellquote.Join(append(args, command...)...)
if workdir != "" {
script = "cd " + shellquote.Join(workdir) + " || exit\n" + script
}
defer common.FlushWriter(c.input.Stdout)
defer common.FlushWriter(c.input.Stderr)
return c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh"}, strings.NewReader(script), c.input.Stdout, c.input.Stderr)
}
}
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
reader, writer := io.Pipe()
defer reader.Close()
go func() { writer.CloseWithError(write(writer)) }()
var stderr bytes.Buffer
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
}
return nil
}
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
return func(ctx context.Context) error {
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
}
}
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
return func(ctx context.Context) error {
return c.extract(ctx, destPath, func(writer io.Writer) error {
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
})
}
}
// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
reader, writer := io.Pipe()
go func() {
var stderr bytes.Buffer
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
if err != nil {
err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
}
writer.CloseWithError(err)
}()
buffered := bufio.NewReader(reader)
if _, err := buffered.Peek(1); err != nil {
reader.Close()
return nil, err
}
return struct {
io.Reader
io.Closer
}{buffered, reader}, nil
}
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(c, srcPath, env, false)
}
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
return func(context.Context) error {
(*env)["PATH"] = cmp.Or((*env)["PATH"], c.pod.path)
return nil
}
}
func (c *kubernetesContainer) Inspect(ctx context.Context) (*Info, error) {
if c.pod.name == "" {
return nil, fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
}
pod, err := c.pod.get(ctx)
if isNotFound(err) {
return nil, fmt.Errorf("pod %s %w", c.pod.name, ErrContainerNotFound)
} else if err != nil {
return nil, err
}
info := &Info{ID: c.pod.name + "/" + c.name, State: "created", Health: HealthNone, Ports: map[string]string{}}
for port := range c.input.ExposedPorts {
info.Ports[port.Port()] = port.Port() // services share the pod's network
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name != c.name {
continue
}
if status.State.Running != nil {
info.State = StateRunning
} else if terminated := status.State.Terminated; terminated != nil {
info.State, info.ExitCode, info.HealthOutput = "exited", terminated.ExitCode, terminated.Message
}
if c.probe != nil {
switch {
case status.Ready:
info.Health = HealthHealthy
case info.State == "exited":
info.Health = HealthUnhealthy
default:
info.Health = HealthStarting
}
}
}
return info, nil
}
func (c *kubernetesContainer) DumpLogs(ctx context.Context) error {
if c.pod.name == "" {
return fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
}
defer common.FlushWriter(c.input.Stdout)
return c.pod.client.do(ctx, http.MethodGet, c.pod.client.path("pods/"+c.pod.name+"/log?container="+c.name), nil, c.input.Stdout)
}
func (c *kubernetesContainer) Remove() common.Executor {
return func(ctx context.Context) error {
if c != c.pod.job {
return nil
}
return errors.Join(c.pod.client.delete(ctx, "pods", c.pod.name), c.pod.client.delete(ctx, "secrets", c.pod.secretName))
}
}
// RemoveOrphanKubernetesResources removes this runner's job pods created before createdBefore and the Secrets named like them, as listing Secrets would reveal all of the namespace's.
func RemoveOrphanKubernetesResources(ctx context.Context, options KubernetesOptions, createdBefore time.Time) error {
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
if err != nil {
return fmt.Errorf("connect to Kubernetes: %w", err)
}
defer client.http.CloseIdleConnections()
var pods struct {
Items []struct {
Metadata struct {
Name string
CreationTimestamp time.Time
}
}
}
if err := client.do(ctx, http.MethodGet, client.path("pods?labelSelector="+url.QueryEscape(runnerUUIDLabel+"="+options.RunnerUUID)), nil, &pods); err != nil {
return fmt.Errorf("list pods: %w", err)
}
var errs []error
for _, item := range pods.Items {
if !item.Metadata.CreationTimestamp.Before(createdBefore) {
continue
}
if err := client.delete(ctx, "secrets", item.Metadata.Name); err != nil {
errs = append(errs, err) // leave the pod for the next cleanup, it is the only way back to its Secret
continue
}
errs = append(errs, client.delete(ctx, "pods", item.Metadata.Name))
}
return errors.Join(errs...)
}
func (c *kubernetesContainer) Close() common.Executor {
return func(context.Context) error {
c.pod.client.http.CloseIdleConnections()
return nil
}
}
func (c *kubernetesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
oldStdout, oldStderr := c.input.Stdout, c.input.Stderr
c.input.Stdout, c.input.Stderr = stdout, stderr
return oldStdout, oldStderr
}
func (c *kubernetesContainer) GetRunnerContext(_ context.Context) map[string]any {
return map[string]any{"os": "Linux", "arch": c.pod.arch, "temp": "/tmp", "tool_cache": DefaultToolCache}
}

View File

@@ -1,400 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"bytes"
"cmp"
"context"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/coder/websocket"
"go.yaml.in/yaml/v4"
)
const (
serviceAccountDir = "/var/run/secrets/kubernetes.io/serviceaccount"
execProtocol = "v5.channel.k8s.io"
)
type kubeClient struct {
server, namespace, token string
tokenFile string // re-read on every request, service account tokens rotate
inCluster bool
http *http.Client
}
type apiError struct {
code int
message string
}
func (e *apiError) Error() string { return e.message }
func isNotFound(err error) bool {
var apiErr *apiError
return errors.As(err, &apiErr) && apiErr.code == http.StatusNotFound
}
func newKubeClient(kubeconfig, namespace string) (*kubeClient, error) {
client := &kubeClient{namespace: namespace}
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}
if host := os.Getenv("KUBERNETES_SERVICE_HOST"); host != "" && kubeconfig == "" {
if err := client.loadInCluster(host, transport.TLSClientConfig); err != nil {
return nil, err
}
} else if err := client.loadKubeconfig(kubeconfig, transport); err != nil {
return nil, err
}
client.http = &http.Client{Transport: transport}
return client, nil
}
func (c *kubeClient) loadInCluster(host string, tlsConfig *tls.Config) error {
c.server = "https://" + net.JoinHostPort(host, os.Getenv("KUBERNETES_SERVICE_PORT"))
c.tokenFile = serviceAccountDir + "/token"
ca, err := os.ReadFile(serviceAccountDir + "/ca.crt")
if err != nil {
return err
}
tlsConfig.RootCAs = x509.NewCertPool()
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
if c.namespace == "" {
namespace, err := os.ReadFile(serviceAccountDir + "/namespace")
if err != nil {
return err
}
c.namespace = strings.TrimSpace(string(namespace))
}
c.inCluster = true
return nil
}
func (c *kubeClient) loadKubeconfig(path string, transport *http.Transport) error {
if paths := filepath.SplitList(os.Getenv("KUBECONFIG")); path == "" && len(paths) > 0 {
path = paths[0]
}
if path == "" {
home, err := os.UserHomeDir()
if err != nil {
return err
}
path = filepath.Join(home, ".kube", "config")
}
content, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("no in-cluster service account and no kubeconfig: %w", err)
}
var config struct {
CurrentContext string `yaml:"current-context"`
Contexts []struct {
Name string
Context struct{ Cluster, User, Namespace string }
}
Clusters []struct {
Name string
Cluster struct {
Server string
TLSServerName string `yaml:"tls-server-name"`
InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify"`
ProxyURL string `yaml:"proxy-url"`
CertificateAuthority string `yaml:"certificate-authority"`
CertificateAuthorityData string `yaml:"certificate-authority-data"`
}
}
Users []struct {
Name string
User struct {
Token string
TokenFile string `yaml:"tokenFile"`
ClientCertificate string `yaml:"client-certificate"`
ClientCertificateData string `yaml:"client-certificate-data"`
ClientKey string `yaml:"client-key"`
ClientKeyData string `yaml:"client-key-data"`
Exec any
AuthProvider any `yaml:"auth-provider"`
}
}
}
if err := yaml.Unmarshal(content, &config); err != nil {
return fmt.Errorf("parse kubeconfig %s: %w", path, err)
}
var current struct{ Cluster, User, Namespace string }
for _, entry := range config.Contexts {
if entry.Name == config.CurrentContext {
current = entry.Context
}
}
if current.Cluster == "" {
return fmt.Errorf("kubeconfig %s has no current context", path)
}
c.namespace = cmp.Or(c.namespace, current.Namespace, "default")
resolve := func(file string) string {
if file == "" || filepath.IsAbs(file) {
return file
}
return filepath.Join(filepath.Dir(path), file)
}
read := func(data, file string) ([]byte, error) {
if data != "" {
return base64.StdEncoding.DecodeString(data)
}
if file == "" {
return nil, nil
}
return os.ReadFile(resolve(file))
}
tlsConfig := transport.TLSClientConfig
for _, cluster := range config.Clusters {
if cluster.Name != current.Cluster {
continue
}
c.server = strings.TrimSuffix(cluster.Cluster.Server, "/")
tlsConfig.ServerName, tlsConfig.InsecureSkipVerify = cluster.Cluster.TLSServerName, cluster.Cluster.InsecureSkipTLSVerify
if cluster.Cluster.ProxyURL != "" {
proxy, err := url.Parse(cluster.Cluster.ProxyURL)
if err != nil {
return fmt.Errorf("kubeconfig %s: %w", path, err)
}
transport.Proxy = http.ProxyURL(proxy)
}
ca, err := read(cluster.Cluster.CertificateAuthorityData, cluster.Cluster.CertificateAuthority)
if err != nil {
return err
}
if ca != nil {
tlsConfig.RootCAs = x509.NewCertPool()
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
}
}
for _, user := range config.Users {
if user.Name != current.User {
continue
}
if user.User.Exec != nil || user.User.AuthProvider != nil {
return fmt.Errorf("kubeconfig %s: user %q authenticates through a credential plugin, which the runner does not support, use a token or client certificate", path, user.Name)
}
c.token, c.tokenFile = user.User.Token, resolve(user.User.TokenFile)
cert, err := read(user.User.ClientCertificateData, user.User.ClientCertificate)
if err != nil {
return err
}
key, err := read(user.User.ClientKeyData, user.User.ClientKey)
if err != nil {
return err
}
if cert != nil {
pair, err := tls.X509KeyPair(cert, key)
if err != nil {
return err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
}
if c.server == "" {
return fmt.Errorf("kubeconfig %s names no server for context %q", path, config.CurrentContext)
}
return nil
}
func (c *kubeClient) authorize(header http.Header) error {
token := c.token
if c.tokenFile != "" {
content, err := os.ReadFile(c.tokenFile)
if err != nil {
return err
}
token = strings.TrimSpace(string(content))
}
if token != "" {
header.Set("Authorization", "Bearer "+token)
}
return nil
}
func (c *kubeClient) path(resource string) string {
return "/api/v1/namespaces/" + c.namespace + "/" + resource
}
func (c *kubeClient) do(ctx context.Context, method, path string, in, out any) error {
var body io.Reader
if in != nil {
content, err := json.Marshal(in)
if err != nil {
return err
}
body = bytes.NewReader(content)
}
req, err := http.NewRequestWithContext(ctx, method, c.server+path, body)
if err != nil {
return err
}
if err := c.authorize(req.Header); err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= http.StatusMultipleChoices {
return statusError(resp)
}
if out == nil {
out = io.Discard // draining lets the connection be reused
}
if writer, ok := out.(io.Writer); ok {
_, err = io.Copy(writer, resp.Body)
return err
}
return json.NewDecoder(resp.Body).Decode(out)
}
func (c *kubeClient) delete(ctx context.Context, resource, name string) error {
if name == "" {
return nil
}
if err := c.do(ctx, http.MethodDelete, c.path(resource+"/"+name), nil, nil); err != nil && !isNotFound(err) {
return fmt.Errorf("delete %s %s: %w", resource, name, err)
}
return nil
}
func statusError(resp *http.Response) error {
content, _ := io.ReadAll(resp.Body)
var status struct{ Message string }
if json.Unmarshal(content, &status) != nil || status.Message == "" {
status.Message = strings.TrimSpace(string(content))
}
return &apiError{code: resp.StatusCode, message: status.Message}
}
func (c *kubeClient) exec(ctx context.Context, pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
query := url.Values{"container": {containerName}, "command": command, "stdout": {"true"}, "stderr": {"true"}}
if stdin != nil {
query.Set("stdin", "true")
}
header := http.Header{}
if err := c.authorize(header); err != nil {
return err
}
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
conn, resp, err := websocket.Dial(ctx, c.server+c.path("pods/"+pod+"/exec?")+query.Encode(), &websocket.DialOptions{
HTTPClient: c.http,
HTTPHeader: header,
Subprotocols: []string{execProtocol},
})
if err != nil {
if resp != nil && resp.StatusCode >= http.StatusMultipleChoices {
defer resp.Body.Close()
err = statusError(resp)
}
return fmt.Errorf("exec in pod %s: %w", pod, err)
}
defer func() { _ = conn.CloseNow() }()
if conn.Subprotocol() != execProtocol {
return fmt.Errorf("the Kubernetes API server does not speak %s, which needs Kubernetes 1.30 or later", execProtocol)
}
conn.SetReadLimit(-1)
if stdin != nil {
go func() {
if err := sendStdin(ctx, conn, stdin); err != nil {
cancel(err)
}
}()
}
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if conn.Ping(ctx) != nil {
return
}
}
}
}()
for {
_, frame, err := conn.Read(ctx)
if err != nil {
return cmp.Or(context.Cause(ctx), fmt.Errorf("exec in pod %s ended without an exit status: %w", pod, err))
}
if len(frame) < 2 {
continue
}
switch frame[0] {
case 1:
_, err = stdout.Write(frame[1:])
case 2:
_, err = stderr.Write(frame[1:])
case 3:
return cmp.Or(context.Cause(ctx), execStatus(frame[1:]))
}
if err != nil {
return err
}
}
}
func sendStdin(ctx context.Context, conn *websocket.Conn, stdin io.Reader) error {
frame := make([]byte, 1+32*1024) // frame[0] is the stdin channel, 0
for {
n, err := stdin.Read(frame[1:])
if n > 0 {
if err := conn.Write(ctx, websocket.MessageBinary, frame[:1+n]); err != nil {
return err
}
}
if errors.Is(err, io.EOF) {
return conn.Write(ctx, websocket.MessageBinary, []byte{255, 0}) // v5 close signal for stdin
}
if err != nil {
return err
}
}
}
func execStatus(content []byte) error {
var status struct {
Status, Message string
Details struct {
Causes []struct{ Reason, Message string }
}
}
if err := json.Unmarshal(content, &status); err != nil {
return fmt.Errorf("decode exec status: %w", err)
}
if status.Status == "Success" {
return nil
}
for _, cause := range status.Details.Causes {
if code, err := strconv.Atoi(cause.Message); cause.Reason == "ExitCode" && err == nil {
return ExitCodeError(code)
}
}
return fmt.Errorf("exec failed: %s", status.Message)
}

View File

@@ -1,374 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/coder/websocket"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type podManifest struct {
Metadata struct{ Labels map[string]string }
Spec struct {
AutomountServiceAccountToken bool
EnableServiceLinks bool
HostAliases []struct{ Hostnames []string }
Volumes []map[string]any
Containers []struct {
Name, ImagePullPolicy string
Resources map[string]any
StartupProbe *struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }
}
}
}
type fakeCluster struct {
mu sync.Mutex
requests map[string][]string
execFailures []http.HandlerFunc
}
func (f *fakeCluster) record(request, content string) {
f.mu.Lock()
defer f.mu.Unlock()
f.requests[request] = append(f.requests[request], content)
}
func (f *fakeCluster) failExec(failures ...http.HandlerFunc) {
f.mu.Lock()
defer f.mu.Unlock()
f.execFailures = failures
}
func (f *fakeCluster) nextExecFailure() http.HandlerFunc {
f.mu.Lock()
defer f.mu.Unlock()
if len(f.execFailures) == 0 {
return nil
}
failure := f.execFailures[0]
f.execFailures = f.execFailures[1:]
return failure
}
// execFailure answers like the API server, code 0 for a failure reported after the websocket upgrade.
func execFailure(t *testing.T, code int, status string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if code != 0 {
w.WriteHeader(code)
_, _ = io.WriteString(w, status)
return
}
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
if err != nil {
t.Error(err)
return
}
defer func() { _ = conn.CloseNow() }()
_ = conn.Write(r.Context(), websocket.MessageBinary, append([]byte{3}, status...))
}
}
func fakeKubernetes(t *testing.T, podStatus string, env ...string) (string, *fakeCluster) {
cluster := &fakeCluster{requests: map[string][]string{}}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer test-token" {
w.WriteHeader(http.StatusUnauthorized)
return
}
body, err := io.ReadAll(r.Body)
assert.NoError(t, err)
cluster.record(r.Method+" "+r.URL.Path, string(body))
pods, secrets := "/api/v1/namespaces/ci/pods", "/api/v1/namespaces/ci/secrets"
switch {
case r.Method == http.MethodGet && r.URL.Path == pods:
cluster.record("list", r.URL.Query().Get("labelSelector"))
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
case strings.HasSuffix(r.URL.Path, "/exec"):
cluster.record("exec", r.URL.RawQuery)
if failure := cluster.nextExecFailure(); failure != nil {
failure(w, r)
} else {
serveLocalExec(t, w, r, env)
}
case strings.HasSuffix(r.URL.Path, "/log"):
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
_, _ = io.WriteString(w, podStatus)
case r.Method == http.MethodPost && (r.URL.Path == pods || r.URL.Path == secrets), r.Method == http.MethodDelete:
default:
w.WriteHeader(http.StatusNotFound)
}
}))
t.Cleanup(server.Close)
kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(kubeconfig), "token"), []byte("test-token\n"), 0o600))
require.NoError(t, os.WriteFile(kubeconfig, fmt.Appendf(nil, `
current-context: test
contexts: [{name: test, context: {cluster: test, user: test, namespace: ci}}]
clusters: [{name: test, cluster: {server: "http://kubernetes.invalid", proxy-url: %q}}]
users: [{name: test, user: {tokenFile: token}}]
`, server.URL), 0o600))
return kubeconfig, cluster
}
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request, env []string) {
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
if err != nil {
t.Error(err)
return
}
defer func() { _ = conn.CloseNow() }()
conn.SetReadLimit(-1)
command := r.URL.Query()["command"]
var stdin bytes.Buffer
for r.URL.Query().Get("stdin") == "true" {
_, frame, err := conn.Read(r.Context())
if err != nil {
t.Error(err)
return
}
if frame[0] == 255 {
break
}
stdin.Write(frame[1:])
}
cmd := exec.Command(command[0], command[1:]...)
cmd.Env = append(append(os.Environ(), "COPYFILE_DISABLE=1"), env...)
var stdout, stderr bytes.Buffer
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
status := `{"status":"Success"}`
var exitErr *exec.ExitError
if err := cmd.Run(); errors.As(err, &exitErr) {
status = fmt.Sprintf(`{"status":"Failure","reason":"NonZeroExitCode","details":{"causes":[{"reason":"ExitCode","message":"%d"}]}}`, exitErr.ExitCode())
} else {
assert.NoError(t, err)
}
for _, frame := range [][]byte{append([]byte{1}, stdout.Bytes()...), append([]byte{2}, stderr.Bytes()...), append([]byte{3}, status...)} {
_ = conn.Write(r.Context(), websocket.MessageBinary, frame)
}
}
func TestKubernetesServiceHostnames(t *testing.T) {
for id, hostnames := range map[string][]string{
"db": {"db"},
"My_DB": {"my-db"},
"devstoreaccount1.azurite.local": {"devstoreaccount1-azurite-local", "devstoreaccount1.azurite.local"},
"DB.Local": {"db-local", "db.local"},
strings.Repeat("a", 64): {strings.Repeat("a", 63), strings.Repeat("a", 64)},
strings.Repeat("a", 254): {strings.Repeat("a", 63)},
} {
assert.Equal(t, hostnames, KubernetesServiceHostnames(id), id)
}
}
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
t.Skip("uses POSIX shell")
}
ctx := t.Context()
workdir, err := filepath.EvalSymlinks(t.TempDir())
require.NoError(t, err)
path := workdir + ":" + os.Getenv("PATH")
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
{"name":"job","ready":true,"state":{"running":{}}},
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`, "PATH="+path)
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
{"spec": map[string]any{
"automountServiceAccountToken": true,
"containers": []any{
map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"cpu": "2"}}},
map[string]any{
"name": "$services", "imagePullPolicy": "Always", "resources": map[string]any{"limits": map[string]any{"memory": "256Mi"}},
"env": []any{map[string]any{"name": "POSTGRES_PASSWORD", "value": "template"}},
},
},
}},
{"spec": map[string]any{
"volumes": []any{map[string]any{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}}},
"containers": []any{map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}}},
}},
}}
pod, err := NewKubernetesPod(options)
require.NoError(t, err)
var jobLog, serviceLog bytes.Buffer
job := pod.JobContainer(&NewContainerInput{Name: "Job-Name", Image: "node", WorkingDir: workdir, Stdout: &jobLog, Stderr: &jobLog})
service := pod.ServiceContainer("My_DB", &NewContainerInput{
Image: "postgres",
Cmd: []string{"postgres", "-D", "$(PGDATA)"},
Env: []string{"POSTGRES_PASSWORD=service-secret"},
WorkflowOptions: "--health-cmd pg_isready --health-interval 2s --health-start-period 4s",
Stdout: &serviceLog,
})
require.NoError(t, job.Create(nil, nil)(ctx))
var manifest podManifest
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
assert.Equal(t, "gitea-runner", manifest.Metadata.Labels["app.kubernetes.io/managed-by"])
assert.Equal(t, "runner-uuid", manifest.Metadata.Labels[runnerUUIDLabel])
assert.True(t, manifest.Spec.AutomountServiceAccountToken)
assert.False(t, manifest.Spec.EnableServiceLinks)
assert.Equal(t, []string{"my-db"}, manifest.Spec.HostAliases[0].Hostnames)
require.Len(t, manifest.Spec.Containers, 2)
assert.Equal(t, map[string]any{"limits": map[string]any{"cpu": "2", "memory": "1Gi"}}, manifest.Spec.Containers[0].Resources)
assert.Empty(t, manifest.Spec.Containers[0].ImagePullPolicy)
assert.Equal(t, "my-db", manifest.Spec.Containers[1].Name)
assert.Equal(t, "Always", manifest.Spec.Containers[1].ImagePullPolicy)
assert.Equal(t, map[string]any{"limits": map[string]any{"memory": "256Mi"}}, manifest.Spec.Containers[1].Resources)
assert.Equal(t, []map[string]any{
{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}},
{"name": "act", "emptyDir": map[string]any{}},
}, manifest.Spec.Volumes)
assert.Equal(t, struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }{2, 30, 5}, *manifest.Spec.Containers[1].StartupProbe)
assert.NotContains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], "service-secret")
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], "service-secret")
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], `"com.gitea.runner.uuid":"runner-uuid"`)
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `{"name":"POSTGRES_PASSWORD","valueFrom":{"secretKeyRef":{"key":"my-db.0","name":"`+pod.secretName+`"}}}`)
assert.Equal(t, []any{"new"}, mergeManifest([]any{"old"}, []any{"new"}, ""))
mount := func(path string) any { return map[string]any{"name": "config", "mountPath": path} }
assert.Equal(t, []any{mount("/a"), mount("/b")}, mergeManifest([]any{mount("/a")}, []any{mount("/b")}, "volumeMounts"))
tmpfs := map[string]any{"name": "workspace", "emptyDir": map[string]any{"medium": "Memory"}}
ephemeral := map[string]any{"name": "workspace", "ephemeral": map[string]any{}}
assert.Equal(t, []any{ephemeral}, mergeManifest([]any{tmpfs}, []any{ephemeral}, "volumes"))
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
cluster.failExec(
execFailure(t, http.StatusInternalServerError, `{"message":"error dialing backend: remote error: tls: internal error"}`),
execFailure(t, 0, `{"status":"Failure","reason":"InternalError","message":"Internal error occurred: error sending request: remote error: tls: internal error"}`),
)
require.NoError(t, job.Start(false)(ctx))
assert.Len(t, cluster.requests["exec"], 3)
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
envArgs, err := os.ReadFile(filepath.Join(workdir, "env-args"))
require.NoError(t, err)
assert.NotContains(t, string(envArgs), "hunter2")
require.NoError(t, job.Exec([]string{"printenv", "INPUT_WHO-TO-GREET"}, stepEnv, "", "")(ctx))
require.NoError(t, job.Exec([]string{"printenv", "UID"}, stepEnv, "", "")(ctx))
require.NoError(t, job.Exec([]string{"pwd", "-P"}, nil, "", workdir)(ctx))
assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
assert.Contains(t, jobLog.String(), "failing\n")
for _, query := range cluster.requests["exec"] {
assert.NotContains(t, query, "hunter2")
}
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
require.NoError(t, err)
reader := tar.NewReader(archive)
header, err := reader.Next()
require.NoError(t, err)
content, err := io.ReadAll(reader)
require.NoError(t, err)
assert.Equal(t, "event.json", header.Name)
assert.JSONEq(t, `{"ref":"main"}`, string(content))
require.NoError(t, archive.Close())
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
require.Error(t, err)
actionDir := filepath.Join(t.TempDir(), "action")
require.NoError(t, os.MkdirAll(actionDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
require.NoError(t, err)
assert.Equal(t, "main()", string(script))
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
pathEnv := map[string]string{}
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
assert.Equal(t, path, pathEnv["PATH"])
info, err := service.Inspect(ctx)
require.NoError(t, err)
assert.Equal(t, HealthHealthy, info.Health)
require.NoError(t, service.DumpLogs(ctx))
assert.Equal(t, "log of my-db", serviceLog.String())
require.NoError(t, job.Remove()(ctx))
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/"+pod.name], 1)
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/"+pod.secretName], 1)
require.NoError(t, RemoveOrphanKubernetesResources(ctx, options, time.Now()))
assert.Equal(t, []string{runnerUUIDLabel + "=runner-uuid"}, cluster.requests["list"])
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/old"], 1)
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/old"], 1)
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/new"])
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
}
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
template := map[string]any{"spec": map[string]any{"containers": []any{map[string]any{"name": "sidecar"}}}}
for _, test := range []struct {
services []string
options, err string
}{
{[]string{"db"}, "--hostname db", "only --health-* options are supported"},
{[]string{"sidecar"}, "", "named like another container"},
{[]string{"my_db", "My-DB"}, "", `service My-DB maps to container "my-db"`},
} {
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci", PodTemplates: []map[string]any{template}})
require.NoError(t, err)
job := pod.JobContainer(&NewContainerInput{Image: "node"})
for _, service := range test.services {
pod.ServiceContainer(service, &NewContainerInput{Image: "postgres", WorkflowOptions: test.options})
}
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
}
running, cluster := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[{"name":"job","state":{"running":{}}}]}}`)
for _, test := range []struct {
failure http.HandlerFunc
err string
}{
{execFailure(t, http.StatusForbidden, `{"message":"pods \"job\" is forbidden"}`), "is forbidden"},
{execFailure(t, 0, `{"status":"Failure","details":{"causes":[{"reason":"ExitCode","message":"126"}]}}`), ExitCodeError(126).Error()},
} {
cluster.failExec(test.failure)
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: running})
require.NoError(t, err)
job := pod.JobContainer(&NewContainerInput{Image: "node"})
require.NoError(t, job.Create(nil, nil)(t.Context()))
require.ErrorContains(t, job.Start(false)(t.Context()), test.err)
}
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
require.NoError(t, err)
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
require.NoError(t, job.Create(nil, nil)(t.Context()))
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
plugin := filepath.Join(t.TempDir(), "kubeconfig")
require.NoError(t, os.WriteFile(plugin, []byte(`
current-context: eks
contexts: [{name: eks, context: {cluster: eks, user: eks}}]
clusters: [{name: eks, cluster: {server: "https://eks.example"}}]
users: [{name: eks, user: {exec: {command: aws}}}]
`), 0o600))
_, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: plugin})
require.ErrorContains(t, err, "credential plugin")
}

View File

@@ -11,28 +11,28 @@ import (
"runtime"
"strings"
"gitea.com/gitea/runner/act/common/logging"
log "github.com/sirupsen/logrus"
)
type LinuxContainerEnvironmentExtensions struct{}
var windowsPathRegex = regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
// Resolves the equivalent host path inside the container
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
// For use in docker volumes and binds
func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
logging.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
return ""
}
abspath, err := filepath.Abs(path)
if err != nil {
logging.Error(err)
log.Error(err)
return ""
}
// Test if the path is a windows path
windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
// Return as-is if no match
@@ -43,7 +43,11 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
// Convert to WSL2-compatible path if it is a windows path
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
return "/mnt/" + strings.ToLower(windowsPathComponents[1]) + "/" + strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
driveLetter := strings.ToLower(windowsPathComponents[1])
translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
// Should make something like /mnt/c/Users/person/My Folder/MyActProject
result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
return result
}
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
@@ -65,41 +69,10 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
const DefaultToolCache = "/opt/hostedtoolcache"
type runnerArchKey struct{}
func WithRunnerArch(ctx context.Context) context.Context { // shares one architecture lookup per run
if ctx.Value(runnerArchKey{}) != nil {
return ctx
}
return context.WithValue(ctx, runnerArchKey{}, &runnerArchCache{lock: make(chan struct{}, 1)})
}
type runnerArchCache struct {
lock chan struct{} // lets cancelled jobs stop waiting for the cache
arch string
}
func cachedRunnerArch(ctx context.Context) string {
cache, ok := ctx.Value(runnerArchKey{}).(*runnerArchCache)
if !ok {
return RunnerArch(ctx)
}
select {
case cache.lock <- struct{}{}:
defer func() { <-cache.lock }()
case <-ctx.Done():
return ""
}
if cache.arch == "" {
cache.arch = RunnerArch(ctx)
}
return cache.arch
}
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
return map[string]any{
"os": "Linux",
"arch": cachedRunnerArch(ctx),
"arch": RunnerArch(ctx),
"temp": "/tmp",
"tool_cache": DefaultToolCache,
}

View File

@@ -5,13 +5,14 @@
package container
import (
"fmt"
"os"
"runtime"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestContainerPath(t *testing.T) {
@@ -21,27 +22,42 @@ func TestContainerPath(t *testing.T) {
workDir string
}
cwd, err := os.Getwd()
require.NoError(t, err)
tests := []containerPathJob{
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
{"/home/act", `/home/act/`, ""},
{cwd, ".", ""},
}
linuxcontainerext := &LinuxContainerEnvironmentExtensions{}
if runtime.GOOS == "windows" {
cwd, err := os.Getwd()
if err != nil {
log.Error(err)
}
rootDrive := os.Getenv("SystemDrive")
tests = []containerPathJob{
rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
for _, v := range []containerPathJob{
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
{"/mnt/f/work/dir", `F:\work\dir`, ""},
{"/mnt/c/windows/to/unix", "windows\\to\\unix", rootDrive + "\\"},
{"/mnt/" + strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + "/act", "act", rootDrive + "\\"},
{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", rootDrive + "\\"},
} {
if v.workDir != "" {
t.Chdir(v.workDir)
}
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
}
}
for _, test := range tests {
if test.workDir != "" {
t.Chdir(test.workDir)
t.Chdir(cwd)
} else {
cwd, err := os.Getwd()
if err != nil {
log.Error(err)
}
for _, v := range []containerPathJob{
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
{"/home/act", `/home/act/`, ""},
{cwd, ".", ""},
} {
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
}
assert.Equal(t, test.destinationPath, (&LinuxContainerEnvironmentExtensions{}).ToContainerPath(test.sourcePath))
}
}
@@ -50,4 +66,5 @@ type typeAssertMockContainer struct {
LinuxContainerEnvironmentExtensions
}
// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment
var _ ExecutionsEnvironment = &typeAssertMockContainer{}

View File

@@ -0,0 +1,73 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"context"
"encoding/json"
"errors"
"fmt"
"os/exec"
"strings"
"time"
)
type macOSVM struct {
Name string `json:"name"`
State string `json:"state"`
Running bool `json:"running"`
AccessedAt string `json:"accessed_at"`
}
// CleanupOrphanMacOSVMs deletes local macOS VMs left behind by this runner that
// have not been accessed since cutoff.
func CleanupOrphanMacOSVMs(ctx context.Context, executorPath, uuid string, cutoff time.Time) error {
if uuid == "" {
return nil
}
output, err := exec.CommandContext(ctx, executorPath, "list", "--format", "json").Output()
if err != nil {
return fmt.Errorf("list macOS VMs: %w", err)
}
var vms []macOSVM
if err := json.Unmarshal(output, &vms); err != nil {
return fmt.Errorf("parse macOS VM list: %w", err)
}
var errs []error
for _, name := range macOSVMsToDelete(vms, uuid, cutoff) {
if err := exec.CommandContext(ctx, executorPath, "destroy", name).Run(); err != nil {
errs = append(errs, fmt.Errorf("destroy macOS VM %s: %w", name, err))
}
}
return errors.Join(errs...)
}
func macOSVMsToDelete(vms []macOSVM, uuid string, cutoff time.Time) []string {
prefix := "GITEA-MACOS-VM-" + uuid + "-"
var stale []string
for _, vm := range vms {
if vm.Running || !strings.HasPrefix(vm.Name, prefix) {
continue
}
accessed, err := parseMacOSVMTime(vm.AccessedAt)
if err != nil || accessed.After(cutoff) {
continue
}
stale = append(stale, vm.Name)
}
return stale
}
func parseMacOSVMTime(raw string) (time.Time, error) {
if raw == "" {
return time.Time{}, errors.New("empty access time")
}
if parsed, err := time.Parse(time.RFC3339Nano, raw); err == nil {
return parsed, nil
}
return time.Parse(time.RFC3339, raw)
}

View File

@@ -0,0 +1,37 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMacOSVMsToDelete(t *testing.T) {
cutoff := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
vms := []macOSVM{
{Name: "GITEA-MACOS-VM-runner-1-stale", AccessedAt: "2026-09-04T10:00:00Z"},
{Name: "GITEA-MACOS-VM-runner-1-running", AccessedAt: "2026-09-04T10:00:00Z", Running: true},
{Name: "GITEA-MACOS-VM-runner-1-fresh", AccessedAt: "2026-09-05T11:00:00Z"},
{Name: "GITEA-MACOS-VM-runner-2-stale", AccessedAt: "2026-09-04T10:00:00Z"},
}
assert.Equal(t, []string{"GITEA-MACOS-VM-runner-1-stale"}, macOSVMsToDelete(vms, "runner-1", cutoff))
}
func TestParseMacOSVMTime(t *testing.T) {
parsed, err := parseMacOSVMTime("2026-09-05T10:00:00Z")
require.NoError(t, err)
assert.Equal(t, time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC), parsed)
parsed, err = parseMacOSVMTime("2026-09-05T10:00:00.123456789Z")
require.NoError(t, err)
assert.Equal(t, time.Date(2026, 9, 5, 10, 0, 0, 123456789, time.UTC), parsed)
_, err = parseMacOSVMTime("")
require.Error(t, err)
}

View File

@@ -0,0 +1,450 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"context"
"errors"
"fmt"
"io"
"maps"
"os/exec"
"path"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
"time"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/filecollector"
"github.com/go-git/go-billy/v5/helper/polyfill"
"github.com/go-git/go-billy/v5/osfs"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
)
const (
defaultMacOSVMBootTimeout = 5 * time.Minute
macOSVMReadyPollInterval = 500 * time.Millisecond
macOSVMStopWaitTimeout = 10 * time.Second
)
// MacOSVMEnvironment runs workflow steps in an ephemeral macOS VM.
type MacOSVMEnvironment struct {
Path string // guest scratch directory
TmpDir string // guest temp directory
ToolCache string // guest tool cache directory
Workdir string // host-side workspace path
GuestWorkdir string // guest-side workspace path
ActPath string // guest act directory
Image string
VMName string
ExecutorPath string
CPU int
Memory int
BootTimeout time.Duration
Stdout io.Writer
Stderr io.Writer
executorPath string
running bool
runCmd *exec.Cmd
runDone chan struct{}
runErr error
}
// MacOSVMEnvironmentInput is the input for NewMacOSVMEnvironment.
type MacOSVMEnvironmentInput struct {
Path string
TmpDir string
ToolCache string
Workdir string
GuestWorkdir string
ActPath string
Image string
VMName string
ExecutorPath string
CPU int
Memory int
BootTimeout time.Duration
Stdout io.Writer
Stderr io.Writer
}
func NewMacOSVMEnvironment(input MacOSVMEnvironmentInput) (*MacOSVMEnvironment, error) {
if input.Image == "" {
return nil, errors.New("macOS VM image is required")
}
if input.VMName == "" {
return nil, errors.New("macOS VM name is required")
}
if input.BootTimeout <= 0 {
input.BootTimeout = defaultMacOSVMBootTimeout
}
if input.ExecutorPath == "" {
input.ExecutorPath = "runner-executor-macos"
}
return &MacOSVMEnvironment{
Path: input.Path,
TmpDir: input.TmpDir,
ToolCache: input.ToolCache,
Workdir: input.Workdir,
GuestWorkdir: input.GuestWorkdir,
ActPath: input.ActPath,
Image: input.Image,
VMName: input.VMName,
ExecutorPath: input.ExecutorPath,
CPU: input.CPU,
Memory: input.Memory,
BootTimeout: input.BootTimeout,
Stdout: input.Stdout,
Stderr: input.Stderr,
executorPath: input.ExecutorPath,
}, nil
}
var _ ExecutionsEnvironment = &MacOSVMEnvironment{}
func (e *MacOSVMEnvironment) Create(_, _ []string) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
// A previous run can leave the deterministic job VM behind. Destroy it first.
_ = e.runExecutor(ctx, nil, io.Discard, io.Discard, "destroy", e.VMName)
args := []string{"provision", e.Image, e.VMName}
if e.CPU > 0 {
args = append(args, "--cpu", strconv.Itoa(e.CPU))
}
if e.Memory > 0 {
args = append(args, "--memory", strconv.Itoa(e.Memory))
}
return e.runExecutor(ctx, nil, e.Stdout, e.Stderr, args...)
}
}
func (e *MacOSVMEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
for _, file := range files {
dest := path.Join(destPath, file.Name)
script := `mkdir -p "$1" && cat > "$2" && chmod "$3" "$2"`
mode := file.Mode
if mode == 0 {
mode = 0o644
}
args := []string{"exec", "--stdin", e.VMName, "/bin/sh", "-c", script, "sh", path.Dir(dest), dest, strconv.FormatInt(mode, 8)}
if err := e.runExecutor(ctx, strings.NewReader(file.Body), e.Stdout, e.Stderr, args...); err != nil {
return err
}
}
return nil
}
}
func (e *MacOSVMEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
pipeReader, pipeWriter := io.Pipe()
writeDone := make(chan error, 1)
go func() {
err := e.writeTar(ctx, pipeWriter, srcPath, useGitIgnore)
_ = pipeWriter.CloseWithError(err)
writeDone <- err
}()
script := `export COPYFILE_DISABLE=1; mkdir -p "$1" && /usr/bin/tar -xf - -C "$1"`
cmdErr := e.runExecutor(ctx, pipeReader, e.Stdout, e.Stderr, "exec", "--stdin", e.VMName, "/bin/sh", "-c", script, "sh", destPath)
_ = pipeReader.Close()
writeErr := <-writeDone
return errors.Join(cmdErr, writeErr)
}
}
func (e *MacOSVMEnvironment) writeTar(ctx context.Context, writer io.Writer, srcPath string, useGitIgnore bool) error {
tarWriter := tar.NewWriter(writer)
defer tarWriter.Close()
srcPrefix := filepath.Dir(srcPath)
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
srcPrefix += string(filepath.Separator)
}
var ignorer gitignore.Matcher
if useGitIgnore {
patterns, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
if err != nil {
common.Logger(ctx).Debugf("Error loading .gitignore: %v", err)
}
ignorer = gitignore.NewMatcher(patterns)
}
collector := &filecollector.FileCollector{
Ignorer: ignorer,
SrcPath: srcPath,
SrcPrefix: srcPrefix,
Handler: &filecollector.TarCollector{TarWriter: tarWriter},
}
return filepath.Walk(srcPath, collector.CollectFiles(ctx, []string{}))
}
func (e *MacOSVMEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
if common.Dryrun(ctx) {
return nil, errors.New("DRYRUN is not supported in GetContainerArchive")
}
cmd := exec.CommandContext(ctx, e.executorPath, "exec", e.VMName, "/usr/bin/env", "COPYFILE_DISABLE=1", "/usr/bin/tar", "-cf", "-", "-C", "/", strings.TrimPrefix(srcPath, "/"))
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
cmd.Stderr = e.Stderr
if err := cmd.Start(); err != nil {
return nil, err
}
return &macOSVMArchiveReader{ReadCloser: stdout, cmd: cmd}, nil
}
type macOSVMArchiveReader struct {
io.ReadCloser
cmd *exec.Cmd
}
func (r *macOSVMArchiveReader) Close() error {
closeErr := r.ReadCloser.Close()
return errors.Join(closeErr, r.cmd.Wait())
}
func (e *MacOSVMEnvironment) Inspect(ctx context.Context) (*Info, error) {
if common.Dryrun(ctx) {
return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
}
state := "exited"
if e.running {
state = StateRunning
}
return &Info{ID: e.VMName, State: state, Health: HealthNone, Ports: map[string]string{}}, nil
}
func (e *MacOSVMEnvironment) DumpLogs(context.Context) error {
return nil
}
func (e *MacOSVMEnvironment) Pull(forcePull bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
return nil
}
}
func (e *MacOSVMEnvironment) Start(bool) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
cmd := exec.Command(e.executorPath, "start", "--no-graphics", e.VMName)
cmd.Stdout = e.Stdout
cmd.Stderr = e.Stderr
if err := cmd.Start(); err != nil {
return err
}
e.runCmd = cmd
e.runDone = make(chan struct{})
go func() {
e.runErr = cmd.Wait()
close(e.runDone)
}()
e.running = true
return e.waitReady(ctx)
}
}
func (e *MacOSVMEnvironment) waitReady(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, e.BootTimeout)
defer cancel()
var lastErr error
for {
if err := e.runExecutor(ctx, nil, io.Discard, io.Discard, "exec", e.VMName, "/usr/bin/true"); err == nil {
return nil
} else {
lastErr = err
}
select {
case <-ctx.Done():
return fmt.Errorf("macOS VM %s did not become ready: %w", e.VMName, lastErr)
case <-e.runDone:
if e.runErr != nil {
return fmt.Errorf("macOS VM start exited before the VM became ready: %w", e.runErr)
}
return errors.New("macOS VM start exited before the VM became ready")
case <-time.After(macOSVMReadyPollInterval):
}
}
}
func (e *MacOSVMEnvironment) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
err := e.runExecutor(ctx, nil, e.Stdout, e.Stderr, macOSVMExecArgs(e.VMName, command, env, e.resolveWorkdir(workdir))...)
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
return ExitCodeError(exitErr.ExitCode())
}
return err
}
}
func macOSVMExecArgs(vmName string, command []string, env map[string]string, workdir string) []string {
args := []string{"exec", vmName, "/usr/bin/env"}
for _, key := range slices.Sorted(maps.Keys(env)) {
args = append(args, key+"="+env[key])
}
args = append(args, "/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", workdir)
return append(args, command...)
}
func (e *MacOSVMEnvironment) resolveWorkdir(workdir string) string {
switch {
case workdir == "":
return e.GuestWorkdir
case strings.HasPrefix(workdir, "/"):
return workdir
default:
return path.Join(e.GuestWorkdir, workdir)
}
}
func (e *MacOSVMEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(e, srcPath, env)
}
func (e *MacOSVMEnvironment) UpdateFromImageEnv(*map[string]string) common.Executor {
return func(context.Context) error {
return nil
}
}
func (e *MacOSVMEnvironment) Remove() common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
cleanCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
if e.runCmd != nil && e.runDone != nil {
select {
case <-e.runDone:
default:
if err := e.runExecutor(cleanCtx, nil, e.Stdout, e.Stderr, "stop", e.VMName, "--timeout", "20"); err != nil {
common.Logger(ctx).Debugf("macOS VM stop failed for VM %s: %v", e.VMName, err)
}
select {
case <-e.runDone:
case <-time.After(macOSVMStopWaitTimeout):
if e.runCmd.Process != nil {
_ = e.runCmd.Process.Kill()
}
common.Logger(ctx).Warnf("timed out waiting for macOS VM start to exit for VM %s", e.VMName)
}
}
}
e.running = false
if err := e.runExecutor(cleanCtx, nil, e.Stdout, e.Stderr, "destroy", e.VMName); err != nil {
common.Logger(ctx).Warnf("failed to destroy macOS VM %s: %v", e.VMName, err)
}
return nil
}
}
func (e *MacOSVMEnvironment) Close() common.Executor {
return func(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
if e.runCmd != nil && e.runCmd.Process != nil {
_ = e.runCmd.Process.Kill()
}
e.runCmd = nil
return nil
}
}
func (e *MacOSVMEnvironment) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
oldOut := e.Stdout
oldErr := e.Stderr
e.Stdout = stdout
e.Stderr = stderr
return oldOut, oldErr
}
func (e *MacOSVMEnvironment) ToContainerPath(hostPath string) string {
if filepath.Clean(hostPath) == filepath.Clean(e.Workdir) {
return e.GuestWorkdir
}
if rel, err := filepath.Rel(e.Workdir, hostPath); err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
return path.Join(e.GuestWorkdir, filepath.ToSlash(rel))
}
return filepath.ToSlash(hostPath)
}
func (e *MacOSVMEnvironment) GetActPath() string {
return e.ActPath
}
func (*MacOSVMEnvironment) GetPathVariableName() string {
return "PATH"
}
func (*MacOSVMEnvironment) DefaultPathVariable() string {
return "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"
}
func (*MacOSVMEnvironment) JoinPathVariable(paths ...string) string {
return strings.Join(paths, ":")
}
func (e *MacOSVMEnvironment) GetRunnerContext(context.Context) map[string]any {
return map[string]any{
"os": "macOS",
"arch": goArchToActionArch(runtime.GOARCH),
"temp": e.TmpDir,
"tool_cache": e.ToolCache,
}
}
func (*MacOSVMEnvironment) IsEnvironmentCaseInsensitive() bool {
return false
}
func (e *MacOSVMEnvironment) runExecutor(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, args ...string) error {
if stdout == nil {
stdout = e.Stdout
}
if stderr == nil {
stderr = e.Stderr
}
cmd := exec.CommandContext(ctx, e.executorPath, args...)
cmd.Stdin = stdin
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("macOS VM executor %s: %w", args[0], err)
}
return nil
}

View File

@@ -0,0 +1,97 @@
//go:build macos_vm_integration
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestMacOSVMEnvironmentIntegration(t *testing.T) {
if runtime.GOOS != "darwin" {
t.Skip("macOS VMs run on macOS")
}
if _, err := exec.LookPath("runner-executor-macos"); err != nil {
t.Skip("runner-executor-macos executable not found")
}
image := os.Getenv("MACOS_VM_TEST_IMAGE")
if image == "" {
t.Skip("set MACOS_VM_TEST_IMAGE to a macOS VM image to run this test")
}
ctx := context.Background()
vmName := fmt.Sprintf("gitea-runner-test-%d", time.Now().UnixNano())
base := "/Users/admin/runner"
if os.Getenv("RUNNER_EXECUTOR_MACOS_DEV_MODE") == "1" {
base = t.TempDir()
}
env, err := NewMacOSVMEnvironment(MacOSVMEnvironmentInput{
Path: base + "/scratch/" + vmName,
TmpDir: base + "/scratch/" + vmName + "/tmp",
ToolCache: base + "/scratch/" + vmName + "/tool_cache",
Workdir: t.TempDir(),
GuestWorkdir: base + "/workspace/" + vmName,
ActPath: base + "/scratch/" + vmName + "/act",
Image: image,
VMName: vmName,
ExecutorPath: "runner-executor-macos",
BootTimeout: 10 * time.Minute,
Stdout: io.Discard,
Stderr: io.Discard,
})
require.NoError(t, err)
t.Cleanup(func() {
_ = env.Remove()(context.Background())
})
require.NoError(t, env.Create(nil, nil)(ctx))
require.NoError(t, env.Start(false)(ctx))
if os.Getenv("RUNNER_EXECUTOR_MACOS_DEV_MODE") == "1" {
require.NoError(t, os.MkdirAll(env.GuestWorkdir, 0o755))
}
require.NoError(t, env.Exec([]string{"/bin/mkdir", "-p", env.GuestWorkdir}, nil, "", "/")(ctx))
require.NoError(t, env.Copy("/tmp", &FileEntry{
Name: "gitea-runner-macos.txt",
Mode: 0o644,
Body: "hello from macos-vm",
})(ctx))
archive, err := env.GetContainerArchive(ctx, "/tmp/gitea-runner-macos.txt")
require.NoError(t, err)
body := readFirstTarFile(t, archive)
require.NoError(t, archive.Close())
require.Equal(t, "hello from macos-vm", body)
var output bytes.Buffer
oldOut, oldErr := env.ReplaceLogWriter(&output, &output)
defer env.ReplaceLogWriter(oldOut, oldErr)
require.NoError(t, env.Exec([]string{"/usr/bin/uname", "-s"}, nil, "", "")(ctx))
require.Contains(t, output.String(), "Darwin")
require.NoError(t, env.Remove()(ctx))
}
func readFirstTarFile(t *testing.T, archive io.Reader) string {
t.Helper()
reader := tar.NewReader(archive)
header, err := reader.Next()
require.NoError(t, err)
require.Contains(t, header.Name, "gitea-runner-macos.txt")
body, err := io.ReadAll(reader)
require.NoError(t, err)
return string(body)
}

View File

@@ -0,0 +1,91 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"context"
"io"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewMacOSVMEnvironment(t *testing.T) {
env, err := NewMacOSVMEnvironment(MacOSVMEnvironmentInput{
Image: "ghcr.io/cirruslabs/macos-sonoma-base:latest",
VMName: "job-vm",
})
require.NoError(t, err)
assert.Equal(t, "ghcr.io/cirruslabs/macos-sonoma-base:latest", env.Image)
assert.Equal(t, "job-vm", env.VMName)
assert.Equal(t, defaultMacOSVMBootTimeout, env.BootTimeout)
assert.Equal(t, "runner-executor-macos", env.executorPath)
_, err = NewMacOSVMEnvironment(MacOSVMEnvironmentInput{VMName: "job-vm"})
require.Error(t, err)
_, err = NewMacOSVMEnvironment(MacOSVMEnvironmentInput{Image: "macos"})
require.Error(t, err)
}
func TestMacOSVMExecArgs(t *testing.T) {
args := macOSVMExecArgs("vm", []string{"bash", "-c", "echo hi"}, map[string]string{"B": "two", "A": "one"}, "/Users/admin/work")
assert.Equal(t, []string{
"exec", "vm", "/usr/bin/env",
"A=one", "B=two",
"/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", "/Users/admin/work",
"bash", "-c", "echo hi",
}, args)
args = macOSVMExecArgs("vm", []string{"true"}, nil, "")
assert.Equal(t, []string{
"exec", "vm", "/usr/bin/env",
"/bin/sh", "-c", `cd "$1" && shift && exec "$@"`, "sh", "",
"true",
}, args)
}
func TestMacOSVMEnvironmentResolveWorkdir(t *testing.T) {
env := &MacOSVMEnvironment{GuestWorkdir: "/Users/admin/work"}
assert.Equal(t, "/Users/admin/work", env.resolveWorkdir(""))
assert.Equal(t, "/Users/admin/work/sub", env.resolveWorkdir("sub"))
assert.Equal(t, "/tmp", env.resolveWorkdir("/tmp"))
}
func TestMacOSVMEnvironmentToContainerPath(t *testing.T) {
env := &MacOSVMEnvironment{Workdir: "/workspace/owner/repo", GuestWorkdir: "/Users/admin/work/owner/repo"}
assert.Equal(t, "/Users/admin/work/owner/repo", env.ToContainerPath("/workspace/owner/repo"))
assert.Equal(t, "/Users/admin/work/owner/repo/sub", env.ToContainerPath("/workspace/owner/repo/sub"))
assert.Equal(t, "/elsewhere", env.ToContainerPath("/elsewhere"))
}
func TestMacOSVMEnvironmentGetRunnerContext(t *testing.T) {
env := &MacOSVMEnvironment{
TmpDir: "/Users/admin/scratch/tmp",
ToolCache: "/Users/admin/scratch/tool_cache",
}
ctx := env.GetRunnerContext(context.Background())
assert.Equal(t, "macOS", ctx["os"])
assert.Equal(t, goArchToActionArch(runtime.GOARCH), ctx["arch"])
assert.Equal(t, "/Users/admin/scratch/tmp", ctx["temp"])
assert.Equal(t, "/Users/admin/scratch/tool_cache", ctx["tool_cache"])
}
func TestMacOSVMEnvironmentReplaceLogWriter(t *testing.T) {
oldOut := io.Discard
oldErr := io.Discard
env := &MacOSVMEnvironment{Stdout: oldOut, Stderr: oldErr}
newOut, newErr := env.ReplaceLogWriter(io.Discard, io.Discard)
assert.Equal(t, oldOut, newOut)
assert.Equal(t, oldErr, newErr)
}
func TestMacOSVMEnvironmentDefaultPathVariable(t *testing.T) {
env := &MacOSVMEnvironment{}
assert.Equal(t, "PATH", env.GetPathVariableName())
assert.Equal(t, "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin", env.DefaultPathVariable())
assert.Equal(t, "/a:/b", env.JoinPathVariable("/a", "/b"))
assert.False(t, env.IsEnvironmentCaseInsensitive())
}

View File

@@ -7,7 +7,6 @@ package container
import (
"archive/tar"
"bufio"
"bytes"
"context"
"fmt"
"io"
@@ -19,8 +18,7 @@ import (
"golang.org/x/text/transform"
)
// parseEnvFile reads a GITHUB_ENV style file, splitting lines like GitHub's runner on Windows or elsewhere.
func parseEnvFile(e Container, srcPath string, env *map[string]string, windows bool) common.Executor {
func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor {
localEnv := *env
return func(ctx context.Context) error {
envTar, err := e.GetContainerArchive(ctx, srcPath)
@@ -40,22 +38,8 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
s := bufio.NewScanner(decoded)
// Default 64 KiB max token size is too small for realistic env-file lines; allow up to 16 MiB.
s.Buffer(make([]byte, 0, 64*1024), 16*1024*1024)
scanned := 0
s.Split(func(data []byte, atEOF bool) (int, []byte, error) {
if index := bytes.IndexByte(data[scanned:], '\n'); index >= 0 {
advance := scanned + index + 1
scanned = 0
return advance, data[:advance], nil
}
scanned = len(data)
if atEOF && len(data) > 0 {
scanned = 0
return len(data), data, nil
}
return 0, nil, nil
})
for s.Scan() {
line := trimEnvLineEnding(s.Text(), windows)
line := s.Text()
// GitHub's runner ignores blank lines
if strings.TrimSpace(line) == "" {
continue
@@ -68,13 +52,10 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
case multiLineEnv != -1:
var multiLineEnvContent []string
multiLineEnvDelimiter := line[multiLineEnv+2:]
if multiLineEnv == 0 || multiLineEnvDelimiter == "" {
return fmt.Errorf("invalid format '%v', name and delimiter must not be empty", line)
}
delimiterFound := false
for s.Scan() {
content := s.Text()
if trimEnvLineEnding(content, windows) == multiLineEnvDelimiter {
if content == multiLineEnvDelimiter {
delimiterFound = true
break
}
@@ -86,7 +67,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
if !delimiterFound {
return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter)
}
localEnv[line[:multiLineEnv]] = trimEnvLineEnding(strings.Join(multiLineEnvContent, ""), windows)
localEnv[line[:multiLineEnv]] = strings.Join(multiLineEnvContent, "\n")
default:
return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line)
}
@@ -94,13 +75,7 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string, windows b
if err := s.Err(); err != nil {
return fmt.Errorf("reading env file: %w", err)
}
env = &localEnv
return nil
}
}
func trimEnvLineEnding(line string, windows bool) string {
if windows && strings.HasSuffix(line, "\r\n") {
return line[:len(line)-2]
}
return strings.TrimSuffix(line, "\n")
}

View File

@@ -5,6 +5,7 @@ package container
import (
"bufio"
"context"
"os"
"path/filepath"
"strings"
@@ -12,6 +13,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
)
@@ -21,78 +23,117 @@ func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
return e, filepath.Join(e.Path, "envfile")
}
func TestParseEnvFile(t *testing.T) {
t.Parallel()
largeValue := strings.Repeat("x", 2*1024*1024)
encodeUTF16 := func(endianness unicode.Endianness) string {
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
require.NoError(t, err)
return string(content)
}
for _, test := range []struct {
name, content string
want map[string]string
}{
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
{"single line CRLF and final bare CR", "FOO=bar\r\nBAZ=qux\r", map[string]string{"FOO": "bar\r", "BAZ": "qux\r"}},
{"single line without final newline", "FOO=bar", map[string]string{"FOO": "bar"}},
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
{"multiline without final newline", "FOO<<EOF\nline1\r\nline2\r\nEOF", map[string]string{"FOO": "line1\r\nline2\r"}},
{"empty multiline without final newline", "FOO<<EOF\nEOF", map[string]string{"FOO": ""}},
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
{"multiline CRLF", "FOO<<EOF\r\nline1\r\nline2\r\nEOF\r\n", map[string]string{"FOO": "line1\r\nline2\r"}},
{"multiline mixed line endings", "FOO<<EOF\nline1\r\nline2\n\r\nEOF\n", map[string]string{"FOO": "line1\r\nline2\n\r"}},
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar\r", "MULTI": "line1\r"}},
} {
t.Run(test.name, func(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env, false)(t.Context()))
assert.Equal(t, test.want, env)
})
}
t.Run("Windows line endings", func(t *testing.T) {
hostEnv, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nline2\nEOF\r\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(hostEnv, envPath, &env, true)(t.Context()))
assert.Equal(t, map[string]string{"FOO": "bar", "MULTI": "line1\r\nline2"}, env)
})
func TestParseEnvFileSingleLine(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\nBAZ=qux\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "qux", env["BAZ"])
}
func TestParseEnvFileMultiLine(t *testing.T) {
e, envPath := newTestHostEnv(t)
content := "FOO<<EOF\nline1\nline2\nEOF\n"
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "line1\nline2", env["FOO"])
}
func TestParseEnvFileLargeValueWithinLimit(t *testing.T) {
e, envPath := newTestHostEnv(t)
big := strings.Repeat("x", 2*1024*1024)
content := "FOO<<EOF\n" + big + "\nEOF\n"
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, big, env["FOO"])
}
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
t.Parallel()
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
tooBig := strings.Repeat("x", 17*1024*1024) // over the 16 MiB cap
content := "FOO<<EOF\n" + tooBig + "\nEOF\n"
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
env := map[string]string{}
err := parseEnvFile(e, envPath, &env, false)(t.Context())
err := parseEnvFile(e, envPath, &env)(context.Background())
require.ErrorIs(t, err, bufio.ErrTooLong)
assert.Contains(t, err.Error(), "reading env file")
}
func TestParseEnvFileInvalidHeredoc(t *testing.T) {
for name, content := range map[string]string{
"missing delimiter": "FOO<<EOF\nline1\nline2\n",
"empty name": "<<EOF\nline1\nEOF\n",
"empty delimiter": "FOO<<\nline1\n\n",
"delimiter with carriage return": "FOO<<EOF\nline1\nEOF\r\n",
} {
t.Run(name, func(t *testing.T) {
// Regression test: a blank line used to fail the job at "Complete Job", after
// every step had already been recorded as successful.
func TestParseEnvFileBlankLines(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("\nFOO=bar\n\n \nBAZ=qux\n\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "qux", env["BAZ"])
}
// blank lines inside a heredoc value are content, not separators
func TestParseEnvFileMultiLineKeepsBlankLines(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\n\nline2\nEOF\n"), 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "line1\n\nline2", env["FOO"])
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n\nline2\nEOF\n"), 0o600))
env = map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "\nline2", env["FOO"])
}
func TestParseEnvFileUTF8BOM(t *testing.T) {
e, envPath := newTestHostEnv(t)
content := append([]byte{0xEF, 0xBB, 0xBF}, []byte("FOO=bar\n")...)
require.NoError(t, os.WriteFile(envPath, content, 0o600))
env := map[string]string{}
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
}
// Windows host mode: PowerShell 5.1 redirection writes UTF-16, which used to be
// unrecognisable as KEY=VALUE, so the writes were silently ignored.
func TestParseEnvFileUTF16(t *testing.T) {
tests := []struct {
name string
encoder *encoding.Encoder
}{
{"little endian", unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder()},
{"big endian", unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder()},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
content, err := tt.encoder.Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
require.NoError(t, err)
require.NoError(t, os.WriteFile(envPath, content, 0o600))
env := map[string]string{}
err := parseEnvFile(e, envPath, &env, false)(t.Context())
require.Error(t, err)
assert.Contains(t, err.Error(), "delimiter")
assert.Empty(t, env)
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
assert.Equal(t, "bar", env["FOO"])
assert.Equal(t, "line1", env["MULTI"])
})
}
}
func TestParseEnvFileMissingDelimiter(t *testing.T) {
e, envPath := newTestHostEnv(t)
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
env := map[string]string{}
err := parseEnvFile(e, envPath, &env)(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "delimiter")
}

View File

@@ -1,55 +0,0 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"archive/tar"
"context"
"io"
"path/filepath"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/filecollector"
)
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
tw := tar.NewWriter(w)
for _, file := range files {
common.Logger(ctx).Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
if err := tw.WriteHeader(&tar.Header{
Name: file.Name,
Mode: file.Mode,
Size: int64(len(file.Body)),
Uid: uid,
Gid: gid,
}); err != nil {
return err
}
if _, err := tw.Write([]byte(file.Body)); err != nil {
return err
}
}
return tw.Close()
}
// writeDirTar prefixes entries with dstPath for extraction at root.
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
logger := common.Logger(ctx)
tw := tar.NewWriter(w)
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
SrcPath: srcPath,
SrcPrefix: srcPrefix,
SkipGitDir: skipGitDir,
UseGitIgnore: useGitIgnore,
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
}).CollectFiles(ctx)); err != nil {
return err
}
return tw.Close()
}

5
act/container/testdata/Dockerfile vendored Normal file
View File

@@ -0,0 +1,5 @@
FROM scratch
ENV PATH="/this/path/does/not/exists/anywhere:/this/either"
ENV SOME_RANDOM_VAR=""
ENV ANOTHER_ONE="BUT_I_HAVE_VALUE"
ENV CONFLICT_VAR="I_EXIST_ONLY_HERE"

View File

@@ -0,0 +1 @@
testfile

Some files were not shown because too many files have changed in this diff Show More