mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-11-04 22:37:13 +08:00
Compare commits
91 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d882311ec6 | ||
|
|
923a0bfb59 | ||
|
|
6f0d4f1b48 | ||
|
|
430f7bb2cf | ||
|
|
befb1b4bb9 | ||
|
|
1a7f09224f | ||
|
|
af2b3f6df5 | ||
|
|
785313cc21 | ||
|
|
41f1e08bec | ||
|
|
4158e05dae | ||
|
|
e356c2ed85 | ||
|
|
ed940db4f0 | ||
|
|
755af87095 | ||
|
|
b9aa47d1a5 | ||
|
|
0d84ec6dab | ||
|
|
b769a9765f | ||
|
|
9c4c66e70f | ||
|
|
42858c61df | ||
|
|
470969dc74 | ||
|
|
be7ddcd15a | ||
|
|
c3c647ef83 | ||
|
|
160e18e705 | ||
|
|
20c8b4abc4 | ||
|
|
ccb9afd7a5 | ||
|
|
49e34813ae | ||
|
|
e0ef637805 | ||
|
|
8502663d0e | ||
|
|
4e8037a93b | ||
|
|
76110557ab | ||
|
|
3bc5cc03f8 | ||
|
|
de961b91eb | ||
|
|
ad9eeb668a | ||
|
|
b741335222 | ||
|
|
20143c9558 | ||
|
|
8b14a38d4a | ||
|
|
179e435cd0 | ||
|
|
f963e9cfe8 | ||
|
|
e5aa9add02 | ||
|
|
329115b148 | ||
|
|
ff30f39b1b | ||
|
|
3802feb3fe | ||
|
|
27e9b4de87 | ||
|
|
4bedadef9d | ||
|
|
83b63250a3 | ||
|
|
2e59141ae9 | ||
|
|
020f0d2861 | ||
|
|
711b72735b | ||
|
|
436d0d2942 | ||
|
|
a284a62d96 | ||
|
|
2b1e64e414 | ||
|
|
ddee2e054a | ||
|
|
629e062f86 | ||
|
|
73e9552f06 | ||
|
|
05162998eb | ||
|
|
568f89d186 | ||
|
|
d93b7d278a | ||
|
|
0cedd98051 | ||
|
|
07ca4bfa7f | ||
|
|
78610b01f6 | ||
|
|
4a8ee4ce57 | ||
|
|
bf03f36d5a | ||
|
|
5a38a3d0c4 | ||
|
|
6618e67645 | ||
|
|
2b599fae7f | ||
|
|
990ebaa2ac | ||
|
|
81add27459 | ||
|
|
ace8653105 | ||
|
|
95daa50d9b | ||
|
|
1952e5cdb6 | ||
|
|
7ad12038d1 | ||
|
|
98816a605c | ||
|
|
e0d00e543d | ||
|
|
ae9ce10f71 | ||
|
|
3962d0d970 | ||
|
|
d83fa77de4 | ||
|
|
58ed5921f3 | ||
|
|
00ff71a879 | ||
|
|
80db5f61ac | ||
|
|
5310d8119a | ||
|
|
b9c6305dee | ||
|
|
699a97fc8c | ||
|
|
19afebc53f | ||
|
|
2cc3000369 | ||
|
|
3d116eb0c2 | ||
|
|
e0ce6776c7 | ||
|
|
69d5c22300 | ||
|
|
54978255f5 | ||
|
|
498282caaa | ||
|
|
2ed8cdb76e | ||
|
|
ff9965e940 | ||
|
|
ba4d3c5b4f |
@@ -19,7 +19,7 @@ jobs:
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
- run: make lint-pr-title
|
||||
|
||||
@@ -11,7 +11,6 @@ on:
|
||||
|
||||
env:
|
||||
DOCKER_ORG: gitea
|
||||
DOCKER_LATEST: nightly
|
||||
|
||||
jobs:
|
||||
goreleaser:
|
||||
@@ -74,10 +73,10 @@ jobs:
|
||||
fetch-depth: 0 # all history for all branches and tags
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
|
||||
|
||||
- name: Set up Docker BuildX
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||
@@ -94,7 +93,7 @@ jobs:
|
||||
echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
|
||||
@@ -69,7 +69,6 @@ jobs:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
env:
|
||||
DOCKER_ORG: gitea
|
||||
DOCKER_LATEST: latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
@@ -77,10 +76,10 @@ jobs:
|
||||
fetch-depth: 0 # all history for all branches and tags
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
|
||||
|
||||
- name: Set up Docker BuildX
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
||||
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
||||
@@ -103,7 +102,7 @@ jobs:
|
||||
suffix=${{ matrix.variant.tag_suffix }},onlatest=true
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
name: checks
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
# The runner image ships a stale docker.io login; point docker at an empty config so
|
||||
@@ -30,8 +28,8 @@ jobs:
|
||||
# daemon retains them between runs, so this is usually a fast manifest re-check.
|
||||
- name: pre-pull test images
|
||||
env:
|
||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:ba849c60be29959425b8734d57b8b4b7d56f98edd9504c9af091d5281095a71e # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
||||
TEST_JOB_IMAGE: node:24-bookworm-slim@sha256:d6aa754f16b3197301076f047b5def2f02ea1dbbc2ca920407d46d7ec7f87b20 # renovate: datasource=docker
|
||||
TEST_SERVICE_IMAGE: nginx:alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2 # renovate: datasource=docker
|
||||
run: |
|
||||
for image in "$TEST_JOB_IMAGE" "$TEST_SERVICE_IMAGE"; do
|
||||
for attempt in 1 2 3; do
|
||||
@@ -53,6 +51,8 @@ jobs:
|
||||
# after `make test` so the images it needs are already present on the host daemon.
|
||||
- name: test against dind image
|
||||
run: make test-dind
|
||||
- name: test against podman
|
||||
run: make test-dind TARGET=podman
|
||||
- name: coverage report
|
||||
run: |
|
||||
make coverage-report
|
||||
|
||||
@@ -39,6 +39,13 @@ linters:
|
||||
deny:
|
||||
- pkg: golang.org/x/exp
|
||||
desc: it's experimental and unreliable
|
||||
logging:
|
||||
files:
|
||||
- $all
|
||||
- "!**/act/common/logging/**"
|
||||
deny:
|
||||
- pkg: log/slog
|
||||
desc: log through gitea.com/gitea/runner/act/common/logging
|
||||
nolintlint:
|
||||
allow-unused: false
|
||||
require-explanation: true
|
||||
|
||||
@@ -28,38 +28,18 @@ builds:
|
||||
ignore:
|
||||
- goos: darwin
|
||||
goarch: arm
|
||||
- goos: darwin
|
||||
goarch: ppc64le
|
||||
- goos: darwin
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: ppc64le
|
||||
- goos: windows
|
||||
goarch: s390x
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "5"
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "6"
|
||||
- goos: windows
|
||||
goarch: arm
|
||||
goarm: "7"
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
- goos: freebsd
|
||||
goarch: ppc64le
|
||||
- goos: freebsd
|
||||
goarch: s390x
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "5"
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "6"
|
||||
- goos: freebsd
|
||||
goarch: arm
|
||||
goarm: "7"
|
||||
- goos: freebsd
|
||||
goarch: arm64
|
||||
flags:
|
||||
|
||||
@@ -28,8 +28,30 @@ depending on the prefix:
|
||||
- **No prefix?** Do not escape, and split multi-line values into one row each.
|
||||
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
|
||||
encoded forms too.
|
||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
|
||||
decodes exactly those two when folding a location into an annotation.
|
||||
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the command
|
||||
handler folds an annotation's location into its decoded message and escapes the result again.
|
||||
|
||||
## Logging
|
||||
|
||||
Log through `act/common/logging`; the linter rejects importing `log/slog` elsewhere.
|
||||
A job's logger comes from `common.Logger(ctx)`; code with no job uses the package functions
|
||||
(`logging.Infof`, `logging.With`). Both take printf-style messages, and `With("key", value)` adds
|
||||
fields.
|
||||
|
||||
- The process-wide logger masks the secrets of every live task, the job logger masks its own.
|
||||
Going around either leaks them.
|
||||
- The job logger's output can reach the job log workflow authors see, the process-wide logger's
|
||||
stays on the runner host, so keep operator-only detail (host paths, internal addresses) off the
|
||||
job logger.
|
||||
- Libraries that log through slog's global logger are forwarded to the process-wide one, so they
|
||||
follow its level and masking too. Avoid dependencies that log through logrus, nothing forwards it.
|
||||
- Tests read what was logged with `logtest.NewLogger()`, or `logtest.SetDefault(t)` for the
|
||||
package functions.
|
||||
|
||||
## Built-in actions
|
||||
|
||||
Each lives in `internal/pkg/action/<name>` and is registered in `builtinActions` in
|
||||
`act/runner/step_action_builtin.go`.
|
||||
|
||||
## End-to-end compatibility tests
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
### BUILDER STAGE
|
||||
#
|
||||
#
|
||||
FROM golang:1.27-alpine3.23 AS builder
|
||||
FROM golang:1.27-alpine AS builder
|
||||
|
||||
# Do not remove `git` here, it is required for getting runner version when executing `make build`
|
||||
RUN apk add --no-cache make git
|
||||
@@ -17,7 +17,7 @@ RUN make clean && make build
|
||||
### DIND VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.7.2-dind AS dind
|
||||
FROM docker:29.9.1-dind AS dind
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -32,12 +32,14 @@ COPY scripts/s6 /etc/s6
|
||||
|
||||
VOLUME /data
|
||||
|
||||
ENV TINI_SUBREAPER=1
|
||||
|
||||
ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
|
||||
### DIND-ROOTLESS VARIANT
|
||||
#
|
||||
#
|
||||
FROM docker:29.7.2-dind-rootless AS dind-rootless
|
||||
FROM docker:29.9.1-dind-rootless AS dind-rootless
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
@@ -56,6 +58,7 @@ VOLUME /data
|
||||
RUN mkdir -p /data && chown -R rootless:rootless /etc/s6 /data
|
||||
|
||||
ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
|
||||
ENV TINI_SUBREAPER=1
|
||||
|
||||
USER rootless
|
||||
ENTRYPOINT ["s6-svscan","/etc/s6"]
|
||||
|
||||
93
Makefile
93
Makefile
@@ -1,25 +1,22 @@
|
||||
DIST := dist
|
||||
EXECUTABLE := gitea-runner
|
||||
DIST_DIRS := $(DIST)/binaries $(DIST)/release
|
||||
GO ?= go
|
||||
SHASUM ?= shasum -a 256
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go
|
||||
XGO_VERSION := go-1.27.x
|
||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.16 # renovate: datasource=go
|
||||
|
||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
||||
WINDOWS_ARCHS ?= windows/amd64
|
||||
PODMAN_TEST_IMAGE ?= quay.io/containers/podman:v5.8.9-immutable@sha256:87f42f37fa85b421664c30b8f5f9cfc23920057f5e5fcadf0b6f68d9edc6d415 # renovate: datasource=docker
|
||||
E2E_JOB_IMAGE ?= docker.gitea.com/runner-images:ubuntu-latest@sha256:0e62e56b382ebf485bff1e51a05cbe27c708545a2e91425467fd31eb3e249521 # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.7-alpine@sha256:892bb2152464f0c5d8972593a5f5afaf8054552580b699d95f8ff4af08c2d33d # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_K3S_IMAGE ?= rancher/k3s:v1.37.1-k3s1@sha256:ca7f37d993d82ef0dcdcfecb2e0e2618ea541dbaffc620c8cedebe01a82acd0d # renovate: datasource=docker
|
||||
|
||||
GOFILES := $(shell find . -type f -name "*.go" -o -name "go.mod" ! -name "generated.*")
|
||||
|
||||
DOCKER_IMAGE ?= gitea/runner
|
||||
DOCKER_TAG ?= nightly
|
||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
||||
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.7.0 # renovate: datasource=go
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.14.0 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.8.0 # renovate: datasource=go
|
||||
|
||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||
|
||||
@@ -32,22 +29,16 @@ endif
|
||||
ifeq ($(HAS_GO), GO)
|
||||
GOPATH ?= $(shell $(GO) env GOPATH)
|
||||
export PATH := $(GOPATH)/bin:$(PATH)
|
||||
|
||||
CGO_EXTRA_CFLAGS := -DSQLITE_MAX_VARIABLE_NUMBER=32766
|
||||
CGO_CFLAGS ?= $(shell $(GO) env CGO_CFLAGS) $(CGO_EXTRA_CFLAGS)
|
||||
endif
|
||||
|
||||
ifeq ($(OS), Windows_NT)
|
||||
GOFLAGS := -v -buildmode=exe
|
||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||
else ifeq ($(OS), Windows)
|
||||
GOFLAGS := -v -buildmode=exe
|
||||
EXECUTABLE ?= $(EXECUTABLE).exe
|
||||
GO_ENV_WINDOWS := set GOOS=windows&&
|
||||
else
|
||||
GOFLAGS := -v
|
||||
EXECUTABLE ?= $(EXECUTABLE)
|
||||
GO_ENV_WINDOWS := GOOS=windows
|
||||
endif
|
||||
|
||||
@@ -96,19 +87,12 @@ go-check:
|
||||
fi
|
||||
|
||||
.PHONY: fmt-check
|
||||
fmt-check: fmt
|
||||
@diff=$$(git diff --color=always -- '*.go'); \
|
||||
if [ -n "$$diff" ]; then \
|
||||
echo "Please run 'make fmt' and commit the result:"; \
|
||||
printf "%s" "$${diff}"; \
|
||||
exit 1; \
|
||||
fi
|
||||
fmt-check: ## check the Go code formatting
|
||||
$(GO) run $(GOLANGCI_LINT_PACKAGE) fmt --diff-colored
|
||||
|
||||
.PHONY: deps-tools
|
||||
deps-tools: ## install tool dependencies
|
||||
$(GO) install $(GOLANGCI_LINT_PACKAGE) & \
|
||||
$(GO) install $(GXZ_PACKAGE) & \
|
||||
$(GO) install $(XGO_PACKAGE) & \
|
||||
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
||||
wait
|
||||
|
||||
@@ -149,13 +133,8 @@ tidy: ## run go mod tidy
|
||||
fi
|
||||
|
||||
.PHONY: tidy-check
|
||||
tidy-check: tidy
|
||||
@diff=$$(git diff --color=always -- go.mod go.sum); \
|
||||
if [ -n "$$diff" ]; then \
|
||||
echo "Please run 'make tidy' and commit the result:"; \
|
||||
printf "%s" "$${diff}"; \
|
||||
exit 1; \
|
||||
fi
|
||||
tidy-check: ## check that go.mod and go.sum are tidy
|
||||
$(GO) mod tidy -diff
|
||||
|
||||
.PHONY: test
|
||||
test: ## test everything (integration tests self-skip without docker/network)
|
||||
@@ -168,17 +147,14 @@ coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
||||
@echo "Wrote .tmp/coverage.md"
|
||||
|
||||
.PHONY: test-dind
|
||||
test-dind: ## run the daemon-facing tests against the built dind image (TARGET=dind|dind-rootless)
|
||||
@./scripts/test-dind.sh $(TARGET)
|
||||
test-dind: ## run the daemon-facing tests against the built dind image or podman (TARGET=dind|dind-rootless|podman)
|
||||
@PODMAN_TEST_IMAGE=$(PODMAN_TEST_IMAGE) ./scripts/test-dind.sh $(TARGET)
|
||||
|
||||
E2E_JOB_IMAGE ?= node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 # renovate: datasource=docker
|
||||
SERVICE_IMAGE ?= nginx:1.31.4-alpine@sha256:db35bfc6b2951e7f8a72db5db120288c127ffaeeb4a6d4b95a26fead017d5913 # renovate: datasource=docker
|
||||
E2E_GITEA_IMAGE ?= gitea/gitea:main-nightly
|
||||
E2E_CONCURRENCY ?= 8
|
||||
|
||||
.PHONY: test-e2e
|
||||
test-e2e: ## run Gitea compatibility tests against E2E_GITEA_IMAGE
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
@E2E_CONCURRENCY=$(E2E_CONCURRENCY) E2E_GITEA_IMAGE=$(E2E_GITEA_IMAGE) E2E_K3S_IMAGE=$(E2E_K3S_IMAGE) E2E_JOB_IMAGE=$(E2E_JOB_IMAGE) GO=$(GO) SERVICE_IMAGE=$(SERVICE_IMAGE) ./tools/test-e2e.sh
|
||||
|
||||
.PHONY: install
|
||||
install: $(GOFILES) ## install the runner binary via `go install`
|
||||
@@ -194,45 +170,6 @@ $(EXECUTABLE): $(GOFILES)
|
||||
deps-backend: ## install backend dependencies
|
||||
$(GO) mod download
|
||||
|
||||
.PHONY: release
|
||||
release: release-windows release-linux release-darwin release-copy release-compress release-check ## build release artifacts
|
||||
|
||||
$(DIST_DIRS):
|
||||
mkdir -p $(DIST_DIRS)
|
||||
|
||||
.PHONY: release-windows
|
||||
release-windows: | $(DIST_DIRS)
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -buildmode exe -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(WINDOWS_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
||||
ifeq ($(CI),true)
|
||||
cp -r /build/* $(DIST)/binaries/
|
||||
endif
|
||||
|
||||
.PHONY: release-linux
|
||||
release-linux: | $(DIST_DIRS)
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -targets '$(LINUX_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
||||
ifeq ($(CI),true)
|
||||
cp -r /build/* $(DIST)/binaries/
|
||||
endif
|
||||
|
||||
.PHONY: release-darwin
|
||||
release-darwin: | $(DIST_DIRS)
|
||||
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) run $(XGO_PACKAGE) -go $(XGO_VERSION) -dest $(DIST)/binaries -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w $(LDFLAGS)' -targets '$(DARWIN_ARCHS)' -out $(EXECUTABLE)-$(VERSION) .
|
||||
ifeq ($(CI),true)
|
||||
cp -r /build/* $(DIST)/binaries/
|
||||
endif
|
||||
|
||||
.PHONY: release-copy
|
||||
release-copy: | $(DIST_DIRS)
|
||||
cd $(DIST); for file in `find . -type f -name "*"`; do cp $${file} ./release/; done;
|
||||
|
||||
.PHONY: release-check
|
||||
release-check: | $(DIST_DIRS)
|
||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done;
|
||||
|
||||
.PHONY: release-compress
|
||||
release-compress: | $(DIST_DIRS)
|
||||
cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && $(GO) run $(GXZ_PACKAGE) -k -9 $${file}; done;
|
||||
|
||||
.PHONY: docker
|
||||
docker: ## build the docker image
|
||||
if ! docker buildx version >/dev/null 2>&1; then \
|
||||
|
||||
183
README.md
183
README.md
@@ -4,7 +4,19 @@
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Docker Engine Community version is required for docker mode. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
On the runner:
|
||||
|
||||
- `git` 2.34.1 or newer on `PATH`, to clone reusable workflows, and actions whose host serves no archive of them.
|
||||
- `docker` or `podman` with `podman.socket` enabled, for docker mode, Docker actions and `docker://` steps. To install Docker CE, follow the official [install instructions](https://docs.docker.com/engine/install/).
|
||||
|
||||
Where steps run, in the job image or on the runner in host mode:
|
||||
|
||||
- `bash` or `sh` for `run` steps, `pwsh` or `powershell` on a Windows host.
|
||||
- `node` for JavaScript actions and `hashFiles()`.
|
||||
- `git` 2.34.1 or newer for [`builtin:checkout`](docs/builtin-actions.md).
|
||||
- `git-lfs` and `ssh` for `builtin:checkout`, only when its `lfs` or `ssh-key` input is used.
|
||||
|
||||
Kubernetes mode needs [a few more tools](docs/kubernetes.md) in job images.
|
||||
|
||||
### Download pre-built binary
|
||||
|
||||
@@ -79,53 +91,24 @@ If the registry succeed, it will run immediately. Next time, you could run the r
|
||||
|
||||
### Run with docker
|
||||
|
||||
```bash
|
||||
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> -v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:nightly
|
||||
```
|
||||
|
||||
Mount a volume on `/data` if you want the registration file and optional config to survive container recreation (see [scripts/run.sh](scripts/run.sh)).
|
||||
|
||||
> **`/data` does not hold the image cache.** It is the runner's working directory and contains only the `.runner` registration file and, optionally, your config file. Images pulled for jobs live in the *Docker daemon's* data root, which for the `dind` flavours is inside the container (`/var/lib/docker`, or `/home/rootless/.local/share/docker` for `dind-rootless`). To keep the image cache across restarts, give that path its own volume as well — otherwise every new container re-pulls the job images. With the `basic` flavour the images live on whichever daemon you point the runner at, so there is nothing extra to persist.
|
||||
|
||||
### Image flavours
|
||||
|
||||
The image is published in three flavours, all built from the single multi-stage [Dockerfile](Dockerfile) in this repository. They differ only in how a Docker daemon is made available to the jobs the runner executes; the `gitea-runner` binary inside them is identical.
|
||||
|
||||
| Tag | Build target | Base image | Docker daemon | Process supervisor | Runs as |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| `latest` (and `<version>`) | `basic` | `alpine` | none — uses an external daemon you provide | [`tini`](https://github.com/krallin/tini) | `root` |
|
||||
| `latest-dind` | `dind` | `docker:dind` | bundled, started inside the container | [`s6`](https://skarnet.org/software/s6/) | `root` (privileged) |
|
||||
| `latest-dind-rootless` | `dind-rootless` | `docker:dind-rootless` | bundled, started rootless inside the container | [`s6`](https://skarnet.org/software/s6/) | `rootless` (UID 1000) |
|
||||
|
||||
#### `latest` — basic
|
||||
|
||||
The default flavour ships only the runner on a minimal Alpine base. It contains **no Docker daemon of its own**: jobs that use `docker://` images need a daemon supplied from outside the container, typically by bind-mounting the host's socket:
|
||||
|
||||
```bash
|
||||
docker run -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock --name my_runner gitea/runner:latest
|
||||
```
|
||||
|
||||
`tini` is the entrypoint (it reaps zombie processes), and it just runs [`scripts/run.sh`](scripts/run.sh), which registers the runner on first start and then execs `gitea-runner daemon`. This flavour does not need `--privileged`. The trade-off is that jobs share the host's daemon, so they can see other containers and images on that daemon.
|
||||
|
||||
#### `latest-dind` — Docker-in-Docker
|
||||
|
||||
This flavour is based on the official `docker:dind` image and bundles its own Docker daemon, so it needs no external socket — only the `--privileged` flag that Docker-in-Docker requires:
|
||||
The recommended Docker-in-Docker image runs its own Docker daemon for jobs. Jobs never get the host's Docker socket, which is equivalent to root access on the host.
|
||||
|
||||
```bash
|
||||
docker run --privileged -e GITEA_INSTANCE_URL=https://your_gitea.com -e GITEA_RUNNER_REGISTRATION_TOKEN=<your_token> \
|
||||
--name my_runner gitea/runner:latest-dind
|
||||
-v "$PWD/data:/data" -v runner-docker:/var/lib/docker --name my_runner gitea/runner:nightly-dind
|
||||
```
|
||||
|
||||
Two processes have to run side by side here (the Docker daemon and the runner), so the entrypoint is the [`s6`](https://skarnet.org/software/s6/) supervision tree under [`scripts/s6`](scripts/s6) instead of `tini`. `s6` starts `dockerd`, and the runner service waits for the daemon to come up (`s6-svwait`) before launching [`run.sh`](scripts/run.sh). Each container has a private daemon isolated from the host's, at the cost of running privileged.
|
||||
The volumes keep the runner's registration and the job image cache across container recreation.
|
||||
|
||||
#### `latest-dind-rootless` — rootless Docker-in-Docker
|
||||
### Image flavours
|
||||
|
||||
Same idea as `dind`, but built on `docker:dind-rootless` so the bundled daemon and the runner run as an unprivileged user (`rootless`, UID 1000) rather than `root`. `DOCKER_HOST` is preset to `unix:///run/user/1000/docker.sock` so the runner talks to the rootless daemon. This reduces the blast radius compared to the privileged `dind` flavour, but rootless Docker carries the usual rootless limitations (networking, cgroups, storage drivers, and some operations that need additional host configuration such as `/etc/subuid` / `/etc/subgid` mappings and unprivileged user-namespace support).
|
||||
| Tag | Docker daemon for jobs |
|
||||
| --- | --- |
|
||||
| `latest-dind` (recommended) | Bundled, needs `--privileged`. |
|
||||
| `latest-dind-rootless` | Bundled, runs as a non-root user (UID 1000) for stronger isolation. Needs `--privileged` and [rootless Docker](https://docs.docker.com/engine/security/rootless/) support on the host. Keeps its image cache in `/home/rootless/.local/share/docker`. |
|
||||
| `latest` | None. Mount the host's socket with `-v /var/run/docker.sock:/var/run/docker.sock`, or set `DOCKER_HOST`. |
|
||||
|
||||
> **The UID is fixed at 1000.** It comes from the `rootless` user baked into the upstream `docker:dind-rootless` base image, and the bundled daemon always listens on `/run/user/1000/docker.sock` inside the container, so running this flavour as a different user (`--user 1001`) does not work. If you need the runner to talk to a *host* rootless daemon that runs under some other UID, use the `basic` flavour instead and bind-mount that daemon's socket (see [examples/vm/rootless-docker.md](examples/vm/rootless-docker.md)); pointing `DOCKER_HOST` at a host socket from inside `dind-rootless` will not work. Changing the UID otherwise means rebuilding the image from a base with a different `rootless` user.
|
||||
|
||||
> **Note on Podman:** these images target the Docker daemon. The bundled `dind`/`dind-rootless` daemons are `dockerd`, not Podman, and the `basic` flavour expects a Docker-compatible socket. Running them under rootless Podman is not a supported configuration, though pointing the `basic` flavour at a Podman socket that emulates the Docker API may work for some workloads.
|
||||
Versioned and `nightly` tags carry the same suffixes, for example `nightly-dind`. The images target Docker, not Podman.
|
||||
|
||||
### Configuration
|
||||
|
||||
@@ -186,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
|
||||
|
||||
#### Environment variables
|
||||
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
|
||||
|
||||
### Labels
|
||||
|
||||
@@ -201,10 +184,10 @@ A label is written as:
|
||||
| Part | Meaning |
|
||||
| --- | --- |
|
||||
| `name` | The name a workflow refers to in `runs-on`, e.g. `ubuntu-latest`. |
|
||||
| `schema` | Either `docker` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` schema: the image to run the job in. |
|
||||
| `schema` | `docker`, `kubernetes` or `host`. Defaults to `host` when omitted. |
|
||||
| `args` | Only used by the `docker` and `kubernetes` schemas: the image to run the job in. |
|
||||
|
||||
Two schemas are supported:
|
||||
Three schemas are supported:
|
||||
|
||||
- **`docker://<image>`** — the job runs inside a container created from `<image>`:
|
||||
|
||||
@@ -212,6 +195,12 @@ Two schemas are supported:
|
||||
ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
|
||||
- **`kubernetes://<image>`** — the job runs as a Kubernetes pod created from `<image>`, see **[docs/kubernetes.md](docs/kubernetes.md)**:
|
||||
|
||||
```text
|
||||
ubuntu-latest:kubernetes://docker.gitea.com/runner-images:ubuntu-latest
|
||||
```
|
||||
|
||||
- **`host`** — the job's steps run directly on the machine the runner is on, using the tools installed there:
|
||||
|
||||
```text
|
||||
@@ -226,9 +215,9 @@ ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,macos:host
|
||||
|
||||
a workflow with `runs-on: ubuntu-latest` is executed in the `runner-images:ubuntu-latest` container, and one with `runs-on: macos` is executed directly on the host.
|
||||
|
||||
Names may themselves contain a colon (for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`); only `host` and `docker` are treated as schemas.
|
||||
Names may themselves contain a colon, for example `pool:e57e18d4-10d4-406f-93bf-60f127221bdd`. Only `host`, `docker` and `kubernetes` are treated as schemas.
|
||||
|
||||
If a job's `runs-on` matches none of the runner's labels, or sets no `runs-on` at all, it still runs: in `runner.default_image` where docker is available, on the host where it is not. Images maintained for this purpose are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
Jobs run on `runner.default_image` when no runner label matches their `runs-on`: as a pod on runners with `kubernetes` labels, otherwise in a container, or on the host when docker is unavailable. Suitable images are listed at [gitea/runner-images](https://gitea.com/gitea/runner-images).
|
||||
|
||||
Labels are chosen at registration time (`--labels`, or the interactive prompt) and can be changed afterwards by editing `runner.labels` in the config file, or in the Gitea UI under the runner's settings.
|
||||
|
||||
@@ -253,7 +242,7 @@ A job's `services` are started before its steps run. When a service's image or i
|
||||
```yaml
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17
|
||||
image: postgres:18
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 5s
|
||||
@@ -268,14 +257,14 @@ Unlike GitHub, a job whose steps run on the host (a `host` label without `contai
|
||||
|
||||
#### Docker from a job (`GITEA_DOCKER_WORKSPACE`)
|
||||
|
||||
A container a job starts through the Docker socket cannot bind-mount the workspace by the job's own path, the daemon does not have it. `GITEA_DOCKER_WORKSPACE` holds the path the daemon sees. Use it as the prefix of workspace binds, with `.` as the fallback for local use, here in a `docker-compose.yaml`:
|
||||
Containers a job starts through the Docker socket can bind-mount the workspace by the job's own path, as on a host, for example `docker run -v "$PWD:/src"`. Without the Docker proxy, for example with a remote `DOCKER_HOST`, use `GITEA_DOCKER_WORKSPACE`, the path the daemon sees, as the prefix of workspace binds, with `.` as the fallback for local use:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ${GITEA_DOCKER_WORKSPACE:-.}/data:/app/data
|
||||
```
|
||||
|
||||
Containers, networks and volumes a job creates through the socket are removed when the job ends. Both apply to jobs in containers, on the host `.` already works.
|
||||
Linux container jobs use a Docker proxy when socket sharing and permissions allow it. Other setups use the daemon socket directly. Host jobs use their existing Docker access, where `.` works. The proxy removes the containers, networks and volumes a job creates, `docker compose` services included, after post steps and the completed hook. Set `container.sweep: false` when jobs deploy services to the runner's Docker daemon.
|
||||
|
||||
#### Proxy
|
||||
|
||||
@@ -291,14 +280,14 @@ The runner uses them for its own requests and gives them to every job, in lower
|
||||
|
||||
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
|
||||
|
||||
- the cache server
|
||||
- the built-in cache server, whose address is assigned at startup
|
||||
- `localhost`, `127.0.0.1` and `::1`
|
||||
- the job's service containers
|
||||
- the Docker daemon, when it is reached over `tcp://`
|
||||
|
||||
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
|
||||
|
||||
To change a value for one job, set it in a step's `env:` or in the job's `container.env`. Setting it at workflow or job level has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
|
||||
For one step, set it in the step's `env:`, for the rest of a job, write it to `$GITHUB_ENV`. Setting it at workflow or job level or in `container.env` has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
|
||||
|
||||
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
|
||||
|
||||
@@ -306,63 +295,68 @@ Dockerfile actions are built with these variables as build arguments, so their `
|
||||
|
||||
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
|
||||
|
||||
#### Built-in actions (`uses: builtin:<name>`)
|
||||
|
||||
Use `uses: builtin:checkout` to check out a repository without downloading `actions/checkout` or needing Node in the job image, see **[docs/builtin-actions.md](docs/builtin-actions.md)**.
|
||||
|
||||
#### Caching (`actions/cache`)
|
||||
|
||||
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
|
||||
Each runner starts its own cache server, so runners do not share cached entries. A runner in a container on the jobs' Docker daemon attaches itself to each job network, so jobs reach its cache without extra setup.
|
||||
|
||||
**Eviction**
|
||||
**Sharing a cache between runners**
|
||||
|
||||
An entry nothing has read or written for `retention` is removed, and a repository past `repo_size_limit` loses its least recently accessed entries until it fits; `size_limit` caps the whole cache the same way. Age alone never retires an entry still in use, and whatever these allow, the cache keeps free space above `health_check.min_free_disk_space_mb` when health checks are enabled.
|
||||
Run one `gitea-runner cache-server` and point every runner at it, using the same secret everywhere, for example from `openssl rand -hex 32`:
|
||||
|
||||
These apply where the cache server runs, so on a shared server they belong in *its* config, not the runners'. See `retention`, `repo_size_limit`, `size_limit` and `sweep_interval` in [config.example.yaml](internal/pkg/config/config.example.yaml) for units and defaults.
|
||||
```yaml
|
||||
# cache-server.yaml
|
||||
cache:
|
||||
dir: /data/actcache
|
||||
port: 8088
|
||||
external_secret: "<secret>" # or external_secret_file: /path/to/secret
|
||||
```
|
||||
|
||||
**Cache service v2**
|
||||
```bash
|
||||
gitea-runner -c cache-server.yaml cache-server
|
||||
```
|
||||
|
||||
`actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with:
|
||||
```yaml
|
||||
# each runner's config
|
||||
cache:
|
||||
external_server: "http://<cache-server-host>:8088/"
|
||||
external_secret: "<secret>"
|
||||
```
|
||||
|
||||
`--dir`, `--host` and `--port` override the matching `cache` keys. Eviction settings take effect on the cache server, not on the runners.
|
||||
|
||||
Runners can also share one `cache.dir` on a file system with working file locks, at the cost of slower cache requests.
|
||||
|
||||
**Storing the cache in an S3-compatible bucket**
|
||||
|
||||
The cache can also be stored in an S3-compatible bucket such as AWS S3, MinIO or RustFS, with no cache server. Runners with the same settings use the same cache:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
v2: false
|
||||
s3:
|
||||
endpoint: "http://minio:9000"
|
||||
region: "us-east-1"
|
||||
bucket: "runner-cache"
|
||||
prefix: "gitea-runner-cache"
|
||||
path_style: true
|
||||
access_key_file: "/run/secrets/cache-s3-access-key"
|
||||
secret_key_file: "/run/secrets/cache-s3-secret-key"
|
||||
```
|
||||
|
||||
Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle on its way into the job, undone whenever the action is downloaded again. A bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork, so it is made whatever `v2` says: that setting only governs the API the runner advertises. Set `runner.patch_actions: false` to leave every bundle exactly as shipped, an escape hatch for an action the edit breaks. The artifact actions then refuse again and the cache client keeps to v1.
|
||||
A cache saved on one runner can be restored on the others within 30 seconds. The eviction settings below do not apply to the bucket, so add lifecycle rules to it that expire old caches and abort incomplete multipart uploads.
|
||||
|
||||
**Shared cache across multiple runners**
|
||||
**Eviction**
|
||||
|
||||
Run one dedicated `gitea-runner cache-server` that all runners point at.
|
||||
Entries not read or written within `retention` (default `168h`) are removed. A repository over `repo_size_limit` (default `10GB`) loses its least recently used entries, and `size_limit` (off by default) caps the whole cache the same way. Entries in use are never removed. The cache also keeps 1024 MiB free on its volume, or `health_check.min_free_disk_space_mb` when health checks are enabled. See [config.example.yaml](internal/pkg/config/config.example.yaml) for all options.
|
||||
|
||||
1. Create a config file for the cache server host:
|
||||
**Cache service v2**
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
dir: /data/actcache
|
||||
port: 8088
|
||||
external_secret: "replace-with-a-strong-random-secret"
|
||||
# external_secret_file: /path/to/secret # secret can also be passed via a file
|
||||
```
|
||||
`actions/cache` v3.4.0, v4.2.0 and later use the cache service v2 API, which the runner serves by default. These actions fall back to v1 on hosts they do not recognize as GitHub, so the runner removes that check from them while a job runs. This also lets the stock `actions/upload-artifact` v4.4.0 and `actions/download-artifact` v4.1.8 and later work without the `gitea-upload-artifact` fork. With `runner.patch_actions: false`, the cache stays on v1 and the artifact actions fail.
|
||||
|
||||
2. Start the server:
|
||||
|
||||
```bash
|
||||
gitea-runner -c cache-server-config.yaml cache-server
|
||||
```
|
||||
|
||||
3. On every runner:
|
||||
|
||||
```yaml
|
||||
cache:
|
||||
external_server: "http://<cache-server-host>:8088/"
|
||||
external_secret: "replace-with-a-strong-random-secret" # must match the server
|
||||
# external_secret_file: /path/to/secret # secret can also be passed via a file
|
||||
```
|
||||
|
||||
Jobs reach the cache server at `external_server`, so when a reverse proxy fronts the server, point `external_server` at the proxy. The cache server itself needs no extra configuration.
|
||||
|
||||
Alternatively, mount the same NFS/CIFS share on every runner and point `cache.dir` at it — simpler, but with weaker isolation between repositories.
|
||||
|
||||
**S3 / MinIO** — mount object storage as a FUSE filesystem (e.g. [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) or [goofys](https://github.com/kahing/goofys)) and set `cache.dir` to the mount point.
|
||||
|
||||
Flags `--dir`, `--host`, and `--port` on `cache-server` override the corresponding `cache.*` YAML keys; all other settings, including `external_secret`, require the config file.
|
||||
With v2, artifact calls also go through the cache server, so jobs must be able to reach it. `cache.v2: false` sends them to Gitea directly, unless the Gitea URL has a path or `runner.insecure` is set with HTTPS.
|
||||
|
||||
#### Official Docker image
|
||||
|
||||
@@ -374,7 +368,8 @@ While the runner is idle it cleans up after earlier jobs:
|
||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- on runners that use docker, per-job networks and volumes left behind by jobs the runner did not live to tear down are removed once unused and 10 minutes old, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- on runners with `kubernetes` labels, job pods with that label older than `runner.workdir_cleanup_age` are removed the same way, together with their Secrets
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||
|
||||
#### Post-task script (`runner.post_task_script`)
|
||||
@@ -397,6 +392,10 @@ Both hooks are synchronous and block the job while they run. Either one exiting
|
||||
|
||||
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
|
||||
|
||||
#### OpenTelemetry (`OTEL_*`)
|
||||
|
||||
Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job, and the runner's metrics, to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**.
|
||||
|
||||
#### Local job logs (`log.job.dir`)
|
||||
|
||||
Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `<start time>-task-<id>.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -15,8 +15,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
// The cache service v2 API. A client on this version talks twirp to
|
||||
@@ -39,24 +37,25 @@ const (
|
||||
|
||||
blobUploadURLTTL = time.Hour
|
||||
|
||||
// twirpInternal is the only error code that is not the client's fault.
|
||||
twirpInternal = "internal"
|
||||
twirpInternal = "internal"
|
||||
twirpUnavailable = "unavailable"
|
||||
twirpUnauthenticated = "unauthenticated"
|
||||
)
|
||||
|
||||
func (h *Handler) registerV2Routes(router *httprouter.Router) {
|
||||
router.POST(cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
||||
router.POST(cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
||||
router.POST(cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
||||
router.PUT(blobPath+"/:id", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
||||
func (h *Handler) registerV2Routes(router *http.ServeMux) {
|
||||
router.HandleFunc("POST "+cacheServiceV2Path+"/CreateCacheEntry", h.bearerAuth(h.v2CreateCacheEntry))
|
||||
router.HandleFunc("POST "+cacheServiceV2Path+"/FinalizeCacheEntryUpload", h.bearerAuth(h.v2FinalizeCacheEntryUpload))
|
||||
router.HandleFunc("POST "+cacheServiceV2Path+"/GetCacheEntryDownloadURL", h.bearerAuth(h.v2GetCacheEntryDownloadURL))
|
||||
router.HandleFunc("PUT "+blobPath+"/{id}", h.signedAuth(blobUploadPurpose, h.v2UploadBlob))
|
||||
}
|
||||
|
||||
// An entry that already exists is reported as not ok, which is how the client learns to skip
|
||||
// the upload.
|
||||
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request) {
|
||||
cred := credFromContext(r.Context())
|
||||
req, err := decodeTwirpRequest[v2CreateRequest](r)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, "malformed_request", err)
|
||||
h.twirpError(w, r, "malformed", err)
|
||||
return
|
||||
}
|
||||
if req.Key == "" || req.Version == "" {
|
||||
@@ -83,16 +82,35 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
||||
return
|
||||
}
|
||||
|
||||
// A second live reservation is finalized by whichever job calls last, against the other's upload.
|
||||
owner := hashedToken(bearerToken(r))
|
||||
if pending, err := findExactCache(db, cred.Repo, req.Key, req.Version, false); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
} else if pending != nil && pending.UsedAt > time.Now().Add(-uploadStallTimeout).Unix() {
|
||||
if pending.Owner != owner {
|
||||
h.twirpNotOK(w, r)
|
||||
return
|
||||
}
|
||||
h.touch(db, pending) // still uploading, so it must not go stale under the sweep
|
||||
h.responseJSON(w, r, http.StatusOK, map[string]any{ // this job retrying its own reservation
|
||||
"ok": true,
|
||||
"signed_upload_url": h.signedURL(cred, blobPath, blobUploadPurpose, pending.ID, time.Now().Add(blobUploadURLTTL)),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
cache := &Cache{
|
||||
Repo: cred.Repo,
|
||||
Key: req.Key,
|
||||
Version: req.Version,
|
||||
Owner: owner,
|
||||
Size: -1, // the size is only known at finalize time
|
||||
CreatedAt: now,
|
||||
UsedAt: now,
|
||||
}
|
||||
if err := insertCache(db, cache); err != nil {
|
||||
if err := h.insertCache(db, cache); err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
}
|
||||
@@ -103,11 +121,11 @@ func (h *Handler) v2CreateCacheEntry(w http.ResponseWriter, r *http.Request, _ h
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Request) {
|
||||
cred := credFromContext(r.Context())
|
||||
req, err := decodeTwirpRequest[v2FinalizeRequest](r)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, "malformed_request", err)
|
||||
h.twirpError(w, r, "malformed", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -119,6 +137,13 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
defer db.Close()
|
||||
|
||||
cache, err := findExactCache(db, cred.Repo, req.Key, req.Version, false)
|
||||
if err == nil && cache != nil && cache.Owner != hashedToken(bearerToken(r)) {
|
||||
cache = nil // not the reservation this job made, so not this job's to commit
|
||||
}
|
||||
if err == nil && cache == nil {
|
||||
// A retry whose first response was lost finds the entry already committed.
|
||||
cache, err = findExactCache(db, cred.Repo, req.Key, req.Version, true)
|
||||
}
|
||||
if err != nil {
|
||||
h.twirpError(w, r, twirpInternal, err)
|
||||
return
|
||||
@@ -127,13 +152,15 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
h.twirpNotOK(w, r)
|
||||
return
|
||||
}
|
||||
db.Close() // commitCache needs the store closed
|
||||
|
||||
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
|
||||
if err := h.commitCache(cache); err != nil {
|
||||
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
|
||||
h.twirpNotOK(w, r)
|
||||
return
|
||||
if !cache.Complete {
|
||||
db.Close() // commitCache needs the store closed
|
||||
cache.Size = int64(cmp.Or(req.SizeBytes, req.SizeBytesCamel))
|
||||
if err := h.commitCache(cache); err != nil {
|
||||
h.logger.Errorf("finalize cache %d (%s): %v", cache.ID, cache.Key, err)
|
||||
h.twirpNotOK(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.responseJSON(w, r, http.StatusOK, map[string]any{
|
||||
@@ -143,11 +170,11 @@ func (h *Handler) v2FinalizeCacheEntryUpload(w http.ResponseWriter, r *http.Requ
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
|
||||
func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Request) {
|
||||
cred := credFromContext(r.Context())
|
||||
req, err := decodeTwirpRequest[v2DownloadRequest](r)
|
||||
if err != nil {
|
||||
h.twirpError(w, r, "malformed_request", err)
|
||||
h.twirpError(w, r, "malformed", err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -178,8 +205,8 @@ func (h *Handler) v2GetCacheEntryDownloadURL(w http.ResponseWriter, r *http.Requ
|
||||
// The archive arrives over the subset of the Azure blob API the toolkit uses: a small
|
||||
// cache is a single PUT, a large one is staged as blocks that a final block list puts
|
||||
// in order.
|
||||
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
||||
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
|
||||
func (h *Handler) v2UploadBlob(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := strconv.ParseUint(r.PathValue("id"), 10, 64)
|
||||
if err != nil {
|
||||
h.responseJSON(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
@@ -230,8 +257,13 @@ func (h *Handler) twirpNotOK(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *Handler) twirpError(w http.ResponseWriter, r *http.Request, code string, err error) {
|
||||
h.logger.Debugf("%s %s: %v", r.Method, r.URL.Path, err)
|
||||
status := http.StatusBadRequest
|
||||
if code == twirpInternal {
|
||||
switch code {
|
||||
case twirpInternal:
|
||||
status = http.StatusInternalServerError
|
||||
case twirpUnavailable:
|
||||
status = http.StatusServiceUnavailable
|
||||
case twirpUnauthenticated:
|
||||
status = http.StatusUnauthorized
|
||||
}
|
||||
h.responseJSON(w, r, status, map[string]any{"code": code, "msg": err.Error()})
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -18,8 +19,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// v2Call posts a twirp request to the cache service and returns the decoded response.
|
||||
// Field names are the proto ones, which is what the toolkit's client sends.
|
||||
func v2Call(t *testing.T, handler *Handler, client *http.Client, method string, request any) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
@@ -36,7 +35,6 @@ func v2Call(t *testing.T, handler *Handler, client *http.Client, method string,
|
||||
return got
|
||||
}
|
||||
|
||||
// putBlob uploads to a signed URL and returns the status, so a test can assert a refusal.
|
||||
func putBlob(t *testing.T, url string, content []byte) int {
|
||||
t.Helper()
|
||||
|
||||
@@ -46,8 +44,6 @@ func putBlob(t *testing.T, url string, content []byte) int {
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusCreated {
|
||||
// The Azure SDK client dereferences this header without checking, so a blob upload that
|
||||
// omits it panics the caller rather than failing it.
|
||||
require.NotEmpty(t, resp.Header.Get("x-ms-request-id"))
|
||||
}
|
||||
return resp.StatusCode
|
||||
@@ -66,8 +62,6 @@ func getURL(t *testing.T, url string) []byte {
|
||||
return body
|
||||
}
|
||||
|
||||
// saveV2 runs the reserve/upload/finalize sequence and returns the finalize response along
|
||||
// with the upload URL it used.
|
||||
func saveV2(t *testing.T, handler *Handler, key, version string, content []byte) (finalized map[string]any, uploadURL string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -83,9 +77,6 @@ func saveV2(t *testing.T, handler *Handler, key, version string, content []byte)
|
||||
}), uploadURL
|
||||
}
|
||||
|
||||
// The whole round trip an actions/cache v2 client makes, plus the guarantees on the signed
|
||||
// URLs it is handed: unsigned requests are refused, an upload URL cannot be replayed to read
|
||||
// or to replace a finalized entry.
|
||||
func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
handler := newTestHandler(t, Policy{})
|
||||
content := []byte("the cached archive")
|
||||
@@ -97,10 +88,8 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
assert.NotEmpty(t, finalized["entry_id"])
|
||||
|
||||
// The upload URL outlives the finalize call, so replaying it must not poison the entry,
|
||||
// and it is an upload URL only: nothing reads a blob back through it.
|
||||
assert.Equal(t, http.StatusBadRequest, putBlob(t, uploadURL, []byte("poisoned")))
|
||||
resp, err := http.Get(uploadURL) //nolint:noctx // the URL is the server under test
|
||||
resp, err := http.Get(uploadURL)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
|
||||
@@ -113,8 +102,6 @@ func TestCacheServiceV2RoundTrip(t *testing.T) {
|
||||
assert.Equal(t, content, getURL(t, downloadURL))
|
||||
}
|
||||
|
||||
// A large archive is staged as blocks and only put in order by the final block list, so
|
||||
// blocks that arrive out of order must still be assembled the way the client asked.
|
||||
func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
handler := newTestHandler(t, Policy{})
|
||||
|
||||
@@ -129,7 +116,6 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
blocks[blockID] = []byte(part)
|
||||
order = append(order, blockID)
|
||||
}
|
||||
// Upload in an order that is not the block list order.
|
||||
for _, blockID := range []string{order[2], order[0], order[1]} {
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=block&blockid="+blockID, blocks[blockID]))
|
||||
}
|
||||
@@ -141,6 +127,17 @@ func TestCacheServiceV2BlockUpload(t *testing.T) {
|
||||
}
|
||||
list.WriteString(`</BlockList>`)
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()))
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, uploadURL+"&comp=blocklist", list.Bytes()),
|
||||
"a client that lost the first answer retries the list it already sent")
|
||||
|
||||
var reordered bytes.Buffer
|
||||
reordered.WriteString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
|
||||
for _, blockID := range []string{order[1], order[0], order[2]} {
|
||||
fmt.Fprintf(&reordered, "<Latest>%s</Latest>", blockID)
|
||||
}
|
||||
reordered.WriteString(`</BlockList>`)
|
||||
assert.Equal(t, http.StatusInternalServerError, putBlob(t, uploadURL+"&comp=blocklist", reordered.Bytes()),
|
||||
"the blocks are already assembled, so a different order would silently disagree with them")
|
||||
|
||||
finalized := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
|
||||
"key": "blocks", "version": "v1", "size_bytes": len("hello world!"),
|
||||
@@ -162,8 +159,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// The toolkit serialises with the proto field names; the camelCase spellings of the same
|
||||
// proto JSON mapping are accepted alongside them.
|
||||
for _, field := range []string{"restore_keys", "restoreKeys"} {
|
||||
t.Run("restore keys match by prefix, spelled "+field, func(t *testing.T) {
|
||||
got := v2Call(t, handler, testClient, "GetCacheEntryDownloadURL", map[string]any{
|
||||
@@ -179,9 +174,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, again["ok"])
|
||||
})
|
||||
|
||||
// A key that is only a prefix of an existing one is a different entry, so the
|
||||
// reservation check must be exact and not a restore-key prefix match, or the shorter
|
||||
// key would be reported as existing and silently never saved.
|
||||
t.Run("a prefix of an existing key is still reserved", func(t *testing.T) {
|
||||
reserved := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "deps", "version": "v1"})
|
||||
require.Equal(t, true, reserved["ok"])
|
||||
@@ -207,7 +199,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// The size the client declares is what Commit validates the assembled archive against.
|
||||
t.Run("finalizing with the wrong size is not ok", func(t *testing.T) {
|
||||
created := v2Call(t, handler, testClient, "CreateCacheEntry", map[string]any{"key": "wrong-size", "version": "v1"})
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, created["signed_upload_url"].(string), []byte("four")))
|
||||
@@ -218,8 +209,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
|
||||
// Both API versions are served from one store, so an entry written through v2 is a hit for
|
||||
// a v1 client asking for the same key and version.
|
||||
t.Run("a v1 client sees an entry written through v2", func(t *testing.T) {
|
||||
resp, err := testClient.Get(fmt.Sprintf("%s%s/cache?keys=deps-abc&version=v1", handler.ExternalURL(), apiPath))
|
||||
require.NoError(t, err)
|
||||
@@ -232,7 +221,6 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.NotEmpty(t, got["archiveLocation"])
|
||||
})
|
||||
|
||||
// The cache of one repository must stay invisible to another, as it does for the v1 API.
|
||||
t.Run("another repository sees nothing", func(t *testing.T) {
|
||||
handler.RegisterJob("other-runtime-token", JobCredential{Repo: "other/repo"})
|
||||
otherClient := &http.Client{Transport: &bearerTransport{token: "other-runtime-token"}}
|
||||
@@ -241,3 +229,93 @@ func TestCacheServiceV2Lookups(t *testing.T) {
|
||||
assert.Equal(t, false, got["ok"])
|
||||
})
|
||||
}
|
||||
|
||||
func TestCacheServiceV2RefusalsAreTwirp(t *testing.T) {
|
||||
handler := newTestHandler(t, Policy{})
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
token string
|
||||
body string
|
||||
status int
|
||||
code string
|
||||
}{
|
||||
{"no bearer at all", "", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
|
||||
{"a bearer nobody registered", "not-a-job", `{"key":"k","version":"v"}`, http.StatusUnauthorized, twirpUnauthenticated},
|
||||
{"a body that is not the request", testToken, `{`, http.StatusBadRequest, "malformed"},
|
||||
{"a request missing its key", testToken, `{"version":"v"}`, http.StatusBadRequest, "invalid_argument"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost,
|
||||
handler.ExternalURL()+cacheServiceV2Path+"/CreateCacheEntry", strings.NewReader(tt.body))
|
||||
require.NoError(t, err)
|
||||
if tt.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+tt.token)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
require.Equal(t, tt.status, resp.StatusCode)
|
||||
got := map[string]string{}
|
||||
require.NoError(t, json.UnmarshalRead(resp.Body, &got))
|
||||
assert.Equal(t, tt.code, got["code"])
|
||||
assert.NotEmpty(t, got["msg"])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheServiceV2ConcurrentSaveAndRetries(t *testing.T) {
|
||||
handler := newTestHandler(t, Policy{})
|
||||
content := []byte("the cached archive")
|
||||
|
||||
const otherJob = "another-jobs-token"
|
||||
defer handler.RegisterJob(otherJob, JobCredential{Repo: testRepo})()
|
||||
reserve := func(client *http.Client) map[string]any {
|
||||
return v2Call(t, handler, client, "CreateCacheEntry", map[string]any{"key": "shared", "version": "v1"})
|
||||
}
|
||||
first := reserve(testClient)
|
||||
require.Equal(t, true, first["ok"])
|
||||
retry := reserve(testClient)
|
||||
assert.Equal(t, true, retry["ok"], "a job retrying its own reservation is not a conflict")
|
||||
entryOf := func(reserved map[string]any) string {
|
||||
parsed, err := url.Parse(reserved["signed_upload_url"].(string))
|
||||
require.NoError(t, err)
|
||||
return parsed.Path
|
||||
}
|
||||
assert.Equal(t, entryOf(first), entryOf(retry), "the same reservation, however freshly its URL is signed")
|
||||
assert.Equal(t, false, reserve(&http.Client{Transport: &bearerTransport{token: otherJob}})["ok"],
|
||||
"another job's reservation would be finalized against the first upload")
|
||||
|
||||
otherClient := &http.Client{Transport: &bearerTransport{token: otherJob}}
|
||||
reserved := v2Call(t, handler, otherClient, "CreateCacheEntry", map[string]any{"key": "owned", "version": "v1"})
|
||||
require.Equal(t, true, reserved["ok"])
|
||||
require.Equal(t, http.StatusCreated, putBlob(t, reserved["signed_upload_url"].(string), content))
|
||||
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
|
||||
map[string]any{"key": "owned", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
|
||||
"another job's upload is not this job's to finalize")
|
||||
|
||||
base := handler.ExternalURL() + apiPath
|
||||
madeByV1 := reserveCache(t, testClient, base, "v1-made", "v1", int64(len(content)))
|
||||
upload, err := http.NewRequestWithContext(t.Context(), http.MethodPatch,
|
||||
fmt.Sprintf("%s/caches/%d", base, madeByV1), bytes.NewReader(content))
|
||||
require.NoError(t, err)
|
||||
upload.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/*", len(content)-1))
|
||||
resp, err := testClient.Do(upload)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
assert.Equal(t, false, v2Call(t, handler, testClient, "FinalizeCacheEntryUpload",
|
||||
map[string]any{"key": "v1-made", "version": "v1", "size_bytes": strconv.Itoa(len(content))})["ok"],
|
||||
"a reservation this job did not make through v2 is not its to commit")
|
||||
|
||||
finalized, _ := saveV2(t, handler, "deps", "v1", content)
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
again := v2Call(t, handler, testClient, "FinalizeCacheEntryUpload", map[string]any{
|
||||
"key": "deps", "version": "v1", "size_bytes": strconv.Itoa(len(content)),
|
||||
})
|
||||
assert.Equal(t, true, again["ok"], "a retry whose first answer was lost must not report a failed save")
|
||||
assert.Equal(t, finalized["entry_id"], again["entry_id"])
|
||||
}
|
||||
|
||||
@@ -11,12 +11,13 @@ type Request struct {
|
||||
}
|
||||
|
||||
type Cache struct {
|
||||
ID uint64 `json:"id" boltholdKey:"ID"`
|
||||
Repo string `json:"repo" boltholdIndex:"Repo"`
|
||||
Key string `json:"key" boltholdIndex:"Key"`
|
||||
Version string `json:"version" boltholdIndex:"Version"`
|
||||
ID uint64 `json:"id"`
|
||||
Repo string `json:"repo"`
|
||||
Key string `json:"key"`
|
||||
Version string `json:"version"`
|
||||
Size int64 `json:"cacheSize"`
|
||||
Complete bool `json:"complete" boltholdIndex:"Complete"`
|
||||
UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"`
|
||||
CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"`
|
||||
Owner string `json:"owner"` // hashed bearer, so a job's own retry is not another job's reservation
|
||||
Complete bool `json:"complete"`
|
||||
UsedAt int64 `json:"usedAt"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
}
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
@@ -15,23 +18,38 @@ import (
|
||||
// Gitea implements only the artifact half of it. Forwarding that half from here makes this origin
|
||||
// the whole service, so ACTIONS_RESULTS_URL can point at it truthfully, which is what the clients
|
||||
// this runner cannot patch need, docker buildx among them.
|
||||
//
|
||||
// The instance to forward to travels with the job registration rather than with configuration, so
|
||||
// a cache server shared between runners serves each of their instances.
|
||||
const artifactServicePath = "/twirp/github.actions.results.api.v1.ArtifactService/"
|
||||
|
||||
// forwardOrNotFound is the router's fallback: the artifact service of the instance the job
|
||||
// registered with, and the 404 the router would have written otherwise.
|
||||
func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
cred, ok := h.lookupCredential(bearerToken(r))
|
||||
if !ok || cred.Results == "" || !strings.HasPrefix(r.URL.Path, artifactServicePath) {
|
||||
if !strings.HasPrefix(r.URL.Path, artifactServicePath) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
cred, ok := h.lookupCredential(bearerToken(r))
|
||||
if !ok {
|
||||
h.twirpError(w, r, twirpUnauthenticated, errors.New("unknown bearer token"))
|
||||
return
|
||||
}
|
||||
if cred.Results == "" {
|
||||
h.twirpError(w, r, twirpInternal, errors.New("no instance is registered for this job"))
|
||||
return
|
||||
}
|
||||
target, err := url.Parse(strings.TrimSuffix(cred.Results, "/"))
|
||||
if err == nil && (target.Hostname() == "" || (target.Scheme != "http" && target.Scheme != "https")) {
|
||||
err = errors.New("not an absolute http or https URL the job could use")
|
||||
}
|
||||
if err != nil {
|
||||
h.logger.Errorf("artifact service forward to %q: %v", cred.Results, err)
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("artifact service address %q is unusable: %w", cred.Results, err))
|
||||
return
|
||||
}
|
||||
if !h.mustProxy(r, cred, target) {
|
||||
redirect := target.JoinPath(r.URL.EscapedPath())
|
||||
redirect.RawQuery = mergeQuery(target.RawQuery, r.URL.RawQuery)
|
||||
h.logger.Debugf("%s %s: redirecting to %s", r.Method, r.URL.Path, target)
|
||||
http.Redirect(w, r, redirect.String(), http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
h.logger.Debugf("%s %s: forwarding to %s", r.Method, r.URL.Path, target)
|
||||
@@ -44,9 +62,9 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
// directly and never through here.
|
||||
r.Out.Host = target.Host
|
||||
},
|
||||
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
||||
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
|
||||
h.logger.Warnf("artifact service forward to %s: %v", target, err)
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
h.twirpError(w, r, twirpInternal, fmt.Errorf("cache server cannot reach the artifact service at %s: %w", target, err))
|
||||
},
|
||||
}
|
||||
if cred.InsecureTLS {
|
||||
@@ -55,5 +73,17 @@ func (h *Handler) forwardOrNotFound(w http.ResponseWriter, r *http.Request) {
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// A protocol switch loses the bearer or the agent, and an untrusted instance needs skipped verification.
|
||||
func (h *Handler) mustProxy(r *http.Request, cred JobCredential, target *url.URL) bool {
|
||||
return cred.InsecureTLS && target.Scheme == "https" || cmp.Or(r.Header.Get("X-Forwarded-Proto"), "http") != target.Scheme
|
||||
}
|
||||
|
||||
func mergeQuery(target, request string) string {
|
||||
if target == "" || request == "" {
|
||||
return target + request
|
||||
}
|
||||
return target + "&" + request
|
||||
}
|
||||
|
||||
// insecureTransport is shared, because a transport per request would pool no connections.
|
||||
var insecureTransport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
|
||||
var insecureTransport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // the runner reaches its instance on the operator's say-so
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"encoding/json/v2"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -14,8 +16,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The artifact half is forwarded under the Host Gitea knows itself by, so the URLs it hands back
|
||||
// still point at Gitea, and nothing else is proxied.
|
||||
func TestFrontResultsService(t *testing.T) {
|
||||
var gotHost, gotPath, gotProto string
|
||||
gitea := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -23,34 +23,112 @@ func TestFrontResultsService(t *testing.T) {
|
||||
_, _ = io.WriteString(w, `{"ok":true}`)
|
||||
}))
|
||||
defer gitea.Close()
|
||||
selfSigned := httptest.NewTLSServer(gitea.Config.Handler)
|
||||
defer selfSigned.Close()
|
||||
|
||||
handler, err := StartHandler(Options{Dir: t.TempDir(), OutboundIP: "127.0.0.1"})
|
||||
require.NoError(t, err)
|
||||
defer handler.Close()
|
||||
const token = "forward-token"
|
||||
const token, rpc = "forward-token", artifactServicePath + "CreateArtifact"
|
||||
|
||||
client := &http.Client{Transport: &bearerTransport{token: token}}
|
||||
post := func(path string) int {
|
||||
noRedirect := func(token string) *http.Client {
|
||||
return &http.Client{
|
||||
Transport: &bearerTransport{token: token},
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
}
|
||||
post := func(path string) (int, map[string]string) {
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+path, nil)
|
||||
require.NoError(t, err)
|
||||
resp, err := client.Do(req)
|
||||
resp, err := noRedirect(token).Do(req)
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
defer resp.Body.Close()
|
||||
body := map[string]string{}
|
||||
_ = json.UnmarshalRead(resp.Body, &body)
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, post(artifactServicePath+"CreateArtifact"),
|
||||
"an unregistered token is forwarded nowhere")
|
||||
status, body := post(rpc)
|
||||
assert.Equal(t, http.StatusUnauthorized, status, "an unregistered token is told so, not left to parse a 404 page")
|
||||
assert.Equal(t, twirpUnauthenticated, body["code"])
|
||||
assert.Contains(t, body["msg"], "bearer")
|
||||
assert.NotEmpty(t, body["msg"])
|
||||
|
||||
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: gitea.URL})()
|
||||
defer handler.RegisterJob(token, JobCredential{Repo: "owner/repo", Results: selfSigned.URL, InsecureTLS: true})()
|
||||
|
||||
assert.Equal(t, http.StatusOK, post(artifactServicePath+"CreateArtifact"))
|
||||
assert.Equal(t, strings.TrimPrefix(gitea.URL, "http://"), gotHost, "Gitea must see the host it mints its URLs from")
|
||||
status, _ = post(rpc)
|
||||
require.Equal(t, http.StatusOK, status, "an http job reaching an https instance must proxy")
|
||||
assert.Equal(t, strings.TrimPrefix(selfSigned.URL, "https://"), gotHost, "Gitea must see the host it mints its URLs from")
|
||||
assert.Empty(t, gotProto, "a forwarded scheme would make an https Gitea mint http URLs")
|
||||
assert.Equal(t, artifactServicePath+"CreateArtifact", gotPath)
|
||||
assert.Equal(t, rpc, gotPath)
|
||||
|
||||
gotPath = ""
|
||||
assert.Equal(t, http.StatusNotFound, post("/twirp/github.actions.results.api.v1.OtherService/Do"))
|
||||
assert.Equal(t, http.StatusNotFound, post("/api/v1/repos/owner/repo"))
|
||||
status, _ = post("/twirp/github.actions.results.api.v1.OtherService/Do")
|
||||
assert.Equal(t, http.StatusNotFound, status)
|
||||
status, _ = post("/api/v1/repos/owner/repo")
|
||||
assert.Equal(t, http.StatusNotFound, status)
|
||||
assert.Empty(t, gotPath, "only the artifact service is forwarded")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cred JobCredential
|
||||
forwardedProto string
|
||||
reqQuery string
|
||||
want int
|
||||
location string
|
||||
wantMsg string
|
||||
}{
|
||||
{name: "an instance the job reaches itself", cred: JobCredential{Results: gitea.URL}, want: http.StatusTemporaryRedirect},
|
||||
{name: "a sub-path instance keeps its prefix", cred: JobCredential{Results: gitea.URL + "/sub"}, want: http.StatusTemporaryRedirect},
|
||||
{name: "an instance with no certificate to distrust", cred: JobCredential{Results: gitea.URL, InsecureTLS: true}, want: http.StatusTemporaryRedirect},
|
||||
{
|
||||
name: "both queries survive", cred: JobCredential{Results: gitea.URL + "?tenant=one"}, reqQuery: "?run=7",
|
||||
want: http.StatusTemporaryRedirect, location: gitea.URL + rpc + "?tenant=one&run=7",
|
||||
},
|
||||
{
|
||||
name: "a trusted https instance is reached by the job itself",
|
||||
cred: JobCredential{Results: selfSigned.URL}, forwardedProto: "https", want: http.StatusTemporaryRedirect,
|
||||
},
|
||||
{
|
||||
name: "the job does not share this server's disregard for the certificate",
|
||||
cred: JobCredential{Results: selfSigned.URL, InsecureTLS: true}, forwardedProto: "https", want: http.StatusOK,
|
||||
},
|
||||
{name: "an http job is not redirected to an https instance", cred: JobCredential{Results: selfSigned.URL}, want: http.StatusInternalServerError},
|
||||
{
|
||||
name: "a forwarded https scheme onto an http instance is proxied",
|
||||
cred: JobCredential{Results: gitea.URL}, forwardedProto: "https", want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "a scheme the job could not use is refused before it is sent one",
|
||||
cred: JobCredential{Results: "ftp://gitea.example"}, want: http.StatusInternalServerError, wantMsg: "unusable",
|
||||
},
|
||||
{
|
||||
name: "an address with no host is refused before it is sent one",
|
||||
cred: JobCredential{Results: "http://:3000"}, want: http.StatusInternalServerError, wantMsg: "unusable",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
defer handler.RegisterJob(tt.name, tt.cred)()
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, handler.ExternalURL()+rpc+tt.reqQuery, nil)
|
||||
require.NoError(t, err)
|
||||
if tt.forwardedProto != "" {
|
||||
req.Header.Set("X-Forwarded-Proto", tt.forwardedProto)
|
||||
}
|
||||
resp, err := noRedirect(tt.name).Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
require.Equal(t, tt.want, resp.StatusCode)
|
||||
switch tt.want {
|
||||
case http.StatusTemporaryRedirect:
|
||||
assert.Equal(t, cmp.Or(tt.location, tt.cred.Results+rpc), resp.Header.Get("Location"))
|
||||
case http.StatusInternalServerError:
|
||||
body := map[string]string{}
|
||||
require.NoError(t, json.UnmarshalRead(resp.Body, &body))
|
||||
assert.Equal(t, twirpInternal, body["code"])
|
||||
assert.Contains(t, body["msg"], cmp.Or(tt.wantMsg, tt.cred.Results))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
348
act/artifactcache/s3_storage.go
Normal file
348
act/artifactcache/s3_storage.go
Normal file
@@ -0,0 +1,348 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/json/v2"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// S3Options configures the bucket that stores completed cache entries.
|
||||
type S3Options struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
Prefix string
|
||||
PathStyle *bool // nil uses virtual-hosted requests on AWS and path-style ones elsewhere
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
}
|
||||
|
||||
type cacheStorage interface {
|
||||
Exist(uint64) (bool, error)
|
||||
Write(uint64, int64, io.Reader) error
|
||||
WriteBlock(uint64, string, io.Reader) error
|
||||
OrderBlocks(uint64, []string) error
|
||||
Commit(uint64, int64) (int64, error)
|
||||
Serve(http.ResponseWriter, *http.Request, uint64)
|
||||
Remove(uint64) error
|
||||
}
|
||||
|
||||
const s3PartSize = 64 << 20
|
||||
|
||||
// s3Storage stages uploads on disk and keeps completed entries and their metadata in a bucket.
|
||||
type s3Storage struct {
|
||||
*Storage
|
||||
client *http.Client
|
||||
bucketURL *url.URL
|
||||
opts S3Options
|
||||
}
|
||||
|
||||
type s3Part struct {
|
||||
PartNumber int
|
||||
ETag string
|
||||
}
|
||||
|
||||
func newS3Storage(staging *Storage, opts S3Options) (*s3Storage, error) {
|
||||
endpoint, err := url.Parse(opts.Endpoint)
|
||||
if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") {
|
||||
return nil, fmt.Errorf("invalid S3 endpoint %q", opts.Endpoint)
|
||||
}
|
||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||
transport = transport.Clone()
|
||||
transport.MaxIdleConnsPerHost = 32 // 4 parts each for the 8 jobs a runner commonly runs
|
||||
transport.ResponseHeaderTimeout = time.Minute // a stalled endpoint must not hold the store forever
|
||||
pathStyle := !strings.HasSuffix(endpoint.Hostname(), ".amazonaws.com")
|
||||
if opts.PathStyle != nil {
|
||||
pathStyle = *opts.PathStyle
|
||||
}
|
||||
if pathStyle {
|
||||
endpoint.Path = "/" + opts.Bucket
|
||||
} else {
|
||||
endpoint.Host = opts.Bucket + "." + endpoint.Host
|
||||
endpoint.Path = "/"
|
||||
}
|
||||
opts.Region = cmp.Or(opts.Region, "us-east-1")
|
||||
return &s3Storage{Storage: staging, client: &http.Client{Transport: transport}, bucketURL: endpoint, opts: opts}, nil
|
||||
}
|
||||
|
||||
func (s *s3Storage) blobKey(id uint64) string {
|
||||
return path.Join(s.opts.Prefix, "blobs", strconv.FormatUint(id, 10))
|
||||
}
|
||||
|
||||
func (s *s3Storage) metadataDir() string {
|
||||
return path.Join(s.opts.Prefix, "metadata") + "/"
|
||||
}
|
||||
|
||||
func (s *s3Storage) metadataKey(id uint64) string {
|
||||
return s.metadataDir() + strconv.FormatUint(id, 10) + ".json"
|
||||
}
|
||||
|
||||
func (s *s3Storage) url(key string, query url.Values) *url.URL {
|
||||
u := *s.bucketURL
|
||||
u.Path = path.Join(u.Path, key)
|
||||
u.RawPath = strings.NewReplacer("+", "%20", "%2F", "/").Replace(url.QueryEscape(u.Path)) // SigV4 encoding
|
||||
u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20")
|
||||
return &u
|
||||
}
|
||||
|
||||
// sign authorizes req with AWS Signature Version 4, leaving the payload unsigned so bodies can stream.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
||||
func (s *s3Storage) sign(req *http.Request, now time.Time) {
|
||||
date := now.UTC().Format("20060102T150405Z")
|
||||
req.Header.Set("X-Amz-Date", date)
|
||||
if req.Header.Get("X-Amz-Content-Sha256") == "" {
|
||||
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
|
||||
}
|
||||
headers := map[string]string{"host": req.URL.Host}
|
||||
for name, values := range req.Header {
|
||||
if name = strings.ToLower(name); strings.HasPrefix(name, "x-amz-") {
|
||||
headers[name] = strings.Join(values, ",")
|
||||
}
|
||||
}
|
||||
names := slices.Sorted(maps.Keys(headers))
|
||||
canonical := []string{req.Method, req.URL.EscapedPath(), req.URL.RawQuery}
|
||||
for _, name := range names {
|
||||
canonical = append(canonical, name+":"+headers[name])
|
||||
}
|
||||
canonical = append(canonical, "", strings.Join(names, ";"), req.Header.Get("X-Amz-Content-Sha256"))
|
||||
scope := date[:8] + "/" + s.opts.Region + "/s3/aws4_request"
|
||||
signature := []byte("AWS4" + s.opts.SecretKey)
|
||||
for _, data := range []string{date[:8], s.opts.Region, "s3", "aws4_request", fmt.Sprintf("AWS4-HMAC-SHA256\n%s\n%s\n%x", date, scope, sha256.Sum256([]byte(strings.Join(canonical, "\n"))))} { // key derivation, the last round signs
|
||||
mac := hmac.New(sha256.New, signature)
|
||||
_, _ = mac.Write([]byte(data))
|
||||
signature = mac.Sum(nil)
|
||||
}
|
||||
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s,SignedHeaders=%s,Signature=%x", s.opts.AccessKey, scope, strings.Join(names, ";"), signature))
|
||||
}
|
||||
|
||||
// do returns the response to a signed request when it succeeded, fs.ErrNotExist for a missing key.
|
||||
func (s *s3Storage) do(ctx context.Context, method, key string, query url.Values, header http.Header, body io.Reader) (*http.Response, error) {
|
||||
var size int64
|
||||
if sized, ok := body.(interface{ Size() int64 }); ok {
|
||||
size = sized.Size()
|
||||
}
|
||||
if size == 0 {
|
||||
body = nil
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, s.url(key, query).String(), body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.ContentLength = size
|
||||
maps.Copy(req.Header, header)
|
||||
s.sign(req, time.Now())
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < http.StatusMultipleChoices {
|
||||
return resp, nil
|
||||
}
|
||||
defer drain(resp)
|
||||
var failure struct{ Code, Message string }
|
||||
_ = xml.NewDecoder(resp.Body).Decode(&failure)
|
||||
if resp.StatusCode == http.StatusNotFound && (failure.Code == "NoSuchKey" || failure.Code == "" && method == http.MethodHead) {
|
||||
return nil, fs.ErrNotExist
|
||||
}
|
||||
return nil, fmt.Errorf("S3 %s %s: %s", method, key, cmp.Or(strings.TrimSpace(failure.Code+" "+failure.Message), resp.Status))
|
||||
}
|
||||
|
||||
func (s *s3Storage) call(ctx context.Context, method, key string, query url.Values, body io.Reader, result any) (http.Header, error) {
|
||||
resp, err := s.do(ctx, method, key, query, nil, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer drain(resp)
|
||||
if result == nil {
|
||||
return resp.Header, nil
|
||||
}
|
||||
return resp.Header, xml.NewDecoder(resp.Body).Decode(result)
|
||||
}
|
||||
|
||||
// drain reads a response to its end before closing it, so its connection gets reused.
|
||||
func drain(resp *http.Response) {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
|
||||
func (s *s3Storage) Exist(id uint64) (bool, error) {
|
||||
_, err := s.call(context.Background(), http.MethodHead, s.blobKey(id), nil, nil, nil)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (s *s3Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
written, err := s.Storage.Commit(id, size)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = s.Storage.Remove(id) }()
|
||||
file, err := os.Open(s.filename(id))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
if err := s.upload(context.Background(), s.blobKey(id), file, written); err != nil {
|
||||
return 0, fmt.Errorf("upload cache to S3: %w", err)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (s *s3Storage) upload(ctx context.Context, key string, file io.ReaderAt, size int64) (uploadErr error) {
|
||||
if size <= s3PartSize {
|
||||
_, err := s.call(ctx, http.MethodPut, key, nil, io.NewSectionReader(file, 0, size), nil)
|
||||
return err
|
||||
}
|
||||
var created struct {
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploads": {""}}, nil, &created); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if uploadErr != nil && created.UploadID != "" {
|
||||
abortCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer cancel()
|
||||
_, _ = s.call(abortCtx, http.MethodDelete, key, url.Values{"uploadId": {created.UploadID}}, nil, nil)
|
||||
}
|
||||
}()
|
||||
var completion struct {
|
||||
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||
Parts []s3Part `xml:"Part"`
|
||||
}
|
||||
partSize := max(int64(s3PartSize), (size-1)/10000+1)
|
||||
completion.Parts = make([]s3Part, (size-1)/partSize+1)
|
||||
workers, workerCtx := errgroup.WithContext(ctx)
|
||||
workers.SetLimit(4)
|
||||
for part := range completion.Parts {
|
||||
workers.Go(func() (err error) {
|
||||
defer common.RecoverPanic(&err)
|
||||
offset := int64(part) * partSize
|
||||
header, err := s.call(workerCtx, http.MethodPut, key, url.Values{"partNumber": {strconv.Itoa(part + 1)}, "uploadId": {created.UploadID}}, io.NewSectionReader(file, offset, min(partSize, size-offset)), nil)
|
||||
completion.Parts[part] = s3Part{part + 1, header.Get("ETag")}
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err := workers.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := xml.Marshal(completion)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var result struct{ Code, Message string } // a completion can fail after its 200 status
|
||||
if _, err := s.call(ctx, http.MethodPost, key, url.Values{"uploadId": {created.UploadID}}, bytes.NewReader(body), &result); err != nil {
|
||||
return err
|
||||
}
|
||||
if result.Code != "" {
|
||||
return fmt.Errorf("S3 complete upload %s: %s %s", key, result.Code, result.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *s3Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||
header := http.Header{}
|
||||
if byteRange := r.Header.Get("Range"); byteRange != "" {
|
||||
header.Set("Range", byteRange)
|
||||
}
|
||||
resp, err := s.do(r.Context(), http.MethodGet, s.blobKey(id), nil, header, nil)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
} else if err != nil {
|
||||
http.Error(w, "read cache from S3", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
for _, name := range []string{"Content-Length", "Content-Range"} {
|
||||
if value := resp.Header.Get(name); value != "" {
|
||||
w.Header().Set(name, value)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// Remove unpublishes an entry before deleting its blob, so no runner imports it without its bytes.
|
||||
func (s *s3Storage) Remove(id uint64) error {
|
||||
for _, key := range []string{s.metadataKey(id), s.blobKey(id)} {
|
||||
if _, err := s.call(context.Background(), http.MethodDelete, key, nil, nil, nil); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("remove %s from S3: %w", key, err)
|
||||
}
|
||||
}
|
||||
return s.Storage.Remove(id)
|
||||
}
|
||||
|
||||
func (s *s3Storage) publish(cache *Cache) error {
|
||||
data, err := json.Marshal(cache)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.call(context.Background(), http.MethodPut, s.metadataKey(cache.ID), nil, bytes.NewReader(data), nil); err != nil {
|
||||
return fmt.Errorf("publish cache to S3: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *s3Storage) published(ctx context.Context) (map[uint64]bool, error) {
|
||||
ids := make(map[uint64]bool)
|
||||
query := url.Values{"list-type": {"2"}, "prefix": {s.metadataDir()}}
|
||||
for {
|
||||
var page struct {
|
||||
Contents []struct{ Key string }
|
||||
IsTruncated bool
|
||||
NextContinuationToken string
|
||||
}
|
||||
if _, err := s.call(ctx, http.MethodGet, "", query, nil, &page); err != nil {
|
||||
return nil, fmt.Errorf("list S3 cache metadata: %w", err)
|
||||
}
|
||||
for _, object := range page.Contents {
|
||||
if id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(object.Key, s.metadataDir()), ".json"), 10, 64); err == nil {
|
||||
ids[id] = true
|
||||
}
|
||||
}
|
||||
if !page.IsTruncated {
|
||||
return ids, nil
|
||||
}
|
||||
query.Set("continuation-token", page.NextContinuationToken)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *s3Storage) fetch(ctx context.Context, id uint64) (*Cache, error) {
|
||||
resp, err := s.do(ctx, http.MethodGet, s.metadataKey(id), nil, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
cache := &Cache{}
|
||||
if err := json.UnmarshalRead(resp.Body, cache); err != nil {
|
||||
return nil, fmt.Errorf("read S3 cache metadata %d: %w", id, err)
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
118
act/artifactcache/s3_storage_test.go
Normal file
118
act/artifactcache/s3_storage_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHandler_S3SharesEntriesAndDropsExpiredOnes(t *testing.T) {
|
||||
var objects sync.Map
|
||||
var uploads atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
key := strings.TrimPrefix(request.URL.Path, "/cache/")
|
||||
switch {
|
||||
case request.Method == http.MethodPut:
|
||||
if strings.Contains(key, "/blobs/") {
|
||||
uploads.Add(1)
|
||||
}
|
||||
body, err := io.ReadAll(request.Body)
|
||||
assert.NoError(t, err)
|
||||
objects.Store(key, body)
|
||||
case request.URL.Query().Has("list-type"):
|
||||
_, _ = io.WriteString(writer, "<ListBucketResult>")
|
||||
objects.Range(func(stored, _ any) bool {
|
||||
_, _ = io.WriteString(writer, "<Contents><Key>"+stored.(string)+"</Key></Contents>")
|
||||
return true
|
||||
})
|
||||
_, _ = io.WriteString(writer, "</ListBucketResult>")
|
||||
default:
|
||||
if body, ok := objects.Load(key); ok {
|
||||
_, _ = writer.Write(body.([]byte))
|
||||
} else {
|
||||
writer.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
opts := Options{S3: &S3Options{Endpoint: server.URL, Bucket: "cache", Prefix: "runners", AccessKey: "key", SecretKey: "secret"}}
|
||||
saver, restorer := startTestHandler(t, opts), startTestHandler(t, opts)
|
||||
entry := map[string]any{"key": "deps", "version": "v1"}
|
||||
|
||||
finalized, _ := saveV2(t, saver, "deps", "v1", []byte("archive"))
|
||||
require.Equal(t, true, finalized["ok"])
|
||||
assert.EqualValues(t, 1, uploads.Load())
|
||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||
found := v2Call(t, restorer, testClient, "GetCacheEntryDownloadURL", entry)
|
||||
require.Equal(t, true, found["ok"])
|
||||
assert.Equal(t, []byte("archive"), getURL(t, found["signed_download_url"].(string)))
|
||||
|
||||
objects.Clear()
|
||||
require.NoError(t, restorer.importMetadata(t.Context()))
|
||||
assert.Equal(t, true, v2Call(t, restorer, testClient, "CreateCacheEntry", entry)["ok"])
|
||||
}
|
||||
|
||||
type s3TestTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (transport s3TestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return transport(req)
|
||||
}
|
||||
|
||||
func TestS3StorageUploadsLargeObjectsInPartsAndAbortsOnFailure(t *testing.T) {
|
||||
for _, failingPart := range []string{"", "2"} {
|
||||
var parts, aborts atomic.Int32
|
||||
var completion struct {
|
||||
Parts []s3Part `xml:"Part"`
|
||||
}
|
||||
storage := &s3Storage{bucketURL: httptest.NewRequest(http.MethodGet, "http://cache.local/bucket", nil).URL}
|
||||
storage.client = &http.Client{Transport: s3TestTransport(func(req *http.Request) (*http.Response, error) {
|
||||
query, response := req.URL.Query(), &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody}
|
||||
switch {
|
||||
case query.Has("uploads"):
|
||||
response.Body = io.NopCloser(strings.NewReader("<InitiateMultipartUploadResult><UploadId>upload</UploadId></InitiateMultipartUploadResult>"))
|
||||
case req.Method == http.MethodPut:
|
||||
parts.Add(1)
|
||||
response.Header.Set("ETag", "etag-"+query.Get("partNumber"))
|
||||
if query.Get("partNumber") == failingPart {
|
||||
response.StatusCode, response.Body = http.StatusBadRequest, io.NopCloser(strings.NewReader("<Error><Code>InvalidPart</Code></Error>"))
|
||||
}
|
||||
case req.Method == http.MethodPost:
|
||||
require.NoError(t, xml.NewDecoder(req.Body).Decode(&completion))
|
||||
response.Body = io.NopCloser(strings.NewReader("<CompleteMultipartUploadResult/>"))
|
||||
case req.Method == http.MethodDelete:
|
||||
aborts.Add(1)
|
||||
}
|
||||
return response, nil
|
||||
})}
|
||||
err := storage.upload(t.Context(), "blobs/1", strings.NewReader(""), 2*int64(s3PartSize)+1)
|
||||
assert.EqualValues(t, 3, parts.Load())
|
||||
if failingPart == "" {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []s3Part{{1, "etag-1"}, {2, "etag-2"}, {3, "etag-3"}}, completion.Parts)
|
||||
assert.Zero(t, aborts.Load())
|
||||
} else {
|
||||
require.ErrorContains(t, err, "InvalidPart")
|
||||
assert.EqualValues(t, 1, aborts.Load())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3StorageSignsLikeAWS(t *testing.T) {
|
||||
storage := &s3Storage{opts: S3Options{Region: "us-east-1", AccessKey: "AKIAIOSFODNN7EXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"}}
|
||||
req := httptest.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||
storage.sign(req, time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
|
||||
assert.Equal(t, "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", req.Header.Get("Authorization"))
|
||||
}
|
||||
@@ -5,8 +5,10 @@
|
||||
package artifactcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/v2"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -30,13 +32,11 @@ func NewStorage(rootDir string) (*Storage, error) {
|
||||
}
|
||||
|
||||
func (s *Storage) Exist(id uint64) (bool, error) {
|
||||
name := s.filename(id)
|
||||
if _, err := os.Stat(name); os.IsNotExist(err) {
|
||||
_, err := os.Stat(s.filename(id))
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
|
||||
@@ -66,6 +66,18 @@ func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error
|
||||
// rename pass is safe because a staged name always carries blockFilePrefix and a target name
|
||||
// never does, so no rename can collide with a block not yet moved.
|
||||
func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
||||
wanted, err := json.Marshal(blockIDs) // a block id is client-chosen, so it cannot be a separator
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
list := filepath.Join(s.tempDir(id), blockFilePrefix+"list")
|
||||
if recorded, err := os.ReadFile(list); err == nil {
|
||||
// A retry repeats the list it already sent; another one would reorder what is now staged.
|
||||
if !bytes.Equal(recorded, wanted) {
|
||||
return fmt.Errorf("cache %d was already assembled from a different block list", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for i, blockID := range blockIDs {
|
||||
if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -74,13 +86,11 @@ func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return os.WriteFile(list, wanted, 0o600)
|
||||
}
|
||||
|
||||
func (s *Storage) Commit(id uint64, size int64) (int64, error) {
|
||||
defer func() {
|
||||
_ = os.RemoveAll(s.tempDir(id))
|
||||
}()
|
||||
defer func() { _ = os.RemoveAll(s.tempDir(id)) }()
|
||||
|
||||
name := s.filename(id)
|
||||
tempNames, err := s.tempNames(id)
|
||||
@@ -139,8 +149,7 @@ func assemble(name string, tempNames []string) (int64, error) {
|
||||
}
|
||||
|
||||
func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) {
|
||||
name := s.filename(id)
|
||||
http.ServeFile(w, r, name)
|
||||
http.ServeFile(w, r, s.filename(id))
|
||||
}
|
||||
|
||||
// Remove deletes an entry's blob and any staged parts. It reports failure so the caller can
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -18,8 +19,6 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
type FileContainerResourceURL struct {
|
||||
@@ -66,25 +65,21 @@ func writeJSON(w http.ResponseWriter, value any) {
|
||||
}
|
||||
}
|
||||
|
||||
func uploads(router *httprouter.Router, baseDir string) {
|
||||
router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
runID := params.ByName("runId")
|
||||
|
||||
func uploads(router *http.ServeMux, baseDir string) {
|
||||
router.HandleFunc("POST /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||
writeJSON(w, FileContainerResourceURL{
|
||||
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, runID),
|
||||
FileContainerResourceURL: fmt.Sprintf("http://%s/upload/%s", req.Host, req.PathValue("runId")),
|
||||
})
|
||||
})
|
||||
|
||||
router.PUT("/upload/:runId", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
router.HandleFunc("PUT /upload/{runId}", func(w http.ResponseWriter, req *http.Request) {
|
||||
itemPath := req.URL.Query().Get("itemPath")
|
||||
runID := params.ByName("runId")
|
||||
|
||||
if req.Header.Get("Content-Encoding") == "gzip" {
|
||||
itemPath += gzipExtension
|
||||
}
|
||||
|
||||
safeRunPath := safeResolve(baseDir, runID)
|
||||
safePath := safeResolve(safeRunPath, itemPath)
|
||||
safePath := safeResolve(safeResolve(baseDir, req.PathValue("runId")), itemPath)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(safePath), os.ModePerm); err != nil {
|
||||
panic(err)
|
||||
@@ -113,20 +108,18 @@ func uploads(router *httprouter.Router, baseDir string) {
|
||||
})
|
||||
})
|
||||
|
||||
router.PATCH("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
router.HandleFunc("PATCH /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, _req *http.Request) {
|
||||
writeJSON(w, ResponseMessage{
|
||||
Message: "success",
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func downloads(router *httprouter.Router, baseDir string) {
|
||||
router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
runID := params.ByName("runId")
|
||||
func downloads(router *http.ServeMux, baseDir string) {
|
||||
router.HandleFunc("GET /_apis/pipelines/workflows/{runId}/artifacts", func(w http.ResponseWriter, req *http.Request) {
|
||||
runID := req.PathValue("runId")
|
||||
|
||||
safePath := safeResolve(baseDir, runID)
|
||||
|
||||
entries, err := os.ReadDir(safePath)
|
||||
entries, err := os.ReadDir(safeResolve(baseDir, runID))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@@ -145,8 +138,8 @@ func downloads(router *httprouter.Router, baseDir string) {
|
||||
})
|
||||
})
|
||||
|
||||
router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
container := params.ByName("container")
|
||||
router.HandleFunc("GET /download/{container}", func(w http.ResponseWriter, req *http.Request) {
|
||||
container := req.PathValue("container")
|
||||
itemPath := req.URL.Query().Get("itemPath")
|
||||
safePath := safeResolve(baseDir, filepath.Join(container, itemPath))
|
||||
|
||||
@@ -160,15 +153,11 @@ func downloads(router *httprouter.Router, baseDir string) {
|
||||
|
||||
// if it was upload as gzip
|
||||
rel = strings.TrimSuffix(rel, gzipExtension)
|
||||
path := filepath.Join(itemPath, rel)
|
||||
|
||||
rel = filepath.ToSlash(rel)
|
||||
path = filepath.ToSlash(path)
|
||||
|
||||
files = append(files, ContainerItem{
|
||||
Path: path,
|
||||
Path: filepath.ToSlash(filepath.Join(itemPath, rel)),
|
||||
ItemType: "file",
|
||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel),
|
||||
ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, filepath.ToSlash(rel)),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
@@ -182,10 +171,8 @@ func downloads(router *httprouter.Router, baseDir string) {
|
||||
})
|
||||
})
|
||||
|
||||
router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
path := params.ByName("path")[1:]
|
||||
|
||||
safePath := safeResolve(baseDir, path)
|
||||
router.HandleFunc("GET /artifact/{path...}", func(w http.ResponseWriter, req *http.Request) {
|
||||
safePath := safeResolve(baseDir, req.PathValue("path"))
|
||||
|
||||
file, err := os.Open(safePath)
|
||||
if err != nil {
|
||||
@@ -213,23 +200,24 @@ func Serve(ctx context.Context, artifactPath, addr, port string) context.CancelF
|
||||
return cancel
|
||||
}
|
||||
|
||||
router := httprouter.New()
|
||||
router := http.NewServeMux()
|
||||
|
||||
logger.Debugf("Artifacts base path '%s'", artifactPath)
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%s", addr, port),
|
||||
Addr: net.JoinHostPort(addr, port),
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
Handler: router,
|
||||
}
|
||||
|
||||
// run server
|
||||
go func() {
|
||||
logger.Infof("Start server on http://%s:%s", addr, port)
|
||||
logger.Infof("Start server on http://%s", server.Addr)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
logger.Fatal(err)
|
||||
logger.Error(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -19,14 +19,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestArtifactFlow(t *testing.T) {
|
||||
artifactPath := t.TempDir()
|
||||
|
||||
router := httprouter.New()
|
||||
router := http.NewServeMux()
|
||||
uploads(router, artifactPath)
|
||||
downloads(router, artifactPath)
|
||||
server := httptest.NewServer(router)
|
||||
|
||||
@@ -4,15 +4,15 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
)
|
||||
|
||||
func TestDryrunContext(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
if Dryrun(ctx) {
|
||||
t.Fatal("plain context should not be dryrun")
|
||||
}
|
||||
@@ -25,7 +25,7 @@ func TestDryrunContext(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestJobErrorContainer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
err := errors.New("job failed")
|
||||
|
||||
SetJobError(ctx, err)
|
||||
@@ -40,34 +40,27 @@ func TestJobErrorContainer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoggerAndHookContext(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
if Logger(ctx) != logrus.StandardLogger() {
|
||||
t.Fatal("plain context should use standard logger")
|
||||
func TestLoggerAndHandlerContext(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
if Logger(ctx) != logging.Default() {
|
||||
t.Fatal("plain context should use the default logger")
|
||||
}
|
||||
if LoggerHook(ctx) != nil {
|
||||
t.Fatal("plain context should not have a logger hook")
|
||||
if LoggerHandler(ctx) != nil {
|
||||
t.Fatal("plain context should not have a logger handler")
|
||||
}
|
||||
if StepIDs(ctx) != nil {
|
||||
t.Fatal("plain context should not have step IDs")
|
||||
}
|
||||
|
||||
logger := logrus.New()
|
||||
logger := logging.New(logtest.New())
|
||||
ctx = WithLogger(ctx, logger)
|
||||
if Logger(ctx) != logger {
|
||||
t.Fatal("WithLogger should set logger")
|
||||
}
|
||||
|
||||
hook := testHook{}
|
||||
ctx = WithLoggerHook(ctx, hook)
|
||||
if LoggerHook(ctx) != hook {
|
||||
t.Fatal("WithLoggerHook should set hook")
|
||||
handler := logtest.New()
|
||||
ctx = WithLoggerHandler(ctx, handler)
|
||||
if LoggerHandler(ctx) != handler {
|
||||
t.Fatal("WithLoggerHandler should set handler")
|
||||
}
|
||||
}
|
||||
|
||||
type testHook struct{}
|
||||
|
||||
func (testHook) Levels() []logrus.Level {
|
||||
return logrus.AllLevels
|
||||
}
|
||||
|
||||
func (testHook) Fire(*logrus.Entry) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -14,13 +14,8 @@ const dryrunContextKeyVal = dryrunContextKey("dryrun")
|
||||
|
||||
// Dryrun returns true if the current context is dryrun
|
||||
func Dryrun(ctx context.Context) bool {
|
||||
val := ctx.Value(dryrunContextKeyVal)
|
||||
if val != nil {
|
||||
if dryrun, ok := val.(bool); ok {
|
||||
return dryrun
|
||||
}
|
||||
}
|
||||
return false
|
||||
dryrun, _ := ctx.Value(dryrunContextKeyVal).(bool)
|
||||
return dryrun
|
||||
}
|
||||
|
||||
// WithDryrun adds a value to the context for dryrun
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
// Executor define contract for the steps of a workflow
|
||||
@@ -21,8 +21,7 @@ type Conditional func(ctx context.Context) bool
|
||||
// NewInfoExecutor is an executor that logs messages
|
||||
func NewInfoExecutor(format string, args ...any) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := Logger(ctx)
|
||||
logger.Infof(format, args...)
|
||||
Logger(ctx).Infof(format, args...)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -30,8 +29,7 @@ func NewInfoExecutor(format string, args ...any) Executor {
|
||||
// NewDebugExecutor is an executor that logs messages
|
||||
func NewDebugExecutor(format string, args ...any) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := Logger(ctx)
|
||||
logger.Debugf(format, args...)
|
||||
Logger(ctx).Debugf(format, args...)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -39,9 +37,7 @@ func NewDebugExecutor(format string, args ...any) Executor {
|
||||
// NewPipelineExecutor creates a new executor from a series of other executors
|
||||
func NewPipelineExecutor(executors ...Executor) Executor {
|
||||
if len(executors) == 0 {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return NewErrorExecutor(nil)
|
||||
}
|
||||
var rtn Executor
|
||||
for _, executor := range executors {
|
||||
@@ -61,6 +57,13 @@ func NewErrorExecutor(err error) Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func RecoverPanic(err *error) { // defer directly in workers to keep stacks in host logs
|
||||
if recovered := recover(); recovered != nil {
|
||||
logging.Errorf("panic: %v\n%s", recovered, debug.Stack())
|
||||
*err = fmt.Errorf("panic: %v", recovered)
|
||||
}
|
||||
}
|
||||
|
||||
// NewParallelExecutor creates a new executor from a parallel of other executors
|
||||
func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||
if len(executors) == 0 {
|
||||
@@ -74,34 +77,29 @@ func NewParallelExecutor(parallel int, executors ...Executor) Executor {
|
||||
errs := make(chan error, len(executors))
|
||||
|
||||
if 1 > parallel {
|
||||
log.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
||||
logging.Debugf("Parallel tasks (%d) below minimum, setting to 1", parallel)
|
||||
parallel = 1
|
||||
}
|
||||
|
||||
log.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||
logging.Infof("NewParallelExecutor: Creating %d workers for %d executors", parallel, len(executors))
|
||||
|
||||
for i := 0; i < parallel; i++ {
|
||||
go func(workerID int, work <-chan Executor, errs chan<- error) {
|
||||
log.Debugf("Worker %d started", workerID)
|
||||
for workerID := range parallel {
|
||||
go func() {
|
||||
logging.Debugf("Worker %d started", workerID)
|
||||
taskCount := 0
|
||||
for executor := range work {
|
||||
taskCount++
|
||||
log.Debugf("Worker %d executing task %d", workerID, taskCount)
|
||||
logging.Debugf("Worker %d executing task %d", workerID, taskCount)
|
||||
// Recover from panics in executors to avoid crashing the worker
|
||||
// goroutine which would leave the runner process hung.
|
||||
// https://gitea.com/gitea/runner/issues/371
|
||||
errs <- func() (err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Errorf("panic in executor: %v\n%s", r, debug.Stack())
|
||||
err = fmt.Errorf("panic: %v", r)
|
||||
}
|
||||
}()
|
||||
defer RecoverPanic(&err)
|
||||
return executor(ctx)
|
||||
}()
|
||||
}
|
||||
log.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||
}(i, work, errs)
|
||||
logging.Debugf("Worker %d finished (%d tasks executed)", workerID, taskCount)
|
||||
}()
|
||||
}
|
||||
|
||||
for i := range executors {
|
||||
@@ -150,12 +148,7 @@ func (e Executor) If(conditional Conditional) Executor {
|
||||
|
||||
// IfNot only runs this executor if conditional is true
|
||||
func (e Executor) IfNot(conditional Conditional) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if !conditional(ctx) {
|
||||
return e(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return e.If(func(ctx context.Context) bool { return !conditional(ctx) })
|
||||
}
|
||||
|
||||
// IfBool only runs this executor if conditional is true
|
||||
|
||||
@@ -7,8 +7,6 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -18,42 +16,25 @@ import (
|
||||
)
|
||||
|
||||
func TestNewWorkflow(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
ctx := t.Context()
|
||||
require.NoError(t, NewPipelineExecutor()(ctx))
|
||||
require.EqualError(t, NewErrorExecutor(errors.New("test error"))(ctx), "test error")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// empty
|
||||
emptyWorkflow := NewPipelineExecutor()
|
||||
assert.NoError(emptyWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// error case
|
||||
errorWorkflow := NewErrorExecutor(errors.New("test error"))
|
||||
assert.Error(errorWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// multiple success case
|
||||
runcount := 0
|
||||
successWorkflow := NewPipelineExecutor(
|
||||
func(ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
},
|
||||
func(ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
})
|
||||
assert.NoError(successWorkflow(ctx)) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(2, runcount)
|
||||
increment := func(_ctx context.Context) error {
|
||||
runcount++
|
||||
return nil
|
||||
}
|
||||
require.NoError(t, NewPipelineExecutor(increment, increment)(ctx))
|
||||
assert.Equal(t, 2, runcount)
|
||||
}
|
||||
|
||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
||||
// find the gate already open so the last one still finishes with no partner left.
|
||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||
var counted, active, peak atomic.Int32
|
||||
var once sync.Once
|
||||
reached := make(chan struct{})
|
||||
|
||||
return func(ctx context.Context) error {
|
||||
return func(_ctx context.Context) error {
|
||||
counted.Add(1)
|
||||
running := active.Add(1)
|
||||
for {
|
||||
@@ -72,93 +53,72 @@ func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic
|
||||
}
|
||||
|
||||
func TestNewParallelExecutor(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
exec, count, maxActive := concurrencyProbe(2)
|
||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
||||
|
||||
// parallelism below 1 falls back to a single worker
|
||||
exec, count, maxActive = concurrencyProbe(1)
|
||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
||||
for name, parallel := range map[string]int{"parallel": 2, "below minimum": 0} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
exec, count, maxActive := concurrencyProbe(int32(max(1, parallel)))
|
||||
require.NoError(t, NewParallelExecutor(parallel, exec, exec, exec)(t.Context()))
|
||||
assert.Equal(t, int32(3), count.Load())
|
||||
assert.Equal(t, int32(max(1, parallel)), maxActive.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
require.NoError(t, NewParallelExecutor(2)(ctx))
|
||||
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
canceledCtx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
err := NewParallelExecutor(2)(canceledCtx)
|
||||
assert.ErrorIs(err, context.Canceled)
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorFailed(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
count := 0
|
||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count++
|
||||
return errors.New("fake error")
|
||||
})
|
||||
err := NewParallelExecutor(1, errorWorkflow)(ctx)
|
||||
assert.Equal(1, count)
|
||||
assert.ErrorIs(context.Canceled, err)
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorCanceled(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
errExpected := errors.New("fake error")
|
||||
|
||||
var count atomic.Int32
|
||||
successWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count.Add(1)
|
||||
return nil
|
||||
})
|
||||
errorWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count.Add(1)
|
||||
return errExpected
|
||||
})
|
||||
err := NewParallelExecutor(3, errorWorkflow, successWorkflow, successWorkflow)(ctx)
|
||||
assert.Equal(int32(3), count.Load())
|
||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
||||
var successCount atomic.Int32
|
||||
executors := make([]Executor, 5)
|
||||
for i := range executors {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
if i == 2 {
|
||||
return errors.New("fake error")
|
||||
func TestNewParallelExecutorRunsAllTasksAfterFailureOrCancellation(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
canceled bool
|
||||
parallel, tasks int
|
||||
}{
|
||||
"failure": {false, 2, 5},
|
||||
"canceled": {true, 2, 5},
|
||||
"canceled single task": {true, 1, 1},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
if tc.canceled {
|
||||
cancel()
|
||||
}
|
||||
var calls atomic.Int32
|
||||
executors := make([]Executor, tc.tasks)
|
||||
for i := range executors {
|
||||
executors[i] = func(_ctx context.Context) error {
|
||||
calls.Add(1)
|
||||
if i == tc.tasks/2 {
|
||||
return errors.New("fake error")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
successCount.Add(1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
|
||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
||||
err := NewParallelExecutor(tc.parallel, executors...)(ctx)
|
||||
if tc.canceled {
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
} else {
|
||||
require.EqualError(t, err, "fake error")
|
||||
}
|
||||
assert.Equal(t, int32(tc.tasks), calls.Load())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
var calls []string
|
||||
record := func(name string) Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return func(_ctx context.Context) error {
|
||||
calls = append(calls, name)
|
||||
return nil
|
||||
}
|
||||
@@ -170,19 +130,13 @@ func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
require.NoError(t, record("if-bool").IfBool(true)(ctx))
|
||||
require.NoError(t, record("main").Finally(record("finally"))(ctx))
|
||||
|
||||
want := []string{"if-true", "if-not", "if-bool", "main", "finally"}
|
||||
if !reflect.DeepEqual(calls, want) {
|
||||
t.Fatalf("calls = %v, want %v", calls, want)
|
||||
}
|
||||
assert.Equal(t, []string{"if-true", "if-not", "if-bool", "main", "finally"}, calls)
|
||||
}
|
||||
|
||||
func TestExecutorFinallyReturnsFinallyErrorWithOriginal(t *testing.T) {
|
||||
mainErr := errors.New("main failed")
|
||||
finalErr := errors.New("cleanup failed")
|
||||
|
||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(context.Background())
|
||||
require.Error(t, err)
|
||||
if !strings.Contains(err.Error(), "cleanup failed") || !strings.Contains(err.Error(), "main failed") {
|
||||
t.Fatalf("finally error = %q, want both cleanup and original error", err)
|
||||
}
|
||||
err := NewErrorExecutor(mainErr).Finally(NewErrorExecutor(finalErr))(t.Context())
|
||||
require.EqualError(t, err, "error occurred running finally: cleanup failed (original error: main failed)")
|
||||
}
|
||||
|
||||
279
act/common/git/fetch_action.go
Normal file
279
act/common/git/fetch_action.go
Normal file
@@ -0,0 +1,279 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
)
|
||||
|
||||
var actionFetches sync.Map
|
||||
|
||||
type actionFetch struct {
|
||||
done chan struct{}
|
||||
dir, sha string
|
||||
err error
|
||||
retry bool // the leader was cancelled or panicked, so followers fetch themselves
|
||||
}
|
||||
|
||||
var archiveRetryDelay = func(retryAfter string) time.Duration { // as actions/runner backs off
|
||||
if seconds, err := strconv.Atoi(retryAfter); err == nil {
|
||||
return min(max(time.Duration(seconds)*time.Second, 10*time.Second), 10*time.Minute)
|
||||
}
|
||||
return time.Duration(10+rand.IntN(21)) * time.Second
|
||||
}
|
||||
|
||||
func FetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) { // input.Dir is one clone URL and ref's, holding a directory per commit
|
||||
if !reflect.ValueOf(input).Comparable() {
|
||||
return fetchAction(ctx, input)
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
call := &actionFetch{done: make(chan struct{}), retry: true}
|
||||
previous, loaded := actionFetches.LoadOrStore(input, call)
|
||||
if !loaded {
|
||||
func() {
|
||||
defer func() { actionFetches.Delete(input); close(call.done) }()
|
||||
call.dir, call.sha, call.err = fetchAction(ctx, input)
|
||||
call.retry = ctx.Err() != nil
|
||||
}()
|
||||
if call.retry {
|
||||
continue
|
||||
}
|
||||
return call.dir, call.sha, call.err
|
||||
}
|
||||
call, _ = previous.(*actionFetch)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-call.done:
|
||||
if !call.retry && ctx.Err() == nil {
|
||||
return call.dir, call.sha, call.err
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", ctx.Err()
|
||||
}
|
||||
|
||||
func fetchAction(ctx context.Context, input NewGitCloneExecutorInput) (string, string, error) {
|
||||
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
defer unlock()
|
||||
if err := os.MkdirAll(input.Dir, 0o755); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
tree := func(sha string) string { return filepath.Join(input.Dir, sha) }
|
||||
record := filepath.Join(input.Dir, "ref")
|
||||
data, _ := os.ReadFile(record)
|
||||
sha, etag, _ := strings.Cut(string(data), "\n")
|
||||
if !isFullObjectID(sha) || !exists(tree(sha)) {
|
||||
sha, etag = "", ""
|
||||
} else if input.OfflineMode || isFullObjectID(input.Ref) {
|
||||
return tree(sha), sha, nil
|
||||
}
|
||||
|
||||
newSha, newEtag, err := downloadArchive(ctx, input, sha, etag)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
common.Logger(ctx).Debugf("Cloning %s@%s: %v", input.URL, input.Ref, err)
|
||||
newEtag = ""
|
||||
newSha, err = exportClone(ctx, input)
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if newSha != sha || newEtag != etag {
|
||||
if err := os.WriteFile(record, []byte(newSha+"\n"+newEtag), 0o600); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
return tree(newSha), newSha, nil
|
||||
}
|
||||
|
||||
func downloadArchive(ctx context.Context, input NewGitCloneExecutorInput, sha, etag string) (string, string, error) {
|
||||
rewritten, err := gitOutput(ctx, "", "ls-remote", "--get-url", input.URL)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
source, err := url.Parse(rewritten)
|
||||
if err != nil || source.Scheme != "http" && source.Scheme != "https" || hexRefRegex.MatchString(input.Ref) && !isFullObjectID(input.Ref) { // only git tells a short commit from a branch
|
||||
return "", "", errors.New("no archive")
|
||||
}
|
||||
archive := source.JoinPath("archive", input.Ref+".tar.gz")
|
||||
if source.Host == "github.com" {
|
||||
source.Host = "codeload.github.com"
|
||||
archive = source.JoinPath("tar.gz", input.Ref)
|
||||
}
|
||||
client := &http.Client{Transport: input.Transport, Timeout: 5 * time.Minute, CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
if request.URL.Host != via[0].URL.Host || request.URL.Scheme != via[0].URL.Scheme { // the token stays on its origin
|
||||
request.Header.Del("Authorization")
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
for attempt := 1; ; attempt++ {
|
||||
newSha, newEtag, retryAfter, err := downloadArchiveOnce(ctx, client, archive.String(), input.Token, sha, etag, input.Dir)
|
||||
if retryAfter == 0 || attempt == 3 || ctx.Err() != nil {
|
||||
return newSha, newEtag, err
|
||||
}
|
||||
common.Logger(ctx).Warnf("Failed to download action '%s'. Error: %v. Back off %s before retry.", archive.Redacted(), err, retryAfter)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", "", ctx.Err()
|
||||
case <-time.After(retryAfter):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func downloadArchiveOnce(ctx context.Context, client *http.Client, archive, token, sha, etag, dir string) (string, string, time.Duration, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, archive, nil)
|
||||
if err != nil {
|
||||
return "", "", 0, err
|
||||
}
|
||||
if etag != "" {
|
||||
request.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
if token != "" {
|
||||
request.SetBasicAuth("token", token)
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
var opErr *net.OpError
|
||||
if errors.As(err, &opErr) && opErr.Op == "dial" {
|
||||
return "", "", 0, err
|
||||
}
|
||||
return "", "", archiveRetryDelay(""), err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch status := response.StatusCode; {
|
||||
case status == http.StatusNotModified && sha != "":
|
||||
return sha, etag, 0, nil
|
||||
case status == http.StatusOK:
|
||||
newEtag := response.Header.Get("ETag")
|
||||
gz, err := gzip.NewReader(response.Body)
|
||||
if err == nil {
|
||||
sha, err = extractTree(gz, dir, !strings.HasSuffix(newEtag, `-false"`)) // Gitea's mark of an unprefixed archive
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.As(err, &netErr) {
|
||||
return "", "", archiveRetryDelay(""), err
|
||||
}
|
||||
return sha, newEtag, 0, err
|
||||
case status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= http.StatusInternalServerError:
|
||||
return "", "", archiveRetryDelay(response.Header.Get("Retry-After")), errors.New(response.Status)
|
||||
default:
|
||||
return "", "", 0, errors.New(response.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func exportClone(ctx context.Context, input NewGitCloneExecutorInput) (string, error) {
|
||||
clone := input
|
||||
clone.Dir = filepath.Join(input.Dir, "git")
|
||||
sha, _, err := resolveClone(ctx, clone)
|
||||
if err != nil || exists(filepath.Join(input.Dir, sha)) {
|
||||
return sha, err
|
||||
}
|
||||
archive, err := runGit(ctx, clone.Dir, nil, "archive", "--format=tar", sha)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = extractTree(bytes.NewReader(archive), input.Dir, false)
|
||||
return sha, err
|
||||
}
|
||||
|
||||
func extractTree(archive io.Reader, dir string, strip bool) (string, error) { // into dir/<commit of the pax header>, strip dropping the directory wrapping the files
|
||||
temp, err := os.MkdirTemp(dir, "tmp-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(temp)
|
||||
root, err := os.OpenRoot(temp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer root.Close()
|
||||
|
||||
sha := ""
|
||||
reader := tar.NewReader(archive)
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if header.Typeflag == tar.TypeXGlobalHeader {
|
||||
if sha = header.PAXRecords["comment"]; isFullObjectID(sha) && exists(filepath.Join(dir, sha)) {
|
||||
return sha, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
name := header.Name
|
||||
if strip {
|
||||
_, name, _ = strings.Cut(name, "/")
|
||||
}
|
||||
if name = strings.TrimSuffix(name, "/"); name == "" {
|
||||
continue
|
||||
}
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
err = root.MkdirAll(name, 0o755)
|
||||
case tar.TypeReg:
|
||||
err = writeTreeFile(root, name, header.Mode, reader)
|
||||
case tar.TypeSymlink:
|
||||
if err = root.Symlink(header.Linkname, name); err != nil && runtime.GOOS == "windows" {
|
||||
err = writeTreeFile(root, name, 0, strings.NewReader(header.Linkname))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if !isFullObjectID(sha) {
|
||||
return "", errors.New("the archive names no commit")
|
||||
}
|
||||
if err := root.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return sha, os.Rename(temp, filepath.Join(dir, sha))
|
||||
}
|
||||
|
||||
func writeTreeFile(root *os.Root, name string, mode int64, content io.Reader) error {
|
||||
perm := os.FileMode(0o644)
|
||||
if mode&0o111 != 0 {
|
||||
perm = 0o755
|
||||
}
|
||||
file, err := root.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, perm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(file, content)
|
||||
return errors.Join(err, file.Close())
|
||||
}
|
||||
|
||||
func exists(name string) bool {
|
||||
_, err := os.Stat(name)
|
||||
return err == nil
|
||||
}
|
||||
315
act/common/git/fetch_action_test.go
Normal file
315
act/common/git/fetch_action_test.go
Normal file
@@ -0,0 +1,315 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFetchAction(t *testing.T) {
|
||||
original := archiveRetryDelay
|
||||
archiveRetryDelay = func(_retryAfter string) time.Duration { return time.Nanosecond }
|
||||
t.Cleanup(func() { archiveRetryDelay = original })
|
||||
|
||||
root := t.TempDir()
|
||||
remote := filepath.Join(root, "org", "action")
|
||||
require.NoError(t, gitCmd("init", "--initial-branch=main", remote))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "action.yml"), []byte("name: action\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(remote, "run.sh"), []byte("#!/bin/sh\n"), 0o755))
|
||||
require.NoError(t, os.Symlink("action.yml", filepath.Join(remote, "link")))
|
||||
require.NoError(t, gitCmd("-C", remote, "add", "."))
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "v1"))
|
||||
|
||||
gitPath, err := exec.LookPath("git")
|
||||
require.NoError(t, err)
|
||||
backend := &cgi.Handler{Path: gitPath, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}}
|
||||
var mu sync.Mutex
|
||||
var archiveRequests []http.Header
|
||||
var mirror *httptest.Server
|
||||
requests, archives, failures := 0, "", 0
|
||||
handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
requests++
|
||||
ref, ok := strings.CutPrefix(request.URL.Path, "/org/action/archive/")
|
||||
if !ok {
|
||||
backend.ServeHTTP(writer, request)
|
||||
return
|
||||
}
|
||||
archiveRequests = append(archiveRequests, request.Header.Clone())
|
||||
switch {
|
||||
case failures > 0:
|
||||
failures--
|
||||
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
case archives == "none":
|
||||
http.NotFound(writer, request)
|
||||
return
|
||||
case archives == "unreachable":
|
||||
http.Redirect(writer, request, "http://127.0.0.1:1"+request.URL.Path, http.StatusFound)
|
||||
return
|
||||
case archives == "redirect" && request.Host != mirror.Listener.Addr().String():
|
||||
http.Redirect(writer, request, mirror.URL+request.URL.Path, http.StatusFound)
|
||||
return
|
||||
}
|
||||
sha, _ := gitOutput(request.Context(), remote, "rev-parse", strings.TrimSuffix(ref, ".tar.gz"))
|
||||
etag := fmt.Sprintf(`W/"%s-tar.gz-%t"`, sha, archives != "unprefixed")
|
||||
if request.Header.Get("If-None-Match") == etag {
|
||||
writer.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
args := []string{"archive", "--format=tar.gz", sha}
|
||||
if archives != "unprefixed" {
|
||||
args = append(args, "--prefix=action/")
|
||||
}
|
||||
archive, _ := runGit(request.Context(), remote, nil, args...)
|
||||
writer.Header().Set("ETag", etag)
|
||||
_, _ = writer.Write(archive)
|
||||
})
|
||||
server := httptest.NewServer(handler)
|
||||
mirror = httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
t.Cleanup(mirror.Close)
|
||||
serve := func(mode string, failing int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
requests, archives, failures, archiveRequests = 0, mode, failing, nil
|
||||
}
|
||||
served := func() (int, []http.Header) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return requests, append([]http.Header(nil), archiveRequests...)
|
||||
}
|
||||
newInput := func(ref string) NewGitCloneExecutorInput {
|
||||
return NewGitCloneExecutorInput{URL: server.URL + "/org/action", Ref: ref, Dir: t.TempDir(), Token: "secret"}
|
||||
}
|
||||
|
||||
t.Run("shares identical fetches and isolates cancellation, panics and differing inputs", func(t *testing.T) {
|
||||
expectedSha := gitRevParse(t, remote, "main")
|
||||
for _, scenario := range []string{"shared", "leader cancelled", "leader panics", "different token"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
input := newInput("main")
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
releaseFetches := sync.OnceFunc(func() { close(release) })
|
||||
t.Cleanup(releaseFetches)
|
||||
var once sync.Once
|
||||
var panicked atomic.Bool
|
||||
transport := actionFetchTransport(func(request *http.Request) (*http.Response, error) {
|
||||
once.Do(func() { close(started) })
|
||||
select {
|
||||
case <-request.Context().Done():
|
||||
return nil, request.Context().Err()
|
||||
case <-release:
|
||||
}
|
||||
if scenario == "leader panics" && panicked.CompareAndSwap(false, true) {
|
||||
panic("broken archive")
|
||||
}
|
||||
return http.DefaultTransport.RoundTrip(request)
|
||||
})
|
||||
input.Transport = &transport
|
||||
leaderCtx, cancelLeader := context.WithCancel(t.Context())
|
||||
defer cancelLeader()
|
||||
waiting := &actionFetchContext{Context: t.Context(), waiting: make(chan struct{})}
|
||||
followerInput := input
|
||||
expectedRequests := 1
|
||||
if scenario == "different token" {
|
||||
followerInput.Token, expectedRequests = "other", 2
|
||||
}
|
||||
leaderDone, followerDone := make(chan error, 1), make(chan error, 1)
|
||||
fetch := func(ctx context.Context, input NewGitCloneExecutorInput, done chan<- error) {
|
||||
var err error
|
||||
defer func() { done <- err }()
|
||||
defer common.RecoverPanic(&err)
|
||||
var tree, sha string
|
||||
if tree, sha, err = FetchAction(ctx, input); err == nil {
|
||||
assert.Equal(t, expectedSha, sha)
|
||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||
}
|
||||
}
|
||||
go fetch(leaderCtx, input, leaderDone)
|
||||
<-started
|
||||
go fetch(waiting, followerInput, followerDone)
|
||||
<-waiting.waiting
|
||||
if scenario == "leader cancelled" {
|
||||
cancelLeader()
|
||||
require.ErrorIs(t, <-leaderDone, context.Canceled)
|
||||
}
|
||||
releaseFetches()
|
||||
switch scenario {
|
||||
case "leader cancelled":
|
||||
case "leader panics":
|
||||
require.ErrorContains(t, <-leaderDone, "panic: broken archive")
|
||||
default:
|
||||
require.NoError(t, <-leaderDone)
|
||||
}
|
||||
require.NoError(t, <-followerDone)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, expectedRequests)
|
||||
if scenario == "different token" {
|
||||
assert.NotEqual(t, headers[0].Get("Authorization"), headers[1].Get("Authorization"))
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("downloads, revalidates and follows a moved tag", func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
input := newInput("v1")
|
||||
tree, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "v1"), sha)
|
||||
assert.Equal(t, filepath.Join(input.Dir, sha), tree)
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"))
|
||||
info, err := os.Stat(filepath.Join(tree, "run.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
|
||||
link, err := os.Readlink(filepath.Join(tree, "link"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", link)
|
||||
|
||||
again, _, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tree, again)
|
||||
|
||||
require.NoError(t, gitCmd("-C", remote, "commit", "--allow-empty", "-m", "moved"))
|
||||
require.NoError(t, gitCmd("-C", remote, "tag", "--force", "v1"))
|
||||
moved, movedSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "v1"), movedSha)
|
||||
assert.NotEqual(t, tree, moved)
|
||||
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||
|
||||
input.OfflineMode = true
|
||||
_, offlineSha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, movedSha, offlineSha)
|
||||
|
||||
requests, headers := served()
|
||||
assert.Equal(t, 3, requests)
|
||||
require.Len(t, headers, 3)
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), headers[0].Get("Authorization"))
|
||||
assert.Empty(t, headers[0].Get("If-None-Match"))
|
||||
assert.Equal(t, fmt.Sprintf(`W/"%s-tar.gz-true"`, sha), headers[1].Get("If-None-Match"))
|
||||
assert.Equal(t, headers[1].Get("If-None-Match"), headers[2].Get("If-None-Match"))
|
||||
})
|
||||
|
||||
t.Run("downloads a pinned commit once and rejects a short one", func(t *testing.T) {
|
||||
serve("prefixed", 0)
|
||||
sha := gitRevParse(t, remote, "main")
|
||||
input := newInput(sha)
|
||||
for range 2 {
|
||||
tree, got, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, sha, got)
|
||||
assert.FileExists(t, filepath.Join(tree, "action.yml"))
|
||||
}
|
||||
_, _, err := FetchAction(t.Context(), newInput(sha[:7]))
|
||||
require.ErrorIs(t, err, ErrShortRef)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 1)
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
})
|
||||
|
||||
t.Run("retries a failing download and drops the token on a redirect to another origin", func(t *testing.T) {
|
||||
serve("redirect", 2)
|
||||
input := newInput("main")
|
||||
_, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
_, headers := served()
|
||||
require.Len(t, headers, 4)
|
||||
assert.NotEmpty(t, headers[2].Get("Authorization"))
|
||||
assert.Empty(t, headers[3].Get("Authorization"))
|
||||
})
|
||||
|
||||
t.Run("keeps an unprefixed archive whole, downloads where insteadOf points and exports a clone without an archive or from an unreachable host", func(t *testing.T) {
|
||||
for _, mode := range []string{"unprefixed", "none", "unreachable"} {
|
||||
serve(mode, 0)
|
||||
tree, sha, err := FetchAction(t.Context(), newInput("main"))
|
||||
require.NoError(t, err, mode)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha, mode)
|
||||
assert.FileExists(t, filepath.Join(tree, "action.yml"), mode)
|
||||
assert.NoDirExists(t, filepath.Join(tree, ".git"), mode)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 1, mode)
|
||||
}
|
||||
|
||||
serve("prefixed", 0)
|
||||
config := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(config, []byte("[url \""+server.URL+"/org/\"]\n\tinsteadOf = https://example.invalid/org/\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", config)
|
||||
input := newInput("main")
|
||||
input.URL = "https://example.invalid/org/action"
|
||||
_, sha, err := FetchAction(t.Context(), input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, gitRevParse(t, remote, "main"), sha)
|
||||
_, headers := served()
|
||||
assert.Len(t, headers, 1)
|
||||
assert.NoDirExists(t, filepath.Join(input.Dir, "git"))
|
||||
})
|
||||
}
|
||||
|
||||
type actionFetchTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (transport actionFetchTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return transport(request)
|
||||
}
|
||||
|
||||
type actionFetchContext struct {
|
||||
context.Context
|
||||
waiting chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (ctx *actionFetchContext) Done() <-chan struct{} {
|
||||
ctx.once.Do(func() { close(ctx.waiting) })
|
||||
return ctx.Context.Done()
|
||||
}
|
||||
|
||||
func TestExtractTreeConfinesEntries(t *testing.T) {
|
||||
outside := t.TempDir()
|
||||
for name, entries := range map[string][]*tar.Header{
|
||||
"parent path": {{Name: "../escape", Typeflag: tar.TypeReg}},
|
||||
"through a symlink": {{Name: "out", Typeflag: tar.TypeSymlink, Linkname: outside}, {Name: "out/escape", Typeflag: tar.TypeReg}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var archive bytes.Buffer
|
||||
writer := tar.NewWriter(&archive)
|
||||
require.NoError(t, writer.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, PAXRecords: map[string]string{"comment": strings.Repeat("a", 40)}}))
|
||||
for _, entry := range entries {
|
||||
require.NoError(t, writer.WriteHeader(entry))
|
||||
}
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := extractTree(&archive, dir, false)
|
||||
require.Error(t, err)
|
||||
assert.NoFileExists(t, filepath.Join(dir, "escape"))
|
||||
assert.NoFileExists(t, filepath.Join(outside, "escape"))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -6,44 +6,67 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/storer"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/mattn/go-isatty"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
codeCommitHTTPRegex = regexp.MustCompile(`^https?://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||
codeCommitSSHRegex = regexp.MustCompile(`ssh://git-codecommit\.(.+)\.amazonaws.com/v1/repos/(.+)$`)
|
||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
||||
hexRefRegex = regexp.MustCompile(`^[0-9a-fA-F]+$`)
|
||||
|
||||
LocalEnvVars = []string{ // `git rev-parse --local-env-vars`, inherited ones point git at another repository
|
||||
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_CONFIG", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_OBJECT_DIRECTORY",
|
||||
"GIT_DIR", "GIT_WORK_TREE", "GIT_IMPLICIT_WORK_TREE", "GIT_GRAFT_FILE", "GIT_INDEX_FILE", "GIT_NO_REPLACE_OBJECTS",
|
||||
"GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_SHALLOW_FILE", "GIT_COMMON_DIR",
|
||||
}
|
||||
|
||||
cloneLocks lock.Keyed[string] // key: clone target directory
|
||||
|
||||
ErrShortRef = errors.New("short SHA references are not supported")
|
||||
)
|
||||
|
||||
// AcquireCloneLock returns an unlock function after locking the per-directory mutex for dir.
|
||||
// Only concurrent operations targeting the same directory are serialized; clones into different directories run in parallel.
|
||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
||||
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
|
||||
func AcquireCloneLock(dir string) func() {
|
||||
return cloneLocks.Lock(dir)
|
||||
// AcquireCloneLock serializes access to dir across runner processes, or within this one where no lock file can be created. Readers of dir must hold it too.
|
||||
func AcquireCloneLock(ctx context.Context, dir string) (func(), error) {
|
||||
dir = filepath.Clean(dir)
|
||||
if runtime.GOOS == "plan9" {
|
||||
return cloneLocks.Lock(dir), nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
release, err := lock.TryLock(dir)
|
||||
if err == nil {
|
||||
return func() { _ = release() }, nil
|
||||
}
|
||||
if !errors.Is(err, lock.ErrLocked) {
|
||||
return cloneLocks.Lock(dir), nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Error struct {
|
||||
@@ -51,196 +74,97 @@ type Error struct {
|
||||
commit string
|
||||
}
|
||||
|
||||
func (e *Error) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
func (e *Error) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func (e *Error) Commit() string {
|
||||
return e.commit
|
||||
}
|
||||
|
||||
// goGitMu serializes go-git repository access across the process. go-git is not safe for
|
||||
// concurrent use of the same repository (even read access decodes packfiles into shared
|
||||
// state), so parallel jobs inspecting the shared workdir repo race without this. The guarded
|
||||
// operations are fast local reads; gitea runs one job per process, so the lock is effectively
|
||||
// uncontended in production.
|
||||
var goGitMu sync.Mutex
|
||||
func (e *Error) Error() string { return e.err.Error() }
|
||||
func (e *Error) Unwrap() error { return e.err }
|
||||
func (e *Error) Commit() string { return e.commit }
|
||||
|
||||
// FindGitRevision get the current git revision
|
||||
func FindGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
return findGitRevision(ctx, file)
|
||||
}
|
||||
|
||||
func findGitRevision(ctx context.Context, file string) (shortSha, sha string, err error) {
|
||||
func FindGitRevision(ctx context.Context, file string) (string, error) {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
gitDir, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
sha, err := gitOutput(ctx, file, "rev-parse", "--verify", "HEAD^{commit}")
|
||||
if err != nil {
|
||||
logger.WithError(err).Error("path", file, "not located inside a git repository")
|
||||
return "", "", err
|
||||
return "", err
|
||||
}
|
||||
|
||||
head, err := gitDir.Reference(plumbing.HEAD, true)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
if len(sha) < 7 {
|
||||
return "", errors.New("HEAD could not be resolved")
|
||||
}
|
||||
|
||||
if head.Hash().IsZero() {
|
||||
return "", "", errors.New("HEAD sha1 could not be resolved")
|
||||
}
|
||||
|
||||
hash := head.Hash().String()
|
||||
|
||||
logger.Debugf("Found revision: %s", hash)
|
||||
return hash[:7], strings.TrimSpace(hash), nil
|
||||
logger.Debugf("Found revision: %s", sha)
|
||||
return sha, nil
|
||||
}
|
||||
|
||||
// FindGitRef get the current git ref
|
||||
func FindGitRef(ctx context.Context, file string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
logger.Debugf("Loading revision from git directory")
|
||||
_, ref, err := findGitRevision(ctx, file)
|
||||
revision, err := FindGitRevision(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
logger.Debugf("HEAD points to '%s'", revision)
|
||||
|
||||
logger.Debugf("HEAD points to '%s'", ref)
|
||||
|
||||
// Prefer the git library to iterate over the references and find a matching tag or branch.
|
||||
refTag := ""
|
||||
refBranch := ""
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
refs, err := gitOutput(ctx, file, "for-each-ref", "--points-at=HEAD", "--format=%(refname)", "refs/tags", "refs/heads")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
iter, err := repo.References()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// find the reference that matches the revision's has
|
||||
err = iter.ForEach(func(r *plumbing.Reference) error {
|
||||
/* tags and branches will have the same hash
|
||||
* when a user checks out a tag, it is not mentioned explicitly
|
||||
* in the go-git package, we must identify the revision
|
||||
* then check if any tag matches that revision,
|
||||
* if so then we checked out a tag
|
||||
* else we look for branches and if matches,
|
||||
* it means we checked out a branch
|
||||
*
|
||||
* If a branches matches first we must continue and check all tags (all references)
|
||||
* in case we match with a tag later in the interation
|
||||
*/
|
||||
if r.Hash().String() == ref {
|
||||
if r.Name().IsTag() {
|
||||
refTag = r.Name().String()
|
||||
}
|
||||
if r.Name().IsBranch() {
|
||||
refBranch = r.Name().String()
|
||||
}
|
||||
var tag, branch string
|
||||
for ref := range strings.FieldsSeq(refs) {
|
||||
switch {
|
||||
case strings.HasPrefix(ref, "refs/tags/"):
|
||||
tag = ref
|
||||
case strings.HasPrefix(ref, "refs/heads/"):
|
||||
branch = ref
|
||||
}
|
||||
|
||||
// we found what we where looking for
|
||||
if refTag != "" && refBranch != "" {
|
||||
return storer.ErrStop
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// order matters here see above comment.
|
||||
if refTag != "" {
|
||||
return refTag, nil
|
||||
if tag != "" {
|
||||
return tag, nil
|
||||
}
|
||||
if refBranch != "" {
|
||||
return refBranch, nil
|
||||
if branch != "" {
|
||||
return branch, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref)
|
||||
return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", revision)
|
||||
}
|
||||
|
||||
// FindGithubRepo get the repo
|
||||
func FindGithubRepo(ctx context.Context, file, githubInstance string) (string, error) {
|
||||
goGitMu.Lock()
|
||||
defer goGitMu.Unlock()
|
||||
|
||||
url, err := findGitRemoteURL(ctx, file, "origin")
|
||||
func FindGithubRepo(ctx context.Context, file string) (string, error) {
|
||||
url, err := findGitRemoteURL(ctx, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, slug := findGitSlug(url, githubInstance)
|
||||
return slug, nil
|
||||
return findGitSlug(url)
|
||||
}
|
||||
|
||||
func findGitRemoteURL(_ context.Context, file, remoteName string) (string, error) {
|
||||
repo, err := git.PlainOpenWithOptions(
|
||||
file,
|
||||
&git.PlainOpenOptions{
|
||||
DetectDotGit: true,
|
||||
EnableDotGitCommonDir: true,
|
||||
},
|
||||
)
|
||||
func findGitRemoteURL(ctx context.Context, file string) (string, error) {
|
||||
urls, err := gitOutput(ctx, file, "config", "--get-all", "remote.origin.url") // `remote get-url` applies insteadOf
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
remote, err := repo.Remote(remoteName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
if urls == "" {
|
||||
return "", errors.New("remote 'origin' exists but has no URL")
|
||||
}
|
||||
|
||||
if len(remote.Config().URLs) < 1 {
|
||||
return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName)
|
||||
}
|
||||
|
||||
return remote.Config().URLs[0], nil
|
||||
url, _, _ := strings.Cut(urls, "\n")
|
||||
return url, nil
|
||||
}
|
||||
|
||||
func findGitSlug(url, githubInstance string) (string, string) {
|
||||
if matches := codeCommitHTTPRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "CodeCommit", matches[2]
|
||||
} else if matches := codeCommitSSHRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "CodeCommit", matches[2]
|
||||
} else if matches := githubHTTPRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
||||
} else if matches := githubSSHRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "GitHub", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
||||
} else if githubInstance != "github.com" {
|
||||
gheHTTPRegex := regexp.MustCompile(fmt.Sprintf(`^https?://%s/(.+)/(.+?)(?:.git)?$`, githubInstance))
|
||||
gheSSHRegex := regexp.MustCompile(githubInstance + "[:/](.+)/(.+?)(?:.git)?$")
|
||||
if matches := gheHTTPRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
||||
} else if matches := gheSSHRegex.FindStringSubmatch(url); matches != nil {
|
||||
return "GitHubEnterprise", fmt.Sprintf("%s/%s", matches[1], matches[2])
|
||||
}
|
||||
func findGitSlug(remoteURL string) (string, error) {
|
||||
if matches := codeCommitHTTPRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||
return matches[2], nil
|
||||
} else if matches := codeCommitSSHRegex.FindStringSubmatch(remoteURL); matches != nil {
|
||||
return matches[2], nil
|
||||
}
|
||||
return "", url
|
||||
if host, repoPath, ok := strings.Cut(remoteURL, ":"); ok && filepath.VolumeName(remoteURL) == "" && !strings.Contains(host, "/") && !strings.HasPrefix(repoPath, "//") {
|
||||
remoteURL = "ssh://" + host + "/" + repoPath // scp-like [user@]host:path
|
||||
}
|
||||
parsed, err := url.Parse(remoteURL)
|
||||
if err != nil {
|
||||
return "", errors.New("cannot parse the origin remote URL")
|
||||
}
|
||||
owner, repo := path.Split(strings.TrimSuffix(strings.Trim(parsed.Path, "/"), ".git"))
|
||||
owner = path.Base(owner)
|
||||
if parsed.Host == "" || repo == "" || owner == "." || owner == "/" {
|
||||
return "", errors.New("cannot determine owner and repository from the origin remote URL")
|
||||
}
|
||||
return owner + "/" + repo, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutorInput the input for the NewGitCloneExecutor
|
||||
@@ -255,277 +179,179 @@ type NewGitCloneExecutorInput struct {
|
||||
// 0 for full clone.
|
||||
Depth int
|
||||
|
||||
// Quiet drops the informational clone line to debug level, for callers that log their own
|
||||
// download summary (the setup section's action report).
|
||||
Quiet bool
|
||||
|
||||
// For Gitea
|
||||
InsecureSkipTLS bool
|
||||
InsecureSkipTLS bool
|
||||
ClientCertFile, ClientKeyFile string // PEM, absolute as git runs in other directories
|
||||
Transport http.RoundTripper // for FetchAction's archive downloads, nil for http.DefaultTransport
|
||||
}
|
||||
|
||||
// CloneIfRequired returns the repository and a boolean indicating whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input NewGitCloneExecutorInput, logger log.FieldLogger) (*git.Repository, bool, error) {
|
||||
r, err := git.PlainOpen(input.Dir)
|
||||
// CloneIfRequired reports whether an existing local clone was reused.
|
||||
func CloneIfRequired(ctx context.Context, input NewGitCloneExecutorInput, logger *logging.Logger) (bool, error) {
|
||||
var origin string
|
||||
_, err := os.Stat(filepath.Join(input.Dir, ".git")) // without it git would discover a repository around Dir
|
||||
if err == nil {
|
||||
// Verify the cached clone still points to the resolved URL before reusing it.
|
||||
remote, err := r.Remote("origin")
|
||||
if err == nil && len(remote.Config().URLs) > 0 && remote.Config().URLs[0] == input.URL {
|
||||
// Reuse existing clone
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
case len(remote.Config().URLs) == 0:
|
||||
logger.Debugf("Removing cached clone at %s because origin has no URL", input.Dir)
|
||||
default:
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, remote.Config().URLs[0], input.URL)
|
||||
}
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return nil, false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
origin, err = findGitRemoteURL(ctx, input.Dir)
|
||||
}
|
||||
|
||||
var progressWriter io.Writer
|
||||
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
|
||||
if entry, ok := logger.(*log.Entry); ok {
|
||||
progressWriter = entry.WriterLevel(log.DebugLevel)
|
||||
} else if lgr, ok := logger.(*log.Logger); ok {
|
||||
progressWriter = lgr.WriterLevel(log.DebugLevel)
|
||||
} else {
|
||||
log.Errorf("Unable to get writer from logger (type=%T)", logger)
|
||||
progressWriter = os.Stdout
|
||||
}
|
||||
if err == nil && origin == input.URL {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
cloneOptions := git.CloneOptions{
|
||||
URL: input.URL,
|
||||
Progress: progressWriter,
|
||||
|
||||
InsecureSkipTLS: input.InsecureSkipTLS, // For Gitea
|
||||
if ctxErr := ctx.Err(); ctxErr != nil { // a cancelled origin lookup must not discard a valid clone
|
||||
return false, ctxErr
|
||||
}
|
||||
if input.Token != "" {
|
||||
cloneOptions.Auth = &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: input.Token,
|
||||
}
|
||||
if err == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin URL changed from %s to %s", input.Dir, origin, input.URL)
|
||||
} else if _, statErr := os.Stat(input.Dir); statErr == nil {
|
||||
logger.Debugf("Removing cached clone at %s because origin cannot be read: %v", input.Dir, err)
|
||||
}
|
||||
|
||||
r, err = cloneAtDepth(ctx, input, cloneOptions, logger)
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to clone %v %s: %v", input.URL, refName, err)
|
||||
return nil, false, err
|
||||
if err := os.RemoveAll(input.Dir); err != nil {
|
||||
return false, fmt.Errorf("remove cached clone %s: %w", input.Dir, err)
|
||||
}
|
||||
|
||||
if err = os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return nil, false, err
|
||||
if err := clone(ctx, input); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return r, false, nil
|
||||
}
|
||||
|
||||
func gitOptions(token string) (fetchOptions git.FetchOptions, pullOptions git.PullOptions) {
|
||||
fetchOptions.RefSpecs = []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}
|
||||
fetchOptions.Force = true
|
||||
pullOptions.Force = true
|
||||
|
||||
if token != "" {
|
||||
auth := &http.BasicAuth{
|
||||
Username: "token",
|
||||
Password: token,
|
||||
}
|
||||
fetchOptions.Auth = auth
|
||||
pullOptions.Auth = auth
|
||||
if err := os.Chmod(input.Dir, 0o755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return fetchOptions, pullOptions
|
||||
}
|
||||
|
||||
// staleRefreshErr reports why a failed refresh must abort: the resolve and
|
||||
// checkout that follow are local and succeed on a cancelled context, which
|
||||
// would hand back the cached revision as if it were fresh.
|
||||
func staleRefreshErr(ctx context.Context, err error) error {
|
||||
if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
return nil
|
||||
}
|
||||
return ctx.Err()
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// NewGitCloneExecutor creates an executor to clone git repos
|
||||
func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
if input.Quiet {
|
||||
logger.Debugf("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||
} else {
|
||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||
}
|
||||
logger.Infof("git clone '%s' # ref=%s", input.URL, input.Ref)
|
||||
logger.Debugf(" cloning %s to %s", input.URL, input.Dir)
|
||||
|
||||
defer AcquireCloneLock(input.Dir)()
|
||||
|
||||
refName := plumbing.ReferenceName("refs/heads/" + input.Ref)
|
||||
r, reused, err := CloneIfRequired(ctx, refName, input, logger)
|
||||
unlock, err := AcquireCloneLock(ctx, input.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
isOfflineMode := input.OfflineMode
|
||||
|
||||
// fetch latest changes
|
||||
fetchOptions, pullOptions := gitOptions(input.Token)
|
||||
|
||||
if input.InsecureSkipTLS { // For Gitea
|
||||
fetchOptions.InsecureSkipTLS = true
|
||||
pullOptions.InsecureSkipTLS = true
|
||||
}
|
||||
|
||||
// Action clones only ever need the tip commit, so keep a shallow cache cheap on update at depth 1 regardless of its original depth
|
||||
// Turning action_shallow_clone off does not convert an existing shallow cache; evict it for a full clone.
|
||||
shallow := isShallow(r)
|
||||
if shallow {
|
||||
fetchOptions.Depth = 1
|
||||
if spec, ok := shallowFetchRefSpec(r, input.Ref); ok {
|
||||
fetchOptions.RefSpecs = []config.RefSpec{spec}
|
||||
}
|
||||
}
|
||||
|
||||
// A just-cloned ref is as current as a fetch would make it, and a commit hash never moves.
|
||||
_, _, present := refRevision(r, input.Ref)
|
||||
refresh := !isOfflineMode && (!present || (reused && !plumbing.IsHash(input.Ref)))
|
||||
|
||||
if refresh {
|
||||
err = r.FetchContext(ctx, &fetchOptions)
|
||||
if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var hash *plumbing.Hash
|
||||
rev := plumbing.Revision(input.Ref)
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
// ResolveRevision returns a nil hash on error, and a branch ref legitimately fails
|
||||
// here (no local refs/heads/<ref>); the duck-typing below resolves it.
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
} else if hash.String() != input.Ref && strings.HasPrefix(hash.String(), input.Ref) {
|
||||
return &Error{
|
||||
err: ErrShortRef,
|
||||
commit: hash.String(),
|
||||
}
|
||||
}
|
||||
|
||||
rev, refType, _ := refRevision(r, input.Ref)
|
||||
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
resolved, reused, err := resolveClone(ctx, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var w *git.Worktree
|
||||
if w, err = r.Worktree(); err != nil {
|
||||
return err
|
||||
if _, err := runGit(ctx, input.Dir, nil, "checkout", "--force", "--detach", resolved); err != nil {
|
||||
return fmt.Errorf("checkout %s: %w", resolved, err)
|
||||
}
|
||||
|
||||
// If the hash resolved doesn't match the ref provided in a workflow then we're
|
||||
// using a branch or tag ref, not a sha
|
||||
//
|
||||
// Repos on disk point to commit hashes, and need to checkout input.Ref before
|
||||
// we try and pull down any changes
|
||||
if hash.String() != input.Ref && refType == "branch" {
|
||||
logger.Debugf("Provided ref is not a sha. Checking out branch before pulling changes")
|
||||
sourceRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", input.Ref))
|
||||
if err = w.Checkout(&git.CheckoutOptions{
|
||||
Branch: sourceRef,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
logger.Errorf("Unable to checkout %s: %v", sourceRef, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
reusedMsg := ""
|
||||
|
||||
switch {
|
||||
case refresh && !shallow:
|
||||
// In shallow mode the depth-limited fetch above already advanced the ref.
|
||||
if err = w.PullContext(ctx, &pullOptions); err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
logger.Debugf("Unable to pull %s: %v", refName, err)
|
||||
}
|
||||
if err := staleRefreshErr(ctx, err); err != nil {
|
||||
return err
|
||||
}
|
||||
case isOfflineMode && reused:
|
||||
if input.OfflineMode && reused {
|
||||
reusedMsg = " (reused in offline mode)"
|
||||
}
|
||||
|
||||
logger.Debugf("Cloned %s to %s%s", input.URL, input.Dir, reusedMsg)
|
||||
|
||||
if hash.String() != input.Ref && refType == "branch" {
|
||||
logger.Debugf("Provided ref is not a sha. Updating branch ref after pull")
|
||||
if hash, err = r.ResolveRevision(rev); err != nil {
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = w.Checkout(&git.CheckoutOptions{
|
||||
Hash: *hash,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
logger.Errorf("Unable to checkout %s: %v", *hash, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err = w.Reset(&git.ResetOptions{
|
||||
Mode: git.HardReset,
|
||||
Commit: *hash,
|
||||
}); err != nil {
|
||||
logger.Errorf("Unable to reset to %s: %v", hash.String(), err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Checked out %s", input.Ref)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// cloneAtDepth falls back to a full clone when the shallow attempt fails.
|
||||
func cloneAtDepth(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions, logger log.FieldLogger) (*git.Repository, error) {
|
||||
if input.Depth > 0 {
|
||||
if plumbing.IsHash(input.Ref) {
|
||||
r, err := fetchPinnedSHA(ctx, input, opts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow fetch of %s at %s failed: %v", input.URL, input.Ref, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
for _, refName := range []plumbing.ReferenceName{
|
||||
plumbing.NewBranchReferenceName(input.Ref),
|
||||
plumbing.NewTagReferenceName(input.Ref),
|
||||
} {
|
||||
shallowOpts := opts
|
||||
shallowOpts.Depth = input.Depth
|
||||
shallowOpts.SingleBranch = true
|
||||
shallowOpts.ReferenceName = refName
|
||||
shallowOpts.Tags = git.NoTags
|
||||
func resolveClone(ctx context.Context, input NewGitCloneExecutorInput) (string, bool, error) { // the caller holds input.Dir's lock
|
||||
logger := common.Logger(ctx)
|
||||
reused, err := CloneIfRequired(ctx, input, logger)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
resolved, err := resolveRef(ctx, input.Dir, input.Ref)
|
||||
if errors.Is(err, ErrShortRef) || (err != nil && input.OfflineMode) {
|
||||
return "", false, err
|
||||
}
|
||||
if !input.OfflineMode && (err != nil || (reused && !isFullHash(input.Ref, resolved))) {
|
||||
if err := fetch(ctx, input); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
resolved, err = resolveRef(ctx, input.Dir, input.Ref)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("Unable to resolve %s: %v", input.Ref, err)
|
||||
}
|
||||
return resolved, reused, err
|
||||
}
|
||||
|
||||
r, err := git.PlainCloneContext(ctx, input.Dir, false, &shallowOpts)
|
||||
if err == nil {
|
||||
return r, nil
|
||||
}
|
||||
logger.Debugf("Shallow clone of %s as %s failed: %v", input.URL, refName, err)
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func clone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if input.Depth > 0 && isFullObjectID(input.Ref) {
|
||||
if err := shallowPinnedClone(ctx, input); err == nil {
|
||||
return nil
|
||||
} else if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
args := []string{"clone", "--no-checkout"}
|
||||
if input.Depth > 0 {
|
||||
args = append(args, "--no-local", "--depth", strconv.Itoa(input.Depth), "--no-tags")
|
||||
if input.Ref != "HEAD" && !strings.HasPrefix(input.Ref, "refs/") {
|
||||
args = append(args, "--branch", input.Ref)
|
||||
}
|
||||
}
|
||||
args = append(args, input.URL, input.Dir)
|
||||
if _, err := runGit(ctx, "", &input, args...); err == nil {
|
||||
return nil
|
||||
} else if input.Depth == 0 {
|
||||
return err
|
||||
}
|
||||
if err := removePartialClone(input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, "", &input, "clone", "--no-checkout", input.URL, input.Dir)
|
||||
return err
|
||||
}
|
||||
|
||||
func shallowPinnedClone(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
if _, err := runGit(ctx, "", &input, "init", "--object-format="+ObjectFormat(input.Ref), input.Dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := runGit(ctx, input.Dir, &input, "remote", "add", "origin", input.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, "fetch", "--depth", strconv.Itoa(input.Depth), "--no-tags", "origin", "+"+input.Ref+":refs/pinned/"+input.Ref)
|
||||
return err
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, input NewGitCloneExecutorInput) error {
|
||||
args := []string{"fetch", "--force", "--no-prune"}
|
||||
if _, err := os.Stat(filepath.Join(input.Dir, ".git", "shallow")); err == nil {
|
||||
args = append(args, "--depth", "1", "--no-tags")
|
||||
}
|
||||
_, err := runGit(ctx, input.Dir, &input, append(args, "origin", "+"+input.Ref+":"+cachedRef(input.Ref))...) // git picks the remote ref, a tag before a branch
|
||||
return err
|
||||
}
|
||||
|
||||
func cachedRef(ref string) string {
|
||||
return fmt.Sprintf("refs/runner/%x", sha256.Sum256([]byte(ref)))
|
||||
}
|
||||
|
||||
func resolveRef(ctx context.Context, dir, ref string) (string, error) {
|
||||
if !isFullObjectID(ref) {
|
||||
for _, name := range []string{cachedRef(ref), "refs/tags/" + ref, "refs/remotes/origin/" + ref} {
|
||||
if sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", name+"^{commit}"); err == nil {
|
||||
return sha, nil
|
||||
}
|
||||
}
|
||||
logger.Debugf("Falling back to a full clone of %s for ref %q", input.URL, input.Ref)
|
||||
}
|
||||
sha, err := gitOutput(ctx, dir, "rev-parse", "--verify", ref+"^{commit}")
|
||||
if err == nil && hexRefRegex.MatchString(ref) && len(sha) != len(ref) {
|
||||
return "", &Error{err: ErrShortRef, commit: sha}
|
||||
}
|
||||
return sha, err
|
||||
}
|
||||
|
||||
return git.PlainCloneContext(ctx, input.Dir, false, &opts)
|
||||
func isFullObjectID(ref string) bool {
|
||||
return ObjectFormat(ref) != ""
|
||||
}
|
||||
|
||||
func isFullHash(ref, resolved string) bool {
|
||||
return isFullObjectID(ref) && len(ref) == len(resolved)
|
||||
}
|
||||
|
||||
// ObjectFormat returns the object format of a full commit hash, empty for anything else.
|
||||
func ObjectFormat(sha string) string {
|
||||
switch {
|
||||
case !hexRefRegex.MatchString(sha):
|
||||
return ""
|
||||
case len(sha) == 40:
|
||||
return "sha1"
|
||||
case len(sha) == 64:
|
||||
return "sha256"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func removePartialClone(dir string) error {
|
||||
@@ -535,71 +361,100 @@ func removePartialClone(dir string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchPinnedSHA(ctx context.Context, input NewGitCloneExecutorInput, opts git.CloneOptions) (*git.Repository, error) {
|
||||
r, err := git.PlainInit(input.Dir, false)
|
||||
func gitOutput(ctx context.Context, dir string, args ...string) (string, error) {
|
||||
output, err := runGit(ctx, dir, nil, args...)
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
func RemoteConfig(token string, insecureSkipTLS bool) []string {
|
||||
var config []string
|
||||
if insecureSkipTLS {
|
||||
config = append(config, "http.sslVerify=false")
|
||||
}
|
||||
if token != "" {
|
||||
config = append(config, "http.extraHeader=", "http.extraHeader=Authorization: Basic "+base64.StdEncoding.EncodeToString([]byte("token:"+token)))
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
// ConfigEnv applies `-c` style entries through the environment, keeping secrets out of argv.
|
||||
func ConfigEnv(config ...string) map[string]string {
|
||||
env := map[string]string{"GIT_CONFIG_COUNT": strconv.Itoa(len(config))}
|
||||
for i, entry := range config {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
env["GIT_CONFIG_KEY_"+strconv.Itoa(i)] = key
|
||||
env["GIT_CONFIG_VALUE_"+strconv.Itoa(i)] = value
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
func ListFiles(ctx context.Context, dir string) ([]string, error) {
|
||||
listed, err := runGit(ctx, dir, nil, "ls-files", "-z", "-t", "--stage", "--others", "--exclude-standard")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{input.URL}}); err != nil {
|
||||
return nil, err
|
||||
var files, repos []string
|
||||
for entry := range strings.SplitSeq(string(listed), "\x00") {
|
||||
tag, file, ok := strings.Cut(entry, " ")
|
||||
if tag != "?" {
|
||||
var meta string
|
||||
meta, file, ok = strings.Cut(file, "\t")
|
||||
if strings.HasPrefix(meta, "160000 ") {
|
||||
file += "/"
|
||||
}
|
||||
}
|
||||
if repo, isRepo := strings.CutSuffix(file, "/"); isRepo {
|
||||
repos = append(repos, repo)
|
||||
} else if ok {
|
||||
files = append(files, file)
|
||||
}
|
||||
}
|
||||
if err := r.FetchContext(ctx, &git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{pinnedRefSpec(input.Ref)},
|
||||
Depth: input.Depth,
|
||||
Tags: git.NoTags,
|
||||
Auth: opts.Auth,
|
||||
Progress: opts.Progress,
|
||||
InsecureSkipTLS: opts.InsecureSkipTLS,
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
slices.Sort(repos)
|
||||
for _, repo := range slices.Compact(repos) {
|
||||
files = append(files, repo)
|
||||
if _, err := os.Lstat(filepath.Join(dir, repo, ".git")); err != nil {
|
||||
continue // git lists an uninitialized submodule as "./" from inside it
|
||||
}
|
||||
nested, err := ListFiles(ctx, filepath.Join(dir, repo))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, file := range nested {
|
||||
files = append(files, repo+"/"+file)
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// pinnedRefSpec keeps a hash-shaped name out of refs/heads and refs/tags.
|
||||
func pinnedRefSpec(sha string) config.RefSpec {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:refs/pinned/%s", sha, sha))
|
||||
}
|
||||
|
||||
// refRevision picks the revision to check out and reports whether it resolves locally.
|
||||
func refRevision(r *git.Repository, ref string) (plumbing.Revision, string, bool) {
|
||||
if plumbing.IsHash(ref) {
|
||||
// git ignores a ref named as 40 hex digits, so a full hash always denotes the commit itself.
|
||||
_, err := r.CommitObject(plumbing.NewHash(ref))
|
||||
return plumbing.Revision(ref), "sha", err == nil
|
||||
}
|
||||
if _, err := r.Tag(ref); err == nil {
|
||||
return plumbing.Revision(path.Join("refs", "tags", ref)), "tag", true
|
||||
}
|
||||
remoteRef := plumbing.ReferenceName(path.Join("refs", "remotes", "origin", ref))
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
return plumbing.Revision(remoteRef), "branch", true
|
||||
}
|
||||
rev := plumbing.Revision(ref)
|
||||
_, err := r.ResolveRevision(rev)
|
||||
return rev, "sha", err == nil
|
||||
}
|
||||
|
||||
// isShallow reports whether the local repository was cloned with a limited depth.
|
||||
func isShallow(r *git.Repository) bool {
|
||||
shallows, err := r.Storer.Shallow()
|
||||
return err == nil && len(shallows) > 0
|
||||
}
|
||||
|
||||
// shallowFetchRefSpec limits a shallow update to the requested ref, falling back to the broad refspec when it is not present locally.
|
||||
func shallowFetchRefSpec(r *git.Repository, ref string) (config.RefSpec, bool) {
|
||||
tagRef := plumbing.NewTagReferenceName(ref)
|
||||
if _, err := r.Reference(tagRef, false); err == nil {
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", tagRef, tagRef)), true
|
||||
}
|
||||
remoteRef := plumbing.NewRemoteReferenceName("origin", ref)
|
||||
if _, err := r.Reference(remoteRef, false); err == nil {
|
||||
branchRef := plumbing.NewBranchReferenceName(ref)
|
||||
return config.RefSpec(fmt.Sprintf("+%s:%s", branchRef, remoteRef)), true
|
||||
}
|
||||
if plumbing.IsHash(ref) {
|
||||
// The broad refspec carries only advertised tips, never a pinned commit.
|
||||
return pinnedRefSpec(ref), true
|
||||
}
|
||||
return "", false
|
||||
func runGit(ctx context.Context, dir string, input *NewGitCloneExecutorInput, args ...string) ([]byte, error) {
|
||||
config := []string{"core.autocrlf=false", "core.eol=lf"} // keep action files LF
|
||||
if input != nil {
|
||||
config = append(config, RemoteConfig(input.Token, input.InsecureSkipTLS)...)
|
||||
if input.ClientCertFile != "" {
|
||||
config = append(config, "http.sslCert="+input.ClientCertFile, "http.sslKey="+input.ClientKeyFile)
|
||||
}
|
||||
}
|
||||
if dir != "" {
|
||||
args = append([]string{"-C", dir}, args...)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Env = slices.DeleteFunc(os.Environ(), func(entry string) bool {
|
||||
name, _, _ := strings.Cut(entry, "=")
|
||||
return slices.Contains(LocalEnvVars, name)
|
||||
})
|
||||
for key, value := range ConfigEnv(config...) {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0")
|
||||
if runtime.GOOS != "windows" {
|
||||
cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } // lets git remove its lock files
|
||||
}
|
||||
cmd.WaitDelay = time.Second
|
||||
var stderr strings.Builder
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return output, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ package git
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -16,48 +16,47 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
gogit "github.com/go-git/go-git/v5"
|
||||
gogitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
gogitclient "github.com/go-git/go-git/v5/plumbing/transport/client"
|
||||
gogitfile "github.com/go-git/go-git/v5/plumbing/transport/file"
|
||||
log "github.com/sirupsen/logrus"
|
||||
logrustest "github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFindGitSlug(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
slugTests := []struct {
|
||||
url string // input
|
||||
provider string // expected result
|
||||
slug string // expected result
|
||||
for _, testcase := range []struct {
|
||||
remoteURL string
|
||||
slug string
|
||||
}{
|
||||
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "CodeCommit", "my-repo-name"},
|
||||
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "CodeCommit", "my-repo"},
|
||||
{"git@github.com:nektos/act.git", "GitHub", "nektos/act"},
|
||||
{"git@github.com:nektos/act", "GitHub", "nektos/act"},
|
||||
{"https://github.com/nektos/act.git", "GitHub", "nektos/act"},
|
||||
{"http://github.com/nektos/act.git", "GitHub", "nektos/act"},
|
||||
{"https://github.com/nektos/act", "GitHub", "nektos/act"},
|
||||
{"http://github.com/nektos/act", "GitHub", "nektos/act"},
|
||||
{"git+ssh://git@github.com/owner/repo.git", "GitHub", "owner/repo"},
|
||||
{"http://myotherrepo.com/act.git", "", "http://myotherrepo.com/act.git"},
|
||||
}
|
||||
|
||||
for _, tt := range slugTests {
|
||||
provider, slug := findGitSlug(tt.url, "github.com")
|
||||
assert.Equal(tt.provider, provider)
|
||||
assert.Equal(tt.slug, slug)
|
||||
{"https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name", "my-repo-name"},
|
||||
{"ssh://git-codecommit.us-west-2.amazonaws.com/v1/repos/my-repo", "my-repo"},
|
||||
{"git@github.com:nektos/act.git", "nektos/act"},
|
||||
{"git@github.com:nektos/act", "nektos/act"},
|
||||
{"https://github.com/nektos/act.git", "nektos/act"},
|
||||
{"http://github.com/nektos/act.git", "nektos/act"},
|
||||
{"https://github.com/nektos/act", "nektos/act"},
|
||||
{"http://github.com/nektos/act", "nektos/act"},
|
||||
{"git+ssh://git@github.com/owner/repo.git", "owner/repo"},
|
||||
{"https://example.com:3000/gitea/owner/repo.git/", "owner/repo"},
|
||||
{"ssh://git@example.com:2222/gitea/owner/repo.git", "owner/repo"},
|
||||
{"example.com:owner/repo.git", "owner/repo"},
|
||||
{"http://myotherrepo.com/act.git", ""},
|
||||
{"https://user:secret@example.com/act.git", ""},
|
||||
{"/tmp/owner/repo.git", ""},
|
||||
{"./owner/repo:branch", ""},
|
||||
} {
|
||||
t.Run(testcase.remoteURL, func(t *testing.T) {
|
||||
slug, err := findGitSlug(testcase.remoteURL)
|
||||
assert.Equal(t, testcase.slug == "", err != nil)
|
||||
assert.NotContains(t, fmt.Sprint(err), "secret")
|
||||
assert.Equal(t, testcase.slug, slug)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,143 +67,118 @@ func TestErrorWrapsCommitAndCause(t *testing.T) {
|
||||
require.Equal(t, "abc123", err.Commit())
|
||||
}
|
||||
|
||||
func cleanGitHooks(dir string) error {
|
||||
hooksDir := filepath.Join(dir, ".git", "hooks")
|
||||
files, err := os.ReadDir(hooksDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.IsDir() {
|
||||
continue
|
||||
}
|
||||
relName := filepath.Join(hooksDir, f.Name())
|
||||
if err := os.Remove(relName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFindGithubRepoUsesOrigin(t *testing.T) {
|
||||
func TestFindGitMetadataOfSHA256RepositoryUsesOrigin(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
const remoteURL = "https://github.com/owner/repo.git"
|
||||
require.NoError(t, gitCmd("init", basedir))
|
||||
require.NoError(t, cleanGitHooks(basedir))
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", basedir))
|
||||
require.NoError(t, os.RemoveAll(filepath.Join(basedir, ".git", "hooks")))
|
||||
require.NoError(t, gitCmd("-C", basedir, "commit", "--allow-empty", "-m", "init"))
|
||||
require.NoError(t, gitCmd("-C", basedir, "remote", "add", "origin", remoteURL))
|
||||
require.NoError(t, gitCmd("-C", basedir, "config", "--add", "remote.origin.url", "https://github.com/other/repo.git"))
|
||||
|
||||
url, err := findGitRemoteURL(context.Background(), basedir, "origin")
|
||||
sha, err := FindGitRevision(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, gitRevParse(t, basedir, "HEAD"), sha)
|
||||
require.Len(t, sha, 64)
|
||||
|
||||
ref, err := FindGitRef(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/main", ref)
|
||||
|
||||
url, err := findGitRemoteURL(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, remoteURL, url)
|
||||
|
||||
slug, err := FindGithubRepo(context.Background(), basedir, "github.com")
|
||||
slug, err := FindGithubRepo(t.Context(), basedir)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "owner/repo", slug)
|
||||
}
|
||||
|
||||
func TestGitFindRef(t *testing.T) {
|
||||
t.Parallel()
|
||||
basedir := t.TempDir()
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Prepare func(t *testing.T, dir string)
|
||||
Assert func(t *testing.T, ref string, err error)
|
||||
commands [][]string
|
||||
ref string
|
||||
}{
|
||||
"new_repo": {
|
||||
Prepare: func(t *testing.T, dir string) {},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.Error(t, err)
|
||||
},
|
||||
},
|
||||
"new_repo": {},
|
||||
"new_repo_with_commit": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/master", ref)
|
||||
},
|
||||
commands: [][]string{{"commit", "--allow-empty", "-m", "msg"}},
|
||||
ref: "refs/heads/master",
|
||||
},
|
||||
"current_head_is_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "commit msg"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.2.3"))
|
||||
require.NoError(t, gitCmd("-C", dir, "checkout", "v1.2.3"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/tags/v1.2.3", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "commit msg"},
|
||||
{"tag", "v1.2.3"},
|
||||
{"checkout", "v1.2.3"},
|
||||
},
|
||||
ref: "refs/tags/v1.2.3",
|
||||
},
|
||||
"current_head_is_same_as_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "1.4.2 release"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/tags/v1.4.2", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "1.4.2 release"},
|
||||
{"tag", "v1.4.2"},
|
||||
},
|
||||
ref: "refs/tags/v1.4.2",
|
||||
},
|
||||
"current_head_is_not_tag": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
require.NoError(t, gitCmd("-C", dir, "tag", "v1.4.2"))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg2"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/master", ref)
|
||||
commands: [][]string{
|
||||
{"commit", "--allow-empty", "-m", "msg"},
|
||||
{"tag", "v1.4.2"},
|
||||
{"commit", "--allow-empty", "-m", "msg2"},
|
||||
},
|
||||
ref: "refs/heads/master",
|
||||
},
|
||||
"current_head_is_another_branch": {
|
||||
Prepare: func(t *testing.T, dir string) {
|
||||
require.NoError(t, gitCmd("-C", dir, "checkout", "-b", "mybranch"))
|
||||
require.NoError(t, gitCmd("-C", dir, "commit", "--allow-empty", "-m", "msg"))
|
||||
},
|
||||
Assert: func(t *testing.T, ref string, err error) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "refs/heads/mybranch", ref)
|
||||
commands: [][]string{
|
||||
{"checkout", "-b", "mybranch"},
|
||||
{"commit", "--allow-empty", "-m", "msg"},
|
||||
},
|
||||
ref: "refs/heads/mybranch",
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := filepath.Join(basedir, name)
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755))
|
||||
require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master"))
|
||||
require.NoError(t, cleanGitHooks(dir))
|
||||
tt.Prepare(t, dir)
|
||||
ref, err := FindGitRef(context.Background(), dir)
|
||||
tt.Assert(t, ref, err)
|
||||
require.NoError(t, os.RemoveAll(filepath.Join(dir, ".git", "hooks")))
|
||||
for _, command := range tt.commands {
|
||||
require.NoError(t, gitCmd(append([]string{"-C", dir}, command...)...))
|
||||
}
|
||||
ref, err := FindGitRef(t.Context(), dir)
|
||||
if tt.ref == "" {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.ref, ref)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutor(t *testing.T) {
|
||||
// Build a local bare "remote" so this runs offline and fast. The cases below mirror
|
||||
// the tag/branch/sha/short-sha ref paths the executor handles, formerly exercised by
|
||||
// cloning actions/checkout and anchore/scan-action over the network.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
remoteDir, workDir := newGitRemote(t)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workDir, "action.sh"), []byte("#!/bin/sh\necho hi\n"), 0o755))
|
||||
require.NoError(t, gitCmd("-C", workDir, "add", "action.sh"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v2"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v2"))
|
||||
|
||||
// A branch with a dash in the name (mirrors the historical scan-action@act-fails case).
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "act-fails"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "branch-commit"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "act-fails"))
|
||||
|
||||
out, err := exec.Command("git", "-C", workDir, "rev-parse", "main").Output()
|
||||
require.NoError(t, err)
|
||||
fullSha := strings.TrimSpace(string(out))
|
||||
fullSha := gitRevParse(t, workDir, "main")
|
||||
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:refs/heads/"+fullSha[:4]))
|
||||
|
||||
hostConfig := filepath.Join(t.TempDir(), "gitconfig")
|
||||
require.NoError(t, os.WriteFile(hostConfig, []byte("[core]\n\tautocrlf = true\n[url \""+remoteDir+"\"]\n\tinsteadOf = https://example.invalid/action\n"), 0o644))
|
||||
t.Setenv("GIT_CONFIG_GLOBAL", hostConfig)
|
||||
t.Setenv("GIT_DIR", t.TempDir())
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Err error
|
||||
@@ -226,34 +200,44 @@ func TestGitCloneExecutor(t *testing.T) {
|
||||
Err: &Error{ErrShortRef, fullSha},
|
||||
Ref: fullSha[:7],
|
||||
},
|
||||
"HEAD": {
|
||||
Ref: "HEAD",
|
||||
},
|
||||
"hex branch named like its own commit": {
|
||||
Ref: fullSha[:4],
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
URL: "https://example.invalid/action",
|
||||
Ref: tt.Ref,
|
||||
Dir: t.TempDir(),
|
||||
Dir: dir,
|
||||
})
|
||||
|
||||
err := clone(context.Background())
|
||||
err := clone(t.Context())
|
||||
if tt.Err != nil {
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, tt.Err, err)
|
||||
} else {
|
||||
assert.Empty(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, tt.Err, clone(t.Context()), "a retry on the cache keeps the error")
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
marker := filepath.Join(dir, "marker")
|
||||
require.NoError(t, os.WriteFile(marker, nil, 0o644))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, marker, "an insteadOf rewrite must not evict the cache")
|
||||
script, err := os.ReadFile(filepath.Join(dir, "action.sh"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "#!/bin/sh\necho hi\n", string(script), "host core.autocrlf must not convert action files")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
createRemote := func(message string) string {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||
remoteDir, workDir := newGitRemote(t, message)
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
return remoteDir
|
||||
@@ -269,16 +253,27 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
nested := filepath.Join(cacheDir, "nested")
|
||||
require.NoError(t, os.Mkdir(nested, 0o755))
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: nested})(t.Context()))
|
||||
assert.DirExists(t, filepath.Join(nested, ".git"), "an empty cache dir inside a clone of the same URL gets its own repository")
|
||||
|
||||
markerPath := filepath.Join(cacheDir, "stale-marker")
|
||||
require.NoError(t, os.WriteFile(markerPath, []byte("stale"), 0o644))
|
||||
|
||||
cancelled, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
_, err := CloneIfRequired(cancelled, NewGitCloneExecutorInput{URL: oldRemoteDir, Ref: "main", Dir: cacheDir}, logging.New(logtest.New()))
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.FileExists(t, markerPath)
|
||||
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: newRemoteDir,
|
||||
Ref: "main",
|
||||
Dir: cacheDir,
|
||||
})(t.Context()))
|
||||
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir, "origin")
|
||||
originURL, err := findGitRemoteURL(t.Context(), cacheDir)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, newRemoteDir, originURL)
|
||||
|
||||
@@ -290,87 +285,58 @@ func TestGitCloneExecutorReclonesWhenOriginURLChanges(t *testing.T) {
|
||||
require.True(t, os.IsNotExist(err), "stale cached directory should be removed before recloning")
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorNonFastForwardRef(t *testing.T) {
|
||||
// Simulate the scenario where a remote ref (e.g. a GitHub PR head ref) changes
|
||||
// non-fast-forward between two fetches. Before the fix, the fetch used Force=false,
|
||||
// causing go-git to return ErrForceNeeded and short-circuit the checkout.
|
||||
|
||||
// Create a bare "remote" repo with an initial commit on main and a feature branch.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
|
||||
// We need a working clone to push commits from.
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
func TestGitCloneExecutorFollowsRemoteRefChanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "initial", "feature-1")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Create a feature branch (simulates refs/pull/N/head).
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "feature"))
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "main", Dir: dir})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
require.NoError(t, gitCmd("-C", dir, "config", "fetch.prune", "true"))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
|
||||
// First clone via the executor — should succeed and cache the repo.
|
||||
cloneDir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: cloneDir,
|
||||
})
|
||||
require.NoError(t, clone(context.Background()))
|
||||
|
||||
// Now force-push the feature branch to a non-fast-forward commit (simulates
|
||||
// a PR rebase). This makes refs/heads/feature non-fast-forward.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "branch", "-D", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "reset", "--hard", "HEAD~1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "feature-rewritten"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "feature"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "main"))
|
||||
require.NoError(t, clone(t.Context()), "a non-fast-forward ref must update")
|
||||
assert.Equal(t, "feature-rewritten", gitHeadSubject(t, dir))
|
||||
|
||||
// Also advance main so we can verify the clone picks up the new commit.
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "second"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
tagDir := t.TempDir()
|
||||
tagClone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remoteDir, Ref: "v9", Dir: tagDir})
|
||||
require.Error(t, tagClone(t.Context()))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v9"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v9"))
|
||||
require.NoError(t, tagClone(t.Context()), "a tag published after a failed lookup must be fetched")
|
||||
|
||||
// Second clone to the same directory — before the fix this returned ErrForceNeeded
|
||||
// and left the working tree at the old commit.
|
||||
err := clone(context.Background())
|
||||
require.NoError(t, err, "fetch with non-fast-forward refs must not fail when Force=true")
|
||||
|
||||
// Verify the working tree was actually updated to the latest main commit.
|
||||
out, err := exec.Command("git", "-C", cloneDir, "log", "--oneline", "-1", "--format=%s").Output()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "second", strings.TrimSpace(string(out)), "working tree should be at the latest commit")
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "tag-moved"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "--force", "v9"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "--force", "origin", "v9"))
|
||||
for range 2 {
|
||||
require.NoError(t, tagClone(t.Context()), "a moved tag must update and stay current")
|
||||
assert.Equal(t, "tag-moved", gitHeadSubject(t, tagDir))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
// Build a local "remote" with a single commit on main.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "initial")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Prime the cache with an online clone of main.
|
||||
cacheDir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: cacheDir,
|
||||
})(context.Background()))
|
||||
})(t.Context()))
|
||||
|
||||
t.Run("cached branch resolves without fetching", func(t *testing.T) {
|
||||
// Offline reuse of a cached branch must succeed even though ResolveRevision(input.Ref)
|
||||
// finds no local refs/heads/<ref>.
|
||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: cacheDir,
|
||||
OfflineMode: true,
|
||||
})(context.Background())
|
||||
})(t.Context())
|
||||
require.NoError(t, err)
|
||||
|
||||
out, err := exec.Command("git", "-C", cacheDir, "log", "--oneline", "-1", "--format=%s").Output()
|
||||
@@ -379,65 +345,19 @@ func TestGitCloneExecutorOfflineMode(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("unresolvable cached ref returns error", func(t *testing.T) {
|
||||
// The ref was never cached; offline mode cannot resolve it and must return an error.
|
||||
err := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "never-fetched",
|
||||
Dir: cacheDir,
|
||||
OfflineMode: true,
|
||||
})(context.Background())
|
||||
})(t.Context())
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorQuietDemotesCloneLine(t *testing.T) {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "initial"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
|
||||
// Quiet callers report the download themselves, so the clone line must not reach the job log.
|
||||
for name, quiet := range map[string]bool{"quiet": true, "not quiet": false} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
logger, hook := logrustest.NewNullLogger()
|
||||
logger.SetLevel(log.InfoLevel)
|
||||
ctx := common.WithLogger(context.Background(), logger.WithField("job", "j1"))
|
||||
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir,
|
||||
Ref: "main",
|
||||
Dir: t.TempDir(),
|
||||
Quiet: quiet,
|
||||
})(ctx))
|
||||
|
||||
var cloneLines int
|
||||
for _, entry := range hook.AllEntries() {
|
||||
if strings.HasPrefix(entry.Message, "git clone ") {
|
||||
cloneLines++
|
||||
}
|
||||
}
|
||||
if quiet {
|
||||
assert.Zero(t, cloneLines)
|
||||
} else {
|
||||
assert.Equal(t, 1, cloneLines)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
// Build a local "remote" with several commits on main plus a tag, so a full clone would pull noticeably more history than a shallow one.
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
for _, m := range []string{"c1", "c2", "c3"} {
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", m))
|
||||
}
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "c1", "c2", "c3")
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
unadvertisedSHA := gitRevParse(t, workDir, "HEAD~1")
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
@@ -445,25 +365,17 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
|
||||
shallowMarker := func(dir string) string { return filepath.Join(dir, ".git", "shallow") }
|
||||
|
||||
t.Run("branch is cloned shallowly", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
})
|
||||
|
||||
t.Run("tag is cloned shallowly", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "v1", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
})
|
||||
for name, ref := range map[string]string{"branch is cloned shallowly": "main", "tag is cloned shallowly": "v1"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: ref, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir), "clone should be shallow")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir), "only the tip commit should be present")
|
||||
assert.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
})
|
||||
}
|
||||
|
||||
unadvertisedRemote := func(t *testing.T, allowUnadvertised bool) string {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
@@ -472,12 +384,11 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
return remote
|
||||
}
|
||||
|
||||
t.Run("commit hash falls back to a full clone when the remote refuses unadvertised objects", func(t *testing.T) {
|
||||
t.Run("commit hash resolves when the remote accepts a direct shallow fetch", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: unadvertisedRemote(t, false), Ref: unadvertisedSHA, Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
assert.NoFileExists(t, shallowMarker(dir))
|
||||
assert.Equal(t, unadvertisedSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
@@ -504,61 +415,47 @@ func TestGitCloneExecutorShallow(t *testing.T) {
|
||||
assert.Equal(t, olderSHA, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
|
||||
t.Run("moving branch updates while staying shallow", func(t *testing.T) {
|
||||
t.Run("moving hexadecimal-named branch updates while staying shallow", func(t *testing.T) {
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "deadbeef", Dir: dir, Depth: 1,
|
||||
})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
require.Equal(t, "c3", gitHeadSubject(t, dir))
|
||||
|
||||
// Advance main on the remote, then reuse the existing shallow clone.
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c4"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "main:deadbeef"))
|
||||
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: "main", Dir: dir, Depth: 1,
|
||||
})(t.Context()))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, "c4", gitHeadSubject(t, dir), "reused shallow clone should update to the new tip")
|
||||
assert.FileExists(t, shallowMarker(dir), "repo should remain shallow after update")
|
||||
assert.Equal(t, 1, gitRevCount(t, dir))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorColdCloneSkipsRefresh(t *testing.T) {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "c1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", "v1"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "-u", "origin", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "push", "origin", "v1"))
|
||||
|
||||
for name, tt := range map[string]struct {
|
||||
Ref string
|
||||
Depth int
|
||||
}{
|
||||
"shallow branch": {"main", 1},
|
||||
"full clone tag": {"v1", 0},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
counter := installCountingTransport(t)
|
||||
require.NoError(t, NewGitCloneExecutor(NewGitCloneExecutorInput{
|
||||
URL: remoteDir, Ref: tt.Ref, Dir: t.TempDir(), Depth: tt.Depth,
|
||||
})(t.Context()))
|
||||
assert.Equal(t, int64(1), counter.sessions.Load())
|
||||
})
|
||||
}
|
||||
t.Run("sha256 full hash is cloned shallowly, reused without the remote, and its 40-character prefix is short", func(t *testing.T) {
|
||||
remote := filepath.Join(t.TempDir(), "remote.git")
|
||||
require.NoError(t, gitCmd("init", "--bare", "--object-format=sha256", "--initial-branch=main", remote))
|
||||
work := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--object-format=sha256", "--initial-branch=main", work))
|
||||
require.NoError(t, gitCmd("-C", work, "remote", "add", "origin", remote))
|
||||
require.NoError(t, gitCmd("-C", work, "commit", "--allow-empty", "-m", "pinned"))
|
||||
require.NoError(t, gitCmd("-C", work, "push", "origin", "main"))
|
||||
sha := gitRevParse(t, work, "HEAD")
|
||||
dir := t.TempDir()
|
||||
clone := NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha, Dir: dir, Depth: 1})
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.FileExists(t, shallowMarker(dir))
|
||||
require.ErrorIs(t, NewGitCloneExecutor(NewGitCloneExecutorInput{URL: remote, Ref: sha[:40], Dir: dir, Depth: 1, OfflineMode: true})(t.Context()), ErrShortRef)
|
||||
require.NoError(t, os.RemoveAll(remote))
|
||||
require.NoError(t, clone(t.Context()))
|
||||
assert.Equal(t, sha, gitRevParse(t, dir, "HEAD"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
remoteDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
workDir := t.TempDir()
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "pinned"))
|
||||
t.Parallel()
|
||||
remoteDir, workDir := newGitRemote(t, "pinned")
|
||||
pinned := gitRevParse(t, workDir, "HEAD")
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", "decoy"))
|
||||
require.NoError(t, gitCmd("-C", workDir, "tag", pinned))
|
||||
@@ -576,23 +473,16 @@ func TestGitCloneExecutorPinnedHashIgnoresShadowingRef(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type countingTransport struct {
|
||||
transport.Transport
|
||||
sessions atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingTransport) NewUploadPackSession(ep *transport.Endpoint, auth transport.AuthMethod) (transport.UploadPackSession, error) {
|
||||
c.sessions.Add(1)
|
||||
return c.Transport.NewUploadPackSession(ep, auth)
|
||||
}
|
||||
|
||||
func installCountingTransport(t *testing.T) *countingTransport {
|
||||
func newGitRemote(t *testing.T, messages ...string) (string, string) {
|
||||
t.Helper()
|
||||
counter := &countingTransport{Transport: gogitfile.DefaultClient}
|
||||
gogitclient.InstallProtocol("file", counter)
|
||||
t.Cleanup(func() { gogitclient.InstallProtocol("file", gogitfile.DefaultClient) })
|
||||
|
||||
return counter
|
||||
remoteDir, workDir := t.TempDir(), t.TempDir()
|
||||
require.NoError(t, gitCmd("init", "--bare", "--initial-branch=main", remoteDir))
|
||||
require.NoError(t, gitCmd("clone", remoteDir, workDir))
|
||||
require.NoError(t, gitCmd("-C", workDir, "checkout", "-b", "main"))
|
||||
for _, message := range messages {
|
||||
require.NoError(t, gitCmd("-C", workDir, "commit", "--allow-empty", "-m", message))
|
||||
}
|
||||
return remoteDir, workDir
|
||||
}
|
||||
|
||||
func gitRevParse(t *testing.T, dir, rev string) string {
|
||||
@@ -622,9 +512,6 @@ func gitCmd(args ...string) error {
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
// Inject a deterministic identity and ignore the host's global/system config so commits
|
||||
// succeed regardless of the host having no user.name/user.email (e.g. CI, GITHUB_ACTIONS
|
||||
// unset) or a global commit.gpgsign, and without mutating the developer's ~/.gitconfig.
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GIT_AUTHOR_NAME=Unit Test",
|
||||
"GIT_AUTHOR_EMAIL=test@test.com",
|
||||
@@ -645,14 +532,42 @@ func gitCmd(args ...string) error {
|
||||
}
|
||||
|
||||
func TestAcquireCloneLock(t *testing.T) {
|
||||
t.Run("blocks on a lock held by another process despite a trailing separator", func(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "cache")
|
||||
release, err := lock.TryLock(dir)
|
||||
require.NoError(t, err)
|
||||
defer func() { require.NoError(t, release()) }()
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
_, err := AcquireCloneLock(ctx, dir+string(filepath.Separator))
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("read-only parent falls back to an in-process lock", func(t *testing.T) {
|
||||
parent := t.TempDir()
|
||||
dir := filepath.Join(parent, "cache")
|
||||
require.NoError(t, os.Mkdir(dir, 0o755))
|
||||
require.NoError(t, os.Chmod(parent, 0o555))
|
||||
defer func() { require.NoError(t, os.Chmod(parent, 0o755)) }()
|
||||
unlock, err := AcquireCloneLock(t.Context(), dir)
|
||||
require.NoError(t, err)
|
||||
unlock()
|
||||
})
|
||||
|
||||
t.Run("same directory serializes", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
unlock1 := AcquireCloneLock(dir)
|
||||
unlock1, err := AcquireCloneLock(t.Context(), dir)
|
||||
require.NoError(t, err)
|
||||
|
||||
secondAcquired := make(chan struct{})
|
||||
go func() {
|
||||
unlock := AcquireCloneLock(dir)
|
||||
unlock, err := AcquireCloneLock(t.Context(), dir)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
close(secondAcquired)
|
||||
unlock()
|
||||
}()
|
||||
@@ -676,12 +591,16 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
dirA := t.TempDir()
|
||||
dirB := t.TempDir()
|
||||
|
||||
unlockA := AcquireCloneLock(dirA)
|
||||
unlockA, err := AcquireCloneLock(t.Context(), dirA)
|
||||
require.NoError(t, err)
|
||||
defer unlockA()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
unlock := AcquireCloneLock(dirB)
|
||||
unlock, err := AcquireCloneLock(t.Context(), dirB)
|
||||
if !assert.NoError(t, err) {
|
||||
return
|
||||
}
|
||||
unlock()
|
||||
close(done)
|
||||
}()
|
||||
@@ -694,13 +613,17 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// An unresponsive remote must not pin a job: the refresh has to be interruptible.
|
||||
func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
func TestNewGitCloneExecutorFetchSendsTokenAndHonoursContext(t *testing.T) {
|
||||
t.Parallel()
|
||||
block := make(chan struct{})
|
||||
reached := make(chan struct{})
|
||||
var once sync.Once
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {
|
||||
once.Do(func() { close(reached) })
|
||||
var authorization string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
|
||||
once.Do(func() {
|
||||
authorization = request.Header.Get("Authorization")
|
||||
close(reached)
|
||||
})
|
||||
<-block
|
||||
}))
|
||||
t.Cleanup(func() {
|
||||
@@ -709,17 +632,15 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
})
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "cached-action")
|
||||
repo, err := gogit.PlainInit(dir, false)
|
||||
require.NoError(t, err)
|
||||
_, err = repo.CreateRemote(&gogitconfig.RemoteConfig{Name: "origin", URLs: []string{server.URL}})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gitCmd("init", dir))
|
||||
require.NoError(t, gitCmd("-C", dir, "remote", "add", "origin", server.URL))
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir})(ctx)
|
||||
done <- NewGitCloneExecutor(NewGitCloneExecutorInput{URL: server.URL, Ref: "main", Dir: dir, Token: "secret"})(ctx)
|
||||
}()
|
||||
|
||||
select {
|
||||
@@ -727,6 +648,7 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the executor never reached the remote")
|
||||
}
|
||||
assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("token:secret")), authorization)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
@@ -736,12 +658,3 @@ func TestNewGitCloneExecutorFetchHonoursContext(t *testing.T) {
|
||||
t.Fatal("fetch ignored context cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleRefreshErr(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
require.NoError(t, staleRefreshErr(ctx, errors.New("remote hung up")))
|
||||
|
||||
cancel()
|
||||
require.ErrorIs(t, staleRefreshErr(ctx, errors.New("remote hung up")), context.Canceled)
|
||||
require.NoError(t, staleRefreshErr(ctx, gogit.NoErrAlreadyUpToDate))
|
||||
}
|
||||
|
||||
@@ -14,13 +14,8 @@ const jobErrorContextKeyVal = jobErrorContextKey("job.error")
|
||||
|
||||
// JobError returns the job error for current context if any
|
||||
func JobError(ctx context.Context) error {
|
||||
val := ctx.Value(jobErrorContextKeyVal)
|
||||
if val != nil {
|
||||
if container, ok := val.(map[string]error); ok {
|
||||
return container["error"]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
container, _ := ctx.Value(jobErrorContextKeyVal).(map[string]error)
|
||||
return container["error"]
|
||||
}
|
||||
|
||||
func SetJobError(ctx context.Context, err error) {
|
||||
@@ -31,6 +26,5 @@ func SetJobError(ctx context.Context, err error) {
|
||||
|
||||
// WithJobErrorContainer adds a value to the context as a container for an error
|
||||
func WithJobErrorContainer(ctx context.Context) context.Context {
|
||||
container := map[string]error{}
|
||||
return context.WithValue(ctx, jobErrorContextKeyVal, container)
|
||||
return context.WithValue(ctx, jobErrorContextKeyVal, map[string]error{})
|
||||
}
|
||||
|
||||
@@ -26,9 +26,7 @@ type lineWriter struct {
|
||||
|
||||
// NewLineWriter creates a new instance of a line writer
|
||||
func NewLineWriter(handlers ...LineHandler) io.Writer {
|
||||
w := new(lineWriter)
|
||||
w.handlers = handlers
|
||||
return w
|
||||
return &lineWriter{handlers: handlers}
|
||||
}
|
||||
|
||||
// FlushWriter flushes w if it implements Flusher. It is a no-op otherwise, so
|
||||
@@ -39,24 +37,15 @@ func FlushWriter(w io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
func (lw *lineWriter) Write(p []byte) (n int, err error) {
|
||||
pBuf := bytes.NewBuffer(p)
|
||||
written := 0
|
||||
for {
|
||||
line, err := pBuf.ReadString('\n')
|
||||
w, _ := lw.buffer.WriteString(line)
|
||||
written += w
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return written, err
|
||||
func (lw *lineWriter) Write(p []byte) (int, error) {
|
||||
for line := range bytes.SplitAfterSeq(p, []byte{'\n'}) {
|
||||
_, _ = lw.buffer.Write(line)
|
||||
if len(line) > 0 && line[len(line)-1] == '\n' {
|
||||
lw.handleLine(lw.buffer.String())
|
||||
lw.buffer.Reset()
|
||||
}
|
||||
lw.handleLine(lw.buffer.String())
|
||||
lw.buffer.Reset()
|
||||
}
|
||||
|
||||
return written, nil
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// Flush emits any buffered, not-yet-newline-terminated content as a final line.
|
||||
@@ -72,8 +61,7 @@ func (lw *lineWriter) Flush() {
|
||||
|
||||
func (lw *lineWriter) handleLine(line string) {
|
||||
for _, h := range lw.handlers {
|
||||
ok := h(line)
|
||||
if !ok {
|
||||
if !h(line) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,24 +9,24 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLineWriter(t *testing.T) {
|
||||
lines := make([]string, 0)
|
||||
lineHandler := func(s string) bool {
|
||||
func TestLineWriterBuffersAndFlushesLines(t *testing.T) {
|
||||
var lines []string
|
||||
lineWriter := NewLineWriter(func(s string) bool {
|
||||
lines = append(lines, s)
|
||||
return true
|
||||
}
|
||||
|
||||
lineWriter := NewLineWriter(lineHandler)
|
||||
})
|
||||
|
||||
assert := assert.New(t)
|
||||
write := func(s string) {
|
||||
n, err := lineWriter.Write([]byte(s))
|
||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(len(s), n, s)
|
||||
}
|
||||
|
||||
write("")
|
||||
write("hello")
|
||||
write(" ")
|
||||
write("world!!\nextra")
|
||||
@@ -34,39 +34,17 @@ func TestLineWriter(t *testing.T) {
|
||||
write(" line\n")
|
||||
write("no newline here...")
|
||||
|
||||
assert.Len(lines, 4)
|
||||
assert.Equal("hello world!!\n", lines[0])
|
||||
assert.Equal("extra line\n", lines[1])
|
||||
assert.Equal(" and another\n", lines[2])
|
||||
assert.Equal("last line\n", lines[3])
|
||||
}
|
||||
expected := []string{"hello world!!\n", "extra line\n", " and another\n", "last line\n"}
|
||||
assert.Equal(expected, lines)
|
||||
|
||||
func TestLineWriterFlush(t *testing.T) {
|
||||
lines := make([]string, 0)
|
||||
lineHandler := func(s string) bool {
|
||||
lines = append(lines, s)
|
||||
return true
|
||||
}
|
||||
|
||||
lineWriter := NewLineWriter(lineHandler)
|
||||
|
||||
assert := assert.New(t)
|
||||
_, err := lineWriter.Write([]byte("complete line\npartial line without newline"))
|
||||
assert.NoError(err) //nolint:testifylint // pre-existing pattern from nektos/act
|
||||
|
||||
// Only the newline-terminated line is emitted before flushing.
|
||||
assert.Equal([]string{"complete line\n"}, lines)
|
||||
|
||||
// Flushing emits the buffered, not-yet-terminated trailing line.
|
||||
FlushWriter(lineWriter)
|
||||
assert.Equal([]string{"complete line\n", "partial line without newline"}, lines)
|
||||
expected = append(expected, "no newline here...")
|
||||
assert.Equal(expected, lines)
|
||||
|
||||
// Flushing again is a no-op: nothing is buffered.
|
||||
FlushWriter(lineWriter)
|
||||
assert.Len(lines, 2)
|
||||
assert.Equal(expected, lines)
|
||||
}
|
||||
|
||||
func TestFlushWriterIgnoresNonFlusher(t *testing.T) {
|
||||
// FlushWriter must be a safe no-op for writers that do not buffer lines.
|
||||
assert.NotPanics(t, func() { FlushWriter(io.Discard) })
|
||||
}
|
||||
|
||||
@@ -7,46 +7,51 @@ package common
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
type loggerContextKey string
|
||||
|
||||
const loggerContextKeyVal = loggerContextKey("logrus.FieldLogger")
|
||||
const loggerContextKeyVal = loggerContextKey("logging.Logger")
|
||||
|
||||
// Logger returns the appropriate logger for current context
|
||||
func Logger(ctx context.Context) logrus.FieldLogger {
|
||||
val := ctx.Value(loggerContextKeyVal)
|
||||
if val != nil {
|
||||
if logger, ok := val.(logrus.FieldLogger); ok {
|
||||
return logger
|
||||
}
|
||||
func Logger(ctx context.Context) *logging.Logger {
|
||||
if logger, ok := ctx.Value(loggerContextKeyVal).(*logging.Logger); ok {
|
||||
return logger
|
||||
}
|
||||
return logrus.StandardLogger()
|
||||
return logging.Default()
|
||||
}
|
||||
|
||||
// WithLogger adds a value to the context for the logger
|
||||
func WithLogger(ctx context.Context, logger logrus.FieldLogger) context.Context {
|
||||
func WithLogger(ctx context.Context, logger *logging.Logger) context.Context {
|
||||
return context.WithValue(ctx, loggerContextKeyVal, logger)
|
||||
}
|
||||
|
||||
type loggerHookKey string
|
||||
type loggerHandlerKey string
|
||||
|
||||
const loggerHookKeyVal = loggerHookKey("logrus.Hook")
|
||||
const loggerHandlerKeyVal = loggerHandlerKey("logging.Handler")
|
||||
|
||||
// LoggerHook returns the appropriate logger hook for current context
|
||||
// the hook affects job logger, not global logger
|
||||
func LoggerHook(ctx context.Context) logrus.Hook {
|
||||
val := ctx.Value(loggerHookKeyVal)
|
||||
if val != nil {
|
||||
if hook, ok := val.(logrus.Hook); ok {
|
||||
return hook
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// LoggerHandler returns the extra handler the job logger sends its records to, or nil.
|
||||
// It affects the job logger, not the global logger.
|
||||
func LoggerHandler(ctx context.Context) logging.Handler {
|
||||
h, _ := ctx.Value(loggerHandlerKeyVal).(logging.Handler)
|
||||
return h
|
||||
}
|
||||
|
||||
// WithLoggerHook adds a value to the context for the logger hook
|
||||
func WithLoggerHook(ctx context.Context, hook logrus.Hook) context.Context {
|
||||
return context.WithValue(ctx, loggerHookKeyVal, hook)
|
||||
// WithLoggerHandler adds the extra handler for the job logger to the context
|
||||
func WithLoggerHandler(ctx context.Context, h logging.Handler) context.Context {
|
||||
return context.WithValue(ctx, loggerHandlerKeyVal, h)
|
||||
}
|
||||
|
||||
type stepIDsContextKey struct{}
|
||||
|
||||
// StepIDs returns the IDs of the step and the composite steps enclosing the current one, outermost first.
|
||||
func StepIDs(ctx context.Context) []string {
|
||||
ids, _ := ctx.Value(stepIDsContextKey{}).([]string)
|
||||
return ids
|
||||
}
|
||||
|
||||
// WithStepIDs replaces the step IDs in the context
|
||||
func WithStepIDs(ctx context.Context, ids []string) context.Context {
|
||||
return context.WithValue(ctx, stepIDsContextKey{}, ids)
|
||||
}
|
||||
|
||||
284
act/common/logging/console.go
Normal file
284
act/common/logging/console.go
Normal file
@@ -0,0 +1,284 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ansiReset = "\x1b[0m"
|
||||
ansiRed = "\x1b[31m"
|
||||
ansiYellow = "\x1b[33m"
|
||||
ansiCyan = "\x1b[36m"
|
||||
ansiDimCyan = "\x1b[2;36m"
|
||||
ansiDimWhite = "\x1b[2;37m"
|
||||
)
|
||||
|
||||
// consoleStart is the zero of the elapsed-seconds stamp used when full timestamps are off.
|
||||
var consoleStart = time.Now()
|
||||
|
||||
// console is the state shared by every consoleHandler derived from one another.
|
||||
type console struct {
|
||||
mu sync.Mutex
|
||||
out io.Writer
|
||||
terminal bool
|
||||
opts TextOptions
|
||||
reportCaller bool
|
||||
level slog.LevelVar
|
||||
}
|
||||
|
||||
func newConsole(out io.Writer) *console {
|
||||
c := &console{}
|
||||
c.setOutput(out)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *console) setOutput(w io.Writer) {
|
||||
terminal := checkIfTerminal(w)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.out, c.terminal = w, terminal
|
||||
}
|
||||
|
||||
// consoleHandler is a slog.Handler writing the text format the runner has always used, so that
|
||||
// moving off logrus changes nothing a user can see: sorted key=value fields on a plain stream,
|
||||
// and a colored prefix with elapsed or full time on a terminal.
|
||||
type consoleHandler struct {
|
||||
c *console
|
||||
attrs []slog.Attr
|
||||
}
|
||||
|
||||
func (h *consoleHandler) Enabled(_ context.Context, level slog.Level) bool {
|
||||
return level >= h.c.level.Level()
|
||||
}
|
||||
|
||||
func (h *consoleHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||
return &consoleHandler{c: h.c, attrs: append(slices.Clone(h.attrs), attrs...)}
|
||||
}
|
||||
|
||||
// WithGroup is a no-op, as only NewSlogHandler wraps a consoleHandler and it never groups.
|
||||
func (h *consoleHandler) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
func (h *consoleHandler) Handle(_ context.Context, r slog.Record) error {
|
||||
c := h.c
|
||||
c.mu.Lock()
|
||||
opts, reportCaller, terminal := c.opts, c.reportCaller, c.terminal
|
||||
c.mu.Unlock()
|
||||
|
||||
data := make(map[string]any, len(h.attrs)+r.NumAttrs())
|
||||
for _, a := range h.attrs {
|
||||
data[a.Key] = a.Value.Resolve().Any()
|
||||
}
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
data[a.Key] = a.Value.Resolve().Any()
|
||||
return true
|
||||
})
|
||||
|
||||
var caller *runtime.Frame
|
||||
if reportCaller && r.PC != 0 {
|
||||
frame, _ := runtime.CallersFrames([]uintptr{r.PC}).Next()
|
||||
caller = &frame
|
||||
}
|
||||
prefixFieldClashes(data, caller != nil)
|
||||
|
||||
keys := slices.Sorted(maps.Keys(data))
|
||||
|
||||
var b bytes.Buffer
|
||||
if terminal && !opts.DisableColors {
|
||||
printColored(&b, opts, r, caller, keys, data)
|
||||
} else {
|
||||
printPlain(&b, opts, r, caller, keys, data)
|
||||
}
|
||||
|
||||
line := b.Bytes()
|
||||
if opts.Filter != nil {
|
||||
line = opts.Filter(line)
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
_, err := c.out.Write(line)
|
||||
return err
|
||||
}
|
||||
|
||||
// prefixFieldClashes moves fields that share a name with a fixed key out of its way.
|
||||
func prefixFieldClashes(data map[string]any, reportCaller bool) {
|
||||
for _, key := range []string{"time", "level", "msg"} {
|
||||
if v, ok := data[key]; ok {
|
||||
data["fields."+key] = v
|
||||
delete(data, key)
|
||||
}
|
||||
}
|
||||
if reportCaller {
|
||||
for _, key := range []string{"func", "file"} {
|
||||
if v, ok := data[key]; ok {
|
||||
data["fields."+key] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printPlain(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||
if !opts.DisableTimestamp {
|
||||
appendKeyValue(b, "time", r.Time.Format(time.RFC3339))
|
||||
}
|
||||
appendKeyValue(b, "level", plainLevel(Level(r.Level)))
|
||||
if r.Message != "" {
|
||||
appendKeyValue(b, "msg", r.Message)
|
||||
}
|
||||
if caller != nil {
|
||||
funcVal, fileVal := callerText(opts, caller)
|
||||
if funcVal != "" {
|
||||
appendKeyValue(b, "func", funcVal)
|
||||
}
|
||||
if fileVal != "" {
|
||||
appendKeyValue(b, "file", fileVal)
|
||||
}
|
||||
}
|
||||
for _, k := range keys {
|
||||
appendKeyValue(b, k, data[k])
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
|
||||
func callerText(opts TextOptions, caller *runtime.Frame) (funcVal, fileVal string) {
|
||||
if opts.CallerPrettyfier != nil {
|
||||
return opts.CallerPrettyfier(caller)
|
||||
}
|
||||
return caller.Function, caller.File + ":" + strconv.Itoa(caller.Line)
|
||||
}
|
||||
|
||||
func printColored(b *bytes.Buffer, opts TextOptions, r slog.Record, caller *runtime.Frame, keys []string, data map[string]any) {
|
||||
msg := strings.TrimSuffix(r.Message, "\n")
|
||||
|
||||
var callerStr string
|
||||
if caller != nil {
|
||||
funcVal, fileVal := callerText(opts, caller)
|
||||
if opts.CallerPrettyfier == nil && caller.Function != "" {
|
||||
funcVal += "()"
|
||||
}
|
||||
switch {
|
||||
case fileVal == "":
|
||||
callerStr = funcVal
|
||||
case funcVal == "":
|
||||
callerStr = fileVal
|
||||
default:
|
||||
callerStr = fileVal + " " + funcVal
|
||||
}
|
||||
}
|
||||
|
||||
level := Level(r.Level)
|
||||
prefix := colorize(level, strings.ToUpper(plainLevel(level))[:4])
|
||||
switch {
|
||||
case opts.DisableTimestamp:
|
||||
fmt.Fprintf(b, "%s%s %-44s ", prefix, callerStr, msg)
|
||||
case !opts.FullTimestamp:
|
||||
fmt.Fprintf(b, "%s[%04d]%s %-44s ", prefix, int(r.Time.Sub(consoleStart)/time.Second), callerStr, msg)
|
||||
default:
|
||||
fmt.Fprintf(b, "%s[%s]%s %-44s ", prefix, r.Time.Format(time.RFC3339), callerStr, msg)
|
||||
}
|
||||
for _, k := range keys {
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(colorize(level, k))
|
||||
b.WriteByte('=')
|
||||
appendValue(b, data[k])
|
||||
}
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
|
||||
// plainLevel spells Warn the way logrus did, as "warning".
|
||||
func plainLevel(l Level) string {
|
||||
if s := l.String(); s != "warn" {
|
||||
return s
|
||||
}
|
||||
return "warning"
|
||||
}
|
||||
|
||||
func colorize(l Level, s string) string {
|
||||
var color string
|
||||
switch {
|
||||
case l <= LevelTrace:
|
||||
color = ansiDimWhite
|
||||
case l <= LevelDebug:
|
||||
color = ansiDimCyan
|
||||
case l <= LevelInfo:
|
||||
color = ansiCyan
|
||||
case l <= LevelWarn:
|
||||
color = ansiYellow
|
||||
default:
|
||||
color = ansiRed
|
||||
}
|
||||
return color + s + ansiReset
|
||||
}
|
||||
|
||||
func appendKeyValue(b *bytes.Buffer, key string, value any) {
|
||||
if b.Len() > 0 {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
appendValue(b, value)
|
||||
}
|
||||
|
||||
func appendValue(b *bytes.Buffer, value any) {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
appendString(b, v)
|
||||
case []byte:
|
||||
appendString(b, string(v))
|
||||
case error:
|
||||
appendFormatted(b, v, v.Error)
|
||||
case fmt.Stringer:
|
||||
appendFormatted(b, v, v.String)
|
||||
default:
|
||||
appendString(b, fmt.Sprint(value))
|
||||
}
|
||||
}
|
||||
|
||||
// appendFormatted writes what get returns, and "<nil>" if that panics on a nil receiver.
|
||||
func appendFormatted(b *bytes.Buffer, v any, get func() string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
if rv := reflect.ValueOf(v); rv.Kind() == reflect.Pointer && rv.IsNil() {
|
||||
appendString(b, "<nil>")
|
||||
} else {
|
||||
appendString(b, fmt.Sprintf("%%!v(PANIC=%v)", r))
|
||||
}
|
||||
}
|
||||
}()
|
||||
appendString(b, get())
|
||||
}
|
||||
|
||||
func appendString(b *bytes.Buffer, s string) {
|
||||
if needsQuoting(s) {
|
||||
b.WriteString(strconv.Quote(s))
|
||||
return
|
||||
}
|
||||
b.WriteString(s)
|
||||
}
|
||||
|
||||
func needsQuoting(s string) bool {
|
||||
for i := range len(s) {
|
||||
c := s[i]
|
||||
switch {
|
||||
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
|
||||
case c == '-', c == '.', c == '_', c == '/', c == '@', c == '^', c == '+':
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
78
act/common/logging/console_test.go
Normal file
78
act/common/logging/console_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func consoleLine(t *testing.T, opts TextOptions, terminal, reportCaller bool, build func(*Logger)) string {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
c := newConsole(&out)
|
||||
c.opts, c.terminal, c.reportCaller = opts, terminal, reportCaller
|
||||
c.level.Set(slog.LevelDebug)
|
||||
build(New(NewSlogHandler(&consoleHandler{c: c})))
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func TestConsolePlain(t *testing.T) {
|
||||
line := consoleLine(t, TextOptions{DisableTimestamp: true}, false, false, func(l *Logger) {
|
||||
l.With("job", "build x", "n", 3).WithError(errors.New("boom")).Warnf("hello world")
|
||||
l.With("msg", "clash", "empty", "").Info("")
|
||||
})
|
||||
assert.Equal(t, "level=warning msg=\"hello world\" error=boom job=\"build x\" n=3\n"+
|
||||
"level=info empty= fields.msg=clash\n", line)
|
||||
}
|
||||
|
||||
func TestConsolePlainTimeFilterAndCaller(t *testing.T) {
|
||||
opts := TextOptions{
|
||||
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
|
||||
return "[fn]", "[" + strings.TrimPrefix(f.Function, "x") + "]"
|
||||
},
|
||||
Filter: func(b []byte) []byte { return bytes.ReplaceAll(b, []byte("secret"), []byte("***")) },
|
||||
}
|
||||
line := consoleLine(t, opts, false, true, func(l *Logger) { l.With("k", "secret").Debugf("a secret") })
|
||||
assert.Regexp(t, `^time="[^"]+" level=debug msg="a \*\*\*" func="\[fn\]" file="\[.*TestConsolePlainTimeFilterAndCaller[^"]*\]" k=\*\*\*\n$`, line)
|
||||
}
|
||||
|
||||
func TestConsoleColored(t *testing.T) {
|
||||
line := consoleLine(t, TextOptions{DisableTimestamp: true}, true, false, func(l *Logger) {
|
||||
l.With("job", "j").Warn("careful")
|
||||
})
|
||||
assert.Equal(t, "\x1b[33mWARN\x1b[0m "+"careful"+strings.Repeat(" ", 44-len("careful"))+" \x1b[33mjob\x1b[0m=j\n", line)
|
||||
|
||||
full := consoleLine(t, TextOptions{FullTimestamp: true}, true, false, func(l *Logger) { l.Error("bad") })
|
||||
assert.Regexp(t, `^\x1b\[31mERRO\x1b\[0m\[\d{4}-\d\d-\d\dT[^\]]+\] bad`, full)
|
||||
|
||||
elapsed := consoleLine(t, TextOptions{}, true, false, func(l *Logger) { l.Info("x") })
|
||||
assert.Regexp(t, `^\x1b\[36mINFO\x1b\[0m\[\d{4}\] x`, elapsed)
|
||||
|
||||
plain := consoleLine(t, TextOptions{DisableColors: true, DisableTimestamp: true}, true, false, func(l *Logger) { l.Info("x") })
|
||||
assert.Equal(t, "level=info msg=x\n", plain)
|
||||
}
|
||||
|
||||
func TestConsoleLevelAndOutput(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
c := newConsole(&out)
|
||||
h := &consoleHandler{c: c}
|
||||
assert.False(t, h.Enabled(t.Context(), slog.LevelDebug)) // info unless raised
|
||||
c.level.Set(slog.LevelError)
|
||||
assert.False(t, h.Enabled(t.Context(), slog.LevelWarn))
|
||||
assert.True(t, h.Enabled(t.Context(), slog.LevelError))
|
||||
|
||||
rec := slog.NewRecord(time.Unix(0, 0), slog.LevelError, "m", 0)
|
||||
rec.AddAttrs(slog.String("c", "3"))
|
||||
require.NoError(t, h.WithAttrs([]slog.Attr{slog.Int("a", 1)}).Handle(t.Context(), rec))
|
||||
assert.Contains(t, out.String(), "a=1 c=3")
|
||||
}
|
||||
83
act/common/logging/global.go
Normal file
83
act/common/logging/global.go
Normal file
@@ -0,0 +1,83 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// TextOptions describes how the process-wide Logger writes a record as text.
|
||||
type TextOptions struct {
|
||||
DisableColors bool
|
||||
DisableTimestamp bool
|
||||
FullTimestamp bool
|
||||
// CallerPrettyfier renames the function and file reported with a record when ReportCaller is on.
|
||||
CallerPrettyfier func(*runtime.Frame) (function, file string)
|
||||
// Filter rewrites each formatted line, for example to mask secrets.
|
||||
Filter func(line []byte) []byte
|
||||
}
|
||||
|
||||
var (
|
||||
std atomic.Pointer[Logger]
|
||||
stdState = newConsole(os.Stderr)
|
||||
)
|
||||
|
||||
func init() {
|
||||
std.Store(New(NewSlogHandler(&consoleHandler{c: stdState})))
|
||||
}
|
||||
|
||||
// Default returns the process-wide Logger.
|
||||
func Default() *Logger { return std.Load() }
|
||||
|
||||
// SetDefault replaces the process-wide Logger, which tests use to capture what the package
|
||||
// functions log. SetLevel, GetLevel, SetOutput and the format functions keep configuring the
|
||||
// initial console Logger.
|
||||
func SetDefault(l *Logger) { std.Store(l) }
|
||||
|
||||
// SetLevel sets the minimum level of the initial process-wide Logger.
|
||||
func SetLevel(level Level) { stdState.level.Set(slog.Level(level)) }
|
||||
|
||||
// GetLevel returns the minimum level of the initial process-wide Logger.
|
||||
func GetLevel() Level { return Level(stdState.level.Level()) }
|
||||
|
||||
// SetOutput sets where the initial process-wide Logger writes.
|
||||
func SetOutput(w io.Writer) { stdState.setOutput(w) }
|
||||
|
||||
// Output returns where the initial process-wide Logger writes.
|
||||
func Output() io.Writer {
|
||||
stdState.mu.Lock()
|
||||
defer stdState.mu.Unlock()
|
||||
return stdState.out
|
||||
}
|
||||
|
||||
// SetTextFormat sets the text format of the initial process-wide Logger.
|
||||
func SetTextFormat(o TextOptions) {
|
||||
stdState.mu.Lock()
|
||||
defer stdState.mu.Unlock()
|
||||
stdState.opts = o
|
||||
}
|
||||
|
||||
// SetReportCaller makes the initial process-wide Logger record where each call came from, which costs time.
|
||||
func SetReportCaller(on bool) {
|
||||
stdState.mu.Lock()
|
||||
defer stdState.mu.Unlock()
|
||||
stdState.reportCaller = on
|
||||
}
|
||||
|
||||
func Enabled(level Level) bool { return Default().Enabled(level) }
|
||||
|
||||
func With(args ...any) *Logger { return Default().With(args...) }
|
||||
func WithError(err error) *Logger { return Default().WithError(err) }
|
||||
func Debugf(format string, args ...any) { Default().logf(LevelDebug, format, args) }
|
||||
func Infof(format string, args ...any) { Default().logf(LevelInfo, format, args) }
|
||||
func Warnf(format string, args ...any) { Default().logf(LevelWarn, format, args) }
|
||||
func Errorf(format string, args ...any) { Default().logf(LevelError, format, args) }
|
||||
func Debug(args ...any) { Default().log(LevelDebug, args) }
|
||||
func Info(args ...any) { Default().log(LevelInfo, args) }
|
||||
func Warn(args ...any) { Default().log(LevelWarn, args) }
|
||||
func Error(args ...any) { Default().log(LevelError, args) }
|
||||
114
act/common/logging/handler.go
Normal file
114
act/common/logging/handler.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attr is a structured key/value pair attached to a log record.
|
||||
type Attr struct {
|
||||
Key string
|
||||
Value any
|
||||
}
|
||||
|
||||
// Record is one log event. Attrs holds the call-site attributes only, the ones added with
|
||||
// Logger.With are accumulated by the Handler.
|
||||
type Record struct {
|
||||
Time time.Time
|
||||
Level Level
|
||||
Message string
|
||||
PC uintptr // program counter of the logging call, 0 if unknown
|
||||
Attrs []Attr
|
||||
}
|
||||
|
||||
// Attr returns the value of the last attribute named key.
|
||||
func (r Record) Attr(key string) (any, bool) {
|
||||
for _, a := range slices.Backward(r.Attrs) {
|
||||
if a.Key == key {
|
||||
return a.Value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Handler consumes records, and is shaped like slog.Handler so that a slog backend can replace
|
||||
// the current one without touching callers. Handle must not modify the Attrs slice it is given.
|
||||
type Handler interface {
|
||||
Enabled(ctx context.Context, level Level) bool
|
||||
Handle(ctx context.Context, r Record) error
|
||||
// WithAttrs returns a Handler that adds attrs to every record it handles.
|
||||
WithAttrs(attrs []Attr) Handler
|
||||
}
|
||||
|
||||
type multiHandler []Handler
|
||||
|
||||
// NewMultiHandler fans every record out to all handlers that have its level enabled.
|
||||
func NewMultiHandler(handlers ...Handler) Handler {
|
||||
return multiHandler(handlers)
|
||||
}
|
||||
|
||||
func (m multiHandler) Enabled(ctx context.Context, level Level) bool {
|
||||
for _, h := range m {
|
||||
if h.Enabled(ctx, level) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m multiHandler) Handle(ctx context.Context, r Record) error {
|
||||
var errs []error
|
||||
for _, h := range m {
|
||||
if h.Enabled(ctx, r.Level) {
|
||||
errs = append(errs, h.Handle(ctx, r))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (m multiHandler) WithAttrs(attrs []Attr) Handler {
|
||||
out := make(multiHandler, len(m))
|
||||
for i, h := range m {
|
||||
out[i] = h.WithAttrs(attrs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type discardHandler struct{}
|
||||
|
||||
// DiscardHandler drops every record.
|
||||
var DiscardHandler Handler = discardHandler{}
|
||||
|
||||
func (discardHandler) Enabled(context.Context, Level) bool { return false }
|
||||
func (discardHandler) Handle(context.Context, Record) error { return nil }
|
||||
func (d discardHandler) WithAttrs([]Attr) Handler { return d }
|
||||
|
||||
type levelHandler struct {
|
||||
h Handler
|
||||
minLevel Level
|
||||
}
|
||||
|
||||
// NewLevelHandler passes only records at or above minLevel on to h.
|
||||
func NewLevelHandler(h Handler, minLevel Level) Handler {
|
||||
return levelHandler{h: h, minLevel: minLevel}
|
||||
}
|
||||
|
||||
func (l levelHandler) Enabled(ctx context.Context, level Level) bool {
|
||||
return level >= l.minLevel && l.h.Enabled(ctx, level)
|
||||
}
|
||||
|
||||
func (l levelHandler) Handle(ctx context.Context, r Record) error {
|
||||
if r.Level < l.minLevel {
|
||||
return nil
|
||||
}
|
||||
return l.h.Handle(ctx, r)
|
||||
}
|
||||
|
||||
func (l levelHandler) WithAttrs(attrs []Attr) Handler {
|
||||
return levelHandler{h: l.h.WithAttrs(attrs), minLevel: l.minLevel}
|
||||
}
|
||||
53
act/common/logging/level.go
Normal file
53
act/common/logging/level.go
Normal file
@@ -0,0 +1,53 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Level is a log severity. The values match log/slog, with Trace added below Debug.
|
||||
type Level int
|
||||
|
||||
const (
|
||||
LevelTrace Level = -8
|
||||
LevelDebug Level = -4
|
||||
LevelInfo Level = 0
|
||||
LevelWarn Level = 4
|
||||
LevelError Level = 8
|
||||
)
|
||||
|
||||
func (l Level) String() string {
|
||||
switch {
|
||||
case l <= LevelTrace:
|
||||
return "trace"
|
||||
case l <= LevelDebug:
|
||||
return "debug"
|
||||
case l <= LevelInfo:
|
||||
return "info"
|
||||
case l <= LevelWarn:
|
||||
return "warn"
|
||||
default:
|
||||
return "error"
|
||||
}
|
||||
}
|
||||
|
||||
// ParseLevel accepts every name logrus does. "fatal" and "panic" map to LevelError because
|
||||
// the Logger has no level above it.
|
||||
func ParseLevel(s string) (Level, error) {
|
||||
switch strings.ToLower(s) {
|
||||
case "trace":
|
||||
return LevelTrace, nil
|
||||
case "debug":
|
||||
return LevelDebug, nil
|
||||
case "info":
|
||||
return LevelInfo, nil
|
||||
case "warn", "warning":
|
||||
return LevelWarn, nil
|
||||
case "error", "fatal", "panic":
|
||||
return LevelError, nil
|
||||
}
|
||||
return LevelInfo, fmt.Errorf("not a valid log level: %q", s)
|
||||
}
|
||||
105
act/common/logging/logger.go
Normal file
105
act/common/logging/logger.go
Normal file
@@ -0,0 +1,105 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
const badKey = "!BADKEY"
|
||||
|
||||
// Logger writes printf-style messages with structured attributes to a Handler.
|
||||
type Logger struct {
|
||||
h Handler
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func New(h Handler) *Logger {
|
||||
return &Logger{h: h, ctx: context.Background()}
|
||||
}
|
||||
|
||||
// With returns a Logger that adds the key/value pairs to every record, like slog.Logger.With.
|
||||
func (l *Logger) With(args ...any) *Logger {
|
||||
if len(args) == 0 {
|
||||
return l
|
||||
}
|
||||
return &Logger{h: l.h.WithAttrs(argsToAttrs(args)), ctx: l.ctx}
|
||||
}
|
||||
|
||||
func (l *Logger) WithError(err error) *Logger {
|
||||
return l.With("error", err)
|
||||
}
|
||||
|
||||
// WithContext returns a Logger that passes ctx to its Handler.
|
||||
func (l *Logger) WithContext(ctx context.Context) *Logger {
|
||||
return &Logger{h: l.h, ctx: ctx}
|
||||
}
|
||||
|
||||
func (l *Logger) Enabled(level Level) bool {
|
||||
return l.h.Enabled(l.ctx, level)
|
||||
}
|
||||
|
||||
func (l *Logger) Debugf(format string, args ...any) { l.logf(LevelDebug, format, args) }
|
||||
func (l *Logger) Infof(format string, args ...any) { l.logf(LevelInfo, format, args) }
|
||||
func (l *Logger) Warnf(format string, args ...any) { l.logf(LevelWarn, format, args) }
|
||||
func (l *Logger) Errorf(format string, args ...any) { l.logf(LevelError, format, args) }
|
||||
|
||||
// Trace and the methods below format their arguments like fmt.Sprint.
|
||||
func (l *Logger) Trace(args ...any) { l.log(LevelTrace, args) }
|
||||
func (l *Logger) Debug(args ...any) { l.log(LevelDebug, args) }
|
||||
func (l *Logger) Info(args ...any) { l.log(LevelInfo, args) }
|
||||
func (l *Logger) Warn(args ...any) { l.log(LevelWarn, args) }
|
||||
func (l *Logger) Error(args ...any) { l.log(LevelError, args) }
|
||||
|
||||
func (l *Logger) logf(level Level, format string, args []any) {
|
||||
if !l.Enabled(level) {
|
||||
return
|
||||
}
|
||||
l.emit(callerPC(4), level, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *Logger) log(level Level, args []any) {
|
||||
if !l.Enabled(level) {
|
||||
return
|
||||
}
|
||||
l.emit(callerPC(4), level, fmt.Sprint(args...))
|
||||
}
|
||||
|
||||
// emit drops the Handler error, as a logger has nowhere to report it.
|
||||
func (l *Logger) emit(pc uintptr, level Level, msg string) {
|
||||
_ = l.h.Handle(l.ctx, Record{Time: time.Now(), Level: level, Message: msg, PC: pc})
|
||||
}
|
||||
|
||||
// callerPC returns the program counter skip frames up, 4 being the caller of an exported method.
|
||||
func callerPC(skip int) uintptr {
|
||||
var pcs [1]uintptr
|
||||
runtime.Callers(skip, pcs[:])
|
||||
return pcs[0]
|
||||
}
|
||||
|
||||
func argsToAttrs(args []any) []Attr {
|
||||
attrs := make([]Attr, 0, len(args)/2+1)
|
||||
for len(args) > 0 {
|
||||
switch k := args[0].(type) {
|
||||
case Attr:
|
||||
attrs = append(attrs, k)
|
||||
args = args[1:]
|
||||
case string:
|
||||
if len(args) == 1 {
|
||||
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||
args = nil
|
||||
} else {
|
||||
attrs = append(attrs, Attr{Key: k, Value: args[1]})
|
||||
args = args[2:]
|
||||
}
|
||||
default:
|
||||
attrs = append(attrs, Attr{Key: badKey, Value: k})
|
||||
args = args[1:]
|
||||
}
|
||||
}
|
||||
return attrs
|
||||
}
|
||||
145
act/common/logging/logging_test.go
Normal file
145
act/common/logging/logging_test.go
Normal file
@@ -0,0 +1,145 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseLevel(t *testing.T) {
|
||||
for in, want := range map[string]logging.Level{
|
||||
"trace": logging.LevelTrace, "DEBUG": logging.LevelDebug, "info": logging.LevelInfo,
|
||||
"warn": logging.LevelWarn, "warning": logging.LevelWarn,
|
||||
"error": logging.LevelError, "fatal": logging.LevelError, "panic": logging.LevelError,
|
||||
} {
|
||||
got, err := logging.ParseLevel(in)
|
||||
require.NoError(t, err, in)
|
||||
assert.Equal(t, want, got, in)
|
||||
}
|
||||
_, err := logging.ParseLevel("loud")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "warn", logging.LevelWarn.String())
|
||||
}
|
||||
|
||||
type countingStringer struct{ n *int }
|
||||
|
||||
func (c countingStringer) String() string { *c.n++; return "x" }
|
||||
|
||||
func TestLogger(t *testing.T) {
|
||||
h := logtest.NewWithLevel(logging.LevelTrace)
|
||||
assert.Nil(t, h.LastRecord())
|
||||
l := logging.New(h)
|
||||
|
||||
l.With("job", "j1").WithError(errors.New("boom")).Warnf("step %d", 2)
|
||||
l.Info("a", 1, 2, "b")
|
||||
l.With("k").With(7).Debug("x")
|
||||
|
||||
recs := h.AllRecords()
|
||||
require.Len(t, recs, 3)
|
||||
assert.Equal(t, "step 2", recs[0].Message)
|
||||
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||
assert.False(t, recs[0].Time.IsZero())
|
||||
job, _ := recs[0].Attr("job")
|
||||
assert.Equal(t, "j1", job)
|
||||
errv, _ := recs[0].Attr("error")
|
||||
require.EqualError(t, errv.(error), "boom")
|
||||
assert.Equal(t, "a1 2b", recs[1].Message)
|
||||
bad, _ := recs[2].Attr("!BADKEY")
|
||||
assert.Equal(t, 7, bad)
|
||||
}
|
||||
|
||||
func TestLoggerSkipsFormattingWhenDisabled(t *testing.T) {
|
||||
n := 0
|
||||
l := logging.New(logging.NewLevelHandler(logtest.NewWithLevel(logging.LevelTrace), logging.LevelWarn))
|
||||
l.Debugf("%v", countingStringer{&n})
|
||||
assert.Zero(t, n)
|
||||
assert.False(t, l.Enabled(logging.LevelInfo))
|
||||
}
|
||||
|
||||
func TestMultiHandler(t *testing.T) {
|
||||
assert.False(t, logging.DiscardHandler.Enabled(t.Context(), logging.LevelError))
|
||||
all, warnOnly := logtest.New(), logtest.NewWithLevel(logging.LevelTrace)
|
||||
l := logging.New(logging.NewMultiHandler(all, logging.NewLevelHandler(warnOnly, logging.LevelWarn), logging.DiscardHandler)).With("a", 1)
|
||||
l.Info("i")
|
||||
l.Error("e")
|
||||
assert.Len(t, all.AllRecords(), 2)
|
||||
require.Len(t, warnOnly.AllRecords(), 1)
|
||||
v, _ := warnOnly.LastRecord().Attr("a")
|
||||
assert.Equal(t, 1, v)
|
||||
}
|
||||
|
||||
func frameOf(t *testing.T, pc uintptr) runtime.Frame {
|
||||
t.Helper()
|
||||
require.NotZero(t, pc)
|
||||
f, _ := runtime.CallersFrames([]uintptr{pc}).Next()
|
||||
return f
|
||||
}
|
||||
|
||||
func TestCallerPC(t *testing.T) {
|
||||
l, h := logtest.NewLogger()
|
||||
l.Infof("method")
|
||||
l.Info("method")
|
||||
|
||||
prev := logging.Default()
|
||||
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||
logging.SetDefault(l)
|
||||
logging.Infof("global")
|
||||
logging.Error("global")
|
||||
|
||||
require.Len(t, h.AllRecords(), 4)
|
||||
for _, r := range h.AllRecords() {
|
||||
assert.True(t, strings.HasSuffix(frameOf(t, r.PC).Function, "TestCallerPC"), r.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultLevel(t *testing.T) {
|
||||
prev := logging.GetLevel()
|
||||
t.Cleanup(func() { logging.SetLevel(prev) })
|
||||
|
||||
logging.SetLevel(logging.LevelWarn)
|
||||
assert.Equal(t, logging.LevelWarn, logging.GetLevel())
|
||||
assert.True(t, logging.Enabled(logging.LevelError))
|
||||
assert.False(t, logging.Enabled(logging.LevelInfo))
|
||||
}
|
||||
|
||||
func TestSlogHandler(t *testing.T) {
|
||||
var buf strings.Builder
|
||||
l := logging.New(logging.NewSlogHandler(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))).With("job", "j1")
|
||||
|
||||
assert.False(t, l.Enabled(logging.LevelTrace))
|
||||
l.WithError(errors.New("boom")).Errorf("fail %s", "x")
|
||||
|
||||
out := buf.String()
|
||||
assert.Contains(t, out, "level=ERROR")
|
||||
assert.Contains(t, out, `msg="fail x"`)
|
||||
assert.Contains(t, out, "job=j1")
|
||||
assert.Contains(t, out, "error=boom")
|
||||
}
|
||||
|
||||
func TestSlogBridge(t *testing.T) {
|
||||
h := logtest.SetDefault(t)
|
||||
slog.With("lib", "actionslib").Warn("from a library", "n", 1)
|
||||
_, _, line, _ := runtime.Caller(0)
|
||||
slog.Debug("dropped")
|
||||
|
||||
recs := h.AllRecords()
|
||||
require.Len(t, recs, 1)
|
||||
assert.Equal(t, "from a library", recs[0].Message)
|
||||
assert.Equal(t, logging.LevelWarn, recs[0].Level)
|
||||
lib, _ := recs[0].Attr("lib")
|
||||
assert.Equal(t, "actionslib", lib)
|
||||
n, _ := recs[0].Attr("n")
|
||||
assert.EqualValues(t, 1, n)
|
||||
assert.Equal(t, line-1, frameOf(t, recs[0].PC).Line)
|
||||
}
|
||||
83
act/common/logging/logtest/logtest.go
Normal file
83
act/common/logging/logtest/logtest.go
Normal file
@@ -0,0 +1,83 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package logtest provides a Handler that keeps records in memory for assertions.
|
||||
package logtest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
type store struct {
|
||||
mu sync.Mutex
|
||||
records []logging.Record
|
||||
}
|
||||
|
||||
// Handler records what it receives at or above its level. Loggers derived with WithAttrs share one store.
|
||||
type Handler struct {
|
||||
store *store
|
||||
attrs []logging.Attr
|
||||
minLevel logging.Level
|
||||
}
|
||||
|
||||
// New records from LevelInfo up, like the logger of logrus's hooks/test.
|
||||
func New() *Handler {
|
||||
return NewWithLevel(logging.LevelInfo)
|
||||
}
|
||||
|
||||
// NewLogger returns a Logger writing to a new Handler from New, and that Handler.
|
||||
func NewLogger() (*logging.Logger, *Handler) {
|
||||
h := New()
|
||||
return logging.New(h), h
|
||||
}
|
||||
|
||||
func NewWithLevel(minLevel logging.Level) *Handler {
|
||||
return &Handler{store: &store{}, minLevel: minLevel}
|
||||
}
|
||||
|
||||
func (h *Handler) Enabled(_ context.Context, level logging.Level) bool { return level >= h.minLevel }
|
||||
|
||||
// Handle stores the record with the attributes accumulated by WithAttrs placed first.
|
||||
func (h *Handler) Handle(_ context.Context, r logging.Record) error {
|
||||
r.Attrs = append(slices.Clone(h.attrs), r.Attrs...)
|
||||
h.store.mu.Lock()
|
||||
defer h.store.mu.Unlock()
|
||||
h.store.records = append(h.store.records, r)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Handler) WithAttrs(attrs []logging.Attr) logging.Handler {
|
||||
return &Handler{store: h.store, attrs: append(slices.Clone(h.attrs), attrs...), minLevel: h.minLevel}
|
||||
}
|
||||
|
||||
func (h *Handler) AllRecords() []logging.Record {
|
||||
h.store.mu.Lock()
|
||||
defer h.store.mu.Unlock()
|
||||
return slices.Clone(h.store.records)
|
||||
}
|
||||
|
||||
// LastRecord returns nil when nothing was logged.
|
||||
func (h *Handler) LastRecord() *logging.Record {
|
||||
h.store.mu.Lock()
|
||||
defer h.store.mu.Unlock()
|
||||
if len(h.store.records) == 0 {
|
||||
return nil
|
||||
}
|
||||
r := h.store.records[len(h.store.records)-1]
|
||||
return &r
|
||||
}
|
||||
|
||||
// SetDefault makes the process-wide Logger record into a new Handler until the test ends.
|
||||
func SetDefault(t testing.TB) *Handler {
|
||||
t.Helper()
|
||||
prev := logging.Default()
|
||||
logger, h := NewLogger()
|
||||
logging.SetDefault(logger)
|
||||
t.Cleanup(func() { logging.SetDefault(prev) })
|
||||
return h
|
||||
}
|
||||
81
act/common/logging/slog.go
Normal file
81
act/common/logging/slog.go
Normal file
@@ -0,0 +1,81 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"slices"
|
||||
)
|
||||
|
||||
func init() {
|
||||
slog.SetDefault(slog.New(slogBridge{}))
|
||||
}
|
||||
|
||||
// slogBridge hands what libraries log through slog's global logger to the process-wide Logger.
|
||||
type slogBridge struct {
|
||||
attrs []Attr
|
||||
}
|
||||
|
||||
func (b slogBridge) Enabled(ctx context.Context, level slog.Level) bool {
|
||||
return Default().h.Enabled(ctx, Level(level))
|
||||
}
|
||||
|
||||
func (b slogBridge) Handle(ctx context.Context, r slog.Record) error {
|
||||
h := Default().h
|
||||
if len(b.attrs) > 0 {
|
||||
h = h.WithAttrs(b.attrs)
|
||||
}
|
||||
attrs := make([]Attr, 0, r.NumAttrs())
|
||||
r.Attrs(func(a slog.Attr) bool {
|
||||
attrs = append(attrs, fromSlogAttr(a))
|
||||
return true
|
||||
})
|
||||
return h.Handle(ctx, Record{Time: r.Time, Level: Level(r.Level), Message: r.Message, PC: r.PC, Attrs: attrs})
|
||||
}
|
||||
|
||||
func (b slogBridge) WithAttrs(attrs []slog.Attr) slog.Handler {
|
||||
out := slices.Clone(b.attrs)
|
||||
for _, a := range attrs {
|
||||
out = append(out, fromSlogAttr(a))
|
||||
}
|
||||
return slogBridge{attrs: out}
|
||||
}
|
||||
|
||||
func (b slogBridge) WithGroup(string) slog.Handler { return b }
|
||||
|
||||
func fromSlogAttr(a slog.Attr) Attr {
|
||||
return Attr{Key: a.Key, Value: a.Value.Resolve().Any()}
|
||||
}
|
||||
|
||||
type slogHandler struct {
|
||||
h slog.Handler
|
||||
}
|
||||
|
||||
// NewSlogHandler writes to h. Levels map one to one, as Level uses slog's values.
|
||||
func NewSlogHandler(h slog.Handler) Handler {
|
||||
return slogHandler{h: h}
|
||||
}
|
||||
|
||||
func (s slogHandler) Enabled(ctx context.Context, level Level) bool {
|
||||
return s.h.Enabled(ctx, slog.Level(level))
|
||||
}
|
||||
|
||||
func (s slogHandler) Handle(ctx context.Context, r Record) error {
|
||||
rec := slog.NewRecord(r.Time, slog.Level(r.Level), r.Message, r.PC)
|
||||
rec.AddAttrs(toSlogAttrs(r.Attrs)...)
|
||||
return s.h.Handle(ctx, rec)
|
||||
}
|
||||
|
||||
func (s slogHandler) WithAttrs(attrs []Attr) Handler {
|
||||
return slogHandler{h: s.h.WithAttrs(toSlogAttrs(attrs))}
|
||||
}
|
||||
|
||||
func toSlogAttrs(attrs []Attr) []slog.Attr {
|
||||
out := make([]slog.Attr, len(attrs))
|
||||
for i, a := range attrs {
|
||||
out[i] = slog.Any(a.Key, a.Value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
20
act/common/logging/terminal_other.go
Normal file
20
act/common/logging/terminal_other.go
Normal file
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !windows
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
func checkIfTerminal(w io.Writer) bool {
|
||||
if f, ok := w.(*os.File); ok {
|
||||
return term.IsTerminal(int(f.Fd()))
|
||||
}
|
||||
return false
|
||||
}
|
||||
27
act/common/logging/terminal_windows.go
Normal file
27
act/common/logging/terminal_windows.go
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build windows
|
||||
|
||||
package logging
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// checkIfTerminal also turns on virtual terminal processing, which the console needs to show colors.
|
||||
func checkIfTerminal(w io.Writer) bool {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
handle := windows.Handle(f.Fd())
|
||||
var mode uint32
|
||||
if err := windows.GetConsoleMode(handle, &mode); err != nil {
|
||||
return false
|
||||
}
|
||||
return windows.SetConsoleMode(handle, mode|windows.ENABLE_VIRTUAL_TERMINAL_PROCESSING) == nil
|
||||
}
|
||||
@@ -10,6 +10,11 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type interfaceIP struct {
|
||||
net.IP
|
||||
Name string
|
||||
}
|
||||
|
||||
// GetOutboundIP returns an outbound IP address of this machine.
|
||||
// It tries to access the internet and returns the local IP address of the connection.
|
||||
// If the machine cannot access the internet, it returns a preferred IP address from network interfaces.
|
||||
@@ -26,11 +31,7 @@ func GetOutboundIP() net.IP {
|
||||
|
||||
// So the machine cannot access the internet. Pick an IP address from network interfaces.
|
||||
if ifs, err := net.Interfaces(); err == nil {
|
||||
type IP struct {
|
||||
net.IP
|
||||
net.Interface
|
||||
}
|
||||
var ips []IP
|
||||
var ips []interfaceIP
|
||||
for _, i := range ifs {
|
||||
if addrs, err := i.Addrs(); err == nil {
|
||||
for _, addr := range addrs {
|
||||
@@ -42,40 +43,41 @@ func GetOutboundIP() net.IP {
|
||||
ip = v.IP
|
||||
}
|
||||
if ip.IsGlobalUnicast() {
|
||||
ips = append(ips, IP{ip, i})
|
||||
ips = append(ips, interfaceIP{ip, i.Name})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(ips) > 1 {
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
ifi := ips[i].Interface
|
||||
ifj := ips[j].Interface
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
ipi := ips[i].IP
|
||||
ipj := ips[j].IP
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// en0 is preferred to en1
|
||||
if ifi.Name != ifj.Name {
|
||||
return ifi.Name < ifj.Name
|
||||
}
|
||||
|
||||
// fallback
|
||||
return ipi.String() < ipj.String()
|
||||
})
|
||||
return ips[0].IP
|
||||
}
|
||||
return preferredIP(ips)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func preferredIP(ips []interfaceIP) net.IP {
|
||||
if len(ips) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(ips, func(i, j int) bool {
|
||||
first, second := ips[i], ips[j]
|
||||
|
||||
// ethernet is preferred
|
||||
if vi, vj := strings.HasPrefix(first.Name, "e"), strings.HasPrefix(second.Name, "e"); vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// IPv4 is preferred
|
||||
if vi, vj := first.To4() != nil, second.To4() != nil; vi != vj {
|
||||
return vi
|
||||
}
|
||||
|
||||
// en0 is preferred to en1
|
||||
if first.Name != second.Name {
|
||||
return first.Name < second.Name
|
||||
}
|
||||
|
||||
// fallback
|
||||
return first.String() < second.String()
|
||||
})
|
||||
return ips[0].IP
|
||||
}
|
||||
|
||||
20
act/common/outbound_ip_test.go
Normal file
20
act/common/outbound_ip_test.go
Normal file
@@ -0,0 +1,20 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestPreferredIP(t *testing.T) {
|
||||
assert.Nil(t, preferredIP(nil))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{{IP: net.ParseIP("2001:db8::1")}}))
|
||||
assert.Equal(t, net.ParseIP("2001:db8::1"), preferredIP([]interfaceIP{
|
||||
{IP: net.ParseIP("192.0.2.1"), Name: "wlan0"},
|
||||
{IP: net.ParseIP("2001:db8::1"), Name: "eth0"},
|
||||
}))
|
||||
}
|
||||
@@ -9,6 +9,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
@@ -71,19 +75,29 @@ const (
|
||||
HealthUnhealthy = string(container.Unhealthy)
|
||||
)
|
||||
|
||||
// marks what a runner creates for its jobs, telling its leftovers from another runner's
|
||||
const runnerUUIDLabel = "com.gitea.runner.uuid"
|
||||
|
||||
// ErrContainerNotFound reports a container the daemon no longer knows. Its text is a
|
||||
// fragment, missingContainerError composes it into the message every operation shares.
|
||||
var ErrContainerNotFound = errors.New("does not exist")
|
||||
|
||||
// DockerProxy is a job's docker socket, fronting the daemon's for the job's lifetime.
|
||||
type DockerProxy struct {
|
||||
Socket string
|
||||
close func(context.Context) error
|
||||
Socket string
|
||||
close func(context.Context) error
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
mounts atomic.Value
|
||||
}
|
||||
|
||||
func (p *DockerProxy) SetMounts(mounts map[string]string) {
|
||||
p.mounts.Store(mounts)
|
||||
}
|
||||
|
||||
// Close removes what the job created through the socket, then stops serving it.
|
||||
func (p *DockerProxy) Close(ctx context.Context) error {
|
||||
return p.close(ctx)
|
||||
p.closeOnce.Do(func() { p.closeErr = p.close(ctx) })
|
||||
return p.closeErr
|
||||
}
|
||||
|
||||
// Info is a snapshot of a container, as of one inspect.
|
||||
@@ -98,6 +112,16 @@ type Info struct {
|
||||
Mounts map[string]string // container path to its source on the daemon
|
||||
}
|
||||
|
||||
// DaemonPath maps target to its daemon path through the nearest mount with a source, empty without one.
|
||||
func (info *Info) DaemonPath(target string) string {
|
||||
for dir := target; dir != "/" && dir != "."; dir = path.Dir(dir) {
|
||||
if source := info.Mounts[dir]; source != "" {
|
||||
return path.Join(source, strings.TrimPrefix(target, dir))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Container for managing docker run containers
|
||||
type Container interface {
|
||||
Create(capAdd, capDrop []string) common.Executor
|
||||
|
||||
@@ -7,20 +7,33 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/distribution/reference"
|
||||
"github.com/docker/cli/cli/config"
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
)
|
||||
|
||||
func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfig, error) {
|
||||
logger := common.Logger(ctx)
|
||||
// config.LoadDefaultConfigFile panics on nil io.Writer when the config
|
||||
// file is malformed; use config.Load to route errors through the logger.
|
||||
cfg, err := config.Load(config.Dir())
|
||||
cfg, err := loadDockerConfig(ctx)
|
||||
if err != nil {
|
||||
logger.Warnf("Could not load docker config: %v", err)
|
||||
return registry.AuthConfig{}, err
|
||||
@@ -34,32 +47,27 @@ func LoadDockerAuthConfig(ctx context.Context, image string) (registry.AuthConfi
|
||||
}
|
||||
}
|
||||
|
||||
authConfig, err := cfg.GetAuthConfig(registryKey)
|
||||
authConfig, err := cfg.authConfig(ctx, registryKey)
|
||||
if err != nil {
|
||||
logger.Warnf("Could not get auth config from docker config: %v", err)
|
||||
return registry.AuthConfig{}, err
|
||||
}
|
||||
|
||||
return registry.AuthConfig(authConfig), nil
|
||||
return authConfig, nil
|
||||
}
|
||||
|
||||
func LoadDockerAuthConfigs(ctx context.Context) map[string]registry.AuthConfig {
|
||||
logger := common.Logger(ctx)
|
||||
cfg, err := config.Load(config.Dir())
|
||||
cfg, err := loadDockerConfig(ctx)
|
||||
if err != nil {
|
||||
logger.Warnf("Could not load docker config: %v", err)
|
||||
return nil
|
||||
}
|
||||
creds, err := cfg.GetAllCredentials()
|
||||
authConfigs, err := cfg.allAuthConfigs(ctx)
|
||||
if err != nil {
|
||||
logger.Warnf("Could not get docker auth configs: %v", err)
|
||||
return nil
|
||||
}
|
||||
authConfigs := make(map[string]registry.AuthConfig, len(creds))
|
||||
for k, v := range creds {
|
||||
authConfigs[k] = registry.AuthConfig(v)
|
||||
}
|
||||
|
||||
return authConfigs
|
||||
}
|
||||
|
||||
@@ -69,3 +77,226 @@ func registryAuthConfigKey(domainName string) string {
|
||||
}
|
||||
return domainName
|
||||
}
|
||||
|
||||
type dockerConfig struct {
|
||||
Auths map[string]registry.AuthConfig `json:"auths"`
|
||||
CredsStore string `json:"credsStore"`
|
||||
CredHelpers map[string]string `json:"credHelpers"`
|
||||
env map[string]registry.AuthConfig // from DOCKER_AUTH_CONFIG, nil when unset or unusable
|
||||
}
|
||||
|
||||
func loadDockerConfig(ctx context.Context) (*dockerConfig, error) {
|
||||
dir := os.Getenv("DOCKER_CONFIG")
|
||||
if dir == "" {
|
||||
home, _ := os.UserHomeDir()
|
||||
if home == "" && runtime.GOOS != "windows" {
|
||||
if current, err := user.Current(); err == nil {
|
||||
home = current.HomeDir
|
||||
}
|
||||
}
|
||||
dir = filepath.Join(home, ".docker")
|
||||
}
|
||||
cfg := &dockerConfig{}
|
||||
filename := filepath.Join(dir, "config.json")
|
||||
file, err := os.Open(filename)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, fmt.Errorf("loading config file: %w", err)
|
||||
}
|
||||
if err == nil {
|
||||
defer file.Close()
|
||||
if err := json.NewDecoder(file).Decode(cfg); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
|
||||
}
|
||||
}
|
||||
for address, auth := range cfg.Auths {
|
||||
if auth.Auth != "" {
|
||||
if auth.Username, auth.Password, err = decodeAuth(auth.Auth); err != nil {
|
||||
return nil, fmt.Errorf("parsing config file (%s): %w", filename, err)
|
||||
}
|
||||
}
|
||||
auth.Auth, auth.ServerAddress = "", address
|
||||
cfg.Auths[address] = auth
|
||||
}
|
||||
if value := os.Getenv("DOCKER_AUTH_CONFIG"); value != "" {
|
||||
if cfg.env, err = parseDockerAuthConfigEnv(value); err != nil {
|
||||
common.Logger(ctx).Warnf("Failed to create credential store from DOCKER_AUTH_CONFIG: %v", err)
|
||||
}
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func decodeAuth(auth string) (string, string, error) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(auth)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
username, password, ok := strings.Cut(string(decoded), ":")
|
||||
if !ok || username == "" {
|
||||
return "", "", errors.New("invalid auth configuration file")
|
||||
}
|
||||
return username, strings.Trim(password, "\x00"), nil
|
||||
}
|
||||
|
||||
func parseDockerAuthConfigEnv(value string) (map[string]registry.AuthConfig, error) {
|
||||
var env struct {
|
||||
Auths map[string]struct {
|
||||
Auth string `json:"auth"`
|
||||
} `json:"auths"`
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(value))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&env); err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
if decoder.More() {
|
||||
return nil, errors.New("DOCKER_AUTH_CONFIG does not support more than one JSON object")
|
||||
}
|
||||
auths := make(map[string]registry.AuthConfig, len(env.Auths))
|
||||
for address, entry := range env.Auths {
|
||||
if entry.Auth == "" {
|
||||
return nil, fmt.Errorf("DOCKER_AUTH_CONFIG environment variable is missing key `auth` for %s", address)
|
||||
}
|
||||
username, password, err := decodeAuth(entry.Auth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auths[address] = registry.AuthConfig{Username: username, Password: password, ServerAddress: address}
|
||||
}
|
||||
return auths, nil
|
||||
}
|
||||
|
||||
func (c *dockerConfig) authConfig(ctx context.Context, key string) (registry.AuthConfig, error) {
|
||||
if auth, ok := c.env[key]; ok {
|
||||
return auth, nil
|
||||
}
|
||||
helper := c.CredsStore
|
||||
if named, ok := c.CredHelpers[key]; ok {
|
||||
helper = named
|
||||
}
|
||||
if helper == "" {
|
||||
return c.fileAuth(key), nil
|
||||
}
|
||||
auth := c.fileAuth(key)
|
||||
stored, err := helperCredentials(ctx, helper, key)
|
||||
if err != nil {
|
||||
return auth, err
|
||||
}
|
||||
auth.Username, auth.Password, auth.IdentityToken, auth.ServerAddress = stored.Username, stored.Password, stored.IdentityToken, stored.ServerAddress
|
||||
return auth, nil
|
||||
}
|
||||
|
||||
func (c *dockerConfig) fileAuth(key string) registry.AuthConfig {
|
||||
if auth, ok := c.Auths[key]; ok {
|
||||
return auth
|
||||
}
|
||||
for address, auth := range c.Auths {
|
||||
if registryHostname(address) == key {
|
||||
return auth
|
||||
}
|
||||
}
|
||||
return registry.AuthConfig{}
|
||||
}
|
||||
|
||||
func (c *dockerConfig) allAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
|
||||
var auths map[string]registry.AuthConfig
|
||||
if c.CredsStore == "" {
|
||||
auths = maps.Clone(c.Auths)
|
||||
} else {
|
||||
var err error
|
||||
if auths, err = c.helperAuthConfigs(ctx); err != nil {
|
||||
if c.env == nil {
|
||||
return nil, err
|
||||
}
|
||||
common.Logger(ctx).Warnf("Could not list the credentials of docker-credential-%s: %v", c.CredsStore, err)
|
||||
}
|
||||
}
|
||||
if auths == nil {
|
||||
auths = map[string]registry.AuthConfig{}
|
||||
}
|
||||
maps.Copy(auths, c.env)
|
||||
if c.CredsStore == "" {
|
||||
c.Auths = maps.Clone(auths) // docker's file store holds the DOCKER_AUTH_CONFIG entries after listing, and credHelpers set to "" read it
|
||||
}
|
||||
for address := range c.CredHelpers {
|
||||
auth, err := c.authConfig(ctx, registryAuthConfigKey(address))
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warnf("Failed to get credentials for registry: %s: %v", address, err)
|
||||
continue
|
||||
}
|
||||
auths[address] = auth
|
||||
}
|
||||
return auths, nil
|
||||
}
|
||||
|
||||
func (c *dockerConfig) helperAuthConfigs(ctx context.Context) (map[string]registry.AuthConfig, error) {
|
||||
out, err := runCredentialHelper(ctx, c.CredsStore, "list", "unused")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error listing credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
var addresses map[string]string
|
||||
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&addresses); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auths := make(map[string]registry.AuthConfig, len(addresses))
|
||||
for address := range addresses {
|
||||
stored, err := helperCredentials(ctx, c.CredsStore, address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
auth := c.Auths[address]
|
||||
auth.Username, auth.Password, auth.IdentityToken = stored.Username, stored.Password, stored.IdentityToken
|
||||
if auth.ServerAddress == "" {
|
||||
auth.ServerAddress = stored.ServerAddress
|
||||
}
|
||||
auths[address] = auth
|
||||
}
|
||||
return auths, nil
|
||||
}
|
||||
|
||||
func helperCredentials(ctx context.Context, helper, address string) (registry.AuthConfig, error) {
|
||||
out, err := runCredentialHelper(ctx, helper, "get", address)
|
||||
if err != nil {
|
||||
if strings.TrimSpace(string(out)) == "credentials not found in native keychain" {
|
||||
return registry.AuthConfig{}, nil
|
||||
}
|
||||
return registry.AuthConfig{}, fmt.Errorf("error getting credentials - err: %w, out: `%s`", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
var stored struct{ Username, Secret string }
|
||||
if err := json.NewDecoder(bytes.NewReader(out)).Decode(&stored); err != nil {
|
||||
return registry.AuthConfig{}, err
|
||||
}
|
||||
if stored.Username == "<token>" {
|
||||
return registry.AuthConfig{IdentityToken: stored.Secret, ServerAddress: address}, nil
|
||||
}
|
||||
return registry.AuthConfig{Username: stored.Username, Password: stored.Secret, ServerAddress: address}, nil
|
||||
}
|
||||
|
||||
func runCredentialHelper(ctx context.Context, helper, action, input string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, "docker-credential-"+helper, action)
|
||||
cmd.Stdin = strings.NewReader(input)
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Output()
|
||||
}
|
||||
|
||||
func registryHostname(address string) string {
|
||||
if _, rest, ok := strings.Cut(address, "://"); ok {
|
||||
if u, err := url.Parse(address); err == nil && u.Hostname() != "" {
|
||||
if u.Port() == "" {
|
||||
return u.Hostname()
|
||||
}
|
||||
return net.JoinHostPort(u.Hostname(), u.Port())
|
||||
}
|
||||
if host, _, _ := strings.Cut(rest, "/"); host != "" {
|
||||
if colon := strings.LastIndex(host, ":"); strings.Count(host, ":") > 1 && !strings.HasPrefix(host, "[") && isDigits(host[colon+1:]) {
|
||||
return net.JoinHostPort(host[:colon], host[colon+1:])
|
||||
}
|
||||
return host
|
||||
}
|
||||
}
|
||||
host, _, _ := strings.Cut(address, "/")
|
||||
return host
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
return s != "" && strings.Trim(s, "0123456789") == ""
|
||||
}
|
||||
|
||||
46
act/container/docker_auth_test.go
Normal file
46
act/container/docker_auth_test.go
Normal file
@@ -0,0 +1,46 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !windows && !(WITHOUT_DOCKER || !(linux || darwin || netbsd))
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadDockerAuthConfigFromFileHelpersAndEnv(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "docker-credential-fake"), []byte("#!/bin/sh\nread -r server\n[ \"$1\" = get ] && [ \"$server\" = helper.example.com ] && exec echo '{\"Username\":\"helped\",\"Secret\":\"pw\"}'\necho 'credentials not found in native keychain'\nexit 1\n"), 0o755))
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
auth := func(credentials string) string { return base64.StdEncoding.EncodeToString([]byte(credentials)) }
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"auths": {"https://legacy.example.com/v1/": {"auth": "`+auth("legacy:pw")+`"}}, "credHelpers": {"helper.example.com": "fake", "gone.example.com": "fake", "env-legacy.example.com": ""}}`), 0o600))
|
||||
t.Setenv("DOCKER_CONFIG", dir)
|
||||
t.Setenv("DOCKER_AUTH_CONFIG", `{"auths": {"env.example.com": {"auth": "`+auth("env:pw")+`"}, "https://env-legacy.example.com/v1/": {"auth": "`+auth("env:pw")+`"}}}`)
|
||||
|
||||
for image, want := range map[string]registry.AuthConfig{
|
||||
"legacy.example.com/app": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
|
||||
"helper.example.com/app": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
|
||||
"env.example.com/app": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
|
||||
"gone.example.com/app": {},
|
||||
} {
|
||||
got, err := LoadDockerAuthConfig(t.Context(), image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, got, image)
|
||||
}
|
||||
assert.Equal(t, map[string]registry.AuthConfig{
|
||||
"https://legacy.example.com/v1/": {Username: "legacy", Password: "pw", ServerAddress: "https://legacy.example.com/v1/"},
|
||||
"helper.example.com": {Username: "helped", Password: "pw", ServerAddress: "helper.example.com"},
|
||||
"gone.example.com": {},
|
||||
"env.example.com": {Username: "env", Password: "pw", ServerAddress: "env.example.com"},
|
||||
"https://env-legacy.example.com/v1/": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
|
||||
"env-legacy.example.com": {Username: "env", Password: "pw", ServerAddress: "https://env-legacy.example.com/v1/"},
|
||||
}, LoadDockerAuthConfigs(t.Context()))
|
||||
}
|
||||
@@ -7,15 +7,18 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/moby/go-archive"
|
||||
"github.com/moby/go-archive/compression"
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/moby/patternmatcher"
|
||||
"github.com/moby/patternmatcher/ignorefile"
|
||||
@@ -43,9 +46,8 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
|
||||
logger.Debugf("Building image from '%v'", input.ContextDir)
|
||||
|
||||
tags := []string{input.ImageTag}
|
||||
options := client.ImageBuildOptions{
|
||||
Tags: tags,
|
||||
Tags: []string{input.ImageTag},
|
||||
Remove: true,
|
||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||
Dockerfile: input.Dockerfile,
|
||||
@@ -73,11 +75,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
logger.Debugf("Creating image from context dir '%s' with tag '%s' and platform '%s'", input.ContextDir, input.ImageTag, input.Platform)
|
||||
resp, err := cli.ImageBuild(ctx, buildContext, options)
|
||||
|
||||
err = logDockerResponse(logger, resp.Body, err != nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return logDockerResponse(logger, resp.Body, err != nil)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,14 +113,140 @@ func createBuildContext(ctx context.Context, contextDir, relDockerfile string) (
|
||||
includes = append(includes, ".dockerignore", relDockerfile)
|
||||
}
|
||||
|
||||
buildCtx, err := archive.TarWithOptions(contextDir, &archive.TarOptions{
|
||||
Compression: compression.None,
|
||||
ExcludePatterns: excludes,
|
||||
IncludeFiles: includes,
|
||||
})
|
||||
pm, err := patternmatcher.New(excludes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buildCtx, nil
|
||||
reader, writer := io.Pipe()
|
||||
go func() { writer.CloseWithError(writeBuildContext(ctx, writer, contextDir, includes, pm)) }()
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func writeBuildContext(ctx context.Context, w io.Writer, contextDir string, includes []string, pm *patternmatcher.PatternMatcher) error {
|
||||
tw := tar.NewWriter(w)
|
||||
seen := map[string]bool{}
|
||||
hardlinks := map[uint64]string{}
|
||||
buffer := make([]byte, 32*1024)
|
||||
for _, include := range includes {
|
||||
include = filepath.FromSlash(include)
|
||||
var parents []parentMatch
|
||||
root := strings.TrimSuffix(contextDir, string(filepath.Separator)) + string(filepath.Separator) + include // unlike filepath.Join, follows a symlinked context dir as docker build does
|
||||
err := filepath.WalkDir(root, func(file string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
rel, err := filepath.Rel(contextDir, file)
|
||||
if err != nil || rel == "." {
|
||||
return nil
|
||||
}
|
||||
if rel != include {
|
||||
for len(parents) > 0 && !strings.HasPrefix(rel, parents[len(parents)-1].dir+string(filepath.Separator)) {
|
||||
parents = parents[:len(parents)-1]
|
||||
}
|
||||
var parentInfo patternmatcher.MatchInfo
|
||||
if len(parents) > 0 {
|
||||
parentInfo = parents[len(parents)-1].info
|
||||
}
|
||||
skip, info, err := pm.MatchesUsingParentResults(rel, parentInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
parents = append(parents, parentMatch{rel, info})
|
||||
}
|
||||
if skip {
|
||||
if entry.IsDir() && !hasExceptionWithin(pm, rel) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if seen[rel] {
|
||||
return nil
|
||||
}
|
||||
seen[rel] = true
|
||||
return addBuildContextEntry(ctx, tw, file, rel, entry, hardlinks, buffer)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tw.Close()
|
||||
}
|
||||
|
||||
type parentMatch struct {
|
||||
dir string
|
||||
info patternmatcher.MatchInfo
|
||||
}
|
||||
|
||||
func hasExceptionWithin(pm *patternmatcher.PatternMatcher, dir string) bool {
|
||||
dirPrefix := dir + string(filepath.Separator)
|
||||
for _, pattern := range pm.Patterns() {
|
||||
if pattern.Exclusion() && strings.HasPrefix(pattern.String()+string(filepath.Separator), dirPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func leaveOut(ctx context.Context, file string, err error) error {
|
||||
common.Logger(ctx).Warnf("Leaving %s out of the build context: %v", file, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
func addBuildContextEntry(ctx context.Context, tw *tar.Writer, file, rel string, entry fs.DirEntry, hardlinks map[uint64]string, buffer []byte) error {
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
if info.Mode()&fs.ModeSocket != 0 {
|
||||
return nil // tar has no socket type
|
||||
}
|
||||
var link string
|
||||
if info.Mode()&fs.ModeSymlink != 0 {
|
||||
if link, err = os.Readlink(file); err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
}
|
||||
header, err := tar.FileInfoHeader(noNameLookups{info}, link)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.ToSlash(rel)
|
||||
inode, linked := hardlinkInode(info)
|
||||
if target, ok := hardlinks[inode]; linked && ok {
|
||||
header.Typeflag, header.Linkname, header.Size = tar.TypeLink, target, 0
|
||||
}
|
||||
var content *os.File
|
||||
if header.Typeflag == tar.TypeReg && header.Size > 0 {
|
||||
if content, err = os.Open(file); err != nil {
|
||||
return leaveOut(ctx, file, err)
|
||||
}
|
||||
defer content.Close()
|
||||
}
|
||||
if linked && header.Typeflag != tar.TypeLink {
|
||||
hardlinks[inode] = header.Name
|
||||
}
|
||||
if info.IsDir() {
|
||||
header.Name += "/"
|
||||
}
|
||||
header.Format = tar.FormatPAX
|
||||
header.ModTime = header.ModTime.Truncate(time.Second)
|
||||
header.AccessTime, header.ChangeTime = time.Time{}, time.Time{}
|
||||
if runtime.GOOS == "windows" {
|
||||
header.Mode = header.Mode&0o755 | 0o111 // Windows has no exec bit, so mark everything executable like docker build
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
if content != nil {
|
||||
_, err = io.CopyBuffer(tw, struct{ io.Reader }{content}, buffer) // hides os.File's WriteTo, which would allocate a buffer per file
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type noNameLookups struct{ fs.FileInfo } // leaves Uname and Gname empty like docker build, so packing never waits on NSS
|
||||
|
||||
func (noNameLookups) Uname() (string, error) { return "", nil }
|
||||
|
||||
func (noNameLookups) Gname() (string, error) { return "", nil }
|
||||
|
||||
49
act/container/docker_build_test.go
Normal file
49
act/container/docker_build_test.go
Normal file
@@ -0,0 +1,49 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCreateBuildContextFollowsDockerignoreAndSymlinkedRootButKeepsDockerfile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
".dockerignore": "*.log\nbuild\n!build/keep.txt\nDockerfile", "Dockerfile": "FROM scratch", "a.txt": "a",
|
||||
"a.log": "log", "build/drop.txt": "drop", "build/keep.txt": "keep",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(dir, "empty"), 0o755))
|
||||
|
||||
contexts := []string{dir}
|
||||
if link := filepath.Join(t.TempDir(), "link"); os.Symlink(dir, link) == nil {
|
||||
contexts = append(contexts, link+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, contextDir := range contexts {
|
||||
buildContext, err := createBuildContext(t.Context(), contextDir, "Dockerfile")
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for reader := tar.NewReader(buildContext); ; {
|
||||
header, err := reader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
names = append(names, header.Name)
|
||||
}
|
||||
require.NoError(t, buildContext.Close())
|
||||
require.Equal(t, []string{".dockerignore", "a.txt", "build/keep.txt", "empty/", "Dockerfile"}, names, contextDir)
|
||||
}
|
||||
}
|
||||
18
act/container/docker_build_unix.go
Normal file
18
act/container/docker_build_unix.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func hardlinkInode(info fs.FileInfo) (uint64, bool) {
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok && !info.IsDir() && stat.Nlink > 1 {
|
||||
return stat.Ino, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
35
act/container/docker_build_unix_test.go
Normal file
35
act/container/docker_build_unix_test.go
Normal file
@@ -0,0 +1,35 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCreateBuildContextKeepsHardlinkedSpecialFilesLinked(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
require.NoError(t, syscall.Mkfifo(filepath.Join(dir, "first"), 0o644))
|
||||
require.NoError(t, os.Link(filepath.Join(dir, "first"), filepath.Join(dir, "second")))
|
||||
|
||||
buildContext, err := createBuildContext(t.Context(), dir, "Dockerfile")
|
||||
require.NoError(t, err)
|
||||
defer buildContext.Close()
|
||||
reader := tar.NewReader(buildContext)
|
||||
first, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, byte(tar.TypeFifo), first.Typeflag)
|
||||
second, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, byte(tar.TypeLink), second.Typeflag)
|
||||
assert.Equal(t, first.Name, second.Linkname)
|
||||
}
|
||||
10
act/container/docker_build_windows.go
Normal file
10
act/container/docker_build_windows.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER
|
||||
|
||||
package container
|
||||
|
||||
import "io/fs"
|
||||
|
||||
func hardlinkInode(_info fs.FileInfo) (uint64, bool) { return 0, false }
|
||||
@@ -4,10 +4,12 @@
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
|
||||
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts.go with:
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts.go with:
|
||||
// * appended with license information
|
||||
// * regexp and loader.ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
|
||||
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
|
||||
// * regexp and ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
|
||||
// * cdiDeviceName in place of the container-device-interface parser's IsQualifiedName
|
||||
// * jsontext in place of json.Compact for the seccomp profile
|
||||
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for nat port types
|
||||
//
|
||||
// docker/cli is licensed under the Apache License, Version 2.0.
|
||||
// See DOCKER_LICENSE for the full license text.
|
||||
@@ -32,14 +34,12 @@ import (
|
||||
"time"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
"github.com/docker/cli/opts"
|
||||
"github.com/docker/go-connections/nat"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/spf13/pflag"
|
||||
cdi "tags.cncf.io/container-device-interface/pkg/parser"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -152,10 +152,8 @@ type containerOptions struct {
|
||||
runtime string
|
||||
autoRemove bool
|
||||
init bool
|
||||
umask opts.UmaskOpt
|
||||
annotations *opts.MapOpts
|
||||
|
||||
Image string
|
||||
Args []string
|
||||
}
|
||||
|
||||
// addFlags adds all command line flags that will be used by parse to the FlagSet
|
||||
@@ -326,6 +324,9 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
|
||||
flags.Var(&copts.shmSize, "shm-size", "Size of /dev/shm")
|
||||
flags.StringVar(&copts.utsMode, "uts", "", "UTS namespace to use")
|
||||
flags.StringVar(&copts.runtime, "runtime", "", "Runtime to use for this container")
|
||||
flags.Var(&copts.umask, "umask", "Set umask for the container")
|
||||
flags.SetAnnotation("umask", "version", []string{"1.56"})
|
||||
flags.SetAnnotation("umask", "ostype", []string{"linux"})
|
||||
|
||||
flags.BoolVar(&copts.init, "init", false, "Run an init inside the container that forwards signals and reaps processes")
|
||||
flags.SetAnnotation("init", "version", []string{"1.25"})
|
||||
@@ -383,7 +384,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
volumes := copts.volumes.GetMap()
|
||||
// add any bind targets to the list of container volumes
|
||||
for bind := range volumes {
|
||||
parsed, err := loader.ParseVolume(bind)
|
||||
parsed, err := ParseVolume(bind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -391,7 +392,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
if parsed.Source != "" {
|
||||
toBind := bind
|
||||
|
||||
if parsed.Type == string(mount.TypeBind) {
|
||||
if parsed.Type == mount.TypeBind {
|
||||
if hostPart, targetPath, ok := strings.Cut(bind, ":"); ok {
|
||||
if !filepath.IsAbs(hostPart) && strings.HasPrefix(hostPart, ".") {
|
||||
if absHostPart, err := filepath.Abs(hostPart); err == nil {
|
||||
@@ -418,12 +419,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
tmpfs[k] = v
|
||||
}
|
||||
|
||||
var runCmd, entrypoint []string
|
||||
|
||||
if len(copts.Args) > 0 {
|
||||
runCmd = copts.Args
|
||||
}
|
||||
|
||||
var entrypoint []string
|
||||
if copts.entrypoint != "" {
|
||||
entrypoint = []string{copts.entrypoint}
|
||||
} else if flags.Changed("entrypoint") {
|
||||
@@ -444,36 +440,15 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
portBindings := network.PortMap{}
|
||||
for port, bindings := range natPortBindings {
|
||||
p, err := network.ParsePort(string(port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
portBindings[p] = []network.PortBinding{}
|
||||
for _, b := range bindings {
|
||||
var hostIP netip.Addr
|
||||
if b.HostIP != "" {
|
||||
hostIP, err = netip.ParseAddr(b.HostIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
portBindings[p] = append(portBindings[p], network.PortBinding{
|
||||
HostIP: hostIP,
|
||||
HostPort: b.HostPort,
|
||||
})
|
||||
}
|
||||
portBindings, err := convertPortMap(natPortBindings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add published ports as exposed ports.
|
||||
exposedPorts := network.PortSet{}
|
||||
for port := range ports {
|
||||
p, err := network.ParsePort(string(port))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exposedPorts[p] = struct{}{}
|
||||
exposedPorts, err := convertPortSet(ports)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Merge in exposed ports to the map of published ports
|
||||
@@ -499,7 +474,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
deviceMappings := make([]container.DeviceMapping, 0, len(devices))
|
||||
cdiDeviceNames := make([]string, 0, len(devices))
|
||||
for _, device := range devices {
|
||||
if cdi.IsQualifiedName(device) {
|
||||
if cdiDeviceName.MatchString(device) {
|
||||
cdiDeviceNames = append(cdiDeviceNames, device)
|
||||
continue
|
||||
}
|
||||
@@ -614,11 +589,10 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
|
||||
deviceRequests := copts.gpus.Value()
|
||||
if len(cdiDeviceNames) > 0 {
|
||||
cdiDeviceRequest := container.DeviceRequest{
|
||||
deviceRequests = append(deviceRequests, container.DeviceRequest{
|
||||
Driver: "cdi",
|
||||
DeviceIDs: cdiDeviceNames,
|
||||
}
|
||||
deviceRequests = append(deviceRequests, cdiDeviceRequest)
|
||||
})
|
||||
}
|
||||
|
||||
resources := container.Resources{
|
||||
@@ -664,8 +638,6 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
AttachStdout: attachStdout,
|
||||
AttachStderr: attachStderr,
|
||||
Env: envVariables,
|
||||
Cmd: runCmd,
|
||||
Image: copts.Image,
|
||||
Volumes: volumes,
|
||||
Entrypoint: entrypoint,
|
||||
WorkingDir: copts.workingDir,
|
||||
@@ -721,6 +693,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
|
||||
MaskedPaths: maskedPaths,
|
||||
ReadonlyPaths: readonlyPaths,
|
||||
Annotations: copts.annotations.GetAll(),
|
||||
Umask: copts.umask.Value(),
|
||||
}
|
||||
|
||||
if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() {
|
||||
@@ -805,10 +778,8 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*network.EndpointSett
|
||||
// For backward compatibility: if no custom options are provided for the network,
|
||||
// and only a single network is specified, omit the endpoint-configuration
|
||||
// on the client (the daemon will still create it when creating the container)
|
||||
if i == 0 && len(copts.netMode.Value()) == 1 {
|
||||
if ep == nil || reflect.ValueOf(*ep).IsZero() {
|
||||
continue
|
||||
}
|
||||
if i == 0 && len(copts.netMode.Value()) == 1 && reflect.ValueOf(*ep).IsZero() {
|
||||
continue
|
||||
}
|
||||
endpoints[n.Target] = ep
|
||||
}
|
||||
@@ -839,14 +810,12 @@ func applyContainerOptions(n *opts.NetworkAttachmentOpts, copts *containerOption
|
||||
return invalidParameter(errors.New("conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses"))
|
||||
}
|
||||
if copts.aliases.Len() > 0 {
|
||||
n.Aliases = make([]string, copts.aliases.Len())
|
||||
copy(n.Aliases, copts.aliases.GetSlice())
|
||||
n.Aliases = slices.Clone(copts.aliases.GetSlice())
|
||||
}
|
||||
// For a user-defined network, "--link" is an endpoint option, it creates an alias. But,
|
||||
// for the default bridge it defines a legacy-link.
|
||||
if container.NetworkMode(n.Target).IsUserDefined() && copts.links.Len() > 0 {
|
||||
n.Links = make([]string, copts.links.Len())
|
||||
copy(n.Links, copts.links.GetSlice())
|
||||
n.Links = slices.Clone(copts.links.GetSlice())
|
||||
}
|
||||
if copts.ipv4Address != nil {
|
||||
if ipv4, ok := netip.AddrFromSlice(copts.ipv4Address.To4()); ok {
|
||||
@@ -885,7 +854,6 @@ func parseNetworkAttachmentOpt(ep opts.NetworkAttachmentOpts) (*network.Endpoint
|
||||
}
|
||||
epConfig.Aliases = append(epConfig.Aliases, ep.Aliases...)
|
||||
if len(ep.DriverOpts) > 0 {
|
||||
epConfig.DriverOpts = make(map[string]string)
|
||||
epConfig.DriverOpts = ep.DriverOpts
|
||||
}
|
||||
if len(ep.Links) > 0 {
|
||||
@@ -929,11 +897,10 @@ func convertToStandardNotation(ports []string) ([]string, error) {
|
||||
}
|
||||
|
||||
func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) {
|
||||
loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts)
|
||||
if loggingDriver == "none" && len(loggingOpts) > 0 {
|
||||
return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver)
|
||||
}
|
||||
return loggingOptsMap, nil
|
||||
return opts.ConvertKVStringsToMap(loggingOpts), nil
|
||||
}
|
||||
|
||||
// takes a local seccomp daemon, reads the file contents for sending to the daemon
|
||||
@@ -1191,6 +1158,8 @@ func convertPortSet(ports nat.PortSet) (network.PortSet, error) {
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
var cdiDeviceName = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?/[a-zA-Z]([a-zA-Z0-9_.-]*[a-zA-Z0-9])?=[a-zA-Z0-9]([a-zA-Z0-9_.:-]*[a-zA-Z0-9])?$`)
|
||||
|
||||
func convertPortMap(bindings nat.PortMap) (network.PortMap, error) {
|
||||
converted := make(network.PortMap, len(bindings))
|
||||
for port, portBindings := range bindings {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Copyright 2022 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts_test.go with:
|
||||
// This file is based on https://github.com/docker/cli/blob/4a63305d74332de5ceba7fcbccbc3cbb7412f5ba/cli/command/container/opts_test.go with:
|
||||
// * appended with license information
|
||||
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter
|
||||
//
|
||||
@@ -25,13 +25,11 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
networktypes "github.com/moby/moby/api/types/network"
|
||||
"github.com/spf13/pflag"
|
||||
"gotest.tools/v3/assert"
|
||||
is "gotest.tools/v3/assert/cmp"
|
||||
"gotest.tools/v3/skip"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func mustParseMAC(s string) networktypes.HardwareAddr {
|
||||
@@ -71,7 +69,6 @@ func parseRun(args []string) (*container.Config, *container.HostConfig, *network
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// TODO(dnephin): fix tests to accept ContainerConfig; see https://github.com/moby/moby/pull/31621
|
||||
containerCfg, err := parse(flags, copts, runtime.GOOS)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
@@ -83,14 +80,13 @@ func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
|
||||
flags := pflag.NewFlagSet("run", pflag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
flags.Usage = nil
|
||||
copts := addFlags(flags)
|
||||
return flags, copts
|
||||
return flags, addFlags(flags)
|
||||
}
|
||||
|
||||
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
|
||||
t.Helper()
|
||||
config, hostConfig, nwConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash"))
|
||||
assert.NilError(t, err)
|
||||
config, hostConfig, nwConfig, err := parseRun(strings.Split(args, " "))
|
||||
require.NoError(t, err)
|
||||
return config, hostConfig, nwConfig
|
||||
}
|
||||
|
||||
@@ -156,65 +152,46 @@ func TestParseRunLinks(t *testing.T) {
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, hostConfig, netConfig := mustParse(t, tc.input)
|
||||
assert.Check(t, is.DeepEqual(hostConfig.Links, tc.expHostConfigLinks))
|
||||
assert.Check(t, is.Len(netConfig.EndpointsConfig, len(tc.expNetConfigLinks)))
|
||||
assert.Equal(t, tc.expHostConfigLinks, hostConfig.Links)
|
||||
assert.Len(t, netConfig.EndpointsConfig, len(tc.expNetConfigLinks))
|
||||
for netName, expLinks := range tc.expNetConfigLinks {
|
||||
nc, ok := netConfig.EndpointsConfig[netName]
|
||||
assert.Assert(t, ok)
|
||||
assert.Check(t, is.DeepEqual(nc.Links, expLinks))
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, expLinks, nc.Links)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRunWithoutUmask(t *testing.T) {
|
||||
_, hostConfig, _, err := parseRun([]string{"ubuntu", "bash"})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, hostConfig.Umask)
|
||||
}
|
||||
|
||||
func TestParseRunUmask(t *testing.T) {
|
||||
_, hostConfig, _, err := parseRun([]string{"--umask", "0022", "ubuntu", "bash"})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, hostConfig.Umask)
|
||||
require.Equal(t, uint32(0o22), *hostConfig.Umask)
|
||||
}
|
||||
|
||||
func TestParseRunAttach(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected container.Config
|
||||
for _, tc := range []struct {
|
||||
input string
|
||||
stdin, stdout, stderr bool
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
expected: container.Config{
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-i",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin -a stdout",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
input: "-a stdin -a stdout -a stderr",
|
||||
expected: container.Config{
|
||||
AttachStdin: true,
|
||||
AttachStdout: true,
|
||||
AttachStderr: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
{"", false, true, true},
|
||||
{"-i", true, true, true},
|
||||
{"-a stdin", true, false, false},
|
||||
{"-a stdin -a stdout", true, true, false},
|
||||
{"-a stdin -a stdout -a stderr", true, true, true},
|
||||
} {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
config, _, _ := mustParse(t, tc.input)
|
||||
assert.Equal(t, config.AttachStdin, tc.expected.AttachStdin)
|
||||
assert.Equal(t, config.AttachStdout, tc.expected.AttachStdout)
|
||||
assert.Equal(t, config.AttachStderr, tc.expected.AttachStderr)
|
||||
require.Equal(t, tc.stdin, config.AttachStdin)
|
||||
require.Equal(t, tc.stdout, config.AttachStdout)
|
||||
require.Equal(t, tc.stderr, config.AttachStderr)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -260,109 +237,52 @@ func TestParseRunWithInvalidArgs(t *testing.T) {
|
||||
flags, _ := setupRunFlags()
|
||||
for _, tc := range tests {
|
||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||
assert.Error(t, flags.Parse(tc.args), tc.error)
|
||||
require.EqualError(t, flags.Parse(tc.args), tc.error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWithVolumes(t *testing.T) { //nolint:gocyclo // verbatim copy from docker/cli tests
|
||||
// A single volume
|
||||
arr, tryit := setupPlatformVolume([]string{`/tmp`}, []string{`c:\tmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %q is missing from volumes. Received %v", tryit, config.Volumes)
|
||||
}
|
||||
|
||||
// Two volumes
|
||||
arr, tryit = setupPlatformVolume([]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds != nil {
|
||||
t.Fatalf("Error parsing volume flags, %q should not mount-bind anything. Received %v", tryit, hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[0]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[0], config.Volumes)
|
||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags, %s is missing from volumes. Received %v", arr[1], config.Volumes)
|
||||
}
|
||||
|
||||
// A single bind mount
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || hostConfig.Binds[0] != arr[0] {
|
||||
t.Fatalf("Error parsing volume flags, %q should mount-bind the path before the colon into the path after the colon. Received %v %v", arr[0], hostConfig.Binds, config.Volumes)
|
||||
}
|
||||
|
||||
// Two bind mounts.
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
}
|
||||
|
||||
// Two bind mounts, first read-only, second read-write.
|
||||
// TODO Windows: The Windows version uses read-write as that's the only mode it supports. Can change this post TP4
|
||||
arr, tryit = setupPlatformVolume(
|
||||
[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`},
|
||||
[]string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
}
|
||||
|
||||
// Similar to previous test but with alternate modes which are only supported by Linux
|
||||
if runtime.GOOS != "windows" {
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, []string{})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
func TestParseWithVolumes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
unix, windows []string
|
||||
binds int
|
||||
}{
|
||||
{[]string{`/tmp`}, []string{`c:\tmp`}, 0},
|
||||
{[]string{`/tmp`, `/var`}, []string{`c:\tmp`, `c:\var`}, 0},
|
||||
{[]string{`/hostTmp:/containerTmp`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`}, 1},
|
||||
{[]string{`/hostTmp:/containerTmp`, `/hostVar:/containerVar`}, []string{os.Getenv("ProgramData") + `:c:\ContainerPD`, os.Getenv("TEMP") + `:c:\containerTmp`}, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:ro`, `/hostVar:/containerVar:rw`}, []string{os.Getenv("TEMP") + `:c:\containerTmp:rw`, os.Getenv("ProgramData") + `:c:\ContainerPD:rw`}, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:ro,Z`, `/hostVar:/containerVar:rw,Z`}, nil, 2},
|
||||
{[]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, nil, 2},
|
||||
{[]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`}, 1},
|
||||
{nil, []string{os.Getenv("SystemDrive") + `\:d:`}, 1},
|
||||
} {
|
||||
volumes := tc.unix
|
||||
if runtime.GOOS == "windows" {
|
||||
volumes = tc.windows
|
||||
}
|
||||
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp:Z`, `/hostVar:/containerVar:z`}, []string{})
|
||||
if _, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || compareRandomizedStrings(hostConfig.Binds[0], hostConfig.Binds[1], arr[0], arr[1]) != nil {
|
||||
t.Fatalf("Error parsing volume flags, `%s and %s` did not mount-bind correctly. Received %v", arr[0], arr[1], hostConfig.Binds)
|
||||
if len(volumes) == 0 {
|
||||
continue
|
||||
}
|
||||
t.Run(strings.Join(volumes, ","), func(t *testing.T) {
|
||||
var args []string
|
||||
for _, volume := range volumes {
|
||||
args = append(args, "-v", volume)
|
||||
}
|
||||
config, hostConfig, _, err := parseRun(args)
|
||||
require.NoError(t, err)
|
||||
if tc.binds == 0 {
|
||||
assert.Nil(t, hostConfig.Binds)
|
||||
} else {
|
||||
assert.ElementsMatch(t, volumes[:tc.binds], hostConfig.Binds)
|
||||
}
|
||||
expectedVolumes := map[string]struct{}{}
|
||||
for _, volume := range volumes[tc.binds:] {
|
||||
expectedVolumes[volume] = struct{}{}
|
||||
}
|
||||
assert.Equal(t, expectedVolumes, config.Volumes)
|
||||
})
|
||||
}
|
||||
|
||||
// One bind mount and one volume
|
||||
arr, tryit = setupPlatformVolume([]string{`/hostTmp:/containerTmp`, `/containerVar`}, []string{os.Getenv("TEMP") + `:c:\containerTmp`, `c:\containerTmp`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] {
|
||||
t.Fatalf("Error parsing volume flags, %s and %s should only one and only one bind mount %s. Received %s", arr[0], arr[1], arr[0], hostConfig.Binds)
|
||||
} else if _, exists := config.Volumes[arr[1]]; !exists {
|
||||
t.Fatalf("Error parsing volume flags %s and %s. %s is missing from volumes. Received %v", arr[0], arr[1], arr[1], config.Volumes)
|
||||
}
|
||||
|
||||
// Root to non-c: drive letter (Windows specific)
|
||||
if runtime.GOOS == "windows" {
|
||||
arr, tryit = setupPlatformVolume([]string{}, []string{os.Getenv("SystemDrive") + `\:d:`})
|
||||
if config, hostConfig, _ := mustParse(t, tryit); hostConfig.Binds == nil || len(hostConfig.Binds) > 1 || hostConfig.Binds[0] != arr[0] || len(config.Volumes) != 0 {
|
||||
t.Fatalf("Error parsing %s. Should have a single bind mount and no volumes", arr[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setupPlatformVolume takes two arrays of volume specs - a Unix style
|
||||
// spec and a Windows style spec. Depending on the platform being unit tested,
|
||||
// it returns one of them, along with a volume string that would be passed
|
||||
// on the docker CLI (e.g. -v /bar -v /foo).
|
||||
func setupPlatformVolume(u, w []string) ([]string, string) {
|
||||
var a []string
|
||||
if runtime.GOOS == "windows" {
|
||||
a = w
|
||||
} else {
|
||||
a = u
|
||||
}
|
||||
s := ""
|
||||
for _, v := range a {
|
||||
s = s + "-v " + v + " "
|
||||
}
|
||||
return a, s
|
||||
}
|
||||
|
||||
// check if (a == c && b == d) || (a == d && b == c)
|
||||
// because maps are randomized
|
||||
func compareRandomizedStrings(a, b, c, d string) error {
|
||||
if a == c && b == d {
|
||||
return nil
|
||||
}
|
||||
if a == d && b == c {
|
||||
return nil
|
||||
}
|
||||
return errors.New("strings don't match")
|
||||
}
|
||||
|
||||
// Simple parse with MacAddress validation
|
||||
@@ -383,66 +303,49 @@ func TestRunFlagsParseWithMemory(t *testing.T) {
|
||||
flags, _ := setupRunFlags()
|
||||
args := []string{"--memory=invalid", "img", "cmd"}
|
||||
err := flags.Parse(args)
|
||||
assert.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
|
||||
require.ErrorContains(t, err, `invalid argument "invalid" for "-m, --memory" flag`)
|
||||
|
||||
_, hostconfig, _ := mustParse(t, "--memory=1G")
|
||||
assert.Check(t, is.Equal(int64(1073741824), hostconfig.Memory))
|
||||
assert.Equal(t, int64(1073741824), hostconfig.Memory)
|
||||
}
|
||||
|
||||
func TestParseWithMemorySwap(t *testing.T) {
|
||||
flags, _ := setupRunFlags()
|
||||
args := []string{"--memory-swap=invalid", "img", "cmd"}
|
||||
err := flags.Parse(args)
|
||||
assert.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
|
||||
require.ErrorContains(t, err, `invalid argument "invalid" for "--memory-swap" flag`)
|
||||
|
||||
_, hostconfig, _ := mustParse(t, "--memory-swap=1G")
|
||||
assert.Check(t, is.Equal(int64(1073741824), hostconfig.MemorySwap))
|
||||
assert.Equal(t, int64(1073741824), hostconfig.MemorySwap)
|
||||
|
||||
_, hostconfig, _ = mustParse(t, "--memory-swap=-1")
|
||||
assert.Check(t, is.Equal(int64(-1), hostconfig.MemorySwap))
|
||||
assert.Equal(t, int64(-1), hostconfig.MemorySwap)
|
||||
}
|
||||
|
||||
func TestParseHostname(t *testing.T) {
|
||||
validHostnames := map[string]string{
|
||||
"hostname": "hostname",
|
||||
"host-name": "host-name",
|
||||
"hostname123": "hostname123",
|
||||
"123hostname": "123hostname",
|
||||
"hostname-of-63-bytes-long-should-be-valid-and-without-any-error": "hostname-of-63-bytes-long-should-be-valid-and-without-any-error",
|
||||
}
|
||||
hostnameWithDomain := "--hostname=hostname.domainname"
|
||||
hostnameWithDomainTld := "--hostname=hostname.domainname.tld"
|
||||
for hostname, expectedHostname := range validHostnames {
|
||||
if config, _, _ := mustParse(t, "--hostname="+hostname); config.Hostname != expectedHostname {
|
||||
t.Fatalf("Expected the config to have 'hostname' as %q, got %q", expectedHostname, config.Hostname)
|
||||
}
|
||||
}
|
||||
if config, _, _ := mustParse(t, hostnameWithDomain); config.Hostname != "hostname.domainname" || config.Domainname != "" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname, got %q", config.Hostname)
|
||||
}
|
||||
if config, _, _ := mustParse(t, hostnameWithDomainTld); config.Hostname != "hostname.domainname.tld" || config.Domainname != "" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as hostname.domainname.tld, got %q", config.Hostname)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHostnameDomainname(t *testing.T) {
|
||||
validDomainnames := map[string]string{
|
||||
"domainname": "domainname",
|
||||
"domain-name": "domain-name",
|
||||
"domainname123": "domainname123",
|
||||
"123domainname": "123domainname",
|
||||
"domainname-63-bytes-long-should-be-valid-and-without-any-errors": "domainname-63-bytes-long-should-be-valid-and-without-any-errors",
|
||||
}
|
||||
for domainname, expectedDomainname := range validDomainnames {
|
||||
if config, _, _ := mustParse(t, "--domainname="+domainname); config.Domainname != expectedDomainname {
|
||||
t.Fatalf("Expected the config to have 'domainname' as %q, got %q", expectedDomainname, config.Domainname)
|
||||
}
|
||||
}
|
||||
if config, _, _ := mustParse(t, "--hostname=some.prefix --domainname=domainname"); config.Hostname != "some.prefix" || config.Domainname != "domainname" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as 'some.prefix' and 'domainname' as 'domainname', got %q and %q", config.Hostname, config.Domainname)
|
||||
}
|
||||
if config, _, _ := mustParse(t, "--hostname=another-prefix --domainname=domainname.tld"); config.Hostname != "another-prefix" || config.Domainname != "domainname.tld" {
|
||||
t.Fatalf("Expected the config to have 'hostname' as 'another-prefix' and 'domainname' as 'domainname.tld', got %q and %q", config.Hostname, config.Domainname)
|
||||
for _, tc := range []struct {
|
||||
input, hostname, domainname string
|
||||
}{
|
||||
{"--hostname=hostname", "hostname", ""},
|
||||
{"--hostname=host-name", "host-name", ""},
|
||||
{"--hostname=hostname123", "hostname123", ""},
|
||||
{"--hostname=123hostname", "123hostname", ""},
|
||||
{"--hostname=hostname-of-63-bytes-long-should-be-valid-and-without-any-error", "hostname-of-63-bytes-long-should-be-valid-and-without-any-error", ""},
|
||||
{"--hostname=hostname.domainname", "hostname.domainname", ""},
|
||||
{"--hostname=hostname.domainname.tld", "hostname.domainname.tld", ""},
|
||||
{"--domainname=domainname", "", "domainname"},
|
||||
{"--domainname=domain-name", "", "domain-name"},
|
||||
{"--domainname=domainname123", "", "domainname123"},
|
||||
{"--domainname=123domainname", "", "123domainname"},
|
||||
{"--domainname=domainname-63-bytes-long-should-be-valid-and-without-any-errors", "", "domainname-63-bytes-long-should-be-valid-and-without-any-errors"},
|
||||
{"--hostname=some.prefix --domainname=domainname", "some.prefix", "domainname"},
|
||||
{"--hostname=another-prefix --domainname=domainname.tld", "another-prefix", "domainname.tld"},
|
||||
} {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
config, _, _ := mustParse(t, tc.input)
|
||||
require.Equal(t, tc.hostname, config.Hostname)
|
||||
require.Equal(t, tc.domainname, config.Domainname)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -461,7 +364,7 @@ func TestParseWithExpose(t *testing.T) {
|
||||
for expose, expectedError := range tests {
|
||||
t.Run(expose, func(t *testing.T) {
|
||||
_, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
||||
assert.Error(t, err, expectedError)
|
||||
require.EqualError(t, err, expectedError)
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -476,30 +379,38 @@ func TestParseWithExpose(t *testing.T) {
|
||||
for expose, exposedPorts := range tests {
|
||||
t.Run(expose, func(t *testing.T) {
|
||||
config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
for _, port := range exposedPorts {
|
||||
_, ok := config.ExposedPorts[port]
|
||||
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("merge with published", func(t *testing.T) {
|
||||
// Merge with actual published port
|
||||
config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"})
|
||||
assert.NilError(t, err)
|
||||
assert.Check(t, is.Len(config.ExposedPorts, 2))
|
||||
config, hostConfig, _, err := parseRun([]string{"--publish=80", "--publish=127.0.0.1:8080:80", "--publish=[::1]:8081:80", "--expose=80-81/tcp"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, networktypes.PortMap{
|
||||
networktypes.MustParsePort("80/tcp"): {
|
||||
{},
|
||||
{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "8080"},
|
||||
{HostIP: netip.MustParseAddr("::1"), HostPort: "8081"},
|
||||
},
|
||||
}, hostConfig.PortBindings)
|
||||
assert.Len(t, config.ExposedPorts, 2)
|
||||
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")}
|
||||
for _, port := range ports {
|
||||
_, ok := config.ExposedPorts[port]
|
||||
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||
assert.True(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseDevice(t *testing.T) {
|
||||
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("Windows and macOS validate server-side")
|
||||
}
|
||||
testCases := []struct {
|
||||
devices []string
|
||||
deviceMapping *container.DeviceMapping
|
||||
@@ -573,17 +484,17 @@ func TestParseDevice(t *testing.T) {
|
||||
|
||||
_, hostconfig, _, err := parseRun(args)
|
||||
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
|
||||
if tc.deviceMapping != nil {
|
||||
if assert.Check(t, is.Len(hostconfig.Devices, 1)) {
|
||||
assert.Check(t, is.DeepEqual(*tc.deviceMapping, hostconfig.Devices[0]))
|
||||
if assert.Len(t, hostconfig.Devices, 1) {
|
||||
assert.Equal(t, *tc.deviceMapping, hostconfig.Devices[0])
|
||||
}
|
||||
} else {
|
||||
assert.Check(t, is.Len(hostconfig.Devices, 0))
|
||||
assert.Empty(t, hostconfig.Devices)
|
||||
}
|
||||
|
||||
assert.Check(t, is.DeepEqual(tc.deviceRequests, hostconfig.DeviceRequests))
|
||||
assert.Equal(t, tc.deviceRequests, hostconfig.DeviceRequests)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -652,11 +563,11 @@ func TestParseDeviceByServerOS(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := parseDevice(tc.device, tc.serverOS)
|
||||
if tc.wantErr != "" {
|
||||
assert.Error(t, err, tc.wantErr)
|
||||
require.EqualError(t, err, tc.wantErr)
|
||||
return
|
||||
}
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, got, tc.want)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -835,13 +746,13 @@ func TestParseNetworkConfig(t *testing.T) {
|
||||
_, hConfig, nwConfig, err := parseRun(tc.flags)
|
||||
|
||||
if tc.expectedErr != "" {
|
||||
assert.Error(t, err, tc.expectedErr)
|
||||
require.EqualError(t, err, tc.expectedErr)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NilError(t, err)
|
||||
assert.DeepEqual(t, hConfig.NetworkMode, tc.expectedHostCfg.NetworkMode)
|
||||
assert.DeepEqual(t, nwConfig.EndpointsConfig, tc.expected, cmpopts.EquateComparable(netip.Addr{}))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expectedHostCfg.NetworkMode, hConfig.NetworkMode)
|
||||
require.Equal(t, tc.expected, nwConfig.EndpointsConfig)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -850,24 +761,24 @@ func TestParseModes(t *testing.T) {
|
||||
// pid ko
|
||||
flags, copts := setupRunFlags()
|
||||
args := []string{"--pid=container:", "img", "cmd"}
|
||||
assert.NilError(t, flags.Parse(args))
|
||||
require.NoError(t, flags.Parse(args))
|
||||
_, err := parse(flags, copts, runtime.GOOS)
|
||||
assert.ErrorContains(t, err, "--pid: invalid PID mode")
|
||||
require.ErrorContains(t, err, "--pid: invalid PID mode")
|
||||
|
||||
// pid ok
|
||||
_, hostconfig, _, err := parseRun([]string{"--pid=host", "img", "cmd"})
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
if !hostconfig.PidMode.Valid() {
|
||||
t.Fatalf("Expected a valid PidMode, got %v", hostconfig.PidMode)
|
||||
}
|
||||
|
||||
// uts ko
|
||||
_, _, _, err = parseRun([]string{"--uts=container:", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
||||
assert.ErrorContains(t, err, "--uts: invalid UTS mode")
|
||||
require.ErrorContains(t, err, "--uts: invalid UTS mode")
|
||||
|
||||
// uts ok
|
||||
_, hostconfig, _, err = parseRun([]string{"--uts=host", "img", "cmd"})
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
if !hostconfig.UTSMode.Valid() {
|
||||
t.Fatalf("Expected a valid UTSMode, got %v", hostconfig.UTSMode)
|
||||
}
|
||||
@@ -879,11 +790,11 @@ func TestRunFlagsParseShmSize(t *testing.T) {
|
||||
args := []string{"--shm-size=a128m", "img", "cmd"}
|
||||
expectedErr := `invalid argument "a128m" for "--shm-size" flag:`
|
||||
err := flags.Parse(args)
|
||||
assert.ErrorContains(t, err, expectedErr)
|
||||
require.ErrorContains(t, err, expectedErr)
|
||||
|
||||
// shm-size ok
|
||||
_, hostconfig, _, err := parseRun([]string{"--shm-size=128m", "img", "cmd"})
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
if hostconfig.ShmSize != 134217728 {
|
||||
t.Fatalf("Expected a valid ShmSize, got %d", hostconfig.ShmSize)
|
||||
}
|
||||
@@ -954,11 +865,11 @@ func TestParseRestartPolicy(t *testing.T) {
|
||||
t.Run(tc.input, func(t *testing.T) {
|
||||
_, hostConfig, _, err := parseRun([]string{"--restart=" + tc.input, "img", "cmd"})
|
||||
if tc.expectedErr != "" {
|
||||
assert.Check(t, is.Error(err, tc.expectedErr))
|
||||
assert.Check(t, is.Nil(hostConfig))
|
||||
assert.Nil(t, hostConfig)
|
||||
assert.EqualError(t, err, tc.expectedErr)
|
||||
} else {
|
||||
assert.NilError(t, err)
|
||||
assert.Check(t, is.DeepEqual(hostConfig.RestartPolicy, tc.expected))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expected, hostConfig.RestartPolicy)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -967,45 +878,33 @@ func TestParseRestartPolicy(t *testing.T) {
|
||||
func TestParseRestartPolicyAutoRemove(t *testing.T) {
|
||||
_, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
|
||||
const expected = "conflicting options: cannot specify both --restart and --rm"
|
||||
assert.Check(t, is.Error(err, expected))
|
||||
assert.EqualError(t, err, expected)
|
||||
}
|
||||
|
||||
func TestParseHealth(t *testing.T) {
|
||||
checkOk := func(args ...string) *container.HealthConfig {
|
||||
config, _, _, err := parseRun(args)
|
||||
if err != nil {
|
||||
t.Fatalf("%#v: %v", args, err)
|
||||
}
|
||||
return config.Healthcheck
|
||||
}
|
||||
checkError := func(expected string, args ...string) {
|
||||
config, _, _, err := parseRun(args)
|
||||
if err == nil {
|
||||
t.Fatalf("Expected error, but got %#v", config)
|
||||
}
|
||||
if err.Error() != expected {
|
||||
t.Fatalf("Expected %#v, got %#v", expected, err)
|
||||
}
|
||||
}
|
||||
health := checkOk("--no-healthcheck", "img", "cmd")
|
||||
if health == nil || len(health.Test) != 1 || health.Test[0] != "NONE" {
|
||||
t.Fatalf("--no-healthcheck failed: %#v", health)
|
||||
}
|
||||
|
||||
health = checkOk("--health-cmd=/check.sh -q", "img", "cmd")
|
||||
if len(health.Test) != 2 || health.Test[0] != "CMD-SHELL" || health.Test[1] != "/check.sh -q" {
|
||||
t.Fatalf("--health-cmd: got %#v", health.Test)
|
||||
}
|
||||
if health.Timeout != 0 {
|
||||
t.Fatalf("--health-cmd: timeout = %s", health.Timeout)
|
||||
}
|
||||
|
||||
checkError("--no-healthcheck conflicts with --health-* options",
|
||||
"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
|
||||
|
||||
health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd")
|
||||
if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second || health.StartInterval != 1*time.Second {
|
||||
t.Fatalf("--health-*: got %#v", health)
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
expected *container.HealthConfig
|
||||
wantErr string
|
||||
}{
|
||||
{[]string{"--no-healthcheck", "img", "cmd"}, &container.HealthConfig{Test: []string{"NONE"}}, ""},
|
||||
{[]string{"--health-cmd=/check.sh -q", "img", "cmd"}, &container.HealthConfig{Test: []string{"CMD-SHELL", "/check.sh -q"}}, ""},
|
||||
{[]string{"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd"}, nil, "--no-healthcheck conflicts with --health-* options"},
|
||||
{
|
||||
[]string{"--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd"},
|
||||
&container.HealthConfig{Timeout: 2 * time.Second, Retries: 3, Interval: 4500 * time.Millisecond, StartPeriod: 5 * time.Second, StartInterval: time.Second},
|
||||
"",
|
||||
},
|
||||
} {
|
||||
t.Run(strings.Join(tc.args, " "), func(t *testing.T) {
|
||||
config, _, _, err := parseRun(tc.args)
|
||||
if tc.wantErr != "" {
|
||||
require.EqualError(t, err, tc.wantErr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.expected, config.Healthcheck)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1024,93 +923,66 @@ func TestParseLoggingOpts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
||||
expErr := "--env-file: open nonexistent: no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expErr = "--env-file: open nonexistent: The system cannot find the file specified."
|
||||
}
|
||||
// env ko
|
||||
if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
||||
}
|
||||
// env ok
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Env) != 1 || config.Env[0] != "ENV1=value1" {
|
||||
t.Fatalf("Expected a config with [ENV1=value1], got %v", config.Env)
|
||||
}
|
||||
config, _, _, err = parseRun([]string{"--env-file=testdata/valid.env", "--env=ENV2=value2", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Env) != 2 || config.Env[0] != "ENV1=value1" || config.Env[1] != "ENV2=value2" {
|
||||
t.Fatalf("Expected a config with [ENV1=value1 ENV2=value2], got %v", config.Env)
|
||||
func TestParseFileVariables(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
flag, file, key string
|
||||
}{
|
||||
{"env", "valid.env", "ENV"},
|
||||
{"label", "valid.label", "LABEL"},
|
||||
} {
|
||||
t.Run(tc.flag, func(t *testing.T) {
|
||||
expectedError := "no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expectedError = "The system cannot find the file specified."
|
||||
}
|
||||
_, _, _, err := parseRun([]string{"--" + tc.flag + "-file=nonexistent"})
|
||||
require.EqualError(t, err, "--"+tc.flag+"-file: open nonexistent: "+expectedError)
|
||||
for _, extra := range []bool{false, true} {
|
||||
args := []string{"--" + tc.flag + "-file=testdata/" + tc.file}
|
||||
expected := []string{tc.key + "1=value1"}
|
||||
if extra {
|
||||
args = append(args, "--"+tc.flag+"="+tc.key+"2=value2")
|
||||
expected = append(expected, tc.key+"2=value2")
|
||||
}
|
||||
config, _, _, err := parseRun(args)
|
||||
require.NoError(t, err)
|
||||
if tc.flag == "env" {
|
||||
assert.Equal(t, expected, config.Env)
|
||||
} else {
|
||||
assert.Len(t, config.Labels, len(expected))
|
||||
for _, label := range expected {
|
||||
key, value, _ := strings.Cut(label, "=")
|
||||
assert.Equal(t, value, config.Labels[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
|
||||
// UTF8 with BOM
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := []string{"FOO=BAR", "HELLO=" + string([]byte{0xe6, 0x82, 0xa8, 0xe5, 0xa5, 0xbd}), "BAR=FOO"}
|
||||
if len(config.Env) != len(env) {
|
||||
t.Fatalf("Expected a config with %d env variables, got %v: %v", len(env), len(config.Env), config.Env)
|
||||
}
|
||||
for i, v := range env {
|
||||
if config.Env[i] != v {
|
||||
t.Fatalf("Expected a config with [%s], got %v", v, []byte(config.Env[i]))
|
||||
}
|
||||
}
|
||||
config, _, _, err := parseRun([]string{"--env-file=testdata/utf8.env"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"FOO=BAR", "HELLO=您好", "BAR=FOO"}, config.Env)
|
||||
|
||||
// UTF16 with BOM
|
||||
e := "invalid utf8 bytes at line"
|
||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
||||
}
|
||||
// UTF16BE with BOM
|
||||
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16be.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", e, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
|
||||
expErr := "--label-file: open nonexistent: no such file or directory"
|
||||
if runtime.GOOS == "windows" {
|
||||
expErr = "--label-file: open nonexistent: The system cannot find the file specified."
|
||||
}
|
||||
// label ko
|
||||
if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr {
|
||||
t.Fatalf("Expected an error with message '%s', got %v", expErr, err)
|
||||
}
|
||||
// label ok
|
||||
config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Labels) != 1 || config.Labels["LABEL1"] != "value1" {
|
||||
t.Fatalf("Expected a config with [LABEL1:value1], got %v", config.Labels)
|
||||
}
|
||||
config, _, _, err = parseRun([]string{"--label-file=testdata/valid.label", "--label=LABEL2=value2", "img", "cmd"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Labels) != 2 || config.Labels["LABEL1"] != "value1" || config.Labels["LABEL2"] != "value2" {
|
||||
t.Fatalf("Expected a config with [LABEL1:value1 LABEL2:value2], got %v", config.Labels)
|
||||
for _, file := range []string{"utf16.env", "utf16be.env"} {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
_, _, _, err := parseRun([]string{"--env-file=testdata/" + file})
|
||||
require.ErrorContains(t, err, "invalid utf8 bytes at line")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEntryPoint(t *testing.T) {
|
||||
config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"})
|
||||
assert.NilError(t, err)
|
||||
assert.Check(t, is.DeepEqual(config.Entrypoint, []string{"anything"}))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"anything"}, config.Entrypoint)
|
||||
}
|
||||
|
||||
func TestValidateDevice(t *testing.T) {
|
||||
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("Windows and macOS validate server-side")
|
||||
}
|
||||
valid := []string{
|
||||
"/home",
|
||||
"/home:/home",
|
||||
@@ -1209,22 +1081,22 @@ func TestValidateDeviceByServerOS(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := validateDevice(tc.value, tc.serverOS)
|
||||
if tc.wantError != "" {
|
||||
assert.Error(t, err, tc.wantError)
|
||||
require.EqualError(t, err, tc.wantError)
|
||||
return
|
||||
}
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, got, tc.want)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
|
||||
got, err := validateDeviceCgroupRule("c 1:3 rwm")
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, got, "c 1:3 rwm")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "c 1:3 rwm", got)
|
||||
|
||||
_, err = validateDeviceCgroupRule("invalid")
|
||||
assert.Error(t, err, "invalid device cgroup format 'invalid'")
|
||||
require.EqualError(t, err, "invalid device cgroup format 'invalid'")
|
||||
|
||||
if invalidParameter(nil) != nil {
|
||||
t.Fatal("invalidParameter(nil) should be nil")
|
||||
@@ -1232,9 +1104,9 @@ func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
|
||||
cause := errors.New("bad input")
|
||||
err = invalidParameter(cause)
|
||||
var invalid interface{ InvalidParameter() }
|
||||
assert.Assert(t, errors.As(err, &invalid))
|
||||
assert.Assert(t, errors.Is(err, cause))
|
||||
assert.Equal(t, invalidParameter(err), err) // already invalid, so not wrapped twice
|
||||
require.ErrorAs(t, err, &invalid)
|
||||
require.ErrorIs(t, err, cause)
|
||||
require.Condition(t, func() bool { return invalidParameter(err) == err })
|
||||
}
|
||||
|
||||
func TestParseSystemPaths(t *testing.T) {
|
||||
@@ -1285,9 +1157,9 @@ func TestParseSystemPaths(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
securityOpts, maskedPaths, readonlyPaths := parseSystemPaths(tc.in)
|
||||
assert.DeepEqual(t, securityOpts, tc.out)
|
||||
assert.DeepEqual(t, maskedPaths, tc.masked)
|
||||
assert.DeepEqual(t, readonlyPaths, tc.readonly)
|
||||
require.Equal(t, tc.out, securityOpts)
|
||||
require.Equal(t, tc.masked, maskedPaths)
|
||||
require.Equal(t, tc.readonly, readonlyPaths)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1309,9 +1181,9 @@ func TestConvertToStandardNotation(t *testing.T) {
|
||||
for key, ports := range valid {
|
||||
convertedPorts, err := convertToStandardNotation(ports)
|
||||
if err != nil {
|
||||
assert.NilError(t, err)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
assert.DeepEqual(t, strings.Split(key, " "), convertedPorts)
|
||||
require.Equal(t, strings.Split(key, " "), convertedPorts)
|
||||
}
|
||||
|
||||
for _, ports := range invalid {
|
||||
|
||||
@@ -8,10 +8,10 @@ package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/client"
|
||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
)
|
||||
|
||||
// ImageExistsLocally returns a boolean indicating if an image with the
|
||||
@@ -30,11 +30,23 @@ func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool,
|
||||
return false, err
|
||||
}
|
||||
|
||||
if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform {
|
||||
if platform == "" || platform == "any" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
requested, err := parsePlatform(platform)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return platformMatches(requested, &specs.Platform{OS: inspectImage.Os, Architecture: inspectImage.Architecture, Variant: inspectImage.Variant}), nil
|
||||
}
|
||||
|
||||
func platformMatches(requested, image *specs.Platform) bool {
|
||||
variant := image.Variant
|
||||
if variant == "" {
|
||||
variant = map[string]string{"amd64": "v1", "arm": "v7", "arm64": "v8"}[image.Architecture]
|
||||
}
|
||||
return image.OS == requested.OS && image.Architecture == requested.Architecture && (requested.Variant == "" || requested.Variant == variant)
|
||||
}
|
||||
|
||||
// RemoveImage removes image from local store, the function is used to run different
|
||||
@@ -46,19 +58,14 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
inspectImage, err := cli.ImageInspect(ctx, imageName)
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
if _, err = cli.ImageRemove(ctx, imageName, client.ImageRemoveOptions{
|
||||
Force: force,
|
||||
PruneChildren: pruneChildren,
|
||||
}); cerrdefs.IsNotFound(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if _, err = cli.ImageRemove(ctx, inspectImage.ID, client.ImageRemoveOptions{
|
||||
Force: force,
|
||||
PruneChildren: pruneChildren,
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -5,20 +5,20 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func init() {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
logging.SetLevel(logging.LevelDebug)
|
||||
}
|
||||
|
||||
// buildScratchImage builds a tiny empty image for the given platform locally (FROM scratch, no
|
||||
@@ -41,7 +41,7 @@ func buildScratchImage(t *testing.T, platform string) string {
|
||||
|
||||
func TestImageExistsLocally(t *testing.T) {
|
||||
requireDocker(t)
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
|
||||
// a non-existent image is reported absent
|
||||
missing, err := ImageExistsLocally(ctx, "library/alpine:this-random-tag-will-never-exist", "linux/amd64")
|
||||
@@ -66,4 +66,11 @@ func TestImageExistsLocally(t *testing.T) {
|
||||
wrongPlatform, err := ImageExistsLocally(ctx, amd64Ref, "linux/arm64")
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.False(t, wrongPlatform)
|
||||
|
||||
removed, err := RemoveImage(ctx, amd64Ref, false, false)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, removed)
|
||||
removed, err = RemoveImage(ctx, amd64Ref, false, false)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, removed)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
type dockerMessage struct {
|
||||
@@ -26,25 +26,16 @@ type dockerMessage struct {
|
||||
Progress string `json:"progress"`
|
||||
}
|
||||
|
||||
func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error {
|
||||
func logDockerResponse(logger *logging.Logger, dockerResponse io.ReadCloser, isError bool) error {
|
||||
if dockerResponse == nil {
|
||||
return nil
|
||||
}
|
||||
defer dockerResponse.Close()
|
||||
|
||||
scanner := bufio.NewScanner(dockerResponse)
|
||||
msg := dockerMessage{}
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Bytes()
|
||||
|
||||
msg.ID = ""
|
||||
msg.Stream = ""
|
||||
msg.Error = ""
|
||||
msg.ErrorDetail.Message = ""
|
||||
msg.Status = ""
|
||||
msg.Progress = ""
|
||||
|
||||
var msg dockerMessage
|
||||
if err := json.Unmarshal(line, &msg); err != nil {
|
||||
writeLog(logger, false, "Unable to unmarshal line [%s] ==> %v", string(line), err)
|
||||
continue
|
||||
@@ -77,7 +68,7 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser,
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeLog(logger logrus.FieldLogger, isError bool, format string, args ...any) {
|
||||
func writeLog(logger *logging.Logger, isError bool, format string, args ...any) {
|
||||
if isError {
|
||||
logger.Errorf(format, args...)
|
||||
} else {
|
||||
|
||||
@@ -10,21 +10,20 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
const (
|
||||
networkCreateAttempts = 3
|
||||
networkCreateRetryDelay = time.Second
|
||||
|
||||
// marks the networks a runner creates for its jobs, so it can tell its own leftovers from
|
||||
// those of another runner sharing the daemon
|
||||
runnerUUIDLabel = "com.gitea.runner.uuid"
|
||||
)
|
||||
|
||||
// RemoveOrphanNetworks removes the networks this runner created for jobs whose teardown did
|
||||
@@ -32,17 +31,17 @@ const (
|
||||
// had an endpoint on it at the time. Each one holds a subnet of the daemon's address pool
|
||||
// until it is removed. Networks created after createdBefore are left alone, so a job starting
|
||||
// while this runs cannot lose the network it has created but not yet attached a container to.
|
||||
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
func RemoveOrphanNetworks(ctx context.Context, runnerUUID, detach string, createdBefore time.Time) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
return removeOrphanNetworks(ctx, cli, runnerUUID, createdBefore)
|
||||
return removeOrphanNetworks(ctx, cli, runnerUUID, detach, createdBefore)
|
||||
}
|
||||
|
||||
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID string, createdBefore time.Time) error {
|
||||
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID, detach string, createdBefore time.Time) error {
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{
|
||||
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID),
|
||||
})
|
||||
@@ -51,18 +50,32 @@ func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID
|
||||
}
|
||||
|
||||
var errs []error
|
||||
networks:
|
||||
for _, n := range networks.Items {
|
||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
// the emptiness check, not the label, is what keeps a live job of another process
|
||||
// sharing this registration safe
|
||||
if len(result.Network.Containers) != 0 || result.Network.Created.After(createdBefore) {
|
||||
if len(result.Network.Containers) > 1 || result.Network.Created.After(createdBefore) {
|
||||
continue
|
||||
}
|
||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
for containerID := range result.Network.Containers {
|
||||
if detach == "" || !strings.HasPrefix(containerID, detach) {
|
||||
continue networks
|
||||
}
|
||||
if err := disconnectNetwork(ctx, cli, n.ID, containerID); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to disconnect container from network %s: %w", n.Name, err))
|
||||
continue networks
|
||||
}
|
||||
}
|
||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); cerrdefs.IsNotFound(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
@@ -78,45 +91,54 @@ func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExec
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return createDockerNetwork(ctx, cli, name, opts)
|
||||
}
|
||||
}
|
||||
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
func createDockerNetwork(ctx context.Context, cli client.APIClient, name string, opts NewDockerNetworkCreateExecutorInput) error {
|
||||
// Only create the network if it doesn't exist
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var options map[string]string
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
// the daemon's mtu only reaches its default bridge, Docker's default is left out to keep default-network-opts
|
||||
if mtu := n.Options["com.docker.network.driver.mtu"]; n.Name == network.NetworkBridge && mtu != "" && mtu != "1500" {
|
||||
options = map[string]string{"com.docker.network.driver.mtu": mtu}
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Options: options,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
common.Logger(ctx).Debugf("Network %v exists", name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
|
||||
func runnerLabels(runnerUUID string) map[string]string {
|
||||
@@ -134,6 +156,50 @@ func isAddressPoolExhausted(err error) bool {
|
||||
strings.Contains(msg, "could not find an available, non-overlapping IPv4 address pool among the defaults") // docker 24 and older
|
||||
}
|
||||
|
||||
func NewDockerNetworkConnectExecutor(networkName, container string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
if err := connectNetwork(ctx, cli, networkName, container); err != nil {
|
||||
common.Logger(ctx).Warnf("cannot connect %s to network %s, so jobs on it cannot reach the cache: %v", container, networkName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func connectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||
_, err := cli.NetworkConnect(ctx, networkName, client.NetworkConnectOptions{
|
||||
Container: container,
|
||||
EndpointConfig: &network.EndpointSettings{Aliases: []string{container}},
|
||||
})
|
||||
if err != nil && (strings.Contains(err.Error(), "already exists in network") || strings.Contains(err.Error(), "already connected")) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func NewDockerNetworkDisconnectExecutor(networkName, container string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
return disconnectNetwork(ctx, cli, networkName, container)
|
||||
}
|
||||
}
|
||||
|
||||
func disconnectNetwork(ctx context.Context, cli client.APIClient, networkName, container string) error {
|
||||
_, err := cli.NetworkDisconnect(ctx, networkName, client.NetworkDisconnectOptions{Container: container, Force: true})
|
||||
if cerrdefs.IsNotFound(err) || (err != nil && strings.Contains(err.Error(), "is not connected")) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
@@ -141,34 +207,79 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
// Make sure that all network of the specified name are removed
|
||||
// cli.NetworkRemove refuses to remove a network if there are duplicates
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
var errs []error
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// it holds a subnet out of the daemon's pool until something reclaims it
|
||||
if len(result.Network.Containers) != 0 {
|
||||
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the idle cleanup reclaims it once they are gone", name)
|
||||
continue
|
||||
}
|
||||
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
return removeNetwork(ctx, cli, name)
|
||||
}
|
||||
}
|
||||
|
||||
func removeNetwork(ctx context.Context, cli client.APIClient, name string) error {
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{}) // NetworkRemove by name refuses duplicates, so remove each by ID
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, n := range networks.Items {
|
||||
if n.Name != name {
|
||||
continue
|
||||
}
|
||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(result.Network.Containers) != 0 {
|
||||
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the orphan cleanup reclaims it once they are gone", name)
|
||||
continue
|
||||
}
|
||||
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// IsolatedCacheContainer returns the container holding addr, when jobs on jobNetwork cannot reach it there.
|
||||
func IsolatedCacheContainer(ctx context.Context, addr netip.Addr, jobNetwork string) (string, error) {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer cli.Close()
|
||||
return isolatedCacheContainer(ctx, cli, addr, jobNetwork)
|
||||
}
|
||||
|
||||
func isolatedCacheContainer(ctx context.Context, cli client.APIClient, addr netip.Addr, jobNetwork string) (string, error) {
|
||||
if jobNetwork == "host" || strings.HasPrefix(jobNetwork, "container:") {
|
||||
return "", nil
|
||||
}
|
||||
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var holderID, holderContainer string
|
||||
for _, summary := range containers.Items {
|
||||
if summary.NetworkSettings == nil {
|
||||
continue
|
||||
}
|
||||
for _, endpoint := range summary.NetworkSettings.Networks {
|
||||
if endpoint != nil && (endpoint.IPAddress == addr || endpoint.GlobalIPv6Address == addr) {
|
||||
holderID = endpoint.NetworkID
|
||||
holderContainer = summary.ID[:12]
|
||||
}
|
||||
}
|
||||
}
|
||||
if holderID == "" {
|
||||
return "", nil
|
||||
}
|
||||
holder, err := cli.NetworkInspect(ctx, holderID, client.NetworkInspectOptions{})
|
||||
if err != nil || holder.Network.Driver != "bridge" {
|
||||
return "", err
|
||||
}
|
||||
if jobNetwork != "" {
|
||||
job, err := cli.NetworkInspect(ctx, jobNetwork, client.NetworkInspectOptions{})
|
||||
if err != nil || job.Network.ID == holderID {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return holderContainer, nil
|
||||
}
|
||||
|
||||
@@ -6,27 +6,118 @@ package container
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsolatedCacheContainer(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cache, lan := netip.MustParseAddr("172.18.0.3"), netip.MustParseAddr("192.168.1.20")
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{}).Return(mobyclient.ContainerListResult{Items: []container.Summary{{}, {
|
||||
ID: "0123456789abcdef",
|
||||
NetworkSettings: &container.NetworkSettingsSummary{Networks: map[string]*network.EndpointSettings{
|
||||
"compose": {NetworkID: "c0ffee", IPAddress: cache},
|
||||
"lan": {NetworkID: "beef", IPAddress: lan},
|
||||
}},
|
||||
}}}, nil)
|
||||
for ref, inspected := range map[string]network.Network{
|
||||
"c0ffee": {ID: "c0ffee", Name: "compose", Driver: "bridge"},
|
||||
"beef": {ID: "beef", Name: "lan", Driver: "macvlan"},
|
||||
"c0f": {ID: "c0ffee"},
|
||||
"jobs": {ID: "f00d"},
|
||||
} {
|
||||
client.On("NetworkInspect", ctx, ref, mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: inspected}}, nil)
|
||||
}
|
||||
check := func(addr netip.Addr, jobNetwork, want string) {
|
||||
got, err := isolatedCacheContainer(ctx, client, addr, jobNetwork)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want, got, jobNetwork)
|
||||
}
|
||||
check(cache, "", "0123456789ab")
|
||||
check(cache, "jobs", "0123456789ab")
|
||||
check(cache, "c0f", "")
|
||||
check(cache, "host", "")
|
||||
check(lan, "", "")
|
||||
check(netip.MustParseAddr("10.0.0.5"), "", "")
|
||||
}
|
||||
|
||||
func TestIsAddressPoolExhausted(t *testing.T) {
|
||||
assert.True(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("Error response from daemon: all predefined address pools have been fully subnetted")))
|
||||
assert.True(t, isAddressPoolExhausted(errors.New("could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network")))
|
||||
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
|
||||
}
|
||||
|
||||
// Of this runner's networks, only the ones nothing is attached to and old enough to predate
|
||||
// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
|
||||
// test on its own, since testify has no expectation to match it against.
|
||||
func TestRemoveOrphanNetworks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
func TestCreateDockerNetworkInheritsDaemonMTU(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
for _, tc := range []struct {
|
||||
bridgeOptions, want map[string]string
|
||||
}{
|
||||
{
|
||||
map[string]string{"com.docker.network.bridge.host_binding_ipv4": "127.0.0.1", "com.docker.network.driver.mtu": "1450"},
|
||||
map[string]string{"com.docker.network.driver.mtu": "1450"},
|
||||
},
|
||||
{map[string]string{"com.docker.network.driver.mtu": "1500"}, nil},
|
||||
} {
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).
|
||||
Return(mobyclient.NetworkListResult{Items: []network.Summary{{Name: network.NetworkBridge, Options: tc.bridgeOptions}}}, nil)
|
||||
client.On("NetworkCreate", ctx, "job-network", mobyclient.NetworkCreateOptions{Driver: "bridge", Scope: "local", Options: tc.want}).
|
||||
Return(mobyclient.NetworkCreateResult{}, nil)
|
||||
require.NoError(t, createDockerNetwork(ctx, client, "job-network", NewDockerNetworkCreateExecutorInput{}))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkConnect(ctx context.Context, id string, opts mobyclient.NetworkConnectOptions) (mobyclient.NetworkConnectResult, error) {
|
||||
return mobyclient.NetworkConnectResult{}, m.Called(ctx, id, opts).Error(0)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkDisconnect(ctx context.Context, id string, opts mobyclient.NetworkDisconnectOptions) (mobyclient.NetworkDisconnectResult, error) {
|
||||
return mobyclient.NetworkDisconnectResult{}, m.Called(ctx, id, opts).Error(0)
|
||||
}
|
||||
|
||||
func TestConnectAndDisconnectNetwork(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
connect := func(answer error) error {
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkConnect", ctx, "jobs", mobyclient.NetworkConnectOptions{
|
||||
Container: "0123456789ab",
|
||||
EndpointConfig: &network.EndpointSettings{Aliases: []string{"0123456789ab"}},
|
||||
}).Return(answer).Once()
|
||||
return connectNetwork(ctx, client, "jobs", "0123456789ab")
|
||||
}
|
||||
require.NoError(t, connect(nil))
|
||||
require.NoError(t, connect(errors.New("endpoint with name runner already exists in network jobs")))
|
||||
require.NoError(t, connect(errors.New("container runner already connected to network jobs")))
|
||||
require.ErrorIs(t, connect(context.Canceled), context.Canceled)
|
||||
|
||||
disconnect := func(answer error) error {
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkDisconnect", ctx, "jobs", mobyclient.NetworkDisconnectOptions{
|
||||
Container: "0123456789ab", Force: true,
|
||||
}).Return(answer).Once()
|
||||
return disconnectNetwork(ctx, client, "jobs", "0123456789ab")
|
||||
}
|
||||
require.NoError(t, disconnect(cerrdefs.ErrNotFound))
|
||||
require.NoError(t, disconnect(errors.New("container runner is not connected to the network jobs")))
|
||||
require.ErrorIs(t, disconnect(context.Canceled), context.Canceled)
|
||||
}
|
||||
|
||||
// Of this runner's networks, only the ones nothing but its own cache container is attached to
|
||||
// and old enough to predate any job now starting are the runner's to reclaim. An unexpected
|
||||
// NetworkRemove fails the test on its own, since testify has no expectation to match it against.
|
||||
func TestRemoveNetworks(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
|
||||
@@ -35,16 +126,46 @@ func TestRemoveOrphanNetworks(t *testing.T) {
|
||||
{ID: "orphan"},
|
||||
{ID: "busy"},
|
||||
{ID: "starting"},
|
||||
{ID: "cached"},
|
||||
{ID: "shared"},
|
||||
{ID: "gone"},
|
||||
{ID: "raced"},
|
||||
}}, nil)
|
||||
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{}, nil)
|
||||
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
|
||||
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil).Twice()
|
||||
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{
|
||||
Network: network.Network{Created: cutoff.Add(time.Second)},
|
||||
Containers: map[string]network.EndpointResource{"0123456789abcdef": {}},
|
||||
}}, nil)
|
||||
client.On("NetworkInspect", ctx, "cached", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}}}}, nil)
|
||||
client.On("NetworkInspect", ctx, "shared", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"0123456789abcdef": {}, "job": {}}}}, nil)
|
||||
client.On("NetworkInspect", ctx, "gone", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{}, cerrdefs.ErrNotFound).Twice()
|
||||
client.On("NetworkInspect", ctx, "raced", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{}, nil).Twice()
|
||||
client.On("NetworkRemove", ctx, "raced", mobyclient.NetworkRemoveOptions{}).
|
||||
Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrNotFound).Twice()
|
||||
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
|
||||
Return(mobyclient.NetworkRemoveResult{}, nil)
|
||||
Return(mobyclient.NetworkRemoveResult{}, nil).Twice()
|
||||
client.On("NetworkDisconnect", ctx, "cached", mobyclient.NetworkDisconnectOptions{Container: "0123456789abcdef", Force: true}).
|
||||
Return(nil).Once()
|
||||
client.On("NetworkRemove", ctx, "cached", mobyclient.NetworkRemoveOptions{}).
|
||||
Return(mobyclient.NetworkRemoveResult{}, nil).Once()
|
||||
|
||||
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
|
||||
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", "0123456789ab", cutoff))
|
||||
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
|
||||
{ID: "orphan", Name: "job"},
|
||||
{ID: "busy", Name: "job"},
|
||||
{ID: "gone", Name: "job"},
|
||||
{ID: "raced", Name: "job"},
|
||||
{ID: "shared", Name: "other"},
|
||||
}}, nil)
|
||||
require.NoError(t, removeNetwork(ctx, client, "job"))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ package container
|
||||
import (
|
||||
"testing"
|
||||
|
||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -17,32 +18,17 @@ func TestParsePlatform(t *testing.T) {
|
||||
assert.Nil(t, got)
|
||||
})
|
||||
|
||||
t.Run("os/arch", func(t *testing.T) {
|
||||
got, err := parsePlatform("linux/amd64")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "amd64", got.Architecture)
|
||||
assert.Empty(t, got.Variant)
|
||||
})
|
||||
|
||||
t.Run("os/arch/variant", func(t *testing.T) {
|
||||
got, err := parsePlatform("linux/arm/v7")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "arm", got.Architecture)
|
||||
assert.Equal(t, "v7", got.Variant)
|
||||
})
|
||||
|
||||
t.Run("input is lowercased", func(t *testing.T) {
|
||||
got, err := parsePlatform("Linux/AMD64/V8")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, "linux", got.OS)
|
||||
assert.Equal(t, "amd64", got.Architecture)
|
||||
assert.Equal(t, "v8", got.Variant)
|
||||
})
|
||||
for input, want := range map[string]specs.Platform{
|
||||
"linux/amd64": {OS: "linux", Architecture: "amd64"},
|
||||
"linux/arm/v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
|
||||
"Linux/AMD64/V8": {OS: "linux", Architecture: "amd64", Variant: "v8"},
|
||||
} {
|
||||
t.Run(input, func(t *testing.T) {
|
||||
got, err := parsePlatform(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &want, got)
|
||||
})
|
||||
}
|
||||
|
||||
for _, bad := range []string{
|
||||
"amd64",
|
||||
@@ -61,3 +47,26 @@ func TestParsePlatform(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlatformMatches(t *testing.T) {
|
||||
for _, testcase := range []struct {
|
||||
requested, image string
|
||||
want bool
|
||||
}{
|
||||
{"linux/amd64", "linux/amd64", true},
|
||||
{"linux/amd64/v1", "linux/amd64", true},
|
||||
{"linux/arm64", "linux/amd64", false},
|
||||
{"linux/arm64/v8", "linux/arm64", true},
|
||||
{"linux/arm/v7", "linux/arm/v7", true},
|
||||
{"linux/arm", "linux/arm/v7", true},
|
||||
{"linux/arm/v6", "linux/arm/v7", false},
|
||||
{"linux/arm", "linux/arm/v6", true},
|
||||
{"windows/arm/v7", "linux/arm/v7", false},
|
||||
} {
|
||||
requested, err := parsePlatform(testcase.requested)
|
||||
require.NoError(t, err)
|
||||
image, err := parsePlatform(testcase.image)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testcase.want, platformMatches(requested, image), "%s on %s", testcase.requested, testcase.image)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,20 +6,24 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -33,66 +37,136 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
jobLabel = "com.gitea.runner.job"
|
||||
jobLabel = "com.gitea.runner.job."
|
||||
maxCreateBody = 8 << 20
|
||||
|
||||
dockerProxyProbeTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
createPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers|networks|volumes)/create$`)
|
||||
rawStreamPath = regexp.MustCompile(`^(/v[0-9.]+)?/(containers/[^/]+/attach|exec/[^/]+/start)$`)
|
||||
proxyProbe struct {
|
||||
sync.Mutex
|
||||
decided bool
|
||||
dir string
|
||||
}
|
||||
)
|
||||
|
||||
// DockerProxyDir returns where job proxy sockets live, "" while undecided or when the daemon cannot open the runner's files.
|
||||
func DockerProxyDir(ctx context.Context) string {
|
||||
proxyProbe.Lock()
|
||||
defer proxyProbe.Unlock()
|
||||
if proxyProbe.decided {
|
||||
return proxyProbe.dir
|
||||
}
|
||||
dir := filepath.Join(os.TempDir(), "gitea-runner-docker")
|
||||
ok, err := daemonSeesDir(ctx, dir)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Debugf("docker proxy probe postponed: %v", err)
|
||||
return ""
|
||||
}
|
||||
proxyProbe.decided = true
|
||||
if ok {
|
||||
proxyProbe.dir = dir
|
||||
} else {
|
||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's filesystem, jobs get the daemon socket directly")
|
||||
}
|
||||
return proxyProbe.dir
|
||||
var dockerProxyDirectories sync.Map
|
||||
|
||||
type dockerProxyDirectory struct {
|
||||
dir, daemonDir string
|
||||
socket os.FileInfo
|
||||
}
|
||||
|
||||
func daemonSeesDir(ctx context.Context, dir string) (bool, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return false, err
|
||||
func NewDockerProxy(ctx context.Context, job string) *DockerProxy {
|
||||
if host := os.Getenv("DOCKER_HOST"); runtime.GOOS != "linux" || host != "" && !strings.HasPrefix(host, "unix://") {
|
||||
return nil
|
||||
}
|
||||
probe := filepath.Join(dir, "probe")
|
||||
if err := os.WriteFile(probe, nil, 0o600); err != nil {
|
||||
return false, err
|
||||
probeCtx, cancel := context.WithTimeout(ctx, dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
cli, err := GetDockerClient(probeCtx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
cli, err := GetDockerClient(ctx)
|
||||
defer cli.Close()
|
||||
daemonSocket, ok := strings.CutPrefix(cli.DaemonHost(), "unix://")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
socket, err := os.Stat(daemonSocket)
|
||||
if err != nil || socket.Mode()&os.ModeSocket == 0 {
|
||||
return nil
|
||||
}
|
||||
workdir, _ := os.Getwd()
|
||||
key := [3]string{cli.DaemonHost(), workdir, os.TempDir()}
|
||||
cached, _ := dockerProxyDirectories.Load(key)
|
||||
directory, seen := cached.(dockerProxyDirectory)
|
||||
seen = seen && os.SameFile(directory.socket, socket) // daemon replacement invalidates cached paths
|
||||
dir, daemonDir := directory.dir, directory.daemonDir
|
||||
if !seen {
|
||||
dir, daemonDir = runnerContainerWorkdir(probeCtx, cli)
|
||||
seen = daemonDir != ""
|
||||
if !seen {
|
||||
if dir, err = filepath.Abs(os.TempDir()); err == nil {
|
||||
daemonDir = dir
|
||||
seen, err = daemonSeesDir(probeCtx, cli, dir, daemonDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
common.Logger(ctx).Infof("docker proxy probe failed, jobs get the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
if !seen {
|
||||
common.Logger(ctx).Infof("the docker daemon cannot reach the runner's temporary or working directory, jobs get the daemon socket directly")
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
proxy, err := StartDockerProxy(daemonSocket, dir, job)
|
||||
if err != nil {
|
||||
dockerProxyDirectories.Delete(key)
|
||||
common.Logger(ctx).Warnf("docker proxy not started, the job gets the daemon socket directly: %v", err)
|
||||
return nil
|
||||
}
|
||||
dockerProxyDirectories.Store(key, dockerProxyDirectory{dir, daemonDir, socket})
|
||||
proxy.Socket = daemonDir + strings.TrimPrefix(proxy.Socket, dir)
|
||||
return proxy
|
||||
}
|
||||
|
||||
// runnerContainerWorkdir looks the runner's container up by hostname to find the daemon's path to its working directory.
|
||||
func runnerContainerWorkdir(ctx context.Context, cli client.APIClient) (workdir, daemonDir string) {
|
||||
workdir, err := os.Getwd()
|
||||
hostname, hostnameErr := os.Hostname()
|
||||
if err != nil || hostnameErr != nil {
|
||||
return "", ""
|
||||
}
|
||||
self, err := cli.ContainerInspect(ctx, hostname, client.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
if daemonDir = containerInfoFromInspect(self.Container).DaemonPath(workdir); daemonDir == "" {
|
||||
return "", ""
|
||||
}
|
||||
if seen, _ := containerSeesMarker(ctx, cli, workdir, self.Container.ID, workdir); !seen { // the hostname may name another container
|
||||
return "", ""
|
||||
}
|
||||
return workdir, daemonDir
|
||||
}
|
||||
|
||||
func containerSeesMarker(ctx context.Context, cli client.APIClient, dir, id, containerDir string) (bool, error) {
|
||||
marker, err := os.CreateTemp(dir, "gitea-runner-probe-")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer cli.Close()
|
||||
defer func() {
|
||||
if err := os.Remove(marker.Name()); err != nil {
|
||||
common.Logger(ctx).Warnf("removing the docker proxy probe marker failed: %v", err)
|
||||
}
|
||||
}()
|
||||
if err := marker.Close(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = cli.ContainerStatPath(ctx, id, client.ContainerStatPathOptions{Path: path.Join(containerDir, filepath.Base(marker.Name()))})
|
||||
if cerrdefs.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// daemonSeesDir reports whether the daemon opens the files the runner writes in dir by their path in daemonDir,
|
||||
// which is what a job's proxy socket mounted from there needs.
|
||||
func daemonSeesDir(ctx context.Context, cli client.APIClient, dir, daemonDir string) (bool, error) {
|
||||
images, err := cli.ImageList(ctx, client.ImageListOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(images.Items) == 0 {
|
||||
return false, errors.New("no image to probe with yet")
|
||||
return false, errors.New("no image available for the docker proxy probe")
|
||||
}
|
||||
// creating validates that a bind source exists on the daemon's side, nothing is started
|
||||
created, err := cli.ContainerCreate(ctx, client.ContainerCreateOptions{
|
||||
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{{Type: mount.TypeBind, Source: probe, Target: "/gitea-runner-probe"}}},
|
||||
Config: &container.Config{Image: images.Items[0].ID, Cmd: []string{"true"}},
|
||||
HostConfig: &container.HostConfig{Mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}, // not the marker itself, podman creates a missing bind source where docker rejects it
|
||||
}},
|
||||
})
|
||||
if cerrdefs.IsInvalidArgument(err) {
|
||||
return false, nil
|
||||
@@ -100,24 +174,38 @@ func daemonSeesDir(ctx context.Context, dir string) (bool, error) {
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = cli.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})
|
||||
return true, err
|
||||
seen, err := containerSeesMarker(ctx, cli, dir, created.ID, "/gitea-runner-probe")
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dockerProxyProbeTimeout)
|
||||
defer cancel()
|
||||
if _, removeErr := cli.ContainerRemove(cleanupCtx, created.ID, client.ContainerRemoveOptions{Force: true, RemoveVolumes: true}); removeErr != nil {
|
||||
return false, fmt.Errorf("removing the docker proxy probe container failed: %w", removeErr)
|
||||
}
|
||||
return seen, err
|
||||
}
|
||||
|
||||
// StartDockerProxy serves a job's docker socket in dir, labelling what the job creates through it.
|
||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
digest := sha256.Sum256([]byte(job))
|
||||
socket := filepath.Join(dir, hex.EncodeToString(digest[:8])+".sock")
|
||||
_ = os.Remove(socket)
|
||||
listener, err := net.Listen("unix", socket)
|
||||
info, err := os.Stat(daemonSocket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info, err := os.Stat(daemonSocket); err == nil {
|
||||
_ = os.Chmod(socket, info.Mode().Perm())
|
||||
if info.Mode()&os.ModeSocket == 0 {
|
||||
return nil, errors.New("docker daemon path is not a Unix socket")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
instance, err := os.MkdirTemp(dir, "p-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
socket := filepath.Join(instance, "docker.sock")
|
||||
listener, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
return nil, errors.Join(err, os.RemoveAll(instance))
|
||||
}
|
||||
if err := copyDockerSocketPermissions(daemonSocket, socket, info); err != nil {
|
||||
return nil, errors.Join(err, listener.Close(), os.RemoveAll(instance))
|
||||
}
|
||||
dial := func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", daemonSocket)
|
||||
@@ -130,61 +218,110 @@ func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
},
|
||||
Transport: transport,
|
||||
}
|
||||
server := &http.Server{ReadHeaderTimeout: 30 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method != http.MethodPost:
|
||||
case createPath.MatchString(r.URL.Path):
|
||||
if err := addLabel(r, job); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
proxy := &DockerProxy{Socket: socket}
|
||||
streams, cancelStreams := context.WithCancel(context.Background())
|
||||
creates, cancelCreates := context.WithCancel(context.Background())
|
||||
var admission sync.Mutex
|
||||
var handlers sync.WaitGroup
|
||||
server := &http.Server{ReadHeaderTimeout: 30 * time.Second, ConnContext: func(ctx context.Context, conn net.Conn) context.Context {
|
||||
return context.WithValue(ctx, dockerProxyConnKey{}, conn)
|
||||
}, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
admission.Lock()
|
||||
if streams.Err() != nil {
|
||||
admission.Unlock()
|
||||
http.Error(w, "docker proxy is closing", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
handlers.Add(1)
|
||||
admission.Unlock()
|
||||
defer handlers.Done()
|
||||
creating := r.Method == http.MethodPost && createPath.MatchString(r.URL.Path)
|
||||
parent, lifetime := r.Context(), streams
|
||||
if creating {
|
||||
parent, lifetime = context.WithoutCancel(parent), creates
|
||||
}
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
defer cancel()
|
||||
stop := context.AfterFunc(lifetime, func() {
|
||||
cancel()
|
||||
if !creating {
|
||||
if conn, ok := parent.Value(dockerProxyConnKey{}).(net.Conn); ok {
|
||||
_ = conn.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
defer stop()
|
||||
r = r.WithContext(ctx)
|
||||
if creating {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxCreateBody)
|
||||
mounts, _ := proxy.mounts.Load().(map[string]string)
|
||||
if err := rewriteCreate(r, job, mounts); err != nil {
|
||||
status := http.StatusBadRequest
|
||||
if _, ok := errors.AsType[*http.MaxBytesError](err); ok {
|
||||
status = http.StatusRequestEntityTooLarge
|
||||
}
|
||||
http.Error(w, err.Error(), status)
|
||||
return
|
||||
}
|
||||
case rawStreamPath.MatchString(r.URL.Path):
|
||||
tunnel(w, r, dial)
|
||||
} else if r.Method == http.MethodPost && rawStreamPath.MatchString(r.URL.Path) {
|
||||
tunnel(w, r, dial, forward)
|
||||
return
|
||||
}
|
||||
forward.ServeHTTP(w, r)
|
||||
})}
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
return &DockerProxy{Socket: socket, close: func(ctx context.Context) error {
|
||||
err := removeJobResources(ctx, job)
|
||||
_ = server.Close()
|
||||
served := make(chan struct{})
|
||||
go func() {
|
||||
defer close(served)
|
||||
_ = server.Serve(listener)
|
||||
}()
|
||||
proxy.close = func(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
admission.Lock()
|
||||
listenerErr := listener.Close()
|
||||
cancelStreams()
|
||||
admission.Unlock()
|
||||
<-served
|
||||
shutdownErr := server.Shutdown(ctx)
|
||||
cancelCreates()
|
||||
serverErr := server.Close()
|
||||
handlers.Wait()
|
||||
transport.CloseIdleConnections()
|
||||
_ = os.Remove(socket)
|
||||
return err
|
||||
}}, nil
|
||||
return errors.Join(ctx.Err(), listenerErr, shutdownErr, serverErr, os.RemoveAll(instance))
|
||||
}
|
||||
return proxy, nil
|
||||
}
|
||||
|
||||
func addLabel(r *http.Request, job string) error {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxCreateBody+1))
|
||||
func rewriteCreate(r *http.Request, job string, mounts map[string]string) error {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(body) > maxCreateBody {
|
||||
return errors.New("create request too large")
|
||||
}
|
||||
if len(bytes.TrimSpace(body)) == 0 {
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
return nil
|
||||
body = []byte("{}")
|
||||
}
|
||||
var fields map[string]json.RawMessage
|
||||
var config struct{ Labels map[string]string }
|
||||
if err := json.Unmarshal(body, &fields); err != nil {
|
||||
return fmt.Errorf("invalid create request: %w", err)
|
||||
}
|
||||
key := "Labels"
|
||||
for name := range fields {
|
||||
if strings.EqualFold(name, key) {
|
||||
key = name
|
||||
break
|
||||
}
|
||||
if err := json.Unmarshal(body, &config); err != nil {
|
||||
return fmt.Errorf("invalid create labels: %w", err)
|
||||
}
|
||||
labels := map[string]string{}
|
||||
if raw := fields[key]; len(raw) > 0 && string(raw) != "null" {
|
||||
if err := json.Unmarshal(raw, &labels); err != nil {
|
||||
return fmt.Errorf("invalid create request: %w", err)
|
||||
}
|
||||
if fields == nil {
|
||||
fields = make(map[string]json.RawMessage)
|
||||
}
|
||||
labels[jobLabel] = job
|
||||
if fields[key], err = json.Marshal(labels); err != nil {
|
||||
if len(mounts) > 0 && !hasAmbiguousFields(body) {
|
||||
translateBinds(fields, createPath.FindStringSubmatch(r.URL.Path)[2], mounts)
|
||||
}
|
||||
maps.DeleteFunc(fields, func(name string, _ json.RawMessage) bool {
|
||||
return strings.EqualFold(name, "Labels")
|
||||
})
|
||||
if config.Labels == nil {
|
||||
config.Labels = make(map[string]string)
|
||||
}
|
||||
config.Labels[jobLabel+job] = "" // per-job key, so nested proxies keep each other's
|
||||
if fields["Labels"], err = json.Marshal(config.Labels); err != nil {
|
||||
return err
|
||||
}
|
||||
if body, err = json.Marshal(fields); err != nil {
|
||||
@@ -196,44 +333,241 @@ func addLabel(r *http.Request, job string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// tunnel splices attach and exec streams, which the daemon hijacks with or without an HTTP upgrade
|
||||
func tunnel(w http.ResponseWriter, r *http.Request, dial func(context.Context, string, string) (net.Conn, error)) {
|
||||
hijacker, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
http.Error(w, "connection cannot be hijacked", http.StatusInternalServerError)
|
||||
return
|
||||
func translateBinds(fields map[string]json.RawMessage, kind string, mounts map[string]string) {
|
||||
cleanedSource := func(source string) string {
|
||||
cleaned := path.Clean(source)
|
||||
if target := jobMount(cleaned, mounts); mounts[target] != "" {
|
||||
return mounts[target] + cleaned[len(target):]
|
||||
}
|
||||
return source
|
||||
}
|
||||
spelledSource := func(source string) string { // dockerd checks these as spelled
|
||||
target := jobMount(path.Clean(source), mounts)
|
||||
if rest, spelled := strings.CutPrefix(source, target); mounts[target] != "" && spelled && (rest == "" || rest[0] == '/') && filepath.IsLocal("."+rest) {
|
||||
return mounts[target] + rest
|
||||
}
|
||||
return cleanedSource(source)
|
||||
}
|
||||
switch kind {
|
||||
case "volumes":
|
||||
var driver string
|
||||
var options map[string]any
|
||||
decodeField(fields, "Driver", &driver)
|
||||
if key := decodeField(fields, "DriverOpts", &options); options != nil {
|
||||
translateDevice(driver, options, spelledSource)
|
||||
if encoded, err := json.Marshal(options); err == nil {
|
||||
fields[key] = encoded
|
||||
}
|
||||
}
|
||||
case "containers":
|
||||
var hostConfig map[string]any
|
||||
key := decodeField(fields, "HostConfig", &hostConfig)
|
||||
binds, _ := field(hostConfig, "Binds").([]any)
|
||||
for i, bind := range binds {
|
||||
bind, _ := bind.(string)
|
||||
source, target, _ := strings.Cut(bind, ":")
|
||||
if translated := cleanedSource(source); strings.HasPrefix(target, "/") && !strings.Contains(translated, ":") {
|
||||
binds[i] = translated + ":" + target
|
||||
}
|
||||
}
|
||||
specs, _ := field(hostConfig, "Mounts").([]any)
|
||||
for _, spec := range specs {
|
||||
spec, _ := spec.(map[string]any)
|
||||
switch field(spec, "Type") {
|
||||
case "bind":
|
||||
if source, ok := field(spec, "Source").(string); ok {
|
||||
spec["Source"] = spelledSource(source)
|
||||
}
|
||||
case "volume":
|
||||
volumeOptions, _ := field(spec, "VolumeOptions").(map[string]any)
|
||||
driverConfig, _ := field(volumeOptions, "DriverConfig").(map[string]any)
|
||||
translateDevice(field(driverConfig, "Name"), field(driverConfig, "Options"), spelledSource)
|
||||
}
|
||||
}
|
||||
if encoded, err := json.Marshal(hostConfig); err == nil && hostConfig != nil {
|
||||
fields[key] = encoded
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func decodeField(fields map[string]json.RawMessage, name string, value any) string {
|
||||
for key, raw := range fields {
|
||||
if strings.EqualFold(key, name) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(raw))
|
||||
decoder.UseNumber()
|
||||
_ = decoder.Decode(value) // wrong types stay unset for dockerd to reject
|
||||
return key
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func translateDevice(driver, options any, translate func(string) string) {
|
||||
optionMap, _ := options.(map[string]any)
|
||||
device, ok := optionMap["device"].(string)
|
||||
flags, _ := optionMap["o"].(string)
|
||||
tokens := strings.Split(flags, ",")
|
||||
local := driver == nil || driver == "" || driver == "local"
|
||||
if ok && local && (slices.Contains(tokens, "bind") || slices.Contains(tokens, "rbind")) && !slices.Contains(tokens, "remount") {
|
||||
optionMap["device"] = translate(device)
|
||||
}
|
||||
}
|
||||
|
||||
// field also renames the matched key to name.
|
||||
func field(object map[string]any, name string) any {
|
||||
for key, value := range object {
|
||||
if strings.EqualFold(key, name) {
|
||||
delete(object, key)
|
||||
object[name] = value
|
||||
return value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
requestFields = []string{"hostconfig", "driver", "driveropts"}
|
||||
asciiFolds = strings.NewReplacer("ſ", "s", "K", "k") // the non-ASCII runes strings.EqualFold matches to ASCII letters
|
||||
)
|
||||
|
||||
// dockerd settles repeated names by order, which re-encoding loses.
|
||||
func hasAmbiguousFields(body []byte) bool {
|
||||
type frame struct {
|
||||
names map[string]bool
|
||||
key string
|
||||
expectKey bool
|
||||
nested bool
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||
decoder.UseNumber()
|
||||
stack := []*frame{{}}
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
return !errors.Is(err, io.EOF)
|
||||
}
|
||||
top := stack[len(stack)-1]
|
||||
if name, ok := token.(string); ok && top.expectKey {
|
||||
top.key, top.expectKey = strings.ToLower(asciiFolds.Replace(name)), false
|
||||
if top.names[top.key] && (top.nested || slices.Contains(requestFields, top.key)) {
|
||||
return true
|
||||
}
|
||||
top.names[top.key] = true
|
||||
continue
|
||||
}
|
||||
top.expectKey = top.names != nil
|
||||
nested := top.nested || len(stack) == 2 && slices.Contains(requestFields, top.key)
|
||||
switch token {
|
||||
case json.Delim('{'):
|
||||
stack = append(stack, &frame{names: map[string]bool{}, expectKey: true, nested: nested})
|
||||
case json.Delim('['):
|
||||
stack = append(stack, &frame{nested: nested})
|
||||
case json.Delim('}'), json.Delim(']'):
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jobMount returns "" also for a path already naming a daemon source.
|
||||
func jobMount(source string, mounts map[string]string) string {
|
||||
target := ""
|
||||
for destination, daemonSource := range mounts {
|
||||
if daemonSource != "" && (source == daemonSource || strings.HasPrefix(source, daemonSource+"/")) {
|
||||
return ""
|
||||
}
|
||||
if len(destination) > len(target) && (source == destination || strings.HasPrefix(source, destination+"/")) {
|
||||
target = destination
|
||||
}
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
type dockerProxyConnKey struct{}
|
||||
|
||||
type dockerProxyResponse struct {
|
||||
response *http.Response
|
||||
}
|
||||
|
||||
func (r dockerProxyResponse) RoundTrip(_ *http.Request) (*http.Response, error) {
|
||||
return r.response, nil
|
||||
}
|
||||
|
||||
// tunnel splices attach and exec streams, which the daemon hijacks with or without an HTTP upgrade
|
||||
func tunnel(w http.ResponseWriter, r *http.Request, dial func(context.Context, string, string) (net.Conn, error), forward *httputil.ReverseProxy) {
|
||||
upstream, err := dial(r.Context(), "", "")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer upstream.Close()
|
||||
stop := context.AfterFunc(r.Context(), func() { _ = upstream.Close() })
|
||||
defer stop()
|
||||
if err := r.Write(upstream); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
downstream, buffered, err := hijacker.Hijack()
|
||||
reader := bufio.NewReader(upstream)
|
||||
var response *http.Response
|
||||
for {
|
||||
response, err = http.ReadResponse(reader, r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if response.StatusCode >= 200 || response.StatusCode == http.StatusSwitchingProtocols {
|
||||
break
|
||||
}
|
||||
maps.Copy(w.Header(), response.Header)
|
||||
w.WriteHeader(response.StatusCode)
|
||||
clear(w.Header())
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
defer func() {
|
||||
_ = upstream.Close()
|
||||
_ = response.Body.Close()
|
||||
}()
|
||||
mediaType, _, _ := mime.ParseMediaType(response.Header.Get("Content-Type"))
|
||||
if response.StatusCode != http.StatusSwitchingProtocols && (response.StatusCode != http.StatusOK || mediaType != "application/vnd.docker.raw-stream") {
|
||||
ordinary := *forward
|
||||
ordinary.Transport = dockerProxyResponse{response: response}
|
||||
ordinary.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
downstream, buffered, err := http.NewResponseController(w).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer downstream.Close()
|
||||
if _, err := io.CopyN(upstream, buffered, int64(buffered.Reader.Buffered())); err != nil {
|
||||
if _, err := fmt.Fprintf(buffered, "%s %s\r\n", response.Proto, response.Status); err != nil {
|
||||
return
|
||||
}
|
||||
done := make(chan struct{}, 2)
|
||||
if err := response.Header.Write(buffered); err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := buffered.WriteString("\r\n"); err != nil {
|
||||
return
|
||||
}
|
||||
if err := buffered.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = io.Copy(upstream, downstream)
|
||||
done <- struct{}{}
|
||||
}()
|
||||
go func() {
|
||||
_, _ = io.Copy(downstream, upstream)
|
||||
done <- struct{}{}
|
||||
defer close(done)
|
||||
if _, err := io.Copy(upstream, io.MultiReader(io.LimitReader(buffered, int64(buffered.Reader.Buffered())), downstream)); err != nil { // Bypass net/http after the prefix so stdin EOF preserves output.
|
||||
_ = upstream.Close()
|
||||
} else if writer, ok := upstream.(interface{ CloseWrite() error }); ok {
|
||||
_ = writer.CloseWrite()
|
||||
} else {
|
||||
_ = upstream.Close()
|
||||
}
|
||||
}()
|
||||
_, _ = io.Copy(downstream, reader)
|
||||
_ = downstream.Close()
|
||||
_ = upstream.Close()
|
||||
<-done
|
||||
}
|
||||
|
||||
func removeJobResources(ctx context.Context, job string) error {
|
||||
func RemoveDockerJobResources(ctx context.Context, job string) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -244,29 +578,24 @@ func removeJobResources(ctx context.Context, job string) error {
|
||||
|
||||
func removeLabelled(ctx context.Context, cli client.APIClient, job string) error {
|
||||
logger := common.Logger(ctx)
|
||||
filters := make(client.Filters).Add("label", jobLabel+"="+job)
|
||||
filters := make(client.Filters).Add("label", jobLabel+job)
|
||||
containers, err := cli.ContainerList(ctx, client.ContainerListOptions{All: true, Filters: filters})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
errs := []error{err}
|
||||
for _, c := range containers.Items {
|
||||
logger.Infof("removing container %s the job left behind", strings.TrimPrefix(strings.Join(c.Names, ","), "/"))
|
||||
errs = append(errs, (&containerReference{cli: cli, id: c.ID}).remove()(ctx))
|
||||
}
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{Filters: filters})
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, err)...)
|
||||
}
|
||||
errs = append(errs, err)
|
||||
for _, n := range networks.Items {
|
||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
|
||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); n.Scope == "swarm" && cerrdefs.IsInvalidArgument(err) { // swarm refuses while a service or its tasks use it
|
||||
logger.Infof("keeping network %s, a swarm service still uses it", n.Name)
|
||||
} else if err != nil && !cerrdefs.IsNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
|
||||
}
|
||||
}
|
||||
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{Filters: filters})
|
||||
if err != nil {
|
||||
return errors.Join(append(errs, err)...)
|
||||
}
|
||||
errs = append(errs, err)
|
||||
for _, v := range volumes.Items {
|
||||
if _, err := cli.VolumeRemove(ctx, v.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", v.Name, err))
|
||||
|
||||
@@ -10,17 +10,24 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/image"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
"github.com/moby/moby/api/types/volume"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
@@ -56,7 +63,13 @@ func daemonSocketPath(t testing.TB, cli mobyclient.APIClient) string {
|
||||
}
|
||||
|
||||
func TestDockerProxy(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix socket ownership is unavailable on Windows")
|
||||
}
|
||||
bodies := make(chan []byte, 8)
|
||||
createStarted, releaseCreate := make(chan struct{}), make(chan struct{})
|
||||
streamsDone := make(chan struct{}, 2)
|
||||
var probes atomic.Int32
|
||||
daemonSocket := filepath.Join(shortTempDir(t), "d.sock")
|
||||
listener, err := net.Listen("unix", daemonSocket)
|
||||
require.NoError(t, err)
|
||||
@@ -64,16 +77,49 @@ func TestDockerProxy(t *testing.T) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/create"):
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if bytes.Contains(body, []byte(`"Image":"probe-image"`)) {
|
||||
_, _ = w.Write([]byte(`{"Id":"probe"}`))
|
||||
return
|
||||
}
|
||||
if r.URL.RawQuery == "wait" {
|
||||
close(createStarted)
|
||||
<-releaseCreate
|
||||
}
|
||||
bodies <- body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.HasSuffix(r.URL.Path, "/start"):
|
||||
case strings.HasSuffix(r.URL.Path, "/images/json"):
|
||||
if probes.Add(1) == 1 {
|
||||
http.Error(w, "probe unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`[{"Id":"probe-image"}]`))
|
||||
case strings.HasSuffix(r.URL.Path, "/json"):
|
||||
http.NotFound(w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/archive"):
|
||||
w.Header().Set("X-Docker-Container-Path-Stat", "e30=")
|
||||
case r.Method == http.MethodDelete:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case r.URL.Path == "/exec/detached/start" || r.URL.Path == "/exec/error/start":
|
||||
if r.URL.Path == "/exec/error/start" {
|
||||
w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
}
|
||||
_, _ = w.Write([]byte("ordinary"))
|
||||
case rawStreamPath.MatchString(r.URL.Path) || r.URL.Path == "/session":
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
conn, buffered, err := w.(http.Hijacker).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\n"))
|
||||
_, _ = io.Copy(conn, buffered)
|
||||
if r.Header.Get("Upgrade") != "" {
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: tcp\r\n\r\nready\n"))
|
||||
} else {
|
||||
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/vnd.docker.raw-stream\r\n\r\nready\n"))
|
||||
}
|
||||
input, _ := io.ReadAll(buffered)
|
||||
_, _ = conn.Write(append([]byte("final:"), input...))
|
||||
streamsDone <- struct{}{}
|
||||
default:
|
||||
w.Header().Set("Api-Version", "1.47")
|
||||
_, _ = w.Write([]byte("OK " + r.Method + " " + r.URL.Path))
|
||||
@@ -83,34 +129,94 @@ func TestDockerProxy(t *testing.T) {
|
||||
t.Cleanup(func() { _ = daemon.Close() })
|
||||
proxy, err := StartDockerProxy(daemonSocket, shortTempDir(t), "job-1")
|
||||
require.NoError(t, err)
|
||||
client := &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
||||
proxy.SetMounts(map[string]string{"/workspace/o/r": "/volumes/job/_data", "/workspace/o/r/tmp": "", "/var/run/docker.sock": "/tmp/p/docker.sock", "/volumes": "/daemon/volumes"})
|
||||
client := &http.Client{Timeout: 5 * time.Second, Transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", proxy.Socket)
|
||||
}}}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
dialProxy := func(t *testing.T) *net.UnixConn {
|
||||
t.Helper()
|
||||
conn, err := net.DialUnix("unix", nil, &net.UnixAddr{Name: proxy.Socket, Net: "unix"})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
require.NoError(t, conn.SetDeadline(time.Now().Add(5*time.Second)))
|
||||
return conn
|
||||
}
|
||||
startStream := func(t *testing.T, path, upgrade string) (*net.UnixConn, *bufio.Reader) {
|
||||
t.Helper()
|
||||
conn := dialProxy(t)
|
||||
_, err := fmt.Fprintf(conn, "POST %s HTTP/1.1\r\nHost: docker\r\n%sContent-Length: 2\r\n\r\n{}input", path, upgrade)
|
||||
require.NoError(t, err)
|
||||
reader := bufio.NewReader(conn)
|
||||
resp, err := http.ReadResponse(reader, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_ = conn.Close()
|
||||
_ = resp.Body.Close()
|
||||
})
|
||||
prefix, err := reader.ReadString('\n')
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "ready\n", prefix)
|
||||
return conn, reader
|
||||
}
|
||||
|
||||
t.Run("labels creates", func(t *testing.T) {
|
||||
for path, body := range map[string]string{
|
||||
"/v1.47/containers/create": `{"Image":"alpine","Labels":{"own":"1"}}`,
|
||||
"/networks/create": `{"Name":"n"}`,
|
||||
"/volumes/create": `{"Name":"v","labels":null}`,
|
||||
for _, testCase := range []struct{ path, body, want string }{
|
||||
{
|
||||
"/v1.47/containers/create", `{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":""}}`,
|
||||
`{"Image":"alpine","Unknown":{"enabled":true},"Labels":{"own":"1","com.gitea.runner.job.job-0":"","com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/containers/create", `{"HostConfig":{"Binds":["/workspace/o/r:/src:ro","/workspace/o/r/../r/sub/:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2"}}`,
|
||||
`{"HostConfig":{"Binds":["/volumes/job/_data:/src:ro","/volumes/job/_data/sub:/sub","workspace/o/r:/relative","/workspace/o/rest:/rest","/workspace/o/r/tmp/x:/tmp","named:/named","/anonymous","/workspace/o/r:ro","/volumes/job/_data/x:/daemon-path"]},"DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"Foo":"1","foo":"2","com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/var/run/docker.sock","Target":"/var/run/docker.sock"},{"type":"bind","source":"/workspace/o/r/sub/","target":"/m"},{"Type":"bind","Source":"/workspace/o/r/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/workspace/o/r/../r","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]}}`,
|
||||
`{"HostConfig":{"Mounts":[{"Type":"bind","Source":"/tmp/p/docker.sock","Target":"/var/run/docker.sock"},{"Type":"bind","Source":"/volumes/job/_data/sub/","target":"/m"},{"Type":"bind","Source":"/volumes/job/_data/sub/../data","Target":"/dotted"},{"Type":"bind","Source":"/volumes/job/_data","Target":"/escaping"},{"Type":"BIND","Source":"/workspace/o/r"},{"Type":"volume","Source":"/workspace/o/r"}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/containers/create", `{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/workspace/o/r/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]}}`,
|
||||
`{"HostConfig":{"Mounts":[{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind","device":"/volumes/job/_data/data"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"unbindable","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Options":{"o":"bind,remount","device":"/workspace/o/r"}}}},{"Type":"volume","VolumeOptions":{"DriverConfig":{"Name":"plugin","Options":{"o":"bind","device":"/workspace/o/r"}}}}]},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/containers/create", `{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null}`,
|
||||
`{"HostConfig":{"privileged":true,"Privileged":false,"Binds":["/workspace/o/r:/src"]},"hoſtconfig":null,"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/networks/create", `{"Name":"n"}`,
|
||||
`{"Name":"n","Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", `{"Name":"v","labels":null,"DriverOpts":{"type":"none","o":"rbind,ro","device":"/workspace/o/r/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
||||
`{"Name":"v","Labels":{"com.gitea.runner.job.job-1":""},"DriverOpts":{"type":"none","o":"rbind,ro","device":"/volumes/job/_data/"},"HostConfig":{"Binds":["/workspace/o/r:/src"]}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", `{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"}}`,
|
||||
`{"Name":"d","DriverOpts":{"o":"bind","device":1,"device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", `{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"}}`,
|
||||
`{"Name":"p","Driver":"plugin","DriverOpts":{"o":"bind","device":"/workspace/o/r"},"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", "",
|
||||
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", "null",
|
||||
`{"Labels":{"com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
{
|
||||
"/volumes/create", `{"Labels":{"discard":"1"},"labels":null,"LABELS":{"own":"1"},"LABELS":{"extra":"2"}}`,
|
||||
`{"Labels":{"own":"1","extra":"2","com.gitea.runner.job.job-1":""}}`,
|
||||
},
|
||||
} {
|
||||
resp, err := client.Post("http://docker"+path, "application/json", strings.NewReader(body))
|
||||
resp, err := client.Post("http://docker"+testCase.path, "application/json", strings.NewReader(testCase.body))
|
||||
require.NoError(t, err)
|
||||
resp.Body.Close()
|
||||
assert.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
var got map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(<-bodies, &got))
|
||||
key := "Labels"
|
||||
if _, ok := got["labels"]; ok {
|
||||
key = "labels"
|
||||
}
|
||||
var labels map[string]string
|
||||
require.NoError(t, json.Unmarshal(got[key], &labels))
|
||||
assert.Equal(t, "job-1", labels[jobLabel], path)
|
||||
if strings.Contains(body, "own") {
|
||||
assert.Equal(t, "1", labels["own"])
|
||||
assert.JSONEq(t, `"alpine"`, string(got["Image"]))
|
||||
}
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
assert.JSONEq(t, testCase.want, string(<-bodies), testCase.body)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -123,58 +229,150 @@ func TestDockerProxy(t *testing.T) {
|
||||
assert.Equal(t, "OK GET /v1.47/_ping", string(body))
|
||||
})
|
||||
|
||||
t.Run("tunnels raw streams", func(t *testing.T) {
|
||||
conn, err := net.Dial("unix", proxy.Socket)
|
||||
require.NoError(t, err)
|
||||
defer conn.Close()
|
||||
_, err = conn.Write([]byte("POST /v1.47/exec/abc/start HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}ping\n"))
|
||||
require.NoError(t, err)
|
||||
t.Run("discovery caches only successes", func(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("automatic proxy discovery requires Linux")
|
||||
}
|
||||
t.Setenv("DOCKER_HOST", "unix://"+daemonSocket)
|
||||
t.Setenv("TMPDIR", shortTempDir(t))
|
||||
require.Nil(t, NewDockerProxy(t.Context(), "failed"))
|
||||
for _, job := range []string{"first", "second"} {
|
||||
proxy := NewDockerProxy(t.Context(), job)
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
}
|
||||
assert.EqualValues(t, 2, probes.Load())
|
||||
dockerProxyDirectories.Range(func(key, value any) bool {
|
||||
directory, _ := value.(dockerProxyDirectory)
|
||||
directory.socket, _ = os.Stat(t.TempDir())
|
||||
dockerProxyDirectories.Store(key, directory)
|
||||
return true
|
||||
})
|
||||
proxy := NewDockerProxy(t.Context(), "replaced daemon")
|
||||
require.NotNil(t, proxy)
|
||||
require.NoError(t, proxy.Close(context.Background()))
|
||||
assert.EqualValues(t, 3, probes.Load())
|
||||
})
|
||||
|
||||
t.Run("tunnels raw streams through stdin EOF", func(t *testing.T) {
|
||||
for _, stream := range []struct{ path, upgrade string }{
|
||||
{path: "/v1.47/exec/abc/start"},
|
||||
{path: "/containers/abc/attach", upgrade: "Connection: Upgrade\r\nUpgrade: tcp\r\n"},
|
||||
} {
|
||||
conn, reader := startStream(t, stream.path, stream.upgrade)
|
||||
require.NoError(t, conn.CloseWrite())
|
||||
output, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "final:input", string(output))
|
||||
<-streamsDone
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("detached and error responses retain keepalive label injection", func(t *testing.T) {
|
||||
conn := dialProxy(t)
|
||||
reader := bufio.NewReader(conn)
|
||||
resp, err := http.ReadResponse(reader, nil)
|
||||
for endpoint, status := range map[string]int{"detached": http.StatusOK, "error": http.StatusBadRequest} {
|
||||
_, err := fmt.Fprintf(conn, "POST /exec/%s/start HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}", endpoint)
|
||||
require.NoError(t, err)
|
||||
resp, err := http.ReadResponse(reader, nil)
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
assert.Equal(t, status, resp.StatusCode)
|
||||
assert.Equal(t, "ordinary", string(body))
|
||||
_, err = io.WriteString(conn, "POST /volumes/create HTTP/1.1\r\nHost: docker\r\nContent-Length: 0\r\n\r\n")
|
||||
require.NoError(t, err)
|
||||
resp, err = http.ReadResponse(reader, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("close joins streams and admitted create", func(t *testing.T) {
|
||||
t.Cleanup(func() { close(releaseCreate) })
|
||||
_, raw := startStream(t, "/exec/live/start", "")
|
||||
_, upgraded := startStream(t, "/session", "Connection: Upgrade\r\nUpgrade: tcp\r\n")
|
||||
creator := dialProxy(t)
|
||||
_, err := io.WriteString(creator, "POST /volumes/create?wait HTTP/1.1\r\nHost: docker\r\nContent-Length: 2\r\n\r\n{}")
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, "application/vnd.docker.raw-stream", resp.Header.Get("Content-Type"))
|
||||
echoed, err := reader.ReadString('\n')
|
||||
<-createStarted
|
||||
require.NoError(t, creator.CloseWrite())
|
||||
closed := make(chan error, 1)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
go func() { closed <- proxy.Close(ctx) }()
|
||||
for _, reader := range []*bufio.Reader{raw, upgraded} {
|
||||
_, err := reader.ReadByte()
|
||||
require.ErrorIs(t, err, io.EOF)
|
||||
<-streamsDone
|
||||
}
|
||||
select {
|
||||
case err := <-closed:
|
||||
t.Fatalf("Close returned before admitted create settled: %v", err)
|
||||
default:
|
||||
}
|
||||
releaseCreate <- struct{}{}
|
||||
resp, err := http.ReadResponse(bufio.NewReader(creator), nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "{}ping\n", echoed)
|
||||
require.NoError(t, resp.Body.Close())
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
assert.JSONEq(t, `{"Labels":{"com.gitea.runner.job.job-1":""}}`, string(<-bodies))
|
||||
require.NoError(t, <-closed)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveLabelledRemovesContainersNetworksAndVolumes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
filters := make(mobyclient.Filters).Add("label", jobLabel+"=job-1")
|
||||
containerFailure := errors.New("container removal failed")
|
||||
listFailure := errors.New("listing failed")
|
||||
volumeFailure := errors.New("volume removal failed")
|
||||
ctx := t.Context()
|
||||
filters := make(mobyclient.Filters).Add("label", jobLabel+"job-1")
|
||||
cli := &mockDockerClient{}
|
||||
cli.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true, Filters: filters}).
|
||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil)
|
||||
cli.On("ContainerKill", ctx, "c1", mock.Anything).Return(mobyclient.ContainerKillResult{}, nil)
|
||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{{ID: "c1", Names: []string{"/app"}}}}, nil).Once()
|
||||
cli.On("ContainerKill", ctx, "c1", mock.Anything).Return(mobyclient.ContainerKillResult{}, nil).Once()
|
||||
cli.On("ContainerRemove", ctx, "c1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
|
||||
Return(mobyclient.ContainerRemoveResult{}, nil)
|
||||
Return(mobyclient.ContainerRemoveResult{}, containerFailure).Once()
|
||||
cli.On("NetworkList", ctx, mobyclient.NetworkListOptions{Filters: filters}).
|
||||
Return(mobyclient.NetworkListResult{Items: []network.Summary{{ID: "n1", Name: "app_default"}}}, nil)
|
||||
cli.On("NetworkRemove", ctx, "n1", mock.Anything).Return(mobyclient.NetworkRemoveResult{}, nil)
|
||||
Return(mobyclient.NetworkListResult{Items: []network.Summary{{ID: "n1", Name: "stack_default", Scope: "swarm"}}}, listFailure).Once()
|
||||
cli.On("NetworkRemove", ctx, "n1", mobyclient.NetworkRemoveOptions{}).Return(mobyclient.NetworkRemoveResult{}, cerrdefs.ErrInvalidArgument).Once()
|
||||
cli.On("VolumeList", ctx, mobyclient.VolumeListOptions{Filters: filters}).
|
||||
Return(mobyclient.VolumeListResult{Items: []volume.Volume{{Name: "app_data"}}}, nil)
|
||||
cli.On("VolumeRemove", ctx, "app_data", mobyclient.VolumeRemoveOptions{}).Return(mobyclient.VolumeRemoveResult{}, nil)
|
||||
Return(mobyclient.VolumeListResult{Items: []volume.Volume{{Name: "app_data"}}}, nil).Once()
|
||||
cli.On("VolumeRemove", ctx, "app_data", mobyclient.VolumeRemoveOptions{}).Return(mobyclient.VolumeRemoveResult{}, volumeFailure).Once()
|
||||
|
||||
require.NoError(t, removeLabelled(ctx, cli, "job-1"))
|
||||
err := removeLabelled(ctx, cli, "job-1")
|
||||
require.ErrorIs(t, err, containerFailure)
|
||||
require.ErrorIs(t, err, listFailure)
|
||||
require.ErrorIs(t, err, volumeFailure)
|
||||
require.NotErrorIs(t, err, cerrdefs.ErrInvalidArgument)
|
||||
require.ErrorContains(t, err, "failed to remove container c1")
|
||||
require.ErrorContains(t, err, "failed to remove volume app_data")
|
||||
cli.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerProxyWithDaemon(t *testing.T) {
|
||||
requireDocker(t)
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
cleanupCtx := context.WithoutCancel(ctx)
|
||||
require.NoError(t, NewDockerPullExecutor(NewDockerPullExecutorInput{Image: "alpine"})(ctx))
|
||||
direct, err := GetDockerClient(ctx)
|
||||
require.NoError(t, err)
|
||||
defer direct.Close()
|
||||
dir := shortTempDir(t)
|
||||
seen, err := daemonSeesDir(ctx, dir)
|
||||
seen, err := daemonSeesDir(ctx, direct, dir, dir)
|
||||
require.NoError(t, err)
|
||||
t.Logf("daemon sees the runner's filesystem: %v", seen)
|
||||
seen, err = daemonSeesDir(ctx, direct, dir, shortTempDir(t))
|
||||
require.NoError(t, err)
|
||||
assert.False(t, seen)
|
||||
|
||||
job := "proxy-test-" + t.Name()
|
||||
job := "proxy-test-" + strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
proxy, err := StartDockerProxy(daemonSocketPath(t, direct), dir, job)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = proxy.Close(context.Background()) })
|
||||
viaProxy, err := mobyclient.New(mobyclient.WithHost("unix://" + proxy.Socket))
|
||||
require.NoError(t, err)
|
||||
defer viaProxy.Close()
|
||||
@@ -183,22 +381,24 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
||||
Config: &container.Config{Image: "alpine", Cmd: []string{"sleep", "300"}, Labels: map[string]string{"own": "1"}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _, _ = direct.ContainerRemove(ctx, created.ID, mobyclient.ContainerRemoveOptions{Force: true}) })
|
||||
t.Cleanup(func() {
|
||||
_, _ = direct.ContainerRemove(cleanupCtx, created.ID, mobyclient.ContainerRemoveOptions{Force: true})
|
||||
})
|
||||
_, err = viaProxy.ContainerStart(ctx, created.ID, mobyclient.ContainerStartOptions{})
|
||||
require.NoError(t, err)
|
||||
net, err := viaProxy.NetworkCreate(ctx, job, mobyclient.NetworkCreateOptions{})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _, _ = direct.NetworkRemove(ctx, net.ID, mobyclient.NetworkRemoveOptions{}) })
|
||||
t.Cleanup(func() { _, _ = direct.NetworkRemove(cleanupCtx, net.ID, mobyclient.NetworkRemoveOptions{}) })
|
||||
_, err = viaProxy.VolumeCreate(ctx, mobyclient.VolumeCreateOptions{Name: job})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _, _ = direct.VolumeRemove(ctx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
|
||||
t.Cleanup(func() { _, _ = direct.VolumeRemove(cleanupCtx, job, mobyclient.VolumeRemoveOptions{Force: true}) })
|
||||
|
||||
inspected, err := direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]string{"own": "1", jobLabel: job}, inspected.Container.Config.Labels)
|
||||
assert.Subset(t, inspected.Container.Config.Labels, map[string]string{"own": "1", jobLabel + job: ""})
|
||||
vol, err := direct.VolumeInspect(ctx, job, mobyclient.VolumeInspectOptions{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, job, vol.Volume.Labels[jobLabel])
|
||||
assert.Contains(t, vol.Volume.Labels, jobLabel+job)
|
||||
|
||||
exec, err := viaProxy.ExecCreate(ctx, created.ID, mobyclient.ExecCreateOptions{Cmd: []string{"cat"}, AttachStdin: true, AttachStdout: true, TTY: true})
|
||||
require.NoError(t, err)
|
||||
@@ -212,6 +412,7 @@ func TestDockerProxyWithDaemon(t *testing.T) {
|
||||
attached.Close()
|
||||
|
||||
require.NoError(t, proxy.Close(ctx))
|
||||
require.NoError(t, RemoveDockerJobResources(ctx, job))
|
||||
_, err = direct.ContainerInspect(ctx, created.ID, mobyclient.ContainerInspectOptions{})
|
||||
assert.True(t, cerrdefs.IsNotFound(err))
|
||||
_, err = direct.NetworkInspect(ctx, net.ID, mobyclient.NetworkInspectOptions{})
|
||||
@@ -268,3 +469,121 @@ func BenchmarkDockerProxy(b *testing.B) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type probeClient struct {
|
||||
mobyclient.APIClient
|
||||
create func(mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error)
|
||||
remove func(context.Context, string, mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error)
|
||||
stat func(string) error
|
||||
mounts []container.MountPoint
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerInspect(context.Context, string, mobyclient.ContainerInspectOptions) (mobyclient.ContainerInspectResult, error) {
|
||||
return mobyclient.ContainerInspectResult{Container: container.InspectResponse{Mounts: c.mounts}}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ImageList(context.Context, mobyclient.ImageListOptions) (mobyclient.ImageListResult, error) {
|
||||
return mobyclient.ImageListResult{Items: []image.Summary{{ID: "probe-image"}}}, nil
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerCreate(_ context.Context, opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
return c.create(opts)
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerRemove(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
return c.remove(ctx, id, opts)
|
||||
}
|
||||
|
||||
func (c *probeClient) ContainerStatPath(_ context.Context, _ string, opts mobyclient.ContainerStatPathOptions) (mobyclient.ContainerStatPathResult, error) {
|
||||
return mobyclient.ContainerStatPathResult{}, c.stat(opts.Path)
|
||||
}
|
||||
|
||||
func TestDaemonSeesDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
markers := make(map[string]bool)
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
daemonDir string
|
||||
removeErr error
|
||||
seen bool
|
||||
removed bool
|
||||
}{
|
||||
{name: "cleanup after cancellation", daemonDir: dir, seen: true, removed: true},
|
||||
{name: "daemon without the directory", daemonDir: filepath.Join(dir, "missing")},
|
||||
{name: "daemon with a different directory at that path", daemonDir: t.TempDir(), removed: true},
|
||||
{name: "cleanup error after cancellation", daemonDir: dir, removeErr: errors.New("cleanup failed"), removed: true},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
removed := false
|
||||
cli := &probeClient{
|
||||
create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
assert.Equal(t, []mount.Mount{{Type: mount.TypeBind, Source: testCase.daemonDir, Target: "/gitea-runner-probe", ReadOnly: true}}, opts.HostConfig.Mounts)
|
||||
cancel()
|
||||
if _, err := os.Stat(testCase.daemonDir); err != nil {
|
||||
return mobyclient.ContainerCreateResult{}, cerrdefs.ErrInvalidArgument
|
||||
}
|
||||
return mobyclient.ContainerCreateResult{ID: "probe"}, nil
|
||||
},
|
||||
stat: func(path string) error {
|
||||
marker := strings.TrimPrefix(path, "/gitea-runner-probe/")
|
||||
assert.False(t, markers[marker])
|
||||
markers[marker] = true
|
||||
if _, err := os.Stat(filepath.Join(testCase.daemonDir, marker)); err != nil {
|
||||
return cerrdefs.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
},
|
||||
remove: func(ctx context.Context, id string, opts mobyclient.ContainerRemoveOptions) (mobyclient.ContainerRemoveResult, error) {
|
||||
removed = true
|
||||
assert.Equal(t, "probe", id)
|
||||
assert.Equal(t, mobyclient.ContainerRemoveOptions{Force: true, RemoveVolumes: true}, opts)
|
||||
require.NoError(t, ctx.Err())
|
||||
_, bounded := ctx.Deadline()
|
||||
assert.True(t, bounded)
|
||||
return mobyclient.ContainerRemoveResult{}, testCase.removeErr
|
||||
},
|
||||
}
|
||||
seen, err := daemonSeesDir(ctx, cli, dir, testCase.daemonDir)
|
||||
require.ErrorIs(t, err, testCase.removeErr)
|
||||
assert.Equal(t, testCase.seen, seen)
|
||||
assert.Equal(t, testCase.removed, removed)
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerContainerWorkdir(t *testing.T) {
|
||||
workdir := t.TempDir()
|
||||
t.Chdir(workdir)
|
||||
volume := container.MountPoint{Type: mount.TypeVolume, Source: "/daemon/_data", Destination: filepath.Dir(workdir)}
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
mount container.MountPoint
|
||||
foreign bool
|
||||
daemonDir string
|
||||
}{
|
||||
{name: "volume above the working directory", mount: volume, daemonDir: filepath.Join(volume.Source, filepath.Base(workdir))},
|
||||
{name: "tmpfs", mount: container.MountPoint{Type: mount.TypeTmpfs, Destination: volume.Destination}},
|
||||
{name: "hostname of another container", mount: volume, foreign: true},
|
||||
} {
|
||||
cli := &probeClient{
|
||||
mounts: []container.MountPoint{testCase.mount},
|
||||
stat: func(path string) error {
|
||||
if _, err := os.Stat(path); err != nil || testCase.foreign {
|
||||
return cerrdefs.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
dir, daemonDir := runnerContainerWorkdir(t.Context(), cli)
|
||||
assert.Equal(t, testCase.daemonDir, daemonDir, testCase.name)
|
||||
assert.Equal(t, testCase.daemonDir != "", dir == workdir, testCase.name)
|
||||
}
|
||||
entries, err := os.ReadDir(workdir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries)
|
||||
}
|
||||
|
||||
48
act/container/docker_proxy_unix.go
Normal file
48
act/container/docker_proxy_unix.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (linux || darwin || netbsd)
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func copyDockerSocketPermissions(daemonSocket, socket string, info os.FileInfo) error {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return errors.New("docker socket ownership is unavailable")
|
||||
}
|
||||
groupErr := os.Chown(socket, int(stat.Uid), int(stat.Gid))
|
||||
if groupErr != nil && (!errors.Is(groupErr, fs.ErrPermission) || int(stat.Uid) != os.Geteuid()) {
|
||||
return groupErr
|
||||
}
|
||||
if err := os.Chown(filepath.Dir(socket), int(stat.Uid), -1); err != nil {
|
||||
return err
|
||||
}
|
||||
mode := uint16(info.Mode().Perm())
|
||||
if err := os.Chmod(socket, fs.FileMode(mode)); err != nil || groupErr == nil {
|
||||
return err
|
||||
}
|
||||
if size, err := unix.Getxattr(daemonSocket, "system.posix_acl_access", nil); err == nil && size > 0 {
|
||||
return groupErr
|
||||
}
|
||||
owner, group, other := mode>>6, mode>>3&7, mode&7
|
||||
acl := binary.LittleEndian.AppendUint32(nil, 2)
|
||||
for _, entry := range []struct {
|
||||
tag, perm uint16
|
||||
id uint32
|
||||
}{{1, owner, math.MaxUint32}, {4, other, math.MaxUint32}, {8, group, stat.Gid}, {16, group | other, math.MaxUint32}, {32, other, math.MaxUint32}} {
|
||||
acl = binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint16(binary.LittleEndian.AppendUint16(acl, entry.tag), entry.perm), entry.id)
|
||||
}
|
||||
return unix.Setxattr(socket, "system.posix_acl_access", acl, 0) // names the group a rootless runner cannot chown to, its own group keeps what others had
|
||||
}
|
||||
15
act/container/docker_proxy_windows.go
Normal file
15
act/container/docker_proxy_windows.go
Normal file
@@ -0,0 +1,15 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func copyDockerSocketPermissions(_, _ string, _ os.FileInfo) error {
|
||||
return errors.New("docker socket ownership cannot be preserved on Windows")
|
||||
}
|
||||
@@ -5,20 +5,19 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/cli/cli/config"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func init() {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
logging.SetLevel(logging.LevelDebug)
|
||||
}
|
||||
|
||||
func TestCleanImage(t *testing.T) {
|
||||
@@ -36,18 +35,15 @@ func TestCleanImage(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
imageOut := cleanImage(context.Background(), table.imageIn)
|
||||
imageOut := cleanImage(t.Context(), table.imageIn)
|
||||
assert.Equal(t, table.imageOut, imageOut)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetImagePullOptions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
|
||||
orig := config.Dir()
|
||||
t.Cleanup(func() { config.SetDir(orig) })
|
||||
|
||||
config.SetDir("/non-existent/docker")
|
||||
t.Setenv("DOCKER_CONFIG", "/non-existent/docker")
|
||||
|
||||
options, err := getImagePullOptions(ctx, NewDockerPullExecutorInput{})
|
||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
@@ -61,7 +57,7 @@ func TestGetImagePullOptions(t *testing.T) {
|
||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZCJ9", options.RegistryAuth, "Username and Password should be provided")
|
||||
|
||||
config.SetDir("testdata/docker-pull-options")
|
||||
t.Setenv("DOCKER_CONFIG", "testdata/docker-pull-options")
|
||||
|
||||
options, err = getImagePullOptions(ctx, NewDockerPullExecutorInput{
|
||||
Image: "nektos/act",
|
||||
@@ -79,7 +75,7 @@ func TestIsPinnedImage(t *testing.T) {
|
||||
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
|
||||
// whichever of the two shapes the daemon used.
|
||||
func TestLogDockerResponseError(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
logger := logging.New(logtest.New())
|
||||
streamErr := func(line string) error {
|
||||
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
|
||||
}
|
||||
|
||||
@@ -7,13 +7,15 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
@@ -24,17 +26,11 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"dario.cat/mergo"
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
"github.com/docker/cli/cli/connhelper"
|
||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/docker/cli/opts"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
@@ -43,7 +39,6 @@ import (
|
||||
"github.com/moby/moby/client"
|
||||
"github.com/moby/moby/client/pkg/versions"
|
||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// drainGracePeriod bounds how long we wait for an output-copy goroutine to
|
||||
@@ -55,15 +50,12 @@ const drainGracePeriod = 2 * time.Second
|
||||
|
||||
// NewContainer creates a reference to a container
|
||||
func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||
cr := new(containerReference)
|
||||
cr.input = input
|
||||
// Resolved up front because the image pull runs before the container is created.
|
||||
cf := createFlagsFromOptions(input.allOptions())
|
||||
if cf.platform != "" {
|
||||
cr.input.Platform = cf.platform
|
||||
input.Platform = cf.platform
|
||||
}
|
||||
cr.pullPolicy = cf.pull
|
||||
return cr
|
||||
return &containerReference{input: input, pullPolicy: cf.pull}
|
||||
}
|
||||
|
||||
// supportsContainerImagePlatform reports whether the Docker server API version
|
||||
@@ -98,8 +90,8 @@ func (cr *containerReference) Start(attach bool) common.Executor {
|
||||
cr.attach().IfBool(attach),
|
||||
cr.start(),
|
||||
cr.wait().IfBool(attach),
|
||||
cr.tryReadUID(),
|
||||
cr.tryReadGID(),
|
||||
cr.tryReadID("-u", func(id int) { cr.UID = id }),
|
||||
cr.tryReadID("-g", func(id int) { cr.GID = id }),
|
||||
func(ctx context.Context) error {
|
||||
// If this fails, then folders have wrong permissions on non root container
|
||||
if cr.UID != 0 || cr.GID != 0 {
|
||||
@@ -257,6 +249,27 @@ func containerInfoFromInspect(inspect container.InspectResponse) *Info {
|
||||
for _, mountPoint := range inspect.Mounts {
|
||||
info.Mounts[mountPoint.Destination] = mountPoint.Source
|
||||
}
|
||||
if hostConfig := inspect.HostConfig; hostConfig != nil { // Mounts omits --tmpfs targets and subpaths
|
||||
for target := range hostConfig.Tmpfs {
|
||||
info.Mounts[path.Clean(target)] = ""
|
||||
}
|
||||
for _, spec := range hostConfig.Mounts {
|
||||
subpath := ""
|
||||
if spec.VolumeOptions != nil {
|
||||
subpath = spec.VolumeOptions.Subpath
|
||||
} else if spec.ImageOptions != nil {
|
||||
subpath = spec.ImageOptions.Subpath
|
||||
}
|
||||
if target := path.Clean(spec.Target); subpath != "" && info.Mounts[target] != "" {
|
||||
info.Mounts[target] = path.Join(info.Mounts[target], subpath)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, target := range []string{"/etc/hosts", "/etc/hostname", "/etc/resolv.conf"} { // specific to the job's network namespace
|
||||
if _, mounted := info.Mounts[target]; !mounted {
|
||||
info.Mounts[target] = ""
|
||||
}
|
||||
}
|
||||
|
||||
if state := inspect.State; state != nil {
|
||||
info.State = string(state.Status)
|
||||
@@ -287,7 +300,25 @@ func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]stri
|
||||
}
|
||||
|
||||
func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
return cr.extractFromImageEnv(env).IfNot(common.Dryrun)
|
||||
envMap := *env
|
||||
return common.Executor(func(ctx context.Context) error {
|
||||
inspect, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
|
||||
if err != nil {
|
||||
common.Logger(ctx).Error(err)
|
||||
return fmt.Errorf("inspect container: %w", err)
|
||||
}
|
||||
if inspect.Container.Config == nil {
|
||||
return nil
|
||||
}
|
||||
for key, value := range opts.ConvertKVStringsToMap(inspect.Container.Config.Env) {
|
||||
if envMap[key] == "" {
|
||||
envMap[key] = value
|
||||
} else if key == "PATH" {
|
||||
envMap[key] += ":" + value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}).IfNot(common.Dryrun)
|
||||
}
|
||||
|
||||
func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor {
|
||||
@@ -336,15 +367,13 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
dockerHost := os.Getenv("DOCKER_HOST")
|
||||
|
||||
if strings.HasPrefix(dockerHost, "ssh://") {
|
||||
var helper *connhelper.ConnectionHelper
|
||||
|
||||
helper, err = connhelper.GetConnectionHelper(dockerHost)
|
||||
if err != nil {
|
||||
var args []string
|
||||
if args, err = sshArgs(dockerHost); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cli, err = client.New(
|
||||
client.WithHost(helper.Host),
|
||||
client.WithDialContext(helper.Dialer),
|
||||
client.WithHost("http://docker.example.com"), // the dialer below ignores the address
|
||||
client.WithDialContext(func(context.Context, string, string) (net.Conn, error) { return dialCommand("ssh", args...) }),
|
||||
)
|
||||
} else {
|
||||
cli, err = client.New(client.FromEnv)
|
||||
@@ -362,17 +391,16 @@ func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) {
|
||||
return cli, nil
|
||||
}
|
||||
|
||||
func GetHostInfo(ctx context.Context) (info system.Info, err error) {
|
||||
var cli client.APIClient
|
||||
cli, err = GetDockerClient(ctx)
|
||||
func GetHostInfo(ctx context.Context) (system.Info, error) {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return info, err
|
||||
return system.Info{}, err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
result, err := cli.Info(ctx, client.InfoOptions{})
|
||||
if err != nil {
|
||||
return info, err
|
||||
return system.Info{}, err
|
||||
}
|
||||
|
||||
return result.Info, nil
|
||||
@@ -444,22 +472,23 @@ func (cr *containerReference) find() common.Executor {
|
||||
}
|
||||
cr.id = ""
|
||||
}
|
||||
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{
|
||||
All: true,
|
||||
})
|
||||
result, err := cr.cli.ContainerInspect(ctx, cr.input.Name, client.ContainerInspectOptions{})
|
||||
if err == nil && result.Container.Name == "/"+cr.input.Name {
|
||||
cr.id = result.Container.ID
|
||||
return nil
|
||||
} else if cerrdefs.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
containers, err := cr.cli.ContainerList(ctx, client.ContainerListOptions{All: true}) // inspect also matches ids, and podman fails it with a 500 for a container removed mid-request
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list containers: %w", err)
|
||||
}
|
||||
|
||||
for _, c := range containers.Items {
|
||||
for _, name := range c.Names {
|
||||
if name[1:] == cr.input.Name {
|
||||
cr.id = c.ID
|
||||
return nil
|
||||
}
|
||||
for _, candidate := range containers.Items {
|
||||
if slices.Contains(candidate.Names, "/"+cr.input.Name) {
|
||||
cr.id = candidate.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -488,14 +517,11 @@ func (cr *containerReference) remove() common.Executor {
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
})
|
||||
switch {
|
||||
case cerrdefs.IsConflict(err):
|
||||
// the daemon's own AutoRemove teardown is running, and it releases the volume
|
||||
// references and the network endpoint only once it finishes
|
||||
cr.waitForRemoval(ctx, idOrName)
|
||||
case err != nil && !cerrdefs.IsNotFound(err):
|
||||
logger.Error(fmt.Errorf("failed to remove container %s: %w", idOrName, err))
|
||||
return nil // keep the id, the container is still there for a later Remove()
|
||||
if cerrdefs.IsConflict(err) {
|
||||
err = cr.waitForRemoval(ctx, idOrName)
|
||||
}
|
||||
if err != nil && !cerrdefs.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to remove container %s: %w", idOrName, err)
|
||||
}
|
||||
|
||||
logger.Debugf("Removed container: %v", idOrName)
|
||||
@@ -504,7 +530,7 @@ func (cr *containerReference) remove() common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) {
|
||||
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) error {
|
||||
// per container, against the one minute the post-job executor allows for the whole
|
||||
// cleanup, so a job with several services can spend most of that budget here
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
@@ -514,18 +540,13 @@ func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName strin
|
||||
Condition: container.WaitConditionRemoved,
|
||||
})
|
||||
select {
|
||||
case <-waitResult.Result:
|
||||
case <-waitResult.Error:
|
||||
case <-ctx.Done():
|
||||
// the client delivers the result over an unbuffered channel, so leave a receiver
|
||||
// behind or its goroutine parks on the send for the lifetime of the process
|
||||
go func() {
|
||||
select {
|
||||
case <-waitResult.Result:
|
||||
case <-waitResult.Error:
|
||||
}
|
||||
}()
|
||||
common.Logger(ctx).Warnf("Timed out waiting for the daemon to remove container %s, its volumes and network may be left behind", idOrName)
|
||||
case result := <-waitResult.Result:
|
||||
if result.Error != nil {
|
||||
return errors.New(result.Error.Message)
|
||||
}
|
||||
return nil
|
||||
case err := <-waitResult.Error:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -534,58 +555,45 @@ func (input *NewContainerInput) allOptions() string {
|
||||
return strings.TrimSpace(input.RunnerOptions + " " + input.WorkflowOptions)
|
||||
}
|
||||
|
||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
|
||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) error {
|
||||
logger := common.Logger(ctx)
|
||||
options := cr.input.allOptions()
|
||||
|
||||
if options == "" {
|
||||
return config, hostConfig, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// For Gitea, checked here because the parse below is what would read those files
|
||||
if err := rejectHostReadingOptions(cr.input.WorkflowOptions); err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
// parse configuration from CLI container.options
|
||||
flags, copts, cf, err := parseContainerOptions(options)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
|
||||
if err := cf.validate(); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
|
||||
// FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment.
|
||||
// In the old fork version, the code is
|
||||
// if len(copts.netMode.Value()) == 0 {
|
||||
// if err = copts.netMode.Set("host"); err != nil {
|
||||
// return nil, nil, fmt.Errorf("cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err)
|
||||
// }
|
||||
// }
|
||||
// And it has been commented with:
|
||||
// If a service container's network is set to `host`, the container will not be able to
|
||||
// connect to the specified network created for the job container and the service containers.
|
||||
// So comment out the following code.
|
||||
// Not the if it's necessary to comment it in the new version,
|
||||
// since it's cr.input.NetworkMode now.
|
||||
|
||||
if len(copts.netMode.Value()) == 0 {
|
||||
if err = copts.netMode.Set(cr.input.NetworkMode); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
||||
return fmt.Errorf("cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err)
|
||||
}
|
||||
}
|
||||
|
||||
// If the `privileged` config has been disabled, `copts.privileged` need to be forced to false,
|
||||
// even if the user specifies `--privileged` in the options string.
|
||||
if !hostConfig.Privileged {
|
||||
if !hostConfig.Privileged && copts.privileged {
|
||||
logger.Warnf("container option %q is not allowed when privileged mode is disabled and will be ignored", "--privileged")
|
||||
copts.privileged = false
|
||||
}
|
||||
|
||||
containerConfig, err := parse(flags, copts, runtime.GOOS)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
return fmt.Errorf("cannot process container options: '%s': '%w'", options, err)
|
||||
}
|
||||
// workflow aliases join the runner's own, deduped so a re-create cannot grow the input
|
||||
for _, endpoint := range containerConfig.NetworkingConfig.EndpointsConfig {
|
||||
@@ -600,29 +608,24 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
if !hostConfig.Privileged {
|
||||
trusted, err := parseOptionsHostConfig(cr.input.RunnerOptions)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return err
|
||||
}
|
||||
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig, trusted)
|
||||
}
|
||||
|
||||
logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config)
|
||||
|
||||
err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot merge container.Config options: '%s': '%w'", options, err)
|
||||
}
|
||||
overlay(reflect.ValueOf(config).Elem(), reflect.ValueOf(containerConfig.Config).Elem(), true)
|
||||
logger.Debugf("Merged container.Config ==> %+v", config)
|
||||
|
||||
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
|
||||
|
||||
cr.sanitizeConfig(ctx, containerConfig.HostConfig) // a dropped volume must not displace the runner's own
|
||||
overlayVolumes(hostConfig, containerConfig.HostConfig)
|
||||
binds := hostConfig.Binds
|
||||
mounts := hostConfig.Mounts
|
||||
networkMode := hostConfig.NetworkMode
|
||||
err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot merge container.HostConfig options: '%s': '%w'", options, err)
|
||||
}
|
||||
overlay(reflect.ValueOf(hostConfig).Elem(), reflect.ValueOf(containerConfig.HostConfig).Elem(), false)
|
||||
hostConfig.Binds = binds
|
||||
hostConfig.Mounts = mounts
|
||||
if cf.name != "" {
|
||||
@@ -632,10 +635,13 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
if flags.Changed("network") || flags.Changed("net") {
|
||||
logger.Warn("--network and --net in the options will be ignored.")
|
||||
}
|
||||
if len(cr.input.Entrypoint) != 0 && flags.Changed("entrypoint") {
|
||||
logger.Warn("--entrypoint in the options will be ignored.")
|
||||
}
|
||||
hostConfig.NetworkMode = networkMode
|
||||
logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig)
|
||||
|
||||
return config, hostConfig, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
@@ -661,17 +667,11 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
ExposedPorts: exposedPorts,
|
||||
Tty: input.AllocatePTY,
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// logger.Debugf("Common container.Config ==> %+v", config)
|
||||
|
||||
if len(input.Cmd) != 0 {
|
||||
config.Cmd = input.Cmd
|
||||
}
|
||||
|
||||
if len(input.Entrypoint) != 0 {
|
||||
config.Entrypoint = input.Entrypoint
|
||||
}
|
||||
|
||||
mounts := make([]mount.Mount, 0)
|
||||
for mountSource, mountTarget := range input.Mounts {
|
||||
mounts = append(mounts, mount.Mount{
|
||||
@@ -706,29 +706,25 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
PortBindings: portBindings,
|
||||
AutoRemove: input.AutoRemove,
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// logger.Debugf("Common container.HostConfig ==> %+v", hostConfig)
|
||||
|
||||
config, hostConfig, err = cr.mergeContainerConfigs(ctx, config, hostConfig)
|
||||
if err != nil {
|
||||
if err := cr.mergeContainerConfigs(ctx, config, hostConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(input.Entrypoint) != 0 {
|
||||
config.Entrypoint = input.Entrypoint
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
config, hostConfig = cr.sanitizeConfig(ctx, config, hostConfig)
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
|
||||
var networkingConfig *network.NetworkingConfig
|
||||
// For Gitea, reduce log noise
|
||||
// logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases)
|
||||
n := hostConfig.NetworkMode
|
||||
// IsUserDefined and IsHost are broken on windows
|
||||
if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 {
|
||||
endpointConfig := &network.EndpointSettings{
|
||||
Aliases: input.NetworkAliases,
|
||||
}
|
||||
networkingConfig = &network.NetworkingConfig{
|
||||
EndpointsConfig: map[string]*network.EndpointSettings{
|
||||
input.NetworkMode: endpointConfig,
|
||||
input.NetworkMode: {Aliases: input.NetworkAliases},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -752,44 +748,6 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) extractFromImageEnv(env *map[string]string) common.Executor {
|
||||
envMap := *env
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
inspect, err := cr.cli.ImageInspect(ctx, cr.input.Image)
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
return fmt.Errorf("inspect image: %w", err)
|
||||
}
|
||||
|
||||
if inspect.Config == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
imageEnv, err := godotenv.Unmarshal(strings.Join(inspect.Config.Env, "\n"))
|
||||
if err != nil {
|
||||
logger.Error(err)
|
||||
return fmt.Errorf("unmarshal image env: %w", err)
|
||||
}
|
||||
|
||||
for k, v := range imageEnv {
|
||||
if k == "PATH" {
|
||||
if envMap[k] == "" {
|
||||
envMap[k] = v
|
||||
} else {
|
||||
envMap[k] += `:` + v
|
||||
}
|
||||
} else if envMap[k] == "" {
|
||||
envMap[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
env = &envMap
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) exec(cmd []string, env map[string]string, user, workdir string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.id == "" {
|
||||
@@ -807,20 +765,11 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
||||
|
||||
logger.Debugf("Exec command '%s'", cmd)
|
||||
isTerminal := cr.input.AllocatePTY
|
||||
envList := make([]string, 0)
|
||||
for k, v := range env {
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
var wd string
|
||||
if workdir != "" {
|
||||
if strings.HasPrefix(workdir, "/") {
|
||||
wd = workdir
|
||||
} else {
|
||||
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
||||
}
|
||||
} else {
|
||||
wd = cr.input.WorkingDir
|
||||
wd := cr.input.WorkingDir
|
||||
if strings.HasPrefix(workdir, "/") {
|
||||
wd = workdir
|
||||
} else if workdir != "" {
|
||||
wd = fmt.Sprintf("%s/%s", cr.input.WorkingDir, workdir)
|
||||
}
|
||||
logger.Debugf("Working directory '%s'", wd)
|
||||
|
||||
@@ -828,7 +777,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
||||
User: user,
|
||||
Cmd: cmd,
|
||||
WorkingDir: wd,
|
||||
Env: envList,
|
||||
Env: getEnvListFromMap(env),
|
||||
TTY: isTerminal,
|
||||
AttachStderr: true,
|
||||
AttachStdout: true,
|
||||
@@ -845,8 +794,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
|
||||
}
|
||||
defer resp.Close()
|
||||
|
||||
err = cr.waitForCommand(ctx, resp.HijackedResponse, idResp, user, workdir)
|
||||
if err != nil {
|
||||
if err := cr.waitForCommand(ctx, resp.HijackedResponse); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -895,15 +843,7 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) tryReadUID() common.Executor {
|
||||
return cr.tryReadID("-u", func(id int) { cr.UID = id })
|
||||
}
|
||||
|
||||
func (cr *containerReference) tryReadGID() common.Executor {
|
||||
return cr.tryReadID("-g", func(id int) { cr.GID = id })
|
||||
}
|
||||
|
||||
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse, _ client.ExecCreateResult, _, _ string) error {
|
||||
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse) error {
|
||||
logger := common.Logger(ctx)
|
||||
|
||||
// Buffered so the copy goroutine never blocks on send if the grace-period
|
||||
@@ -945,6 +885,7 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
|
||||
}
|
||||
|
||||
func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
dstPath = path.Clean(dstPath)
|
||||
return func(ctx context.Context) error {
|
||||
if cr.id == "" {
|
||||
return cr.missingContainerError("copy directory to %s", dstPath)
|
||||
@@ -955,69 +896,27 @@ func (cr *containerReference) copyDir(dstPath, srcPath string, useGitIgnore, ski
|
||||
return err
|
||||
}
|
||||
logger.Debugf("Writing tarball %s from %s", tarFile.Name(), srcPath)
|
||||
defer func(tarFile *os.File) {
|
||||
name := tarFile.Name()
|
||||
err := tarFile.Close()
|
||||
if !errors.Is(err, os.ErrClosed) {
|
||||
defer func() {
|
||||
if err := tarFile.Close(); !errors.Is(err, os.ErrClosed) {
|
||||
logger.Error(err)
|
||||
}
|
||||
err = os.Remove(name)
|
||||
if err != nil {
|
||||
if err := os.Remove(tarFile.Name()); err != nil {
|
||||
logger.Error(err)
|
||||
}
|
||||
}(tarFile)
|
||||
tw := tar.NewWriter(tarFile)
|
||||
|
||||
srcPrefix := filepath.Dir(srcPath)
|
||||
if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) {
|
||||
srcPrefix += string(filepath.Separator)
|
||||
}()
|
||||
extractionRoot := "/"
|
||||
if actPath := cr.GetActPath(); strings.HasPrefix(dstPath+"/", actPath+"/") {
|
||||
extractionRoot = actPath // keeps tar entries from crossing the image's /var/run symlink
|
||||
}
|
||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||
|
||||
var ignorer gitignore.Matcher
|
||||
if useGitIgnore {
|
||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
||||
if err != nil {
|
||||
logger.Debugf("Error loading .gitignore: %v", err)
|
||||
}
|
||||
|
||||
ignorer = gitignore.NewMatcher(ps)
|
||||
}
|
||||
|
||||
fc := &filecollector.FileCollector{
|
||||
Ignorer: ignorer,
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
Handler: &filecollector.TarCollector{
|
||||
TarWriter: tw,
|
||||
UID: cr.UID,
|
||||
GID: cr.GID,
|
||||
DstDir: dstPath[1:],
|
||||
},
|
||||
}
|
||||
|
||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
if err := writeDirTar(ctx, tarFile, path.Join("/", strings.TrimPrefix(dstPath, extractionRoot)), srcPath, useGitIgnore, skipGitDir, cr.UID, cr.GID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Extracting content from '%s' to '%s'", tarFile.Name(), dstPath)
|
||||
_, err = tarFile.Seek(0, 0)
|
||||
if err != nil {
|
||||
if _, err := tarFile.Seek(0, 0); err != nil {
|
||||
return fmt.Errorf("failed to seek tar archive: %w", err)
|
||||
}
|
||||
_, err = cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
|
||||
DestinationPath: "/",
|
||||
Content: tarFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy content to container: %w", err)
|
||||
}
|
||||
return nil
|
||||
return cr.upload(ctx, extractionRoot, tarFile)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1026,41 +925,23 @@ func (cr *containerReference) copyContent(dstPath string, files ...*FileEntry) c
|
||||
if cr.id == "" {
|
||||
return cr.missingContainerError("copy to %s", dstPath)
|
||||
}
|
||||
logger := common.Logger(ctx)
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
for _, file := range files {
|
||||
logger.Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
|
||||
hdr := &tar.Header{
|
||||
Name: file.Name,
|
||||
Mode: file.Mode,
|
||||
Size: int64(len(file.Body)),
|
||||
Uid: cr.UID,
|
||||
Gid: cr.GID,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(file.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
if err := writeFilesTar(ctx, &buf, cr.UID, cr.GID, files...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Debugf("Extracting content to '%s'", dstPath)
|
||||
_, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{
|
||||
DestinationPath: dstPath,
|
||||
Content: &buf,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy content to container: %w", err)
|
||||
}
|
||||
return nil
|
||||
common.Logger(ctx).Debugf("Extracting content to '%s'", dstPath)
|
||||
return cr.upload(ctx, dstPath, &buf)
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) upload(ctx context.Context, dstPath string, content io.Reader) error {
|
||||
if _, err := cr.cli.CopyToContainer(ctx, cr.id, client.CopyToContainerOptions{DestinationPath: dstPath, Content: content}); err != nil {
|
||||
return fmt.Errorf("failed to copy content to container: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cr *containerReference) attach() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
out, err := cr.cli.ContainerAttach(ctx, cr.id, client.ContainerAttachOptions{
|
||||
@@ -1139,7 +1020,7 @@ func (cr *containerReference) wait() common.Executor {
|
||||
// sanitizeOptionsHostConfig takes back everything a workflow could escape the container with,
|
||||
// setting each field to trusted, which is what the runner's own options parse to on their own.
|
||||
// Only for unprivileged mode, since privileged mode grants host access anyway.
|
||||
func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *container.HostConfig) {
|
||||
func sanitizeOptionsHostConfig(logger *logging.Logger, hostConfig, trusted *container.HostConfig) {
|
||||
resetOption(logger, "--pid", &hostConfig.PidMode, trusted.PidMode)
|
||||
resetOption(logger, "--ipc", &hostConfig.IpcMode, trusted.IpcMode)
|
||||
resetOption(logger, "--uts", &hostConfig.UTSMode, trusted.UTSMode)
|
||||
@@ -1159,6 +1040,10 @@ func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *c
|
||||
// systempaths=unconfined lands in these two rather than in SecurityOpt
|
||||
resetOption(logger, "--security-opt", &hostConfig.MaskedPaths, trusted.MaskedPaths)
|
||||
resetOption(logger, "--security-opt", &hostConfig.ReadonlyPaths, trusted.ReadonlyPaths)
|
||||
resetOption(logger, "--log-driver", &hostConfig.LogConfig.Type, trusted.LogConfig.Type) // drivers run in the daemon, egressing from its network
|
||||
resetOption(logger, "--log-opt", &hostConfig.LogConfig.Config, trusted.LogConfig.Config)
|
||||
resetOption(logger, "--oom-score-adj", &hostConfig.OomScoreAdj, trusted.OomScoreAdj) // -1000 exempts it from the host OOM killer
|
||||
resetOption(logger, "--oom-kill-disable", &hostConfig.OomKillDisable, trusted.OomKillDisable)
|
||||
|
||||
// a driver mounts what it likes, e.g. local with device= binds any host path, which
|
||||
// valid_volumes never gets to see
|
||||
@@ -1174,7 +1059,7 @@ func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig, trusted *c
|
||||
|
||||
// resetOption puts a field back to the runner's own value. It compares the values rather than
|
||||
// the flags, so a field that more than one option feeds cannot slip through.
|
||||
func resetOption[T any](logger logrus.FieldLogger, option string, field *T, trusted T) {
|
||||
func resetOption[T any](logger *logging.Logger, option string, field *T, trusted T) {
|
||||
if reflect.DeepEqual(*field, trusted) {
|
||||
return
|
||||
}
|
||||
@@ -1196,17 +1081,15 @@ func parseOptionsHostConfig(options string) (*container.HostConfig, error) {
|
||||
return containerConfig.HostConfig, nil
|
||||
}
|
||||
|
||||
// For Gitea
|
||||
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
|
||||
func (cr *containerReference) sanitizeConfig(ctx context.Context, hostConfig *container.HostConfig) {
|
||||
logger := common.Logger(ctx)
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
|
||||
if len(cr.input.ValidVolumes) > 0 {
|
||||
matcher := newValidVolumeMatcher(ctx, cr.input.ValidVolumes)
|
||||
// sanitize binds
|
||||
sanitizedBinds := make([]string, 0, len(hostConfig.Binds))
|
||||
for _, bind := range hostConfig.Binds {
|
||||
parsed, err := loader.ParseVolume(bind)
|
||||
parsed, err := ParseVolume(bind)
|
||||
if err != nil {
|
||||
logger.Warnf("parse volume [%s] error: %v", bind, err)
|
||||
continue
|
||||
@@ -1216,40 +1099,96 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
continue
|
||||
}
|
||||
if matcher.isValid(parsed.Source, mount.Type(parsed.Type)) {
|
||||
if matcher.isValid(parsed.Source, parsed.Type, parsed.ReadOnly) {
|
||||
if matcher.needsNoCopy(parsed.Source, parsed.Type) && !parsed.NoCopy {
|
||||
bind += ",nocopy"
|
||||
}
|
||||
sanitizedBinds = append(sanitizedBinds, bind)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", parsed.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Binds = sanitizedBinds
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
if matcher.isValid(mt.Source, mt.Type) {
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
} else {
|
||||
for _, bind := range hostConfig.Binds {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", bind)
|
||||
}
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
hostConfig.Binds = []string{}
|
||||
}
|
||||
// sanitize mounts
|
||||
sanitizedMounts := make([]mount.Mount, 0, len(hostConfig.Mounts))
|
||||
for _, mt := range hostConfig.Mounts {
|
||||
readOnly := mt.ReadOnly && (mt.BindOptions == nil || !mt.BindOptions.ReadOnlyNonRecursive) // bind-recursive=writable leaves submounts writable
|
||||
if matcher.isValid(mt.Source, mt.Type, readOnly) {
|
||||
if matcher.needsNoCopy(mt.Source, mt.Type) {
|
||||
mt.VolumeOptions = cmp.Or(mt.VolumeOptions, &mount.VolumeOptions{})
|
||||
mt.VolumeOptions.NoCopy = true
|
||||
}
|
||||
sanitizedMounts = append(sanitizedMounts, mt)
|
||||
} else {
|
||||
logger.Warnf("[%s] is not a valid volume, will be ignored", mt.Source)
|
||||
}
|
||||
hostConfig.Binds = []string{}
|
||||
hostConfig.Mounts = []mount.Mount{}
|
||||
}
|
||||
hostConfig.Mounts = sanitizedMounts
|
||||
}
|
||||
|
||||
return config, hostConfig
|
||||
func SanitizeBinds(ctx context.Context, validVolumes, binds []string) []string {
|
||||
cr := &containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}
|
||||
hostConfig := &container.HostConfig{Binds: binds}
|
||||
cr.sanitizeConfig(ctx, hostConfig)
|
||||
return hostConfig.Binds
|
||||
}
|
||||
|
||||
func overlay(dst, src reflect.Value, appendSlices bool) {
|
||||
switch dst.Kind() {
|
||||
case reflect.Struct:
|
||||
opaque := true
|
||||
for i := range dst.NumField() {
|
||||
if dst.Type().Field(i).IsExported() {
|
||||
opaque = false
|
||||
overlay(dst.Field(i), src.Field(i), appendSlices)
|
||||
}
|
||||
}
|
||||
if opaque {
|
||||
dst.Set(src)
|
||||
}
|
||||
case reflect.Map:
|
||||
if src.IsNil() {
|
||||
return
|
||||
}
|
||||
if dst.IsNil() {
|
||||
dst.Set(reflect.MakeMap(dst.Type()))
|
||||
}
|
||||
for key, value := range src.Seq2() {
|
||||
dst.SetMapIndex(key, value)
|
||||
}
|
||||
case reflect.Slice:
|
||||
if appendSlices {
|
||||
dst.Set(reflect.AppendSlice(dst, src))
|
||||
} else if src.Len() > 0 {
|
||||
dst.Set(src)
|
||||
}
|
||||
case reflect.Pointer:
|
||||
if src.IsNil() {
|
||||
return
|
||||
}
|
||||
if dst.IsNil() {
|
||||
dst.Set(src)
|
||||
return
|
||||
}
|
||||
overlay(dst.Elem(), src.Elem(), appendSlices)
|
||||
case reflect.Array:
|
||||
dst.Set(src)
|
||||
default:
|
||||
if !src.IsZero() {
|
||||
dst.Set(src)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
|
||||
func bindTarget(bind string) string {
|
||||
parsed, err := loader.ParseVolume(bind)
|
||||
parsed, err := ParseVolume(bind)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -1276,28 +1215,39 @@ func overlayVolumes(dst, src *container.HostConfig) {
|
||||
}
|
||||
|
||||
type validVolumeMatcher struct {
|
||||
allowAll bool
|
||||
named []glob.Glob
|
||||
host []glob.Glob
|
||||
allowAll bool
|
||||
allowAllReadOnly bool
|
||||
named []validVolumeRule
|
||||
host []validVolumeRule
|
||||
}
|
||||
|
||||
type validVolumeRule struct {
|
||||
pattern string
|
||||
readOnly bool
|
||||
}
|
||||
|
||||
func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolumeMatcher {
|
||||
logger := common.Logger(ctx)
|
||||
ret := validVolumeMatcher{
|
||||
named: make([]glob.Glob, 0, len(validVolumes)),
|
||||
host: make([]glob.Glob, 0, len(validVolumes)),
|
||||
named: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
host: make([]validVolumeRule, 0, len(validVolumes)),
|
||||
}
|
||||
|
||||
for _, v := range validVolumes {
|
||||
v, readOnly := strings.CutSuffix(v, ":ro")
|
||||
if v == "**" {
|
||||
ret.allowAll = true
|
||||
if readOnly {
|
||||
ret.allowAllReadOnly = true
|
||||
} else {
|
||||
ret.allowAll = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !isHostVolumePattern(v) {
|
||||
if g, err := glob.Compile(v); err != nil {
|
||||
logger.Errorf("create glob from %s error: %v", v, err)
|
||||
if doublestar.ValidatePattern(v) {
|
||||
ret.named = append(ret.named, validVolumeRule{pattern: v, readOnly: readOnly})
|
||||
} else {
|
||||
ret.named = append(ret.named, g)
|
||||
logger.Errorf("invalid volume pattern %s", v)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -1306,40 +1256,44 @@ func newValidVolumeMatcher(ctx context.Context, validVolumes []string) validVolu
|
||||
logger.Errorf("normalize volume pattern %s error: %v", v, err)
|
||||
continue
|
||||
}
|
||||
if g, err := glob.Compile(normalized); err != nil {
|
||||
logger.Errorf("create glob from %s error: %v", normalized, err)
|
||||
if doublestar.ValidatePathPattern(normalized) {
|
||||
ret.host = append(ret.host, validVolumeRule{pattern: normalized, readOnly: readOnly})
|
||||
} else {
|
||||
ret.host = append(ret.host, g)
|
||||
logger.Errorf("invalid volume pattern %s", normalized)
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type) bool {
|
||||
if m.allowAll {
|
||||
func (m validVolumeMatcher) isValid(source string, sourceType mount.Type, readOnly bool) bool {
|
||||
if m.allowAll || (m.allowAllReadOnly && readOnly) {
|
||||
return true
|
||||
}
|
||||
if sourceType == mount.TypeTmpfs && source == "" { // backed by memory, exposes nothing from the host, same as --tmpfs
|
||||
return true
|
||||
}
|
||||
rules, match := m.named, doublestar.MatchUnvalidated
|
||||
if isHostVolumeSource(source, sourceType) {
|
||||
normalized, err := normalizeHostVolumePath(source)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, g := range m.host {
|
||||
if g.Match(normalized) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
rules, match, source = m.host, doublestar.PathMatchUnvalidated, normalized
|
||||
}
|
||||
for _, g := range m.named {
|
||||
if g.Match(source) {
|
||||
for _, rule := range rules {
|
||||
if (readOnly || !rule.readOnly) && match(rule.pattern, source) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// needsNoCopy reports a volume only `:ro` rules allow, as dockerd fills an empty volume from the image even when mounted read-only.
|
||||
func (m validVolumeMatcher) needsNoCopy(source string, sourceType mount.Type) bool {
|
||||
return sourceType == mount.TypeVolume && !m.isValid(source, sourceType, false)
|
||||
}
|
||||
|
||||
func isHostVolumePattern(pattern string) bool {
|
||||
return filepath.IsAbs(pattern) ||
|
||||
strings.HasPrefix(pattern, "."+string(filepath.Separator)) ||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
@@ -12,12 +13,18 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
"gitea.com/gitea/runner/act/common/logging/logtest"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
@@ -25,50 +32,33 @@ import (
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDocker(t *testing.T) {
|
||||
requireDocker(t)
|
||||
ctx := context.Background()
|
||||
client, err := GetDockerClient(ctx)
|
||||
require.NoError(t, err)
|
||||
defer client.Close()
|
||||
|
||||
dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{
|
||||
ContextDir: "testdata",
|
||||
ImageTag: "envmergetest",
|
||||
})
|
||||
|
||||
err = dockerBuild(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
cr := &containerReference{
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "envmergetest",
|
||||
},
|
||||
}
|
||||
func TestDockerEnvironmentUsesContainerPath(t *testing.T) {
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", mock.Anything, "amd64-job", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{Config: &container.Config{Env: []string{
|
||||
"PATH=/opt/node/x64/bin:/usr/bin:/bin", "CONFLICT_VAR=container-value", "SOME_RANDOM_VAR=",
|
||||
"ANOTHER_ONE=old-value", "ANOTHER_ONE=BUT_I_HAVE_VALUE", "LITERAL='quoted'=$HOME\nsecond line",
|
||||
}}}}, nil)
|
||||
env := map[string]string{
|
||||
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin",
|
||||
"PATH": "/custom/bin",
|
||||
"RANDOM_VAR": "WITH_VALUE",
|
||||
"ANOTHER_VAR": "",
|
||||
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
|
||||
"CONFLICT_VAR": "step-value",
|
||||
}
|
||||
|
||||
envExecutor := cr.extractFromImageEnv(&env)
|
||||
err = envExecutor(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, (&containerReference{cli: client, id: "amd64-job"}).UpdateFromImageEnv(&env)(t.Context()))
|
||||
assert.Equal(t, map[string]string{
|
||||
"PATH": "/usr/local/bin:/usr/bin:/usr/sbin:/bin:/sbin:/this/path/does/not/exists/anywhere:/this/either",
|
||||
"PATH": "/custom/bin:/opt/node/x64/bin:/usr/bin:/bin",
|
||||
"RANDOM_VAR": "WITH_VALUE",
|
||||
"ANOTHER_VAR": "",
|
||||
"SOME_RANDOM_VAR": "",
|
||||
"ANOTHER_ONE": "BUT_I_HAVE_VALUE",
|
||||
"CONFLICT_VAR": "I_EXIST_IN_MULTIPLE_PLACES",
|
||||
"CONFLICT_VAR": "step-value",
|
||||
"LITERAL": "'quoted'=$HOME\nsecond line",
|
||||
}, env)
|
||||
}
|
||||
|
||||
@@ -137,6 +127,11 @@ func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.Netw
|
||||
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkCreate(ctx context.Context, name string, opts mobyclient.NetworkCreateOptions) (mobyclient.NetworkCreateResult, error) {
|
||||
args := m.Called(ctx, name, opts)
|
||||
return args.Get(0).(mobyclient.NetworkCreateResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
|
||||
@@ -175,7 +170,7 @@ func (m *mockConn) Close() (err error) {
|
||||
}
|
||||
|
||||
func TestDockerExecAbort(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
|
||||
reader := &interruptReader{started: make(chan struct{}), interrupted: make(chan struct{}), stopped: make(chan struct{})}
|
||||
conn := &mockConn{}
|
||||
@@ -191,11 +186,9 @@ func TestDockerExecAbort(t *testing.T) {
|
||||
}, nil)
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{},
|
||||
}
|
||||
|
||||
channel := make(chan error)
|
||||
@@ -216,14 +209,12 @@ func TestDockerExecAbort(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDockerExecFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
conn := &mockConn{}
|
||||
ctx := t.Context()
|
||||
|
||||
client := &mockDockerClient{}
|
||||
client.On("ExecCreate", ctx, "123", mock.AnythingOfType("client.ExecCreateOptions")).Return(mobyclient.ExecCreateResult{ID: "id"}, nil)
|
||||
client.On("ExecAttach", ctx, "id", mock.AnythingOfType("client.ExecAttachOptions")).Return(mobyclient.ExecAttachResult{
|
||||
Conn: conn,
|
||||
Conn: &mockConn{},
|
||||
Reader: bufio.NewReader(strings.NewReader("output")),
|
||||
}, nil)
|
||||
client.On("ExecInspect", ctx, "id", mobyclient.ExecInspectOptions{}).Return(mobyclient.ExecInspectResult{
|
||||
@@ -231,11 +222,9 @@ func TestDockerExecFailure(t *testing.T) {
|
||||
}, nil)
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{},
|
||||
}
|
||||
|
||||
err := cr.exec([]string{""}, map[string]string{}, "user", "workdir")(ctx)
|
||||
@@ -244,7 +233,6 @@ func TestDockerExecFailure(t *testing.T) {
|
||||
assert.Equal(t, ExitCodeError(1), exitErr)
|
||||
assert.Equal(t, "Process completed with exit code 1.", err.Error())
|
||||
|
||||
conn.AssertExpectations(t)
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
@@ -259,11 +247,8 @@ func stdcopyFrame(stream stdcopy.StdType, payload string) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
// TestDockerAttachFlushesTrailingLine verifies that wait() blocks until the
|
||||
// attach() streaming goroutine has drained and flushed the container's output,
|
||||
// so a final line without a trailing newline is not lost.
|
||||
func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
|
||||
framed := bytes.NewBuffer(stdcopyFrame(stdcopy.Stdout, "line one\nlast line without newline"))
|
||||
|
||||
@@ -282,18 +267,13 @@ func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
statusCh <- container.WaitResponse{StatusCode: 0}
|
||||
errCh := make(chan error, 1)
|
||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||
Return(mobyclient.ContainerWaitResult{
|
||||
Result: (<-chan container.WaitResponse)(statusCh),
|
||||
Error: (<-chan error)(errCh),
|
||||
})
|
||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
Stdout: logWriter,
|
||||
Stderr: logWriter,
|
||||
},
|
||||
@@ -302,38 +282,21 @@ func TestDockerAttachFlushesTrailingLine(t *testing.T) {
|
||||
require.NoError(t, cr.attach()(ctx))
|
||||
require.NoError(t, cr.wait()(ctx))
|
||||
|
||||
// wait() must have blocked until the goroutine drained AND flushed; the
|
||||
// trailing, non-newline-terminated line must therefore be present. Reading
|
||||
// lines here is race-free because wait() synchronizes on attachDone, which
|
||||
// the goroutine closes after the final append.
|
||||
assert.Equal(t, []string{"line one\n", "last line without newline"}, lines)
|
||||
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestDockerWaitFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
|
||||
statusCh := make(chan container.WaitResponse, 1)
|
||||
statusCh <- container.WaitResponse{StatusCode: 2}
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerWait", ctx, "123", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}).
|
||||
Return(mobyclient.ContainerWaitResult{
|
||||
Result: (<-chan container.WaitResponse)(statusCh),
|
||||
Error: (<-chan error)(errCh),
|
||||
})
|
||||
Return(mobyclient.ContainerWaitResult{Result: statusCh})
|
||||
|
||||
cr := &containerReference{
|
||||
id: "123",
|
||||
cli: client,
|
||||
input: &NewContainerInput{
|
||||
Image: "image",
|
||||
},
|
||||
}
|
||||
|
||||
err := cr.wait()(ctx)
|
||||
err := (&containerReference{id: "123", cli: client}).wait()(ctx)
|
||||
var exitErr ExitCodeError
|
||||
require.ErrorAs(t, err, &exitErr)
|
||||
assert.Equal(t, ExitCodeError(2), exitErr)
|
||||
@@ -345,42 +308,49 @@ func TestDockerWaitFailure(t *testing.T) {
|
||||
// A remove that raced the daemon's AutoRemove teardown is not a failure and must not
|
||||
// be logged as one.
|
||||
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
||||
removeFailure := errors.New("driver failed to remove root filesystem")
|
||||
removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}
|
||||
killOpts := mobyclient.ContainerKillOptions{Signal: "SIGKILL"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
wantWait bool
|
||||
wantFailure bool
|
||||
name string
|
||||
err error
|
||||
wantWait bool
|
||||
waitErr error
|
||||
wantErr error
|
||||
}{
|
||||
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress"), wantWait: true},
|
||||
{name: "wait canceled", err: cerrdefs.ErrConflict, wantWait: true, waitErr: context.Canceled, wantErr: context.Canceled},
|
||||
{name: "removed during wait", err: cerrdefs.ErrConflict, wantWait: true, waitErr: cerrdefs.ErrNotFound},
|
||||
{name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")},
|
||||
{name: "removed cleanly", err: nil},
|
||||
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantFailure: true},
|
||||
{name: "real failure", err: removeFailure, wantErr: removeFailure},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logger, hook := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
logger, hook := logtest.NewLogger()
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
|
||||
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
|
||||
if tc.wantWait {
|
||||
removed := make(chan container.WaitResponse, 1)
|
||||
removed <- container.WaitResponse{}
|
||||
waitErrors := make(chan error, 1)
|
||||
if tc.waitErr == nil {
|
||||
removed <- container.WaitResponse{}
|
||||
} else {
|
||||
waitErrors <- tc.waitErr
|
||||
}
|
||||
client.On("ContainerWait", mock.Anything, "abc", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionRemoved}).
|
||||
Return(mobyclient.ContainerWaitResult{Result: removed})
|
||||
Return(mobyclient.ContainerWaitResult{Result: removed, Error: waitErrors})
|
||||
}
|
||||
cr := &containerReference{id: "abc", cli: client}
|
||||
|
||||
require.NoError(t, cr.remove()(ctx))
|
||||
// a failure keeps the id, so a later Remove() can retry it
|
||||
if tc.wantFailure {
|
||||
require.ErrorIs(t, cr.remove()(ctx), tc.wantErr)
|
||||
if tc.wantErr != nil {
|
||||
assert.Equal(t, "abc", cr.id)
|
||||
assert.Len(t, hook.AllEntries(), 1)
|
||||
} else {
|
||||
assert.Empty(t, cr.id)
|
||||
assert.Empty(t, hook.AllEntries())
|
||||
}
|
||||
assert.Empty(t, hook.AllRecords())
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
@@ -390,7 +360,7 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
||||
// create() lost its reply, must still be removed rather than leaking with its network. It
|
||||
// was never started here, so it is not worth a kill of its own.
|
||||
func TestRemoveWithoutIDUsesName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
|
||||
Return(mobyclient.ContainerRemoveResult{}, nil)
|
||||
@@ -400,70 +370,81 @@ func TestRemoveWithoutIDUsesName(t *testing.T) {
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// find() must drop a stale cached id so later Copy/Exec don't hit the
|
||||
// daemon with a torn-down container.
|
||||
func TestFindRevalidatesStaleID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||
notFound := cerrdefs.ErrNotFound.WithMessage("No such container")
|
||||
boom := errors.New("daemon unreachable")
|
||||
newCR := func(id string) (*containerReference, *mockDockerClient) {
|
||||
client := &mockDockerClient{}
|
||||
return &containerReference{id: id, cli: client, input: &NewContainerInput{Name: "job-1"}}, client
|
||||
for _, tc := range []struct {
|
||||
name, cachedID string
|
||||
cachedErr error
|
||||
lookup container.InspectResponse
|
||||
lookupErr error
|
||||
listed []container.Summary
|
||||
wantID string
|
||||
retryList bool
|
||||
}{
|
||||
{name: "stale id cleared, name lookup empty", cachedID: "stale", cachedErr: notFound, lookupErr: notFound},
|
||||
{name: "stale id cleared, name lookup repopulates", cachedID: "stale", cachedErr: notFound, lookup: container.InspectResponse{ID: "fresh", Name: "/job-1"}, wantID: "fresh"},
|
||||
{name: "id match falls back to listing", lookup: container.InspectResponse{ID: "job-1abc", Name: "/somebody-else"}, listed: []container.Summary{{ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh"},
|
||||
{name: "live id kept", cachedID: "live", wantID: "live"},
|
||||
{name: "transient inspect error trusts cache", cachedID: "live", cachedErr: boom, wantID: "live"},
|
||||
{name: "name lookup error falls back to listing", lookupErr: boom, listed: []container.Summary{{ID: "other", Names: []string{"/somebody-else"}}, {ID: "fresh", Names: []string{"/job-1"}}}, wantID: "fresh", retryList: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &mockDockerClient{}
|
||||
cr := &containerReference{id: tc.cachedID, cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
if tc.cachedID != "" {
|
||||
client.On("ContainerInspect", ctx, tc.cachedID, inspectOpts).Return(mobyclient.ContainerInspectResult{}, tc.cachedErr)
|
||||
}
|
||||
if tc.cachedID == "" || cerrdefs.IsNotFound(tc.cachedErr) {
|
||||
client.On("ContainerInspect", ctx, "job-1", inspectOpts).Return(mobyclient.ContainerInspectResult{Container: tc.lookup}, tc.lookupErr)
|
||||
}
|
||||
if tc.listed != nil {
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{Items: tc.listed}, nil).Once()
|
||||
}
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, tc.wantID, cr.id)
|
||||
if tc.retryList {
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, boom)
|
||||
cr.id = ""
|
||||
require.ErrorIs(t, cr.find()(ctx), boom)
|
||||
}
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
listOpts := mobyclient.ContainerListOptions{All: true}
|
||||
inspectOpts := mobyclient.ContainerInspectOptions{}
|
||||
}
|
||||
|
||||
t.Run("stale id cleared, name lookup empty", func(t *testing.T) {
|
||||
cr, client := newCR("stale")
|
||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Empty(t, cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("stale id cleared, name lookup repopulates", func(t *testing.T) {
|
||||
cr, client := newCR("stale")
|
||||
client.On("ContainerInspect", ctx, "stale", inspectOpts).Return(mobyclient.ContainerInspectResult{}, notFound)
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
||||
{ID: "other", Names: []string{"/somebody-else"}},
|
||||
{ID: "fresh", Names: []string{"/job-1"}},
|
||||
}}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "fresh", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("live id kept", func(t *testing.T) {
|
||||
cr, client := newCR("live")
|
||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, nil)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "live", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("transient inspect error trusts cache", func(t *testing.T) {
|
||||
cr, client := newCR("live")
|
||||
client.On("ContainerInspect", ctx, "live", inspectOpts).Return(mobyclient.ContainerInspectResult{}, boom)
|
||||
require.NoError(t, cr.find()(ctx))
|
||||
assert.Equal(t, "live", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
|
||||
t.Run("list error propagates", func(t *testing.T) {
|
||||
cr, client := newCR("")
|
||||
client.On("ContainerList", ctx, listOpts).Return(mobyclient.ContainerListResult{}, boom)
|
||||
require.ErrorIs(t, cr.find()(ctx), boom)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
func TestRunnerArchCachedPerRun(t *testing.T) {
|
||||
var infos atomic.Int32
|
||||
daemon := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
writer.Header().Set("API-Version", "1.47")
|
||||
if strings.HasSuffix(request.URL.Path, "/info") {
|
||||
infos.Add(1)
|
||||
_, _ = io.WriteString(writer, `{"Architecture":"aarch64"}`)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(daemon.Close)
|
||||
t.Setenv("DOCKER_HOST", daemon.URL)
|
||||
extensions := &LinuxContainerEnvironmentExtensions{}
|
||||
run := WithRunnerArch(t.Context())
|
||||
var lookups sync.WaitGroup
|
||||
for range 4 {
|
||||
lookups.Go(func() { assert.Equal(t, "ARM64", extensions.GetRunnerContext(run)["arch"]) })
|
||||
}
|
||||
lookups.Wait()
|
||||
assert.Equal(t, "ARM64", extensions.GetRunnerContext(t.Context())["arch"])
|
||||
assert.EqualValues(t, 2, infos.Load())
|
||||
}
|
||||
|
||||
// Every daemon entry point fails fast with a clear, container-named
|
||||
// error when no live cr.id is known.
|
||||
func TestRejectsMissingContainer(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).Return(mobyclient.ContainerListResult{}, nil)
|
||||
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
|
||||
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
check := func(op string, err error) {
|
||||
t.Helper()
|
||||
@@ -486,36 +467,53 @@ func TestRejectsMissingContainer(t *testing.T) {
|
||||
check("Inspect after removal", err)
|
||||
}
|
||||
|
||||
// End-to-end: a stale cr.id is cleared, repopulated from name lookup,
|
||||
// and the Copy completes against the fresh id.
|
||||
func TestPublicCopyPipelineHandlesStaleID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
||||
client.On("ContainerList", ctx, mobyclient.ContainerListOptions{All: true}).
|
||||
Return(mobyclient.ContainerListResult{Items: []container.Summary{
|
||||
{ID: "fresh", Names: []string{"/job-1"}},
|
||||
}}, nil)
|
||||
client.On("CopyToContainer", ctx, "fresh", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
|
||||
return opts.DestinationPath == "/var/run/act"
|
||||
})).Return(mobyclient.CopyToContainerResult{}, nil)
|
||||
for _, testcase := range []struct {
|
||||
name, destination, extractionRoot, entry string
|
||||
directory bool
|
||||
}{
|
||||
{name: "files", destination: "/var/run/act", extractionRoot: "/var/run/act", entry: "x"},
|
||||
{name: "action below act volume", directory: true, destination: "/var/run/act/actions/checkout/", extractionRoot: "/var/run/act", entry: "actions/checkout/x"},
|
||||
{name: "sibling prefix", directory: true, destination: "/var/run/actor/action", extractionRoot: "/", entry: "var/run/actor/action/x"},
|
||||
} {
|
||||
t.Run(testcase.name, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerInspect", ctx, "stale", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound.WithMessage("gone"))
|
||||
client.On("ContainerInspect", ctx, "job-1", mobyclient.ContainerInspectOptions{}).
|
||||
Return(mobyclient.ContainerInspectResult{Container: container.InspectResponse{ID: "fresh", Name: "/job-1"}}, nil)
|
||||
client.On("CopyToContainer", ctx, "fresh", mock.Anything).
|
||||
Run(func(args mock.Arguments) {
|
||||
opts := args.Get(2).(mobyclient.CopyToContainerOptions)
|
||||
assert.Equal(t, testcase.extractionRoot, opts.DestinationPath)
|
||||
header, err := tar.NewReader(opts.Content).Next()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testcase.entry, header.Name)
|
||||
}).Return(mobyclient.CopyToContainerResult{}, nil).Once()
|
||||
|
||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
require.NoError(t, cr.Copy("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||
assert.Equal(t, "fresh", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
cr := &containerReference{id: "stale", cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
if testcase.directory {
|
||||
source := t.TempDir()
|
||||
require.NoError(t, os.WriteFile(filepath.Join(source, "x"), nil, 0o644))
|
||||
require.NoError(t, cr.CopyDir(testcase.destination, source+string(filepath.Separator), false, true)(ctx))
|
||||
} else {
|
||||
require.NoError(t, cr.Copy(testcase.destination, &FileEntry{Name: "x", Mode: 0o644})(ctx))
|
||||
}
|
||||
assert.Equal(t, "fresh", cr.id)
|
||||
client.AssertExpectations(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Type assert containerReference implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &containerReference{}
|
||||
|
||||
func TestCheckVolumes(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
desc string
|
||||
validVolumes []string
|
||||
binds []string
|
||||
expectedBinds []string
|
||||
mounts []mount.Mount
|
||||
expectedMounts []mount.Mount
|
||||
}{
|
||||
{
|
||||
desc: "match all volumes",
|
||||
@@ -553,11 +551,14 @@ func TestCheckVolumes(t *testing.T) {
|
||||
"shared_volume",
|
||||
"/home/test/data",
|
||||
"/etc/conf.d/*.json",
|
||||
"/opt/**",
|
||||
},
|
||||
binds: []string{
|
||||
"shared_volume:/shared_volume",
|
||||
"/home/test/data:/test_data",
|
||||
"/etc/conf.d/base.json:/config/base.json",
|
||||
"/etc/conf.d/sub/nested.json:/config/nested.json",
|
||||
"/opt/app/cache:/cache",
|
||||
"sql_data:/sql_data",
|
||||
"/secrets/keys:/keys",
|
||||
},
|
||||
@@ -565,21 +566,45 @@ func TestCheckVolumes(t *testing.T) {
|
||||
"shared_volume:/shared_volume",
|
||||
"/home/test/data:/test_data",
|
||||
"/etc/conf.d/base.json:/config/base.json",
|
||||
"/opt/app/cache:/cache",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only rules allow only read-only mounts and copy nothing into volumes",
|
||||
validVolumes: []string{"/etc/ssl:ro", "data:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "/etc/ssl:/ssl", "data:/data:ro", "data:/data", "shared:/shared:ro"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "data:/data:ro,nocopy", "shared:/shared:ro"},
|
||||
mounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true, BindOptions: &mount.BindOptions{ReadOnlyNonRecursive: true}},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true},
|
||||
},
|
||||
expectedMounts: []mount.Mount{
|
||||
{Type: mount.TypeBind, Source: "/etc/ssl", Target: "/ssl", ReadOnly: true},
|
||||
{Type: mount.TypeVolume, Source: "data", Target: "/data", ReadOnly: true, VolumeOptions: &mount.VolumeOptions{NoCopy: true}},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "read-only wildcard keeps writable rules writable",
|
||||
validVolumes: []string{"**:ro", "shared"},
|
||||
binds: []string{"/etc/ssl:/ssl:ro", "data:/data", "shared:/shared"},
|
||||
expectedBinds: []string{"/etc/ssl:/ssl:ro", "shared:/shared"},
|
||||
},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
logger, hook := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
logger, hook := logtest.NewLogger()
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{
|
||||
ValidVolumes: tc.validVolumes,
|
||||
},
|
||||
}
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{Binds: tc.binds})
|
||||
hostConf := &container.HostConfig{Binds: tc.binds, Mounts: tc.mounts}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Equal(t, tc.expectedBinds, hostConf.Binds)
|
||||
assert.Len(t, hook.AllEntries(), len(tc.binds)-len(tc.expectedBinds)) // every drop is warned about
|
||||
assert.ElementsMatch(t, tc.expectedMounts, hostConf.Mounts)
|
||||
assert.Len(t, hook.AllRecords(), len(tc.binds)+len(tc.mounts)-len(tc.expectedBinds)-len(tc.expectedMounts))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -599,41 +624,35 @@ func TestMergeContainerConfigsDropsVolumeDriversFromWorkflows(t *testing.T) {
|
||||
assert.Equal(t, "job-escape", hostConfig.Mounts[0].Source)
|
||||
}
|
||||
|
||||
// Both of these are read here, on the runner, so a workflow could read the runner's files
|
||||
// and environment with them.
|
||||
func TestMergeContainerConfigsKeepsTheRunnersFilesAndEnvToItself(t *testing.T) {
|
||||
hostFile := filepath.Join(t.TempDir(), "host.env")
|
||||
require.NoError(t, os.WriteFile(hostFile, []byte("STOLEN=from-the-host\n"), 0o600))
|
||||
t.Setenv("RUNNER_SECRET", "s3cr3t")
|
||||
|
||||
for _, option := range []string{"--env-file " + hostFile, "--label-file " + hostFile} {
|
||||
logger, _ := test.NewNullLogger()
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{NetworkMode: "bridge", WorkflowOptions: option}}
|
||||
|
||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.ErrorContains(t, err, "not allowed in a workflow")
|
||||
require.ErrorContains(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}), "not allowed in a workflow")
|
||||
|
||||
// the runner reading its own files is what those options are for
|
||||
cr = &containerReference{input: &NewContainerInput{NetworkMode: "bridge", RunnerOptions: option}}
|
||||
_, _, err = cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||
}
|
||||
|
||||
// a bare name is no longer resolved from the runner's environment, for either source
|
||||
logger, _ := test.NewNullLogger()
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
NetworkMode: "bridge",
|
||||
RunnerOptions: "--env RUNNER_SECRET",
|
||||
WorkflowOptions: "--env RUNNER_SECRET --env GIVEN=value",
|
||||
}}
|
||||
|
||||
config, _, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
config := &container.Config{}
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), config, &container.HostConfig{}))
|
||||
assert.Equal(t, []string{"RUNNER_SECRET", "RUNNER_SECRET", "GIVEN=value"}, config.Env)
|
||||
}
|
||||
|
||||
func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
logger := logging.New(logtest.New())
|
||||
|
||||
// every field the sanitizer resets, so a reset dropped in a refactor fails here
|
||||
hostConfig := &container.HostConfig{
|
||||
@@ -655,6 +674,9 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
DeviceCgroupRules: []string{"a *:* rwm"},
|
||||
DeviceRequests: []container.DeviceRequest{{Count: -1, Capabilities: [][]string{{"gpu"}}}},
|
||||
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
|
||||
LogConfig: container.LogConfig{Type: "syslog", Config: map[string]string{"syslog-address": "tcp://169.254.169.254:514"}},
|
||||
OomScoreAdj: -1000,
|
||||
OomKillDisable: new(true),
|
||||
}
|
||||
|
||||
sanitizeOptionsHostConfig(logger, hostConfig, &container.HostConfig{})
|
||||
@@ -662,23 +684,23 @@ func TestSanitizeOptionsHostConfig(t *testing.T) {
|
||||
assert.Equal(t, &container.HostConfig{}, hostConfig)
|
||||
}
|
||||
|
||||
// mergeOptions merges both option sources into a bare container, returning the result and its log.
|
||||
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *test.Hook) {
|
||||
func mergeOptions(t *testing.T, runnerOptions, workflowOptions string, privileged bool) (*container.HostConfig, *logtest.Handler) {
|
||||
t.Helper()
|
||||
logger, hook := test.NewNullLogger()
|
||||
logger, hook := logtest.NewLogger()
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
RunnerOptions: runnerOptions,
|
||||
WorkflowOptions: workflowOptions,
|
||||
NetworkMode: "bridge",
|
||||
UsernsMode: "private",
|
||||
ValidVolumes: []string{"**"},
|
||||
}}
|
||||
|
||||
_, hostConfig, err := cr.mergeContainerConfigs(common.WithLogger(context.Background(), logger), &container.Config{}, &container.HostConfig{
|
||||
hostConfig := &container.HostConfig{
|
||||
Privileged: privileged,
|
||||
UsernsMode: container.UsernsMode("private"),
|
||||
NetworkMode: container.NetworkMode("bridge"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, hostConfig))
|
||||
return hostConfig, hook
|
||||
}
|
||||
|
||||
@@ -687,7 +709,9 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
|
||||
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
|
||||
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
|
||||
"--security-opt apparmor=unconfined --volumes-from other --isolation process " +
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
|
||||
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1 " +
|
||||
"--log-driver=syslog --log-opt syslog-address=tcp://169.254.169.254:514 " +
|
||||
"--oom-score-adj=-1000 --oom-kill-disable"
|
||||
|
||||
// whatever the workflow adds, an unprivileged container comes out exactly as the runner's
|
||||
// own options alone describe it, field for field
|
||||
@@ -714,8 +738,8 @@ func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.
|
||||
}
|
||||
|
||||
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
logger := logging.New(logtest.New())
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
|
||||
base := t.TempDir()
|
||||
allowed := filepath.Join(base, "allowed")
|
||||
@@ -730,26 +754,46 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
}
|
||||
|
||||
escapingPath := allowed + string(filepath.Separator) + ".." + string(filepath.Separator) + "denied"
|
||||
_, hostConf := cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf := &container.HostConfig{
|
||||
Binds: []string{escapingPath + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
|
||||
linkPath := filepath.Join(allowed, "link")
|
||||
if err := os.Symlink(denied, linkPath); err != nil {
|
||||
t.Skipf("cannot create symlink: %v", err)
|
||||
}
|
||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf = &container.HostConfig{
|
||||
Binds: []string{linkPath + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
|
||||
_, hostConf = cr.sanitizeConfig(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf = &container.HostConfig{
|
||||
Binds: []string{filepath.Join(linkPath, "missing") + ":/mnt"},
|
||||
})
|
||||
}
|
||||
cr.sanitizeConfig(ctx, hostConf)
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
}
|
||||
|
||||
func TestSanitizeConfigKeepsTmpfsMounts(t *testing.T) {
|
||||
hostConfig, _ := mergeOptions(t, "", "--mount type=tmpfs,dst=/var/lib/mysql", false)
|
||||
require.Len(t, hostConfig.Mounts, 1)
|
||||
|
||||
for _, validVolumes := range [][]string{nil, {"data"}, {"**:ro"}} {
|
||||
logger, hook := logtest.NewLogger()
|
||||
sanitized := &container.HostConfig{Mounts: []mount.Mount{
|
||||
hostConfig.Mounts[0],
|
||||
{Type: mount.TypeVolume, Target: "/anonymous"},
|
||||
{Type: mount.TypeTmpfs, Source: "/etc", Target: "/etc-copy"},
|
||||
}}
|
||||
(&containerReference{input: &NewContainerInput{ValidVolumes: validVolumes}}).sanitizeConfig(common.WithLogger(t.Context(), logger), sanitized)
|
||||
assert.Equal(t, hostConfig.Mounts, sanitized.Mounts)
|
||||
assert.Len(t, hook.AllRecords(), 2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainerInfoFromInspect(t *testing.T) {
|
||||
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
|
||||
info := containerInfoFromInspect(container.InspectResponse{
|
||||
@@ -807,31 +851,44 @@ func TestContainerInfoFromInspect(t *testing.T) {
|
||||
Mounts: []container.MountPoint{
|
||||
{Type: mount.TypeVolume, Name: "job", Source: "/var/lib/docker/volumes/job/_data", Destination: "/workspace/owner/repo"},
|
||||
{Type: mount.TypeBind, Source: "/var/run/docker.sock", Destination: "/var/run/docker.sock"},
|
||||
{Type: mount.TypeVolume, Name: "cache", Source: "/var/lib/docker/volumes/cache/_data", Destination: "/cache"},
|
||||
{Type: mount.TypeBind, Source: "/custom/resolv.conf", Destination: "/etc/resolv.conf"},
|
||||
},
|
||||
HostConfig: &container.HostConfig{
|
||||
Tmpfs: map[string]string{"/workspace/owner/repo//tmp/": "size=1m"},
|
||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "cache", Target: "/cache/", VolumeOptions: &mount.VolumeOptions{Subpath: "project"}}},
|
||||
},
|
||||
})
|
||||
|
||||
assert.Equal(t, map[string]string{
|
||||
"/workspace/owner/repo": "/var/lib/docker/volumes/job/_data",
|
||||
"/var/run/docker.sock": "/var/run/docker.sock",
|
||||
"/workspace/owner/repo": "/var/lib/docker/volumes/job/_data",
|
||||
"/workspace/owner/repo/tmp": "",
|
||||
"/var/run/docker.sock": "/var/run/docker.sock",
|
||||
"/cache": "/var/lib/docker/volumes/cache/_data/project",
|
||||
"/etc/resolv.conf": "/custom/resolv.conf",
|
||||
"/etc/hosts": "",
|
||||
"/etc/hostname": "",
|
||||
}, info.Mounts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
logger := logging.New(logtest.New())
|
||||
ctx := common.WithLogger(t.Context(), logger)
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{
|
||||
NetworkMode: "bridge",
|
||||
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
|
||||
NetworkMode: "bridge",
|
||||
RunnerOptions: "--volume /host/tools:/opt/hostedtoolcache",
|
||||
WorkflowOptions: "--volume /unapproved:/var/run/docker.sock",
|
||||
ValidVolumes: []string{"/host/tools"},
|
||||
},
|
||||
}
|
||||
|
||||
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
|
||||
hostConf := &container.HostConfig{
|
||||
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
|
||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, cr.mergeContainerConfigs(ctx, &container.Config{}, hostConf))
|
||||
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
|
||||
assert.Empty(t, hostConf.Mounts)
|
||||
}
|
||||
@@ -839,22 +896,51 @@ func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
||||
func TestMergeContainerConfigsWarnsOnlyAboutOptionsThatWereGiven(t *testing.T) {
|
||||
warnings := func(runnerOptions, workflowOptions string) int {
|
||||
_, hook := mergeOptions(t, runnerOptions, workflowOptions, false)
|
||||
return len(hook.AllEntries())
|
||||
return len(hook.AllRecords())
|
||||
}
|
||||
|
||||
assert.Zero(t, warnings("--volume /host/tools:/opt/hostedtoolcache", ""))
|
||||
assert.Zero(t, warnings("", "--shm-size 1g"))
|
||||
assert.Equal(t, 1, warnings("--network host", ""))
|
||||
assert.Equal(t, 1, warnings("", "--privileged"))
|
||||
}
|
||||
|
||||
func TestDockerCreateKeepsInputEntrypoint(t *testing.T) {
|
||||
for _, entrypoint := range [][]string{nil, {"sleep", "3600"}} {
|
||||
for _, optionEntrypoint := range []string{"/sbin/dumb-init", ""} {
|
||||
logger, hook := logtest.NewLogger()
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{
|
||||
NetworkMode: "bridge",
|
||||
Entrypoint: entrypoint,
|
||||
RunnerOptions: "--entrypoint=runner-entrypoint",
|
||||
WorkflowOptions: "--entrypoint=" + optionEntrypoint + " --init",
|
||||
},
|
||||
cli: &probeClient{create: func(opts mobyclient.ContainerCreateOptions) (mobyclient.ContainerCreateResult, error) {
|
||||
if len(entrypoint) > 0 {
|
||||
assert.Equal(t, entrypoint, opts.Config.Entrypoint)
|
||||
} else {
|
||||
assert.Equal(t, []string{optionEntrypoint}, opts.Config.Entrypoint)
|
||||
}
|
||||
require.NotNil(t, opts.HostConfig.Init)
|
||||
assert.True(t, *opts.HostConfig.Init)
|
||||
return mobyclient.ContainerCreateResult{ID: "created"}, nil
|
||||
}},
|
||||
}
|
||||
require.NoError(t, cr.create(nil, nil)(common.WithLogger(t.Context(), logger)))
|
||||
assert.Equal(t, "created", cr.id)
|
||||
assert.Equal(t, len(entrypoint) > 0, len(hook.AllRecords()) == 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsKeepsNetworkAliasesFromOptions(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
logger := logging.New(logtest.New())
|
||||
cr := &containerReference{input: &NewContainerInput{
|
||||
NetworkMode: "job-network", NetworkAliases: []string{"redis"}, WorkflowOptions: "--network-alias redis-primary",
|
||||
}}
|
||||
|
||||
_, _, err := cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, cr.mergeContainerConfigs(common.WithLogger(t.Context(), logger), &container.Config{}, &container.HostConfig{}))
|
||||
assert.Equal(t, []string{"redis", "redis-primary"}, cr.input.NetworkAliases)
|
||||
}
|
||||
|
||||
|
||||
111
act/container/docker_ssh.go
Normal file
111
act/container/docker_ssh.go
Normal file
@@ -0,0 +1,111 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/docker/cli/cli/connhelper/ssh"
|
||||
)
|
||||
|
||||
func sshArgs(daemonURL string) ([]string, error) {
|
||||
spec, err := ssh.ParseURL(daemonURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := []string{"docker", "system", "dial-stdio"}
|
||||
if strings.Trim(spec.Path, "/") != "" {
|
||||
remote = []string{"docker", "--host=unix://" + spec.Path, "system", "dial-stdio"}
|
||||
}
|
||||
return spec.Command([]string{"-o", "ConnectTimeout=30", "-T"}, remote...)
|
||||
}
|
||||
|
||||
func dialCommand(name string, args ...string) (net.Conn, error) {
|
||||
conn := &commandConn{cmd: exec.Command(name, args...)}
|
||||
conn.cmd.SysProcAttr = sshSysProcAttr() // a session of its own like docker's, so a ProxyCommand never reads the runner's terminal: https://github.com/docker/cli/issues/1707
|
||||
conn.cmd.Stderr = &conn.stderr
|
||||
conn.cmd.WaitDelay = time.Second // a ProxyCommand child keeping stderr open would otherwise block Wait after ssh exits
|
||||
conn.wait = sync.OnceValue(func() error {
|
||||
if err := conn.cmd.Wait(); err != nil && !errors.Is(err, exec.ErrWaitDelay) {
|
||||
return fmt.Errorf("%s: %w: %s", strings.Join(conn.cmd.Args, " "), err, strings.TrimSpace(conn.stderr.String()))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
var err error
|
||||
if conn.stdin, err = conn.cmd.StdinPipe(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if conn.stdout, err = conn.cmd.StdoutPipe(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conn, conn.cmd.Start()
|
||||
}
|
||||
|
||||
type commandConn struct {
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
stdout io.ReadCloser
|
||||
stderr cappedBuffer
|
||||
wait func() error
|
||||
}
|
||||
|
||||
func (c *commandConn) waitWithin(timeout time.Duration) error {
|
||||
timer := time.AfterFunc(timeout, func() { _ = c.cmd.Process.Kill() })
|
||||
defer timer.Stop()
|
||||
return c.wait()
|
||||
}
|
||||
|
||||
func (c *commandConn) Read(p []byte) (int, error) {
|
||||
n, err := c.stdout.Read(p)
|
||||
if errors.Is(err, io.EOF) {
|
||||
if waitErr := c.waitWithin(10 * time.Second); waitErr != nil {
|
||||
return n, waitErr
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (c *commandConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
|
||||
|
||||
// CloseWrite lets the daemon see the end of a hijacked stream, like a TCP half-close.
|
||||
func (c *commandConn) CloseWrite() error { return c.stdin.Close() }
|
||||
|
||||
func (c *commandConn) Close() error {
|
||||
_ = c.stdin.Close()
|
||||
if c.cmd.Process.Signal(syscall.SIGTERM) != nil { // SIGTERM lets ssh take its ProxyCommand down, Windows only kills
|
||||
_ = c.cmd.Process.Kill()
|
||||
}
|
||||
_ = c.waitWithin(3 * time.Second)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *commandConn) LocalAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
|
||||
|
||||
func (c *commandConn) RemoteAddr() net.Addr { return &net.UnixAddr{Name: "stdio", Net: "unix"} }
|
||||
|
||||
func (c *commandConn) SetDeadline(_deadline time.Time) error { return nil }
|
||||
|
||||
func (c *commandConn) SetReadDeadline(_deadline time.Time) error { return nil }
|
||||
|
||||
func (c *commandConn) SetWriteDeadline(_deadline time.Time) error { return nil }
|
||||
|
||||
type cappedBuffer struct{ bytes.Buffer }
|
||||
|
||||
func (b *cappedBuffer) Write(p []byte) (int, error) {
|
||||
if b.Len() > 4096 {
|
||||
b.Reset()
|
||||
}
|
||||
return b.Buffer.Write(p)
|
||||
}
|
||||
12
act/container/docker_ssh_linux.go
Normal file
12
act/container/docker_ssh_linux.go
Normal file
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER
|
||||
|
||||
package container
|
||||
|
||||
import "syscall"
|
||||
|
||||
func sshSysProcAttr() *syscall.SysProcAttr {
|
||||
return &syscall.SysProcAttr{Setsid: true, Pdeathsig: syscall.SIGKILL}
|
||||
}
|
||||
51
act/container/docker_ssh_test.go
Normal file
51
act/container/docker_ssh_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSSHArgs(t *testing.T) {
|
||||
args, err := sshArgs("ssh://me@host:2222/run/user/1000/my docker.sock")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"-l", "me", "-p", "2222", "-o", "ConnectTimeout=30", "-T", "--", "host", "docker '--host=unix:///run/user/1000/my docker.sock' system dial-stdio"}, args)
|
||||
|
||||
args, err = sshArgs("ssh://[::1]")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"-o", "ConnectTimeout=30", "-T", "--", "::1", "docker system dial-stdio"}, args)
|
||||
|
||||
for _, invalid := range []string{"ssh://me:secret@host", "ssh://host?x=1", "ssh://host#x", "ssh://"} {
|
||||
_, err := sshArgs(invalid)
|
||||
assert.Error(t, err, invalid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialCommandStreamsStdioAndReportsExit(t *testing.T) {
|
||||
if _, err := exec.LookPath("sh"); err != nil {
|
||||
t.Skip("needs a POSIX shell")
|
||||
}
|
||||
conn, err := dialCommand("sh", "-c", "cat")
|
||||
require.NoError(t, err)
|
||||
_, err = conn.Write([]byte("ping"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, conn.(interface{ CloseWrite() error }).CloseWrite())
|
||||
echoed, err := io.ReadAll(conn)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "ping", string(echoed))
|
||||
require.NoError(t, conn.Close())
|
||||
|
||||
conn, err = dialCommand("sh", "-c", "exec 3<&0; cat <&3 >/dev/null & echo 'no route to host' >&2; exit 255")
|
||||
require.NoError(t, err)
|
||||
_, err = io.ReadAll(conn)
|
||||
require.ErrorContains(t, err, "no route to host")
|
||||
require.NoError(t, conn.Close())
|
||||
}
|
||||
12
act/container/docker_ssh_unix.go
Normal file
12
act/container/docker_ssh_unix.go
Normal file
@@ -0,0 +1,12 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER && (darwin || netbsd)
|
||||
|
||||
package container
|
||||
|
||||
import "syscall"
|
||||
|
||||
func sshSysProcAttr() *syscall.SysProcAttr {
|
||||
return &syscall.SysProcAttr{Setsid: true}
|
||||
}
|
||||
10
act/container/docker_ssh_windows.go
Normal file
10
act/container/docker_ssh_windows.go
Normal file
@@ -0,0 +1,10 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
//go:build !WITHOUT_DOCKER
|
||||
|
||||
package container
|
||||
|
||||
import "syscall"
|
||||
|
||||
func sshSysProcAttr() *syscall.SysProcAttr { return nil }
|
||||
@@ -9,6 +9,7 @@ package container
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/netip"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
@@ -17,6 +18,10 @@ import (
|
||||
"github.com/moby/moby/api/types/system"
|
||||
)
|
||||
|
||||
func IsolatedCacheContainer(context.Context, netip.Addr, string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// ImageExistsLocally returns a boolean indicating if an image with the
|
||||
// requested name, tag and architecture exists in the local docker image store
|
||||
func ImageExistsLocally(ctx context.Context, imageName, platform string) (bool, error) {
|
||||
@@ -29,8 +34,12 @@ func RemoveImage(ctx context.Context, imageName string, force, pruneChildren boo
|
||||
return false, errors.New("Unsupported Operation")
|
||||
}
|
||||
|
||||
func DockerProxyDir(ctx context.Context) string {
|
||||
return ""
|
||||
func RemoveDockerJobResources(_ context.Context, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewDockerProxy(_ context.Context, _ string) *DockerProxy {
|
||||
return nil
|
||||
}
|
||||
|
||||
func StartDockerProxy(daemonSocket, dir, job string) (*DockerProxy, error) {
|
||||
@@ -82,6 +91,30 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
func NewDockerNetworkConnectExecutor(_, _ string) common.Executor {
|
||||
return func(_ context.Context) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func NewDockerNetworkDisconnectExecutor(_, _ string) common.Executor {
|
||||
return func(_ context.Context) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func RemoveOrphanNetworks(_ context.Context, _, _ string, _ time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SanitizeBinds(_ context.Context, _, binds []string) []string {
|
||||
return binds
|
||||
}
|
||||
|
||||
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,40 +8,67 @@ package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
list, err := cli.VolumeList(ctx, client.VolumeListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, vol := range list.Items {
|
||||
if vol.Name == volumeName {
|
||||
return removeExecutor(volumeName, force)(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Volume not found - do nothing
|
||||
return nil
|
||||
func CreateJobVolumes(ctx context.Context, runnerUUID string, volumeNames []string) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
for _, volumeName := range volumeNames {
|
||||
if _, err := cli.VolumeCreate(ctx, client.VolumeCreateOptions{
|
||||
Name: volumeName,
|
||||
Labels: runnerLabels(runnerUUID),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeExecutor(volume string, force bool) common.Executor {
|
||||
func RemoveOrphanJobVolumes(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
volumes, err := cli.VolumeList(ctx, client.VolumeListOptions{
|
||||
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID).Add("dangling", "true"),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
for _, item := range volumes.Items {
|
||||
created, err := time.Parse(time.RFC3339, item.CreatedAt)
|
||||
// an unreadable or recent timestamp may belong to a job that is still starting up
|
||||
if err != nil || created.After(createdBefore) || item.Labels[runnerUUIDLabel] != runnerUUID {
|
||||
continue
|
||||
}
|
||||
if _, err := cli.VolumeRemove(ctx, item.Name, client.VolumeRemoveOptions{}); err != nil && !cerrdefs.IsNotFound(err) {
|
||||
errs = append(errs, fmt.Errorf("failed to remove volume %s: %w", item.Name, err))
|
||||
continue
|
||||
}
|
||||
common.Logger(ctx).Infof("removed docker volume %s left behind by an earlier job", item.Name)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func NewDockerVolumeRemoveExecutor(volumeName string, force bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
logger.Debugf("docker volume rm %s", volume)
|
||||
common.Logger(ctx).Debugf("docker volume rm %s", volumeName)
|
||||
|
||||
if common.Dryrun(ctx) {
|
||||
return nil
|
||||
@@ -53,7 +80,10 @@ func removeExecutor(volume string, force bool) common.Executor {
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
_, err = cli.VolumeRemove(ctx, volume, client.VolumeRemoveOptions{Force: force})
|
||||
_, err = cli.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: force})
|
||||
if cerrdefs.IsNotFound(err) { // already gone is the outcome we wanted
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
@@ -15,7 +14,7 @@ import (
|
||||
// so the daemon has to be pinged explicitly here to decide whether to skip.
|
||||
func requireDocker(t *testing.T) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
t.Skipf("skipping: docker client unavailable: %v", err)
|
||||
|
||||
@@ -7,6 +7,7 @@ package container
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -27,9 +28,6 @@ import (
|
||||
"gitea.com/gitea/runner/internal/pkg/process"
|
||||
|
||||
"github.com/creack/pty"
|
||||
"github.com/go-git/go-billy/v5/helper/polyfill"
|
||||
"github.com/go-git/go-billy/v5/osfs"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
@@ -67,15 +65,11 @@ func (e *HostEnvironment) processGroup(ctx context.Context) *process.Group {
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Create(_, _ []string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Close() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
@@ -100,25 +94,16 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore, skipGi
|
||||
srcPrefix += string(filepath.Separator)
|
||||
}
|
||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||
var ignorer gitignore.Matcher
|
||||
if useGitIgnore {
|
||||
ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil)
|
||||
if err != nil {
|
||||
logger.Debugf("Error loading .gitignore: %v", err)
|
||||
}
|
||||
|
||||
ignorer = gitignore.NewMatcher(ps)
|
||||
}
|
||||
fc := &filecollector.FileCollector{
|
||||
Ignorer: ignorer,
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
UseGitIgnore: useGitIgnore,
|
||||
Handler: &filecollector.CopyCollector{
|
||||
DstDir: destPath,
|
||||
},
|
||||
}
|
||||
return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
||||
return filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,7 +137,7 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
||||
SrcPrefix: srcPrefix,
|
||||
Handler: tc,
|
||||
}
|
||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{}))
|
||||
err = filepath.Walk(srcPath, fc.CollectFiles(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -180,15 +165,11 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Pull(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Start(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
type ptyWriter struct {
|
||||
@@ -205,7 +186,6 @@ func (w *ptyWriter) Write(buf []byte) (int, error) {
|
||||
}
|
||||
if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' {
|
||||
_, _ = w.Out.Write([]byte("\n"))
|
||||
return n, io.EOF
|
||||
}
|
||||
return n, io.EOF
|
||||
}
|
||||
@@ -254,40 +234,25 @@ func writeKeepAlive(ppty io.Writer) {
|
||||
}
|
||||
}
|
||||
|
||||
func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) {
|
||||
defer func() {
|
||||
finishLog()
|
||||
}()
|
||||
if _, err := io.Copy(writer, ppty); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) UpdateFromImageEnv(_ *map[string]string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
return common.NewPipelineExecutor()
|
||||
}
|
||||
|
||||
func getEnvListFromMap(env map[string]string) []string {
|
||||
envList := make([]string, 0)
|
||||
for k, v := range env {
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
envList := make([]string, 0, len(env))
|
||||
for key, value := range env {
|
||||
envList = append(envList, key+"="+value)
|
||||
}
|
||||
return envList
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error {
|
||||
envList := getEnvListFromMap(env)
|
||||
var wd string
|
||||
func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, workdir string) error {
|
||||
wd := e.Path
|
||||
if workdir != "" {
|
||||
if filepath.IsAbs(workdir) {
|
||||
wd = workdir
|
||||
} else {
|
||||
wd = workdir
|
||||
if !filepath.IsAbs(workdir) {
|
||||
wd = filepath.Join(e.Path, workdir)
|
||||
}
|
||||
} else {
|
||||
wd = e.Path
|
||||
}
|
||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||
@@ -300,9 +265,8 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
cmd := exec.CommandContext(ctx, f)
|
||||
cmd.Path = f
|
||||
cmd.Args = command
|
||||
cmd.Stdin = nil
|
||||
cmd.Stdout = e.StdOut
|
||||
cmd.Env = envList
|
||||
cmd.Env = getEnvListFromMap(env)
|
||||
cmd.Stderr = e.StdOut
|
||||
cmd.Dir = wd
|
||||
cmd.SysProcAttr = process.SysProcAttr(cmdline, false)
|
||||
@@ -334,7 +298,10 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
writer = &ptyWriter{Out: e.StdOut}
|
||||
var finishLog context.CancelFunc
|
||||
logctx, finishLog = context.WithCancel(context.Background())
|
||||
go copyPtyOutput(writer, ppty, finishLog)
|
||||
go func() {
|
||||
defer finishLog()
|
||||
_, _ = io.Copy(writer, ppty)
|
||||
}()
|
||||
go writeKeepAlive(ppty)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
@@ -350,7 +317,7 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
} else {
|
||||
defer k.Close()
|
||||
}
|
||||
err = cmd.Wait()
|
||||
err = treeKill.Wait(ctx, cmd)
|
||||
if err != nil {
|
||||
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
|
||||
return ExitCodeError(exitErr.ExitCode())
|
||||
@@ -369,13 +336,13 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor {
|
||||
return e.ExecWithCmdLine(command, "", env, user, workdir)
|
||||
func (e *HostEnvironment) Exec(command []string, env map[string]string, _user, workdir string) common.Executor {
|
||||
return e.ExecWithCmdLine(command, "", env, workdir)
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor {
|
||||
func (e *HostEnvironment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, workdir string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil {
|
||||
if err := e.exec(ctx, command, cmdline, env, workdir); err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("this step has been cancelled: %w", err)
|
||||
@@ -617,8 +584,7 @@ func (*HostEnvironment) GetPathVariableName() string {
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) DefaultPathVariable() string {
|
||||
v, _ := os.LookupEnv(e.GetPathVariableName())
|
||||
return v
|
||||
return os.Getenv(e.GetPathVariableName())
|
||||
}
|
||||
|
||||
func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
||||
@@ -628,30 +594,22 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string {
|
||||
// Reference for Arch values for runner.arch
|
||||
// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context
|
||||
func goArchToActionArch(arch string) string {
|
||||
archMapper := map[string]string{
|
||||
return cmp.Or(map[string]string{
|
||||
"x86_64": "X64",
|
||||
"amd64": "X64",
|
||||
"386": "X86",
|
||||
"arm": "ARM",
|
||||
"aarch64": "ARM64",
|
||||
"arm64": "ARM64",
|
||||
}
|
||||
if arch, ok := archMapper[arch]; ok {
|
||||
return arch
|
||||
}
|
||||
return arch
|
||||
}[arch], arch)
|
||||
}
|
||||
|
||||
func goOsToActionOs(os string) string {
|
||||
osMapper := map[string]string{
|
||||
return cmp.Or(map[string]string{
|
||||
"linux": "Linux",
|
||||
"darwin": "macOS",
|
||||
"windows": "Windows",
|
||||
}
|
||||
if os, ok := osMapper[os]; ok {
|
||||
return os
|
||||
}
|
||||
return os
|
||||
}[os], os)
|
||||
}
|
||||
|
||||
func (e *HostEnvironment) GetRunnerContext(_ context.Context) map[string]any {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -18,13 +17,12 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Type assert HostEnvironment implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &HostEnvironment{}
|
||||
|
||||
func TestActionPlatformNames(t *testing.T) {
|
||||
@@ -33,38 +31,25 @@ func TestActionPlatformNames(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCopyDir(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
StdOut: os.Stdout,
|
||||
Workdir: path.Join("testdata", "scratch"),
|
||||
}
|
||||
_ = os.MkdirAll(e.Path, 0o700)
|
||||
_ = os.MkdirAll(e.TmpDir, 0o700)
|
||||
_ = os.MkdirAll(e.ToolCache, 0o700)
|
||||
_ = os.MkdirAll(e.ActPath, 0o700)
|
||||
err := e.CopyDir(e.Workdir, e.Path, true, false)(ctx)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCopyDirSkipGitDir(t *testing.T) {
|
||||
for name, skipGitDir := range map[string]bool{"kept for a workspace copy": false, "skipped for an action copy": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
useGitIgnore, skipGitDir bool
|
||||
}{
|
||||
{name: "git dir kept for a workspace copy"},
|
||||
{name: "git dir skipped for an action copy", skipGitDir: true},
|
||||
{name: "unfiltered outside a git repository", useGitIgnore: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(src, ".git", "objects"), 0o700))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, ".git", "objects", "pack"), []byte("x"), 0o600))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(src, "action.yml"), []byte("name: x\n"), 0o600))
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "dest")
|
||||
e := &HostEnvironment{StdOut: os.Stdout}
|
||||
require.NoError(t, e.CopyDir(dest, src+string(filepath.Separator), false, skipGitDir)(context.Background()))
|
||||
require.NoError(t, (&HostEnvironment{}).CopyDir(dest, src+string(filepath.Separator), test.useGitIgnore, test.skipGitDir)(t.Context()))
|
||||
|
||||
assert.FileExists(t, filepath.Join(dest, "action.yml"))
|
||||
if skipGitDir {
|
||||
if test.skipGitDir {
|
||||
assert.NoDirExists(t, filepath.Join(dest, ".git"))
|
||||
} else {
|
||||
assert.FileExists(t, filepath.Join(dest, ".git", "objects", "pack"))
|
||||
@@ -74,32 +59,20 @@ func TestCopyDirSkipGitDir(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetContainerArchive(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
StdOut: os.Stdout,
|
||||
Workdir: path.Join("testdata", "scratch"),
|
||||
}
|
||||
_ = os.MkdirAll(e.Path, 0o700)
|
||||
_ = os.MkdirAll(e.TmpDir, 0o700)
|
||||
_ = os.MkdirAll(e.ToolCache, 0o700)
|
||||
_ = os.MkdirAll(e.ActPath, 0o700)
|
||||
ctx := t.Context()
|
||||
e := &HostEnvironment{Path: t.TempDir()}
|
||||
expectedContent := []byte("sdde/7sh")
|
||||
err := os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0o600)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
archive, err := e.GetContainerArchive(ctx, e.Path)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
defer archive.Close()
|
||||
reader := tar.NewReader(archive)
|
||||
h, err := reader.Next()
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "action.yml", h.Name)
|
||||
content, err := io.ReadAll(reader)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedContent, content)
|
||||
_, err = reader.Next()
|
||||
assert.ErrorIs(t, err, io.EOF)
|
||||
@@ -109,19 +82,8 @@ func TestHostEnvironmentExecExitCode(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
StdOut: io.Discard,
|
||||
Workdir: filepath.Join(dir, "path"),
|
||||
}
|
||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
||||
assert.NoError(t, os.MkdirAll(p, 0o700)) //nolint:testifylint // test setup
|
||||
}
|
||||
ctx := t.Context()
|
||||
e := &HostEnvironment{Path: t.TempDir(), StdOut: io.Discard}
|
||||
|
||||
err := e.Exec([]string{"sh", "-c", "exit 3"}, map[string]string{"PATH": os.Getenv("PATH")}, "", "")(ctx)
|
||||
var exitErr ExitCodeError
|
||||
@@ -143,25 +105,17 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||
{name: "on", allocPTY: true, expect: "TTY"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
buf := &bytes.Buffer{}
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(dir, "path"),
|
||||
TmpDir: filepath.Join(dir, "tmp"),
|
||||
ToolCache: filepath.Join(dir, "tool_cache"),
|
||||
ActPath: filepath.Join(dir, "act_path"),
|
||||
Path: t.TempDir(),
|
||||
StdOut: buf,
|
||||
Workdir: filepath.Join(dir, "path"),
|
||||
AllocatePTY: tc.allocPTY,
|
||||
}
|
||||
for _, p := range []string{e.Path, e.TmpDir, e.ToolCache, e.ActPath} {
|
||||
require.NoError(t, os.MkdirAll(p, 0o700))
|
||||
}
|
||||
|
||||
err := e.Exec(
|
||||
[]string{"sh", "-c", "[ -t 1 ] && printf TTY || printf NOTTY"},
|
||||
map[string]string{"PATH": os.Getenv("PATH")}, "", "",
|
||||
)(context.Background())
|
||||
)(t.Context())
|
||||
require.NoError(t, err)
|
||||
got := strings.TrimSpace(strings.ReplaceAll(buf.String(), "\r", ""))
|
||||
assert.Equal(t, tc.expect, got)
|
||||
@@ -169,51 +123,26 @@ func TestHostEnvironmentAllocatePTY(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemovePreservesWorkdirByDefault(t *testing.T) {
|
||||
logger := logrus.New()
|
||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
||||
base := t.TempDir()
|
||||
miscRoot := filepath.Join(base, "misc")
|
||||
path := filepath.Join(miscRoot, "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
workdir := filepath.Join(base, "workspace", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
Workdir: workdir,
|
||||
CleanUp: func() {
|
||||
_ = os.RemoveAll(miscRoot)
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
func TestHostEnvironmentRemoveWorkdir(t *testing.T) {
|
||||
for name, owned := range map[string]bool{"preserves caller workdir": false, "cleans owned workdir": true} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
miscRoot, workdir := t.TempDir(), t.TempDir()
|
||||
e := &HostEnvironment{
|
||||
Path: filepath.Join(miscRoot, "hostexecutor"),
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: owned,
|
||||
CleanUp: func() { _ = os.RemoveAll(miscRoot) },
|
||||
}
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
require.NoError(t, e.Remove()(t.Context()))
|
||||
_, err := os.Stat(workdir)
|
||||
if owned {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
require.NoError(t, e.Remove()(ctx))
|
||||
_, err := os.Stat(workdir)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestHostEnvironmentRemoveCleansWorkdirWhenOwned(t *testing.T) {
|
||||
logger := logrus.New()
|
||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
||||
base := t.TempDir()
|
||||
miscRoot := filepath.Join(base, "misc")
|
||||
path := filepath.Join(miscRoot, "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
workdir := filepath.Join(base, "workspace", "123", "owner", "repo")
|
||||
require.NoError(t, os.MkdirAll(workdir, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
Workdir: workdir,
|
||||
CleanWorkdir: true,
|
||||
CleanUp: func() {
|
||||
_ = os.RemoveAll(miscRoot)
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
require.NoError(t, e.Remove()(ctx))
|
||||
_, err := os.Stat(workdir)
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||
@@ -226,105 +155,55 @@ func TestRemoveAllWithContextDoesNotHangOnStuckDelete(t *testing.T) {
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
// removeAllWithContext intentionally leaks the delete goroutine on timeout,
|
||||
// and that goroutine still references removeAll. Unblock it and wait for it
|
||||
// to return before restoring the var, so the restore can't race the read.
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
removeAll = orig
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := removeAllWithContext(ctx, t.TempDir())
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp guards against a stalled
|
||||
// CleanUp callback (e.g. an os.RemoveAll blocked by an AV/EDR filter driver or
|
||||
// an unresponsive mount) wedging the runner slot forever at "Cleaning up
|
||||
// container". Remove must time out the callback and complete job teardown.
|
||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckCleanUp(t *testing.T) {
|
||||
// Keep the suite fast: shrink the per-phase teardown timeout for this test.
|
||||
orig := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = orig })
|
||||
|
||||
logger := logrus.New()
|
||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
||||
base := t.TempDir()
|
||||
path := filepath.Join(base, "misc", "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) }) // unblock the leaked goroutine at test end
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
CleanUp: func() {
|
||||
<-release // simulate a delete syscall stuck indefinitely
|
||||
},
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck CleanUp callback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval guards against a
|
||||
// stalled os.RemoveAll on the misc/workspace paths (same AV/EDR wedge as
|
||||
// #1023) wedging job completion after the CleanUp callback has already timed
|
||||
// out or finished.
|
||||
func TestHostEnvironmentRemoveDoesNotHangOnStuckPathRemoval(t *testing.T) {
|
||||
origTimeout := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||
|
||||
release := make(chan struct{})
|
||||
stubDone := make(chan struct{})
|
||||
|
||||
origRemoveAll := removeAll
|
||||
removeAll = func(string) error {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
return nil
|
||||
}
|
||||
// The stuck delete goroutine outlives the timed-out Remove and still reads
|
||||
// removeAll; unblock it and wait before restoring to avoid a restore/read race.
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
removeAll = origRemoveAll
|
||||
})
|
||||
|
||||
logger := logrus.New()
|
||||
ctx := common.WithLogger(context.Background(), logrus.NewEntry(logger))
|
||||
base := t.TempDir()
|
||||
path := filepath.Join(base, "misc", "hostexecutor")
|
||||
require.NoError(t, os.MkdirAll(path, 0o700))
|
||||
|
||||
e := &HostEnvironment{
|
||||
Path: path,
|
||||
StdOut: os.Stdout,
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(ctx) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("Remove() hung on a stuck path removal")
|
||||
func TestHostEnvironmentRemoveDoesNotHang(t *testing.T) {
|
||||
for _, phase := range []string{"CleanUp callback", "path removal"} {
|
||||
t.Run(phase, func(t *testing.T) {
|
||||
origTimeout := hostCleanupTimeout
|
||||
hostCleanupTimeout = 100 * time.Millisecond
|
||||
t.Cleanup(func() { hostCleanupTimeout = origTimeout })
|
||||
release, stubDone := make(chan struct{}), make(chan struct{})
|
||||
block := func() {
|
||||
defer close(stubDone)
|
||||
<-release
|
||||
}
|
||||
e := &HostEnvironment{Path: filepath.Join(t.TempDir(), "misc", "hostexecutor"), StdOut: os.Stdout}
|
||||
if phase == "CleanUp callback" {
|
||||
e.CleanUp = block
|
||||
} else {
|
||||
origRemoveAll := removeAll
|
||||
removeAll = func(string) error {
|
||||
block()
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeAll = origRemoveAll })
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
close(release)
|
||||
<-stubDone
|
||||
})
|
||||
require.NoError(t, os.MkdirAll(e.Path, 0o700))
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- e.Remove()(common.WithLogger(t.Context(), logging.Default())) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("Remove() hung on a stuck %s", phase)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -332,17 +211,11 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
t.Run("single dir", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\workspace\job1`})
|
||||
assert.Contains(t, s, `$paths = @('C:\workspace\job1')`)
|
||||
// Self-PID guard is essential — without it the script could taskkill
|
||||
// the PowerShell process running it.
|
||||
assert.Contains(t, s, "$selfPid = $PID")
|
||||
assert.Contains(t, s, "$_.ProcessId -eq $selfPid")
|
||||
// Must match both ExecutablePath (binaries from the workspace) and
|
||||
// CommandLine (system binaries invoked with workspace paths in args),
|
||||
// both bounded by dir+separator so a name-prefix sibling is spared.
|
||||
assert.Contains(t, s, `$prefix = $p + '\'`)
|
||||
assert.Contains(t, s, "$_.ExecutablePath.StartsWith($prefix")
|
||||
assert.Contains(t, s, "$_.CommandLine.IndexOf($prefix")
|
||||
// Each matched PID must be tree-killed, not just stopped.
|
||||
assert.Contains(t, s, "taskkill.exe /PID $_.ProcessId /T /F")
|
||||
})
|
||||
|
||||
@@ -355,25 +228,16 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
assert.Contains(t, s, `'C:\work\path'`)
|
||||
assert.Contains(t, s, `'C:\work\workdir'`)
|
||||
assert.Contains(t, s, `'C:\Users\runner\AppData\Local\Temp\job-42'`)
|
||||
// Commas between entries — no trailing comma, no leading comma.
|
||||
assert.Contains(t, s, `'C:\work\path','C:\work\workdir',`)
|
||||
})
|
||||
|
||||
t.Run("path with single quote is escaped", func(t *testing.T) {
|
||||
// In PowerShell single-quoted strings the only special char is the
|
||||
// quote itself, escaped by doubling. A workspace path that ever
|
||||
// contained `'` would inject a command into the script otherwise.
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\it's\path`})
|
||||
assert.Contains(t, s, `'C:\work\it''s\path'`)
|
||||
// And it must NOT appear unescaped — otherwise the quote would
|
||||
// terminate the literal early.
|
||||
assert.NotContains(t, s, `'C:\work\it's\path'`)
|
||||
})
|
||||
|
||||
t.Run("path with wildcard metacharacters is matched literally", func(t *testing.T) {
|
||||
// A path containing [ ] ? * must be embedded verbatim and matched with
|
||||
// ordinal String methods, not -like, otherwise the metacharacters would
|
||||
// be interpreted as wildcards and the leftover process could escape.
|
||||
s := buildWindowsWorkspaceKillScript([]string{`C:\work\[job]?1`})
|
||||
assert.Contains(t, s, `'C:\work\[job]?1'`)
|
||||
assert.NotContains(t, s, "-like")
|
||||
@@ -383,7 +247,6 @@ func TestBuildWindowsWorkspaceKillScript(t *testing.T) {
|
||||
|
||||
t.Run("empty dir list still produces a valid script", func(t *testing.T) {
|
||||
s := buildWindowsWorkspaceKillScript(nil)
|
||||
// Empty array literal — script runs, matches nothing, is a no-op.
|
||||
assert.Contains(t, s, "$paths = @()")
|
||||
assert.Contains(t, s, "Get-CimInstance Win32_Process")
|
||||
})
|
||||
|
||||
589
act/container/kubernetes.go
Normal file
589
act/container/kubernetes.go
Normal file
@@ -0,0 +1,589 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/kballard/go-shellquote"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// KubernetesOptions configures the pod a job runs in.
|
||||
type KubernetesOptions struct {
|
||||
Kubeconfig string
|
||||
Namespace string
|
||||
PodTemplates []map[string]any // merged in order, a later one winning
|
||||
MaxLifetime time.Duration
|
||||
ForcePull bool
|
||||
RunnerUUID string
|
||||
}
|
||||
|
||||
// KubernetesPod runs a job container and its service containers as one pod.
|
||||
type KubernetesPod struct {
|
||||
options KubernetesOptions
|
||||
client *kubeClient
|
||||
name, secretName string
|
||||
job *kubernetesContainer
|
||||
services []*kubernetesContainer
|
||||
arch, path string
|
||||
}
|
||||
|
||||
type kubernetesContainer struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
pod *KubernetesPod
|
||||
id string // the workflow's service id, name is its DNS label form
|
||||
name string
|
||||
input *NewContainerInput
|
||||
probe map[string]any
|
||||
}
|
||||
|
||||
type containerStatus struct {
|
||||
Name string
|
||||
Ready bool
|
||||
State struct {
|
||||
Waiting *struct{ Reason, Message string }
|
||||
Running *struct{}
|
||||
Terminated *struct {
|
||||
ExitCode int
|
||||
Message string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type podStatus struct {
|
||||
Spec struct{ Containers []struct{} }
|
||||
Status struct {
|
||||
Phase, Reason, Message string
|
||||
Conditions []struct{ Type, Status, Message string }
|
||||
InitContainerStatuses, ContainerStatuses []containerStatus
|
||||
}
|
||||
}
|
||||
|
||||
func NewKubernetesPod(options KubernetesOptions) (*KubernetesPod, error) {
|
||||
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to Kubernetes: %w", err)
|
||||
}
|
||||
return &KubernetesPod{options: options, client: client}, nil
|
||||
}
|
||||
|
||||
// JobContainer returns the container of the pod the job's steps run in.
|
||||
func (p *KubernetesPod) JobContainer(input *NewContainerInput) ExecutionsEnvironment {
|
||||
p.job = &kubernetesContainer{pod: p, name: "job", input: input}
|
||||
return p.job
|
||||
}
|
||||
|
||||
var (
|
||||
invalidLabelChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
||||
dnsSubdomain = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
|
||||
)
|
||||
|
||||
// kubernetesServiceName returns the container name of service id, made a DNS label as it must be.
|
||||
func kubernetesServiceName(id string) string {
|
||||
name := strings.Trim(invalidLabelChars.ReplaceAllString(strings.ToLower(id), "-"), "-")
|
||||
return strings.TrimRight(name[:min(63, len(name))], "-")
|
||||
}
|
||||
|
||||
// KubernetesServiceHostnames returns the hostnames of service id, its container name and, where hostAliases accept it as a DNS subdomain, id lowercased.
|
||||
func KubernetesServiceHostnames(id string) []string {
|
||||
name, host := kubernetesServiceName(id), strings.ToLower(id)
|
||||
if host != name && len(host) <= 253 && dnsSubdomain.MatchString(host) { // https://github.com/kubernetes/apimachinery/blob/v0.35.1/pkg/util/validation/validation.go#L198-L207
|
||||
return []string{name, host}
|
||||
}
|
||||
return []string{name}
|
||||
}
|
||||
|
||||
// ServiceContainer returns a container of the pod the job reaches at KubernetesServiceHostnames(id).
|
||||
func (p *KubernetesPod) ServiceContainer(id string, input *NewContainerInput) ExecutionsEnvironment {
|
||||
service := &kubernetesContainer{pod: p, id: id, name: kubernetesServiceName(id), input: input}
|
||||
p.services = append(p.services, service)
|
||||
return service
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) create(ctx context.Context) error {
|
||||
prefix := strings.Trim(strings.ToLower(p.job.input.Name), "-")
|
||||
prefix = cmp.Or(strings.TrimRight(prefix[:min(40, len(prefix))], "-"), "gitea-runner") + "-"
|
||||
name := prefix + strings.ToLower(rand.Text()[:8]) // chosen here, so cleanup knows it even when a create response is lost
|
||||
labels := map[string]any{"app.kubernetes.io/managed-by": "gitea-runner"}
|
||||
if p.options.RunnerUUID != "" {
|
||||
labels[runnerUUIDLabel] = p.options.RunnerUUID
|
||||
}
|
||||
metadata := map[string]any{"name": name, "labels": labels}
|
||||
if hostname, err := os.Hostname(); p.client.inCluster && err == nil {
|
||||
var runner struct{ Metadata struct{ Name, UID string } }
|
||||
if err := p.client.do(ctx, http.MethodGet, p.client.path("pods/"+hostname), nil, &runner); err == nil {
|
||||
metadata["ownerReferences"] = []any{map[string]any{"apiVersion": "v1", "kind": "Pod", "name": runner.Metadata.Name, "uid": runner.Metadata.UID}}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("The runner is not pod %s, job pods get no owner: %v", hostname, err)
|
||||
}
|
||||
}
|
||||
secretData := map[string]string{}
|
||||
manifest, err := p.manifest(name, secretData, metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, service := range p.services {
|
||||
if !slices.ContainsFunc(KubernetesServiceHostnames(service.id), func(hostname string) bool { return strings.EqualFold(hostname, service.id) }) { // glibc and Go match /etc/hosts case-insensitively
|
||||
common.Logger(ctx).Warnf("Service %s is reachable as %s or localhost, as a Kubernetes hostname cannot be %q", service.id, service.name, service.id)
|
||||
}
|
||||
}
|
||||
if len(secretData) > 0 {
|
||||
p.secretName = name // named like the pod, RemoveOrphanKubernetesResources finds it by that
|
||||
if err := p.client.do(ctx, http.MethodPost, p.client.path("secrets"), map[string]any{"metadata": metadata, "stringData": secretData}, nil); err != nil {
|
||||
return fmt.Errorf("create secret: %w", err)
|
||||
}
|
||||
}
|
||||
p.name = name
|
||||
if err := p.client.do(ctx, http.MethodPost, p.client.path("pods"), manifest, nil); err != nil {
|
||||
return fmt.Errorf("create pod: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) manifest(secret string, secretData map[string]string, metadata map[string]any) (any, error) {
|
||||
template := map[string]any{"spec": map[string]any{
|
||||
"automountServiceAccountToken": false,
|
||||
"enableServiceLinks": false,
|
||||
"terminationGracePeriodSeconds": 0, // the job container's tail ignores SIGTERM
|
||||
}}
|
||||
for _, layer := range p.options.PodTemplates {
|
||||
template, _ = mergeManifest(template, layer, "").(map[string]any)
|
||||
}
|
||||
templateSpec, _ := template["spec"].(map[string]any)
|
||||
templateContainers, _ := templateSpec["containers"].([]any)
|
||||
templateVolumes, _ := templateSpec["volumes"].([]any)
|
||||
var serviceTemplate any
|
||||
if index := slices.IndexFunc(templateContainers, func(container any) bool { return manifestField(container, "name") == "$services" }); index >= 0 {
|
||||
serviceTemplate = templateContainers[index]
|
||||
templateSpec["containers"] = slices.Delete(slices.Clone(templateContainers), index, index+1)
|
||||
}
|
||||
var containers, volumes []any
|
||||
var hostnames []string
|
||||
names := map[string]bool{}
|
||||
for _, container := range templateContainers {
|
||||
names[manifestField(container, "name")] = true
|
||||
}
|
||||
for _, c := range append([]*kubernetesContainer{p.job}, p.services...) {
|
||||
if c != p.job && (c.name == "" || names[c.name]) {
|
||||
return nil, fmt.Errorf("service %s maps to container %q, which is empty or named like another container of the pod", c.id, c.name)
|
||||
}
|
||||
names[c.name] = true
|
||||
spec, err := c.spec(secret, secretData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c == p.job {
|
||||
containers = append(containers, spec)
|
||||
continue
|
||||
}
|
||||
containers = append(containers, mergeManifest(serviceTemplate, spec, ""))
|
||||
hostnames = append(hostnames, KubernetesServiceHostnames(c.id)...)
|
||||
}
|
||||
for _, name := range []string{"workspace", "act"} {
|
||||
if !slices.ContainsFunc(templateVolumes, func(volume any) bool { return manifestField(volume, "name") == name }) {
|
||||
volumes = append(volumes, map[string]any{"name": name, "emptyDir": map[string]any{}})
|
||||
}
|
||||
}
|
||||
spec := map[string]any{"restartPolicy": "Never", "containers": containers, "volumes": volumes}
|
||||
if len(hostnames) > 0 {
|
||||
spec["hostAliases"] = []any{map[string]any{"ip": "127.0.0.1", "hostnames": hostnames}}
|
||||
}
|
||||
if p.options.MaxLifetime > 0 {
|
||||
spec["activeDeadlineSeconds"] = max(1, int64(p.options.MaxLifetime.Seconds()))
|
||||
}
|
||||
return mergeManifest(template, map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}, ""), nil
|
||||
}
|
||||
|
||||
// mergeManifest merges over onto base like a strategic merge patch: maps key by key, map list items by name or volumeMounts by mountPath, env entries, volumes and other values replaced.
|
||||
func mergeManifest(base, over any, field string) any {
|
||||
switch over := over.(type) {
|
||||
case map[string]any:
|
||||
merged := map[string]any{}
|
||||
if base, ok := base.(map[string]any); ok {
|
||||
maps.Copy(merged, base)
|
||||
}
|
||||
for key, value := range over {
|
||||
merged[key] = mergeManifest(merged[key], value, key)
|
||||
}
|
||||
return merged
|
||||
case []any:
|
||||
key := cmp.Or(map[string]string{"volumeMounts": "mountPath"}[field], "name")
|
||||
baseList, _ := base.([]any)
|
||||
merged := slices.Clone(baseList)
|
||||
for _, item := range over {
|
||||
if _, ok := item.(map[string]any); !ok {
|
||||
return over
|
||||
}
|
||||
name := manifestField(item, key)
|
||||
index := slices.IndexFunc(merged, func(existing any) bool { return name != "" && manifestField(existing, key) == name })
|
||||
switch {
|
||||
case index < 0:
|
||||
merged = append(merged, item)
|
||||
case field == "env" || field == "volumes": // each holds one source, merging two makes a pod the API rejects
|
||||
merged[index] = item
|
||||
default:
|
||||
merged[index] = mergeManifest(merged[index], item, "")
|
||||
}
|
||||
}
|
||||
return merged
|
||||
}
|
||||
return over
|
||||
}
|
||||
|
||||
func manifestField(item any, key string) string {
|
||||
fields, _ := item.(map[string]any)
|
||||
value, _ := fields[key].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) spec(secret string, secretData map[string]string) (map[string]any, error) {
|
||||
env := []any{}
|
||||
for index, entry := range c.input.Env {
|
||||
key, value, _ := strings.Cut(entry, "=")
|
||||
if c == c.pod.job {
|
||||
env = append(env, map[string]any{"name": key, "value": value})
|
||||
continue
|
||||
}
|
||||
secretKey := fmt.Sprintf("%s.%d", c.name, index)
|
||||
secretData[secretKey] = value
|
||||
env = append(env, map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": secret, "key": secretKey}}})
|
||||
}
|
||||
spec := map[string]any{
|
||||
"name": c.name, "image": c.input.Image, "command": escapeExpansion(c.input.Entrypoint), "args": escapeExpansion(c.input.Cmd),
|
||||
"workingDir": c.input.WorkingDir, "env": env,
|
||||
}
|
||||
if c.pod.options.ForcePull {
|
||||
spec["imagePullPolicy"] = "Always"
|
||||
}
|
||||
if c == c.pod.job {
|
||||
spec["volumeMounts"] = []any{
|
||||
map[string]any{"name": "workspace", "mountPath": c.input.WorkingDir},
|
||||
map[string]any{"name": "act", "mountPath": c.GetActPath()},
|
||||
}
|
||||
}
|
||||
if err := c.parseOptions(); err != nil {
|
||||
return nil, fmt.Errorf("service %s: %w", c.name, err)
|
||||
}
|
||||
if c.probe != nil {
|
||||
spec["startupProbe"] = c.probe
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
// escapeExpansion keeps kubelet from expanding $(VAR) and $$ in command and args, which docker passes as is.
|
||||
func escapeExpansion(args []string) []string {
|
||||
args = slices.Clone(args)
|
||||
for index := range args {
|
||||
args[index] = strings.ReplaceAll(args[index], "$", "$$")
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// parseOptions turns docker health options, the only ones a pod supports, into a startup probe, which like docker gives up after the retries.
|
||||
func (c *kubernetesContainer) parseOptions() error {
|
||||
args, err := shellquote.Split(c.input.WorkflowOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flags := pflag.NewFlagSet("options", pflag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
command := flags.String("health-cmd", "", "")
|
||||
interval := flags.Duration("health-interval", 0, "")
|
||||
timeout := flags.Duration("health-timeout", 0, "")
|
||||
retries := flags.Int("health-retries", 0, "")
|
||||
startPeriod := flags.Duration("health-start-period", 0, "")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return fmt.Errorf("options %q: %w, only --health-* options are supported on Kubernetes", c.input.WorkflowOptions, err)
|
||||
}
|
||||
if *command == "" {
|
||||
return nil
|
||||
}
|
||||
period := max(1, int(cmp.Or(*interval, 30*time.Second).Seconds()))
|
||||
c.probe = map[string]any{
|
||||
"exec": map[string]any{"command": []string{"/bin/sh", "-c", *command}},
|
||||
"periodSeconds": period,
|
||||
"timeoutSeconds": max(1, int(cmp.Or(*timeout, 30*time.Second).Seconds())),
|
||||
"failureThreshold": max(1, cmp.Or(*retries, 3)) + int(startPeriod.Seconds())/period, // docker does not count failures in the start period
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) get(ctx context.Context) (*podStatus, error) {
|
||||
var pod podStatus
|
||||
return &pod, p.client.do(ctx, http.MethodGet, p.client.path("pods/"+p.name), nil, &pod)
|
||||
}
|
||||
|
||||
func (p *KubernetesPod) waitStarted(ctx context.Context) error {
|
||||
var unscheduled string
|
||||
for delay := 100 * time.Millisecond; ; delay = min(2*delay, time.Second) {
|
||||
pod, err := p.get(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if pod.Status.Phase == "Failed" {
|
||||
return fmt.Errorf("pod %s failed: %s", p.name, cmp.Or(pod.Status.Message, pod.Status.Reason))
|
||||
}
|
||||
started := len(pod.Status.ContainerStatuses) == len(pod.Spec.Containers)
|
||||
for _, status := range append(pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses...) {
|
||||
if waiting := status.State.Waiting; waiting != nil {
|
||||
switch waiting.Reason {
|
||||
case "ErrImagePull", "ImagePullBackOff", "InvalidImageName", "ErrImageNeverPull", "CreateContainerConfigError", "CreateContainerError", "RunContainerError":
|
||||
return fmt.Errorf("container %s of pod %s: %s: %s", status.Name, p.name, waiting.Reason, waiting.Message)
|
||||
}
|
||||
started = false
|
||||
}
|
||||
}
|
||||
if started {
|
||||
return nil
|
||||
}
|
||||
for _, condition := range pod.Status.Conditions {
|
||||
if condition.Type == "PodScheduled" && condition.Status == "False" && condition.Message != unscheduled {
|
||||
unscheduled = condition.Message
|
||||
common.Logger(ctx).Infof("Waiting for pod %s to be scheduled: %s", p.name, unscheduled)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Create(_, _ []string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
return c.pod.create(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Start(_ bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
if err := c.pod.waitStarted(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `printf '%s\n%s' "$(uname -m)" "$PATH"`}, nil, &stdout, io.Discard); err != nil {
|
||||
return err
|
||||
}
|
||||
machine, path, _ := strings.Cut(stdout.String(), "\n")
|
||||
c.pod.arch, c.pod.path = goArchToActionArch(machine), path
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (*kubernetesContainer) Pull(bool) common.Executor {
|
||||
return func(context.Context) error { return nil }
|
||||
}
|
||||
|
||||
var (
|
||||
shellName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
bashReadonly = []string{"BASHOPTS", "BASH_VERSINFO", "EUID", "PPID", "SHELLOPTS", "UID"} // bash as sh cannot export these
|
||||
)
|
||||
|
||||
// Exec feeds the command to sh on stdin, which exports the env so it stays out of the request URL and, where sh can export a name, out of process arguments.
|
||||
func (c *kubernetesContainer) Exec(command []string, env map[string]string, _, workdir string) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
script, args := "", []string{"env", "--"}
|
||||
for name, value := range env {
|
||||
if shellName.MatchString(name) && !slices.Contains(bashReadonly, name) {
|
||||
script += "command export " + shellquote.Join(name+"="+value) + "\n"
|
||||
} else {
|
||||
args = append(args, name+"="+value)
|
||||
}
|
||||
}
|
||||
script += "exec " + shellquote.Join(append(args, command...)...)
|
||||
if workdir != "" {
|
||||
script = "cd " + shellquote.Join(workdir) + " || exit\n" + script
|
||||
}
|
||||
defer common.FlushWriter(c.input.Stdout)
|
||||
defer common.FlushWriter(c.input.Stderr)
|
||||
return c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh"}, strings.NewReader(script), c.input.Stdout, c.input.Stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) extract(ctx context.Context, destPath string, write func(io.Writer) error) error {
|
||||
reader, writer := io.Pipe()
|
||||
defer reader.Close()
|
||||
go func() { writer.CloseWithError(write(writer)) }()
|
||||
var stderr bytes.Buffer
|
||||
if err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `mkdir -p "$1" && tar -xf - -C "$1"`, "sh", destPath}, reader, io.Discard, &stderr); err != nil {
|
||||
return fmt.Errorf("extract to %s: %w: %s", destPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Copy(destPath string, files ...*FileEntry) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error { return writeFilesTar(ctx, writer, 0, 0, files...) })
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) CopyDir(destPath, srcPath string, useGitIgnore, skipGitDir bool) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
return c.extract(ctx, destPath, func(writer io.Writer) error {
|
||||
return writeDirTar(ctx, writer, "/", srcPath, useGitIgnore, skipGitDir, 0, 0)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// GetContainerArchive streams the archive, checking the path first as tar writes an empty archive for a missing one.
|
||||
func (c *kubernetesContainer) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
|
||||
reader, writer := io.Pipe()
|
||||
go func() {
|
||||
var stderr bytes.Buffer
|
||||
err := c.pod.client.exec(ctx, c.pod.name, c.name, []string{"sh", "-c", `test -e "$1" -o -L "$1" && exec tar -cf - -C "$2" "$3"`, "sh", srcPath, path.Dir(srcPath), path.Base(srcPath)}, nil, writer, &stderr)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("archive %s: %w: %s", srcPath, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
writer.CloseWithError(err)
|
||||
}()
|
||||
buffered := bufio.NewReader(reader)
|
||||
if _, err := buffered.Peek(1); err != nil {
|
||||
reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{buffered, reader}, nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
|
||||
return parseEnvFile(c, srcPath, env)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) UpdateFromImageEnv(env *map[string]string) common.Executor {
|
||||
return func(context.Context) error {
|
||||
(*env)["PATH"] = cmp.Or((*env)["PATH"], c.pod.path)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Inspect(ctx context.Context) (*Info, error) {
|
||||
if c.pod.name == "" {
|
||||
return nil, fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
|
||||
}
|
||||
pod, err := c.pod.get(ctx)
|
||||
if isNotFound(err) {
|
||||
return nil, fmt.Errorf("pod %s %w", c.pod.name, ErrContainerNotFound)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info := &Info{ID: c.pod.name + "/" + c.name, State: "created", Health: HealthNone, Ports: map[string]string{}}
|
||||
for port := range c.input.ExposedPorts {
|
||||
info.Ports[port.Port()] = port.Port() // services share the pod's network
|
||||
}
|
||||
for _, status := range pod.Status.ContainerStatuses {
|
||||
if status.Name != c.name {
|
||||
continue
|
||||
}
|
||||
if status.State.Running != nil {
|
||||
info.State = StateRunning
|
||||
} else if terminated := status.State.Terminated; terminated != nil {
|
||||
info.State, info.ExitCode, info.HealthOutput = "exited", terminated.ExitCode, terminated.Message
|
||||
}
|
||||
if c.probe != nil {
|
||||
switch {
|
||||
case status.Ready:
|
||||
info.Health = HealthHealthy
|
||||
case info.State == "exited":
|
||||
info.Health = HealthUnhealthy
|
||||
default:
|
||||
info.Health = HealthStarting
|
||||
}
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) DumpLogs(ctx context.Context) error {
|
||||
if c.pod.name == "" {
|
||||
return fmt.Errorf("container %s %w", c.name, ErrContainerNotFound)
|
||||
}
|
||||
defer common.FlushWriter(c.input.Stdout)
|
||||
return c.pod.client.do(ctx, http.MethodGet, c.pod.client.path("pods/"+c.pod.name+"/log?container="+c.name), nil, c.input.Stdout)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if c != c.pod.job {
|
||||
return nil
|
||||
}
|
||||
return errors.Join(c.pod.client.delete(ctx, "pods", c.pod.name), c.pod.client.delete(ctx, "secrets", c.pod.secretName))
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveOrphanKubernetesResources removes this runner's job pods created before createdBefore and the Secrets named like them, as listing Secrets would reveal all of the namespace's.
|
||||
func RemoveOrphanKubernetesResources(ctx context.Context, options KubernetesOptions, createdBefore time.Time) error {
|
||||
client, err := newKubeClient(options.Kubeconfig, options.Namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect to Kubernetes: %w", err)
|
||||
}
|
||||
defer client.http.CloseIdleConnections()
|
||||
var pods struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string
|
||||
CreationTimestamp time.Time
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := client.do(ctx, http.MethodGet, client.path("pods?labelSelector="+url.QueryEscape(runnerUUIDLabel+"="+options.RunnerUUID)), nil, &pods); err != nil {
|
||||
return fmt.Errorf("list pods: %w", err)
|
||||
}
|
||||
var errs []error
|
||||
for _, item := range pods.Items {
|
||||
if !item.Metadata.CreationTimestamp.Before(createdBefore) {
|
||||
continue
|
||||
}
|
||||
if err := client.delete(ctx, "secrets", item.Metadata.Name); err != nil {
|
||||
errs = append(errs, err) // leave the pod for the next cleanup, it is the only way back to its Secret
|
||||
continue
|
||||
}
|
||||
errs = append(errs, client.delete(ctx, "pods", item.Metadata.Name))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) Close() common.Executor {
|
||||
return func(context.Context) error {
|
||||
c.pod.client.http.CloseIdleConnections()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) {
|
||||
oldStdout, oldStderr := c.input.Stdout, c.input.Stderr
|
||||
c.input.Stdout, c.input.Stderr = stdout, stderr
|
||||
return oldStdout, oldStderr
|
||||
}
|
||||
|
||||
func (c *kubernetesContainer) GetRunnerContext(_ context.Context) map[string]any {
|
||||
return map[string]any{"os": "Linux", "arch": c.pod.arch, "temp": "/tmp", "tool_cache": DefaultToolCache}
|
||||
}
|
||||
400
act/container/kubernetes_client.go
Normal file
400
act/container/kubernetes_client.go
Normal file
@@ -0,0 +1,400 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"go.yaml.in/yaml/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
serviceAccountDir = "/var/run/secrets/kubernetes.io/serviceaccount"
|
||||
execProtocol = "v5.channel.k8s.io"
|
||||
)
|
||||
|
||||
type kubeClient struct {
|
||||
server, namespace, token string
|
||||
tokenFile string // re-read on every request, service account tokens rotate
|
||||
inCluster bool
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
code int
|
||||
message string
|
||||
}
|
||||
|
||||
func (e *apiError) Error() string { return e.message }
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var apiErr *apiError
|
||||
return errors.As(err, &apiErr) && apiErr.code == http.StatusNotFound
|
||||
}
|
||||
|
||||
func newKubeClient(kubeconfig, namespace string) (*kubeClient, error) {
|
||||
client := &kubeClient{namespace: namespace}
|
||||
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}
|
||||
if host := os.Getenv("KUBERNETES_SERVICE_HOST"); host != "" && kubeconfig == "" {
|
||||
if err := client.loadInCluster(host, transport.TLSClientConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err := client.loadKubeconfig(kubeconfig, transport); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.http = &http.Client{Transport: transport}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) loadInCluster(host string, tlsConfig *tls.Config) error {
|
||||
c.server = "https://" + net.JoinHostPort(host, os.Getenv("KUBERNETES_SERVICE_PORT"))
|
||||
c.tokenFile = serviceAccountDir + "/token"
|
||||
ca, err := os.ReadFile(serviceAccountDir + "/ca.crt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tlsConfig.RootCAs = x509.NewCertPool()
|
||||
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
|
||||
if c.namespace == "" {
|
||||
namespace, err := os.ReadFile(serviceAccountDir + "/namespace")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.namespace = strings.TrimSpace(string(namespace))
|
||||
}
|
||||
c.inCluster = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) loadKubeconfig(path string, transport *http.Transport) error {
|
||||
if paths := filepath.SplitList(os.Getenv("KUBECONFIG")); path == "" && len(paths) > 0 {
|
||||
path = paths[0]
|
||||
}
|
||||
if path == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path = filepath.Join(home, ".kube", "config")
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("no in-cluster service account and no kubeconfig: %w", err)
|
||||
}
|
||||
var config struct {
|
||||
CurrentContext string `yaml:"current-context"`
|
||||
Contexts []struct {
|
||||
Name string
|
||||
Context struct{ Cluster, User, Namespace string }
|
||||
}
|
||||
Clusters []struct {
|
||||
Name string
|
||||
Cluster struct {
|
||||
Server string
|
||||
TLSServerName string `yaml:"tls-server-name"`
|
||||
InsecureSkipTLSVerify bool `yaml:"insecure-skip-tls-verify"`
|
||||
ProxyURL string `yaml:"proxy-url"`
|
||||
CertificateAuthority string `yaml:"certificate-authority"`
|
||||
CertificateAuthorityData string `yaml:"certificate-authority-data"`
|
||||
}
|
||||
}
|
||||
Users []struct {
|
||||
Name string
|
||||
User struct {
|
||||
Token string
|
||||
TokenFile string `yaml:"tokenFile"`
|
||||
ClientCertificate string `yaml:"client-certificate"`
|
||||
ClientCertificateData string `yaml:"client-certificate-data"`
|
||||
ClientKey string `yaml:"client-key"`
|
||||
ClientKeyData string `yaml:"client-key-data"`
|
||||
Exec any
|
||||
AuthProvider any `yaml:"auth-provider"`
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := yaml.Unmarshal(content, &config); err != nil {
|
||||
return fmt.Errorf("parse kubeconfig %s: %w", path, err)
|
||||
}
|
||||
var current struct{ Cluster, User, Namespace string }
|
||||
for _, entry := range config.Contexts {
|
||||
if entry.Name == config.CurrentContext {
|
||||
current = entry.Context
|
||||
}
|
||||
}
|
||||
if current.Cluster == "" {
|
||||
return fmt.Errorf("kubeconfig %s has no current context", path)
|
||||
}
|
||||
c.namespace = cmp.Or(c.namespace, current.Namespace, "default")
|
||||
|
||||
resolve := func(file string) string {
|
||||
if file == "" || filepath.IsAbs(file) {
|
||||
return file
|
||||
}
|
||||
return filepath.Join(filepath.Dir(path), file)
|
||||
}
|
||||
read := func(data, file string) ([]byte, error) {
|
||||
if data != "" {
|
||||
return base64.StdEncoding.DecodeString(data)
|
||||
}
|
||||
if file == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return os.ReadFile(resolve(file))
|
||||
}
|
||||
|
||||
tlsConfig := transport.TLSClientConfig
|
||||
for _, cluster := range config.Clusters {
|
||||
if cluster.Name != current.Cluster {
|
||||
continue
|
||||
}
|
||||
c.server = strings.TrimSuffix(cluster.Cluster.Server, "/")
|
||||
tlsConfig.ServerName, tlsConfig.InsecureSkipVerify = cluster.Cluster.TLSServerName, cluster.Cluster.InsecureSkipTLSVerify
|
||||
if cluster.Cluster.ProxyURL != "" {
|
||||
proxy, err := url.Parse(cluster.Cluster.ProxyURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("kubeconfig %s: %w", path, err)
|
||||
}
|
||||
transport.Proxy = http.ProxyURL(proxy)
|
||||
}
|
||||
ca, err := read(cluster.Cluster.CertificateAuthorityData, cluster.Cluster.CertificateAuthority)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ca != nil {
|
||||
tlsConfig.RootCAs = x509.NewCertPool()
|
||||
tlsConfig.RootCAs.AppendCertsFromPEM(ca)
|
||||
}
|
||||
}
|
||||
for _, user := range config.Users {
|
||||
if user.Name != current.User {
|
||||
continue
|
||||
}
|
||||
if user.User.Exec != nil || user.User.AuthProvider != nil {
|
||||
return fmt.Errorf("kubeconfig %s: user %q authenticates through a credential plugin, which the runner does not support, use a token or client certificate", path, user.Name)
|
||||
}
|
||||
c.token, c.tokenFile = user.User.Token, resolve(user.User.TokenFile)
|
||||
cert, err := read(user.User.ClientCertificateData, user.User.ClientCertificate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := read(user.User.ClientKeyData, user.User.ClientKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cert != nil {
|
||||
pair, err := tls.X509KeyPair(cert, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tlsConfig.Certificates = []tls.Certificate{pair}
|
||||
}
|
||||
}
|
||||
if c.server == "" {
|
||||
return fmt.Errorf("kubeconfig %s names no server for context %q", path, config.CurrentContext)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) authorize(header http.Header) error {
|
||||
token := c.token
|
||||
if c.tokenFile != "" {
|
||||
content, err := os.ReadFile(c.tokenFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token = strings.TrimSpace(string(content))
|
||||
}
|
||||
if token != "" {
|
||||
header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *kubeClient) path(resource string) string {
|
||||
return "/api/v1/namespaces/" + c.namespace + "/" + resource
|
||||
}
|
||||
|
||||
func (c *kubeClient) do(ctx context.Context, method, path string, in, out any) error {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
content, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(content)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.server+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.authorize(req.Header); err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= http.StatusMultipleChoices {
|
||||
return statusError(resp)
|
||||
}
|
||||
if out == nil {
|
||||
out = io.Discard // draining lets the connection be reused
|
||||
}
|
||||
if writer, ok := out.(io.Writer); ok {
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return err
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
func (c *kubeClient) delete(ctx context.Context, resource, name string) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if err := c.do(ctx, http.MethodDelete, c.path(resource+"/"+name), nil, nil); err != nil && !isNotFound(err) {
|
||||
return fmt.Errorf("delete %s %s: %w", resource, name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func statusError(resp *http.Response) error {
|
||||
content, _ := io.ReadAll(resp.Body)
|
||||
var status struct{ Message string }
|
||||
if json.Unmarshal(content, &status) != nil || status.Message == "" {
|
||||
status.Message = strings.TrimSpace(string(content))
|
||||
}
|
||||
return &apiError{code: resp.StatusCode, message: status.Message}
|
||||
}
|
||||
|
||||
func (c *kubeClient) exec(ctx context.Context, pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
|
||||
query := url.Values{"container": {containerName}, "command": command, "stdout": {"true"}, "stderr": {"true"}}
|
||||
if stdin != nil {
|
||||
query.Set("stdin", "true")
|
||||
}
|
||||
header := http.Header{}
|
||||
if err := c.authorize(header); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
defer cancel(nil)
|
||||
conn, resp, err := websocket.Dial(ctx, c.server+c.path("pods/"+pod+"/exec?")+query.Encode(), &websocket.DialOptions{
|
||||
HTTPClient: c.http,
|
||||
HTTPHeader: header,
|
||||
Subprotocols: []string{execProtocol},
|
||||
})
|
||||
if err != nil {
|
||||
if resp != nil && resp.StatusCode >= http.StatusMultipleChoices {
|
||||
defer resp.Body.Close()
|
||||
err = statusError(resp)
|
||||
}
|
||||
return fmt.Errorf("exec in pod %s: %w", pod, err)
|
||||
}
|
||||
defer func() { _ = conn.CloseNow() }()
|
||||
if conn.Subprotocol() != execProtocol {
|
||||
return fmt.Errorf("the Kubernetes API server does not speak %s, which needs Kubernetes 1.30 or later", execProtocol)
|
||||
}
|
||||
conn.SetReadLimit(-1)
|
||||
|
||||
if stdin != nil {
|
||||
go func() {
|
||||
if err := sendStdin(ctx, conn, stdin); err != nil {
|
||||
cancel(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if conn.Ping(ctx) != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
_, frame, err := conn.Read(ctx)
|
||||
if err != nil {
|
||||
return cmp.Or(context.Cause(ctx), fmt.Errorf("exec in pod %s ended without an exit status: %w", pod, err))
|
||||
}
|
||||
if len(frame) < 2 {
|
||||
continue
|
||||
}
|
||||
switch frame[0] {
|
||||
case 1:
|
||||
_, err = stdout.Write(frame[1:])
|
||||
case 2:
|
||||
_, err = stderr.Write(frame[1:])
|
||||
case 3:
|
||||
return cmp.Or(context.Cause(ctx), execStatus(frame[1:]))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sendStdin(ctx context.Context, conn *websocket.Conn, stdin io.Reader) error {
|
||||
frame := make([]byte, 1+32*1024) // frame[0] is the stdin channel, 0
|
||||
for {
|
||||
n, err := stdin.Read(frame[1:])
|
||||
if n > 0 {
|
||||
if err := conn.Write(ctx, websocket.MessageBinary, frame[:1+n]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
return conn.Write(ctx, websocket.MessageBinary, []byte{255, 0}) // v5 close signal for stdin
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func execStatus(content []byte) error {
|
||||
var status struct {
|
||||
Status, Message string
|
||||
Details struct {
|
||||
Causes []struct{ Reason, Message string }
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal(content, &status); err != nil {
|
||||
return fmt.Errorf("decode exec status: %w", err)
|
||||
}
|
||||
if status.Status == "Success" {
|
||||
return nil
|
||||
}
|
||||
for _, cause := range status.Details.Causes {
|
||||
if code, err := strconv.Atoi(cause.Message); cause.Reason == "ExitCode" && err == nil {
|
||||
return ExitCodeError(code)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("exec failed: %s", status.Message)
|
||||
}
|
||||
313
act/container/kubernetes_test.go
Normal file
313
act/container/kubernetes_test.go
Normal file
@@ -0,0 +1,313 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/coder/websocket"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type podManifest struct {
|
||||
Metadata struct{ Labels map[string]string }
|
||||
Spec struct {
|
||||
AutomountServiceAccountToken bool
|
||||
EnableServiceLinks bool
|
||||
HostAliases []struct{ Hostnames []string }
|
||||
Volumes []map[string]any
|
||||
Containers []struct {
|
||||
Name, ImagePullPolicy string
|
||||
Resources map[string]any
|
||||
StartupProbe *struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeCluster struct {
|
||||
mu sync.Mutex
|
||||
requests map[string][]string
|
||||
}
|
||||
|
||||
func (f *fakeCluster) record(request, content string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.requests[request] = append(f.requests[request], content)
|
||||
}
|
||||
|
||||
func fakeKubernetes(t *testing.T, podStatus string) (string, *fakeCluster) {
|
||||
cluster := &fakeCluster{requests: map[string][]string{}}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer test-token" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
cluster.record(r.Method+" "+r.URL.Path, string(body))
|
||||
pods, secrets := "/api/v1/namespaces/ci/pods", "/api/v1/namespaces/ci/secrets"
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == pods:
|
||||
cluster.record("list", r.URL.Query().Get("labelSelector"))
|
||||
_, _ = io.WriteString(w, `{"items":[{"metadata":{"name":"old","creationTimestamp":"2020-01-01T00:00:00Z"}},{"metadata":{"name":"new","creationTimestamp":"2999-01-01T00:00:00Z"}}]}`)
|
||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
||||
cluster.record("exec", r.URL.RawQuery)
|
||||
serveLocalExec(t, w, r)
|
||||
case strings.HasSuffix(r.URL.Path, "/log"):
|
||||
_, _ = io.WriteString(w, "log of "+r.URL.Query().Get("container"))
|
||||
case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, pods+"/"):
|
||||
_, _ = io.WriteString(w, podStatus)
|
||||
case r.Method == http.MethodPost && (r.URL.Path == pods || r.URL.Path == secrets), r.Method == http.MethodDelete:
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
kubeconfig := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(kubeconfig), "token"), []byte("test-token\n"), 0o600))
|
||||
require.NoError(t, os.WriteFile(kubeconfig, fmt.Appendf(nil, `
|
||||
current-context: test
|
||||
contexts: [{name: test, context: {cluster: test, user: test, namespace: ci}}]
|
||||
clusters: [{name: test, cluster: {server: "http://kubernetes.invalid", proxy-url: %q}}]
|
||||
users: [{name: test, user: {tokenFile: token}}]
|
||||
`, server.URL), 0o600))
|
||||
return kubeconfig, cluster
|
||||
}
|
||||
|
||||
func serveLocalExec(t *testing.T, w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{Subprotocols: []string{execProtocol}})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = conn.CloseNow() }()
|
||||
conn.SetReadLimit(-1)
|
||||
command := r.URL.Query()["command"]
|
||||
var stdin bytes.Buffer
|
||||
for r.URL.Query().Get("stdin") == "true" {
|
||||
_, frame, err := conn.Read(r.Context())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if frame[0] == 255 {
|
||||
break
|
||||
}
|
||||
stdin.Write(frame[1:])
|
||||
}
|
||||
cmd := exec.Command(command[0], command[1:]...)
|
||||
cmd.Env = append(os.Environ(), "COPYFILE_DISABLE=1")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdin, cmd.Stdout, cmd.Stderr = &stdin, &stdout, &stderr
|
||||
status := `{"status":"Success"}`
|
||||
var exitErr *exec.ExitError
|
||||
if err := cmd.Run(); errors.As(err, &exitErr) {
|
||||
status = fmt.Sprintf(`{"status":"Failure","reason":"NonZeroExitCode","details":{"causes":[{"reason":"ExitCode","message":"%d"}]}}`, exitErr.ExitCode())
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
for _, frame := range [][]byte{append([]byte{1}, stdout.Bytes()...), append([]byte{2}, stderr.Bytes()...), append([]byte{3}, status...)} {
|
||||
_ = conn.Write(r.Context(), websocket.MessageBinary, frame)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesServiceHostnames(t *testing.T) {
|
||||
for id, hostnames := range map[string][]string{
|
||||
"db": {"db"},
|
||||
"My_DB": {"my-db"},
|
||||
"devstoreaccount1.azurite.local": {"devstoreaccount1-azurite-local", "devstoreaccount1.azurite.local"},
|
||||
"DB.Local": {"db-local", "db.local"},
|
||||
strings.Repeat("a", 64): {strings.Repeat("a", 63), strings.Repeat("a", 64)},
|
||||
strings.Repeat("a", 254): {strings.Repeat("a", 63)},
|
||||
} {
|
||||
assert.Equal(t, hostnames, KubernetesServiceHostnames(id), id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesPodRunsJobThroughAPI(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("uses POSIX shell")
|
||||
}
|
||||
ctx := t.Context()
|
||||
kubeconfig, cluster := fakeKubernetes(t, `{"spec":{"containers":[{},{}]},"status":{"phase":"Running","containerStatuses":[
|
||||
{"name":"job","ready":true,"state":{"running":{}}},
|
||||
{"name":"my-db","ready":true,"state":{"running":{}}}]}}`)
|
||||
workdir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
options := KubernetesOptions{Kubeconfig: kubeconfig, RunnerUUID: "runner-uuid", PodTemplates: []map[string]any{
|
||||
{"spec": map[string]any{
|
||||
"automountServiceAccountToken": true,
|
||||
"containers": []any{
|
||||
map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"cpu": "2"}}},
|
||||
map[string]any{
|
||||
"name": "$services", "imagePullPolicy": "Always", "resources": map[string]any{"limits": map[string]any{"memory": "256Mi"}},
|
||||
"env": []any{map[string]any{"name": "POSTGRES_PASSWORD", "value": "template"}},
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"spec": map[string]any{
|
||||
"volumes": []any{map[string]any{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}}},
|
||||
"containers": []any{map[string]any{"name": "job", "resources": map[string]any{"limits": map[string]any{"memory": "1Gi"}}}},
|
||||
}},
|
||||
}}
|
||||
pod, err := NewKubernetesPod(options)
|
||||
require.NoError(t, err)
|
||||
var jobLog, serviceLog bytes.Buffer
|
||||
job := pod.JobContainer(&NewContainerInput{Name: "Job-Name", Image: "node", WorkingDir: workdir, Stdout: &jobLog, Stderr: &jobLog})
|
||||
service := pod.ServiceContainer("My_DB", &NewContainerInput{
|
||||
Image: "postgres",
|
||||
Cmd: []string{"postgres", "-D", "$(PGDATA)"},
|
||||
Env: []string{"POSTGRES_PASSWORD=service-secret"},
|
||||
WorkflowOptions: "--health-cmd pg_isready --health-interval 2s --health-start-period 4s",
|
||||
Stdout: &serviceLog,
|
||||
})
|
||||
|
||||
require.NoError(t, job.Create(nil, nil)(ctx))
|
||||
var manifest podManifest
|
||||
require.NoError(t, json.Unmarshal([]byte(cluster.requests["POST /api/v1/namespaces/ci/pods"][0]), &manifest))
|
||||
assert.Equal(t, "gitea-runner", manifest.Metadata.Labels["app.kubernetes.io/managed-by"])
|
||||
assert.Equal(t, "runner-uuid", manifest.Metadata.Labels[runnerUUIDLabel])
|
||||
assert.True(t, manifest.Spec.AutomountServiceAccountToken)
|
||||
assert.False(t, manifest.Spec.EnableServiceLinks)
|
||||
assert.Equal(t, []string{"my-db"}, manifest.Spec.HostAliases[0].Hostnames)
|
||||
require.Len(t, manifest.Spec.Containers, 2)
|
||||
assert.Equal(t, map[string]any{"limits": map[string]any{"cpu": "2", "memory": "1Gi"}}, manifest.Spec.Containers[0].Resources)
|
||||
assert.Empty(t, manifest.Spec.Containers[0].ImagePullPolicy)
|
||||
assert.Equal(t, "my-db", manifest.Spec.Containers[1].Name)
|
||||
assert.Equal(t, "Always", manifest.Spec.Containers[1].ImagePullPolicy)
|
||||
assert.Equal(t, map[string]any{"limits": map[string]any{"memory": "256Mi"}}, manifest.Spec.Containers[1].Resources)
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"name": "workspace", "emptyDir": map[string]any{"sizeLimit": "1Gi"}},
|
||||
{"name": "act", "emptyDir": map[string]any{}},
|
||||
}, manifest.Spec.Volumes)
|
||||
assert.Equal(t, struct{ PeriodSeconds, TimeoutSeconds, FailureThreshold int }{2, 30, 5}, *manifest.Spec.Containers[1].StartupProbe)
|
||||
assert.NotContains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], "service-secret")
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], "service-secret")
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/secrets"][0], `"com.gitea.runner.uuid":"runner-uuid"`)
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `{"name":"POSTGRES_PASSWORD","valueFrom":{"secretKeyRef":{"key":"my-db.0","name":"`+pod.secretName+`"}}}`)
|
||||
assert.Equal(t, []any{"new"}, mergeManifest([]any{"old"}, []any{"new"}, ""))
|
||||
mount := func(path string) any { return map[string]any{"name": "config", "mountPath": path} }
|
||||
assert.Equal(t, []any{mount("/a"), mount("/b")}, mergeManifest([]any{mount("/a")}, []any{mount("/b")}, "volumeMounts"))
|
||||
tmpfs := map[string]any{"name": "workspace", "emptyDir": map[string]any{"medium": "Memory"}}
|
||||
ephemeral := map[string]any{"name": "workspace", "ephemeral": map[string]any{}}
|
||||
assert.Equal(t, []any{ephemeral}, mergeManifest([]any{tmpfs}, []any{ephemeral}, "volumes"))
|
||||
assert.Contains(t, cluster.requests["POST /api/v1/namespaces/ci/pods"][0], `"args":["postgres","-D","$$(PGDATA)"]`)
|
||||
|
||||
require.NoError(t, os.WriteFile(filepath.Join(workdir, "env"), []byte("#!/bin/sh\necho \"$@\" > \"$0-args\"\nexec /usr/bin/env \"$@\"\n"), 0o755))
|
||||
t.Setenv("PATH", workdir+":"+os.Getenv("PATH"))
|
||||
require.NoError(t, job.Start(false)(ctx))
|
||||
|
||||
stepEnv := map[string]string{"INPUT_WHO-TO-GREET": "it's\nme", "SECRET": "hunter2", "UID": "5"}
|
||||
require.NoError(t, job.Exec([]string{"printenv", "SECRET"}, stepEnv, "", "")(ctx))
|
||||
envArgs, err := os.ReadFile(filepath.Join(workdir, "env-args"))
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(envArgs), "hunter2")
|
||||
require.NoError(t, job.Exec([]string{"printenv", "INPUT_WHO-TO-GREET"}, stepEnv, "", "")(ctx))
|
||||
require.NoError(t, job.Exec([]string{"printenv", "UID"}, stepEnv, "", "")(ctx))
|
||||
require.NoError(t, job.Exec([]string{"pwd", "-P"}, nil, "", workdir)(ctx))
|
||||
assert.Equal(t, "hunter2\nit's\nme\n5\n"+workdir+"\n", jobLog.String())
|
||||
assert.Equal(t, ExitCodeError(3), job.Exec([]string{"sh", "-c", "echo failing >&2; exit 3"}, nil, "", "")(ctx))
|
||||
assert.Contains(t, jobLog.String(), "failing\n")
|
||||
for _, query := range cluster.requests["exec"] {
|
||||
assert.NotContains(t, query, "hunter2")
|
||||
}
|
||||
|
||||
require.NoError(t, job.Copy(workdir+"/act/", &FileEntry{Name: "workflow/event.json", Mode: 0o644, Body: `{"ref":"main"}`})(ctx))
|
||||
archive, err := job.GetContainerArchive(ctx, workdir+"/act/workflow/event.json")
|
||||
require.NoError(t, err)
|
||||
reader := tar.NewReader(archive)
|
||||
header, err := reader.Next()
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "event.json", header.Name)
|
||||
assert.JSONEq(t, `{"ref":"main"}`, string(content))
|
||||
require.NoError(t, archive.Close())
|
||||
_, err = job.GetContainerArchive(ctx, workdir+"/missing")
|
||||
require.Error(t, err)
|
||||
|
||||
actionDir := filepath.Join(t.TempDir(), "action")
|
||||
require.NoError(t, os.MkdirAll(actionDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(actionDir, "index.js"), []byte("main()"), 0o644))
|
||||
require.NoError(t, job.CopyDir(workdir+"/act/actions/sha/", actionDir+"/", false, false)(ctx))
|
||||
script, err := os.ReadFile(workdir + "/act/actions/sha/index.js")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "main()", string(script))
|
||||
assert.Contains(t, cluster.requests["exec"][len(cluster.requests["exec"])-1], "command="+url.QueryEscape(workdir+"/act/actions/sha/"))
|
||||
|
||||
pathEnv := map[string]string{}
|
||||
require.NoError(t, job.UpdateFromImageEnv(&pathEnv)(ctx))
|
||||
assert.Equal(t, os.Getenv("PATH"), pathEnv["PATH"])
|
||||
|
||||
info, err := service.Inspect(ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, HealthHealthy, info.Health)
|
||||
require.NoError(t, service.DumpLogs(ctx))
|
||||
assert.Equal(t, "log of my-db", serviceLog.String())
|
||||
|
||||
require.NoError(t, job.Remove()(ctx))
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/"+pod.name], 1)
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/"+pod.secretName], 1)
|
||||
|
||||
require.NoError(t, RemoveOrphanKubernetesResources(ctx, options, time.Now()))
|
||||
assert.Equal(t, []string{runnerUUIDLabel + "=runner-uuid"}, cluster.requests["list"])
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/old"], 1)
|
||||
assert.Len(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/old"], 1)
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/pods/new"])
|
||||
assert.Empty(t, cluster.requests["DELETE /api/v1/namespaces/ci/secrets/new"])
|
||||
}
|
||||
|
||||
func TestKubernetesPodFailsBeforeSteps(t *testing.T) {
|
||||
kubeconfig, _ := fakeKubernetes(t, `{"spec":{"containers":[{}]},"status":{"containerStatuses":[
|
||||
{"name":"job","state":{"waiting":{"reason":"ErrImagePull","message":"no such image"}}}]}}`)
|
||||
template := map[string]any{"spec": map[string]any{"containers": []any{map[string]any{"name": "sidecar"}}}}
|
||||
for _, test := range []struct {
|
||||
services []string
|
||||
options, err string
|
||||
}{
|
||||
{[]string{"db"}, "--hostname db", "only --health-* options are supported"},
|
||||
{[]string{"sidecar"}, "", "named like another container"},
|
||||
{[]string{"my_db", "My-DB"}, "", `service My-DB maps to container "my-db"`},
|
||||
} {
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci", PodTemplates: []map[string]any{template}})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "node"})
|
||||
for _, service := range test.services {
|
||||
pod.ServiceContainer(service, &NewContainerInput{Image: "postgres", WorkflowOptions: test.options})
|
||||
}
|
||||
require.ErrorContains(t, job.Create(nil, nil)(t.Context()), test.err)
|
||||
}
|
||||
pod, err := NewKubernetesPod(KubernetesOptions{Kubeconfig: kubeconfig, Namespace: "ci"})
|
||||
require.NoError(t, err)
|
||||
job := pod.JobContainer(&NewContainerInput{Image: "missing"})
|
||||
require.NoError(t, job.Create(nil, nil)(t.Context()))
|
||||
require.ErrorContains(t, job.Start(false)(t.Context()), "ErrImagePull: no such image")
|
||||
|
||||
plugin := filepath.Join(t.TempDir(), "kubeconfig")
|
||||
require.NoError(t, os.WriteFile(plugin, []byte(`
|
||||
current-context: eks
|
||||
contexts: [{name: eks, context: {cluster: eks, user: eks}}]
|
||||
clusters: [{name: eks, cluster: {server: "https://eks.example"}}]
|
||||
users: [{name: eks, user: {exec: {command: aws}}}]
|
||||
`), 0o600))
|
||||
_, err = NewKubernetesPod(KubernetesOptions{Kubeconfig: plugin})
|
||||
require.ErrorContains(t, err, "credential plugin")
|
||||
}
|
||||
@@ -11,28 +11,28 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gitea.com/gitea/runner/act/common/logging"
|
||||
)
|
||||
|
||||
type LinuxContainerEnvironmentExtensions struct{}
|
||||
|
||||
var windowsPathRegex = regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
||||
|
||||
// Resolves the equivalent host path inside the container
|
||||
// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject
|
||||
// For use in docker volumes and binds
|
||||
func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string {
|
||||
if runtime.GOOS == "windows" && strings.Contains(path, "/") {
|
||||
log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
|
||||
logging.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.")
|
||||
return ""
|
||||
}
|
||||
|
||||
abspath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
logging.Error(err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Test if the path is a windows path
|
||||
windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`)
|
||||
windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath)
|
||||
|
||||
// Return as-is if no match
|
||||
@@ -43,11 +43,7 @@ func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string
|
||||
// Convert to WSL2-compatible path if it is a windows path
|
||||
// NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows
|
||||
// based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work.
|
||||
driveLetter := strings.ToLower(windowsPathComponents[1])
|
||||
translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
||||
// Should make something like /mnt/c/Users/person/My Folder/MyActProject
|
||||
result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`)
|
||||
return result
|
||||
return "/mnt/" + strings.ToLower(windowsPathComponents[1]) + "/" + strings.ReplaceAll(windowsPathComponents[2], `\`, `/`)
|
||||
}
|
||||
|
||||
func (*LinuxContainerEnvironmentExtensions) GetActPath() string {
|
||||
@@ -69,10 +65,41 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
|
||||
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
|
||||
const DefaultToolCache = "/opt/hostedtoolcache"
|
||||
|
||||
type runnerArchKey struct{}
|
||||
|
||||
func WithRunnerArch(ctx context.Context) context.Context { // shares one architecture lookup per run
|
||||
if ctx.Value(runnerArchKey{}) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, runnerArchKey{}, &runnerArchCache{lock: make(chan struct{}, 1)})
|
||||
}
|
||||
|
||||
type runnerArchCache struct {
|
||||
lock chan struct{} // lets cancelled jobs stop waiting for the cache
|
||||
arch string
|
||||
}
|
||||
|
||||
func cachedRunnerArch(ctx context.Context) string {
|
||||
cache, ok := ctx.Value(runnerArchKey{}).(*runnerArchCache)
|
||||
if !ok {
|
||||
return RunnerArch(ctx)
|
||||
}
|
||||
select {
|
||||
case cache.lock <- struct{}{}:
|
||||
defer func() { <-cache.lock }()
|
||||
case <-ctx.Done():
|
||||
return ""
|
||||
}
|
||||
if cache.arch == "" {
|
||||
cache.arch = RunnerArch(ctx)
|
||||
}
|
||||
return cache.arch
|
||||
}
|
||||
|
||||
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
|
||||
return map[string]any{
|
||||
"os": "Linux",
|
||||
"arch": RunnerArch(ctx),
|
||||
"arch": cachedRunnerArch(ctx),
|
||||
"temp": "/tmp",
|
||||
"tool_cache": DefaultToolCache,
|
||||
}
|
||||
|
||||
@@ -5,14 +5,13 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestContainerPath(t *testing.T) {
|
||||
@@ -22,42 +21,27 @@ func TestContainerPath(t *testing.T) {
|
||||
workDir string
|
||||
}
|
||||
|
||||
linuxcontainerext := &LinuxContainerEnvironmentExtensions{}
|
||||
|
||||
cwd, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
tests := []containerPathJob{
|
||||
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
||||
{"/home/act", `/home/act/`, ""},
|
||||
{cwd, ".", ""},
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
|
||||
rootDrive := os.Getenv("SystemDrive")
|
||||
rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "")
|
||||
for _, v := range []containerPathJob{
|
||||
tests = []containerPathJob{
|
||||
{"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""},
|
||||
{"/mnt/f/work/dir", `F:\work\dir`, ""},
|
||||
{"/mnt/c/windows/to/unix", "windows\\to\\unix", rootDrive + "\\"},
|
||||
{fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", rootDrive + "\\"},
|
||||
} {
|
||||
if v.workDir != "" {
|
||||
t.Chdir(v.workDir)
|
||||
}
|
||||
|
||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
||||
{"/mnt/" + strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + "/act", "act", rootDrive + "\\"},
|
||||
}
|
||||
|
||||
t.Chdir(cwd)
|
||||
} else {
|
||||
cwd, err := os.Getwd()
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
for _, v := range []containerPathJob{
|
||||
{"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""},
|
||||
{"/home/act", `/home/act/`, ""},
|
||||
{cwd, ".", ""},
|
||||
} {
|
||||
assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath))
|
||||
}
|
||||
for _, test := range tests {
|
||||
if test.workDir != "" {
|
||||
t.Chdir(test.workDir)
|
||||
}
|
||||
assert.Equal(t, test.destinationPath, (&LinuxContainerEnvironmentExtensions{}).ToContainerPath(test.sourcePath))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,5 +50,4 @@ type typeAssertMockContainer struct {
|
||||
LinuxContainerEnvironmentExtensions
|
||||
}
|
||||
|
||||
// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment
|
||||
var _ ExecutionsEnvironment = &typeAssertMockContainer{}
|
||||
|
||||
@@ -75,7 +75,6 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
|
||||
if err := s.Err(); err != nil {
|
||||
return fmt.Errorf("reading env file: %w", err)
|
||||
}
|
||||
env = &localEnv
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ package container
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
)
|
||||
|
||||
@@ -23,117 +21,53 @@ func newTestHostEnv(t *testing.T) (*HostEnvironment, string) {
|
||||
return e, filepath.Join(e.Path, "envfile")
|
||||
}
|
||||
|
||||
func TestParseEnvFileSingleLine(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO=bar\nBAZ=qux\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "qux", env["BAZ"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileMultiLine(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content := "FOO<<EOF\nline1\nline2\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "line1\nline2", env["FOO"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileLargeValueWithinLimit(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
big := strings.Repeat("x", 2*1024*1024)
|
||||
content := "FOO<<EOF\n" + big + "\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, big, env["FOO"])
|
||||
func TestParseEnvFile(t *testing.T) {
|
||||
largeValue := strings.Repeat("x", 2*1024*1024)
|
||||
encodeUTF16 := func(endianness unicode.Endianness) string {
|
||||
content, err := unicode.UTF16(endianness, unicode.UseBOM).NewEncoder().Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||
require.NoError(t, err)
|
||||
return string(content)
|
||||
}
|
||||
for _, test := range []struct {
|
||||
name, content string
|
||||
want map[string]string
|
||||
}{
|
||||
{"single line", "FOO=bar\nBAZ=qux\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
{"blank lines", "\nFOO=bar\n\n \nBAZ=qux\n\n", map[string]string{"FOO": "bar", "BAZ": "qux"}},
|
||||
{"multiline", "FOO<<EOF\nline1\nline2\nEOF\n", map[string]string{"FOO": "line1\nline2"}},
|
||||
{"multiline blank lines", "FOO<<EOF\nline1\n\nline2\nEOF\n", map[string]string{"FOO": "line1\n\nline2"}},
|
||||
{"multiline leading blank line", "FOO<<EOF\n\nline2\nEOF\n", map[string]string{"FOO": "\nline2"}},
|
||||
{"large value within limit", "FOO<<EOF\n" + largeValue + "\nEOF\n", map[string]string{"FOO": largeValue}},
|
||||
{"UTF8 BOM", "\xef\xbb\xbfFOO=bar\n", map[string]string{"FOO": "bar"}},
|
||||
{"UTF16 little endian", encodeUTF16(unicode.LittleEndian), map[string]string{"FOO": "bar", "MULTI": "line1"}},
|
||||
{"UTF16 big endian", encodeUTF16(unicode.BigEndian), map[string]string{"FOO": "bar", "MULTI": "line1"}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(test.content), 0o600))
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(t.Context()))
|
||||
assert.Equal(t, test.want, env)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvFileLineExceedsBufferReportsScannerError(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
tooBig := strings.Repeat("x", 17*1024*1024) // over the 16 MiB cap
|
||||
content := "FOO<<EOF\n" + tooBig + "\nEOF\n"
|
||||
require.NoError(t, os.WriteFile(envPath, []byte(content), 0o600))
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n"+strings.Repeat("x", 17*1024*1024)+"\nEOF\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env)(context.Background())
|
||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
||||
require.ErrorIs(t, err, bufio.ErrTooLong)
|
||||
assert.Contains(t, err.Error(), "reading env file")
|
||||
}
|
||||
|
||||
// Regression test: a blank line used to fail the job at "Complete Job", after
|
||||
// every step had already been recorded as successful.
|
||||
func TestParseEnvFileBlankLines(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("\nFOO=bar\n\n \nBAZ=qux\n\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "qux", env["BAZ"])
|
||||
}
|
||||
|
||||
// blank lines inside a heredoc value are content, not separators
|
||||
func TestParseEnvFileMultiLineKeepsBlankLines(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\n\nline2\nEOF\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "line1\n\nline2", env["FOO"])
|
||||
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\n\nline2\nEOF\n"), 0o600))
|
||||
env = map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "\nline2", env["FOO"])
|
||||
}
|
||||
|
||||
func TestParseEnvFileUTF8BOM(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content := append([]byte{0xEF, 0xBB, 0xBF}, []byte("FOO=bar\n")...)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
}
|
||||
|
||||
// Windows host mode: PowerShell 5.1 redirection writes UTF-16, which used to be
|
||||
// unrecognisable as KEY=VALUE, so the writes were silently ignored.
|
||||
func TestParseEnvFileUTF16(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
encoder *encoding.Encoder
|
||||
}{
|
||||
{"little endian", unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder()},
|
||||
{"big endian", unicode.UTF16(unicode.BigEndian, unicode.UseBOM).NewEncoder()},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
content, err := tt.encoder.Bytes([]byte("FOO=bar\r\nMULTI<<EOF\r\nline1\r\nEOF\r\n"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(envPath, content, 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
require.NoError(t, parseEnvFile(e, envPath, &env)(context.Background()))
|
||||
assert.Equal(t, "bar", env["FOO"])
|
||||
assert.Equal(t, "line1", env["MULTI"])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseEnvFileMissingDelimiter(t *testing.T) {
|
||||
e, envPath := newTestHostEnv(t)
|
||||
require.NoError(t, os.WriteFile(envPath, []byte("FOO<<EOF\nline1\nline2\n"), 0o600))
|
||||
|
||||
env := map[string]string{}
|
||||
err := parseEnvFile(e, envPath, &env)(context.Background())
|
||||
err := parseEnvFile(e, envPath, &env)(t.Context())
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "delimiter")
|
||||
}
|
||||
|
||||
55
act/container/tar.go
Normal file
55
act/container/tar.go
Normal file
@@ -0,0 +1,55 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// Copyright 2020 The nektos/act Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/filecollector"
|
||||
)
|
||||
|
||||
func writeFilesTar(ctx context.Context, w io.Writer, uid, gid int, files ...*FileEntry) error {
|
||||
tw := tar.NewWriter(w)
|
||||
for _, file := range files {
|
||||
common.Logger(ctx).Debugf("Writing entry to tarball %s len:%d", file.Name, len(file.Body))
|
||||
if err := tw.WriteHeader(&tar.Header{
|
||||
Name: file.Name,
|
||||
Mode: file.Mode,
|
||||
Size: int64(len(file.Body)),
|
||||
Uid: uid,
|
||||
Gid: gid,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write([]byte(file.Body)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tw.Close()
|
||||
}
|
||||
|
||||
// writeDirTar prefixes entries with dstPath for extraction at root.
|
||||
func writeDirTar(ctx context.Context, w io.Writer, dstPath, srcPath string, useGitIgnore, skipGitDir bool, uid, gid int) error {
|
||||
logger := common.Logger(ctx)
|
||||
tw := tar.NewWriter(w)
|
||||
srcPrefix := strings.TrimSuffix(filepath.Dir(srcPath), string(filepath.Separator)) + string(filepath.Separator)
|
||||
logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath)
|
||||
|
||||
if err := filepath.Walk(srcPath, (&filecollector.FileCollector{
|
||||
SrcPath: srcPath,
|
||||
SrcPrefix: srcPrefix,
|
||||
SkipGitDir: skipGitDir,
|
||||
UseGitIgnore: useGitIgnore,
|
||||
Handler: &filecollector.TarCollector{TarWriter: tw, UID: uid, GID: gid, DstDir: dstPath[1:]},
|
||||
}).CollectFiles(ctx)); err != nil {
|
||||
return err
|
||||
}
|
||||
return tw.Close()
|
||||
}
|
||||
5
act/container/testdata/Dockerfile
vendored
5
act/container/testdata/Dockerfile
vendored
@@ -1,5 +0,0 @@
|
||||
FROM scratch
|
||||
ENV PATH="/this/path/does/not/exists/anywhere:/this/either"
|
||||
ENV SOME_RANDOM_VAR=""
|
||||
ENV ANOTHER_ONE="BUT_I_HAVE_VALUE"
|
||||
ENV CONFLICT_VAR="I_EXIST_ONLY_HERE"
|
||||
1
act/container/testdata/scratch/test.txt
vendored
1
act/container/testdata/scratch/test.txt
vendored
@@ -1 +0,0 @@
|
||||
testfile
|
||||
85
act/container/volume_spec.go
Normal file
85
act/container/volume_spec.go
Normal file
@@ -0,0 +1,85 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
)
|
||||
|
||||
type VolumeSpec struct {
|
||||
Type mount.Type
|
||||
Source string
|
||||
Target string
|
||||
ReadOnly bool
|
||||
NoCopy bool
|
||||
}
|
||||
|
||||
// ParseVolume parses spec like `docker run -v` does, without knowing the target platform.
|
||||
func ParseVolume(spec string) (VolumeSpec, error) {
|
||||
switch len(spec) {
|
||||
case 0:
|
||||
return VolumeSpec{}, errors.New("invalid empty volume spec")
|
||||
case 1, 2:
|
||||
return VolumeSpec{Type: mount.TypeVolume, Target: spec}, nil
|
||||
}
|
||||
|
||||
spec = string([]rune(spec)) // each invalid UTF-8 byte becomes U+FFFD, as in the JSON the daemon receives
|
||||
var parts []string
|
||||
for rest := spec; ; {
|
||||
start := 0
|
||||
if hasDriveLetter(rest) {
|
||||
start = strings.IndexByte(rest, ':') + 1
|
||||
}
|
||||
end := strings.IndexByte(rest[start:], ':')
|
||||
if end < 0 {
|
||||
parts = append(parts, rest)
|
||||
break
|
||||
}
|
||||
parts = append(parts, rest[:start+end])
|
||||
rest = rest[start+end+1:]
|
||||
}
|
||||
if len(parts) > 3 {
|
||||
return VolumeSpec{}, fmt.Errorf("invalid spec: %s: too many colons", spec)
|
||||
}
|
||||
if slices.Contains(parts, "") {
|
||||
return VolumeSpec{}, fmt.Errorf("invalid spec: %s: empty section between colons", spec)
|
||||
}
|
||||
|
||||
volume := VolumeSpec{Type: mount.TypeVolume, Target: parts[0]}
|
||||
if len(parts) > 1 {
|
||||
volume.Source, volume.Target = parts[0], parts[1]
|
||||
if isHostPath(volume.Source) {
|
||||
volume.Type = mount.TypeBind
|
||||
}
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
for option := range strings.SplitSeq(parts[2], ",") {
|
||||
switch option {
|
||||
case "ro":
|
||||
volume.ReadOnly = true
|
||||
case "rw":
|
||||
volume.ReadOnly = false
|
||||
case "nocopy":
|
||||
volume.NoCopy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return volume, nil
|
||||
}
|
||||
|
||||
func hasDriveLetter(s string) bool {
|
||||
letter, size := utf8.DecodeRuneInString(s)
|
||||
return size < len(s) && s[size] == ':' && unicode.IsLetter(letter)
|
||||
}
|
||||
|
||||
func isHostPath(source string) bool {
|
||||
return strings.ContainsAny(source[:1], "./~") || strings.HasPrefix(source, `\\`) || hasDriveLetter(source)
|
||||
}
|
||||
18
act/container/volume_spec_test.go
Normal file
18
act/container/volume_spec_test.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestParseVolumeReplacesInvalidUTF8LikeTheDaemonSees(t *testing.T) {
|
||||
parsed, err := ParseVolume("/data/\xff\xfe:/mnt:ro")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, VolumeSpec{Type: mount.TypeBind, Source: "/data/<2F><>", Target: "/mnt", ReadOnly: true}, parsed)
|
||||
}
|
||||
@@ -16,10 +16,10 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
git "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/index"
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
type Handler interface {
|
||||
@@ -43,7 +43,6 @@ func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
|
||||
// update the name to correctly reflect the desired destination when untaring
|
||||
header.Name = path.Join(tc.DstDir, fpath)
|
||||
header.Mode = int64(fi.Mode())
|
||||
header.ModTime = fi.ModTime()
|
||||
header.Uid = tc.UID
|
||||
header.Gid = tc.GID
|
||||
|
||||
@@ -58,10 +57,8 @@ func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string,
|
||||
}
|
||||
|
||||
// copy file data into tar writer
|
||||
if _, err := io.Copy(tc.TarWriter, f); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err = io.Copy(tc.TarWriter, f)
|
||||
return err
|
||||
}
|
||||
|
||||
type CopyCollector struct {
|
||||
@@ -87,80 +84,67 @@ func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string
|
||||
return err
|
||||
}
|
||||
defer df.Close()
|
||||
if _, err := io.Copy(df, f); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
_, err = io.Copy(df, f)
|
||||
return err
|
||||
}
|
||||
|
||||
type FileCollector struct {
|
||||
Ignorer gitignore.Matcher
|
||||
SrcPath string
|
||||
SrcPrefix string
|
||||
SkipGitDir bool
|
||||
Handler Handler
|
||||
SrcPath string
|
||||
SrcPrefix string
|
||||
SkipGitDir bool
|
||||
UseGitIgnore bool
|
||||
Handler Handler
|
||||
}
|
||||
|
||||
func openGitIndex(path string) (*index.Index, error) {
|
||||
repo, err := git.PlainOpen(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// foldPath compares git's spelling of a path with the filesystem's, which may differ in case or Unicode normalization.
|
||||
func foldPath(name string) string {
|
||||
return strings.ToLower(norm.NFC.String(name))
|
||||
}
|
||||
|
||||
func (fc *FileCollector) gitFiles(ctx context.Context) map[string]bool {
|
||||
if !fc.UseGitIgnore {
|
||||
return nil
|
||||
}
|
||||
return repo.Storer.Index()
|
||||
files, err := git.ListFiles(ctx, fc.SrcPath)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Debugf("Copying %s without applying .gitignore: %v", fc.SrcPath, err)
|
||||
return nil
|
||||
}
|
||||
keep := map[string]bool{".": true}
|
||||
for _, file := range files {
|
||||
for file = foldPath(file); !keep[file]; file = path.Dir(file) {
|
||||
keep[file] = true
|
||||
}
|
||||
}
|
||||
return keep
|
||||
}
|
||||
|
||||
func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc {
|
||||
i, _ := openGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...)))
|
||||
func (fc *FileCollector) CollectFiles(ctx context.Context) filepath.WalkFunc {
|
||||
keep := fc.gitFiles(ctx)
|
||||
return func(file string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
if ctx.Err() != nil {
|
||||
return errors.New("copy cancelled")
|
||||
}
|
||||
|
||||
sansPrefix := strings.TrimPrefix(file, fc.SrcPrefix)
|
||||
split := strings.Split(sansPrefix, string(filepath.Separator))
|
||||
// The root folders should be skipped, submodules only have the last path component set to "." by filepath.Walk
|
||||
if fi.IsDir() && len(split) > 0 && split[len(split)-1] == "." {
|
||||
return nil
|
||||
}
|
||||
if fc.SkipGitDir && split[len(split)-1] == ".git" {
|
||||
if fc.SkipGitDir && fi.Name() == ".git" {
|
||||
if fi.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var entry *index.Entry
|
||||
if i != nil {
|
||||
entry, err = i.Entry(strings.Join(split[len(submodulePath):], "/"))
|
||||
} else {
|
||||
err = index.ErrEntryNotFound
|
||||
}
|
||||
if err != nil && fc.Ignorer != nil && fc.Ignorer.Match(split, fi.IsDir()) {
|
||||
if fi.IsDir() {
|
||||
if i != nil {
|
||||
ms, err := i.Glob(strings.Join(append(split[len(submodulePath):], "**"), "/"))
|
||||
if err != nil || len(ms) == 0 {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
} else {
|
||||
if keep != nil {
|
||||
rel, _ := filepath.Rel(fc.SrcPath, file)
|
||||
if rel = filepath.ToSlash(rel); !keep[foldPath(rel)] && !strings.Contains("/"+rel+"/", "/.git/") {
|
||||
if fi.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err == nil && entry.Mode == filemode.Submodule {
|
||||
err = filepath.Walk(file, fc.CollectFiles(ctx, split))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filepath.SkipDir
|
||||
}
|
||||
path := filepath.ToSlash(sansPrefix)
|
||||
path := filepath.ToSlash(strings.TrimPrefix(file, fc.SrcPrefix))
|
||||
|
||||
// return on non-regular files (thanks to [kumo](https://medium.com/@komuw/just-like-you-did-fbdd7df829d3) for this suggested update)
|
||||
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
linkName, err := os.Readlink(file)
|
||||
if err != nil {
|
||||
@@ -171,17 +155,12 @@ func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []strin
|
||||
return nil
|
||||
}
|
||||
|
||||
// open file
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if ctx != nil {
|
||||
stop := context.AfterFunc(ctx, func() { _ = f.Close() })
|
||||
defer stop()
|
||||
}
|
||||
defer context.AfterFunc(ctx, func() { _ = f.Close() })()
|
||||
|
||||
return fc.Handler.WriteFile(path, fi, "", f)
|
||||
}
|
||||
|
||||
@@ -9,106 +9,78 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
git "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIgnoredTrackedfile(t *testing.T) {
|
||||
func gitCmd(t *testing.T, args ...string) {
|
||||
t.Helper()
|
||||
out, err := exec.Command("git", args...).CombinedOutput()
|
||||
require.NoError(t, err, string(out))
|
||||
}
|
||||
|
||||
func TestCollectFilesKeepsWhatGitTracksOrDoesNotIgnore(t *testing.T) {
|
||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
||||
repo, err := git.PlainInit(repoDir, false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte(".*\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
|
||||
worktree, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
_, err = worktree.Add(".gitignore")
|
||||
require.NoError(t, err)
|
||||
for _, dir := range []string{repoDir, filepath.Join(repoDir, "sub"), filepath.Join(repoDir, "nested")} {
|
||||
gitCmd(t, "init", "-q", dir)
|
||||
}
|
||||
for name, body := range map[string]string{
|
||||
".gitignore": ".*\nign/\n",
|
||||
".env": "",
|
||||
"Renamed": "",
|
||||
"ign/tracked": "",
|
||||
"ign/untracked": "",
|
||||
"sub/.gitignore": "dropped\n",
|
||||
"sub/dropped": "",
|
||||
"sub/kept": "",
|
||||
"nested/kept": "",
|
||||
} {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(filepath.Join(repoDir, name)), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, name), []byte(body), 0o644))
|
||||
}
|
||||
require.NoError(t, os.Mkdir(filepath.Join(repoDir, "uninitialized"), 0o755))
|
||||
gitCmd(t, "-C", repoDir, "add", "-f", ".gitignore", "Renamed", "ign/tracked")
|
||||
require.NoError(t, os.Rename(filepath.Join(repoDir, "Renamed"), filepath.Join(repoDir, "renamed")))
|
||||
for _, gitlink := range []string{"sub", "uninitialized"} {
|
||||
gitCmd(t, "-C", repoDir, "update-index", "--add", "--cacheinfo", "160000,4b825dc642cb6eb9a060e54bf8d69288fbee4904,"+gitlink)
|
||||
}
|
||||
expected := map[string]string{".gitignore": "", "ign/tracked": "", "nested/kept": "", "renamed": "", "sub/.gitignore": "", "sub/kept": ""}
|
||||
if runtime.GOOS != "windows" {
|
||||
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
||||
expected["test.env"] = ".env"
|
||||
}
|
||||
|
||||
var archive bytes.Buffer
|
||||
tw := tar.NewWriter(&archive)
|
||||
patterns, err := gitignore.ReadPatterns(worktree.Filesystem, nil)
|
||||
require.NoError(t, err)
|
||||
ignorer := gitignore.NewMatcher(patterns)
|
||||
fc := &FileCollector{
|
||||
Ignorer: ignorer,
|
||||
SrcPath: repoDir,
|
||||
SrcPrefix: repoDir + string(filepath.Separator),
|
||||
Handler: &TarCollector{
|
||||
TarWriter: tw,
|
||||
},
|
||||
}
|
||||
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
|
||||
assert.NoError(t, err, "successfully collect files")
|
||||
fc := &FileCollector{SrcPath: repoDir, SrcPrefix: repoDir + string(filepath.Separator), UseGitIgnore: true, Handler: &TarCollector{TarWriter: tw}}
|
||||
require.NoError(t, filepath.Walk(repoDir, fc.CollectFiles(t.Context())))
|
||||
require.NoError(t, tw.Close())
|
||||
|
||||
files := map[string]string{}
|
||||
tr := tar.NewReader(&archive)
|
||||
h, err := tr.Next()
|
||||
assert.NoError(t, err, "tar must not be empty") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, ".gitignore", h.Name)
|
||||
_, err = tr.Next()
|
||||
assert.ErrorIs(t, err, io.EOF, "tar must only contain one element")
|
||||
for header, err := tr.Next(); err != io.EOF; header, err = tr.Next() {
|
||||
require.NoError(t, err)
|
||||
files[header.Name] = header.Linkname
|
||||
}
|
||||
assert.Contains(t, files, ".git/HEAD")
|
||||
maps.DeleteFunc(files, func(name, _ string) bool { return strings.Contains("/"+name, "/.git/") })
|
||||
assert.Equal(t, expected, files)
|
||||
}
|
||||
|
||||
func TestSymlinks(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("creating symlinks requires elevated privileges on Windows")
|
||||
}
|
||||
repoDir := filepath.Join(t.TempDir(), "mygitrepo")
|
||||
repo, err := git.PlainInit(repoDir, false)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".env"), []byte("test=val1\n"), 0o644))
|
||||
require.NoError(t, os.Symlink(".env", filepath.Join(repoDir, "test.env")))
|
||||
worktree, err := repo.Worktree()
|
||||
require.NoError(t, err)
|
||||
_, err = worktree.Add("test.env")
|
||||
require.NoError(t, err)
|
||||
|
||||
var archive bytes.Buffer
|
||||
tw := tar.NewWriter(&archive)
|
||||
fc := &FileCollector{
|
||||
SrcPath: repoDir,
|
||||
SrcPrefix: repoDir + string(filepath.Separator),
|
||||
Handler: &TarCollector{
|
||||
TarWriter: tw,
|
||||
},
|
||||
}
|
||||
err = filepath.Walk(repoDir, fc.CollectFiles(context.Background(), nil))
|
||||
assert.NoError(t, err, "successfully collect files")
|
||||
require.NoError(t, tw.Close())
|
||||
tr := tar.NewReader(&archive)
|
||||
h, err := tr.Next()
|
||||
files := map[string]tar.Header{}
|
||||
for err == nil {
|
||||
files[h.Name] = *h
|
||||
h, err = tr.Next()
|
||||
}
|
||||
|
||||
assert.Equal(t, ".env", files[".env"].Name)
|
||||
assert.Equal(t, "test.env", files["test.env"].Name)
|
||||
assert.Equal(t, ".env", files["test.env"].Linkname)
|
||||
assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF")
|
||||
}
|
||||
|
||||
// Regression for https://gitea.com/gitea/runner/issues/876 and /941:
|
||||
// re-copying an action directory must overwrite a pre-existing read-only
|
||||
// file (e.g. a git pack .idx at mode 0444) instead of failing with EACCES
|
||||
// on macOS or "Access is denied" on Windows.
|
||||
func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
|
||||
dst := t.TempDir()
|
||||
target := filepath.Join(dst, "sub", "pack.idx")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o755))
|
||||
require.NoError(t, os.WriteFile(target, []byte("old"), 0o444))
|
||||
|
||||
src := filepath.Join(t.TempDir(), "pack.idx")
|
||||
require.NoError(t, os.WriteFile(src, []byte("new"), 0o444))
|
||||
fi, err := os.Stat(src)
|
||||
fi, err := os.Stat(target)
|
||||
require.NoError(t, err)
|
||||
|
||||
cc := &CopyCollector{DstDir: dst}
|
||||
@@ -119,8 +91,6 @@ func TestCopyCollectorWriteFileOverwritesReadOnlyFile(t *testing.T) {
|
||||
assert.Equal(t, "new", string(got))
|
||||
}
|
||||
|
||||
// Without the destination removal, os.Symlink fails with EEXIST when the
|
||||
// path already holds a regular file from an earlier copy of the action.
|
||||
func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("creating symlinks requires elevated privileges on Windows")
|
||||
@@ -129,11 +99,8 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
||||
target := filepath.Join(dst, "link")
|
||||
require.NoError(t, os.WriteFile(target, []byte("stale"), 0o644))
|
||||
|
||||
fi, err := os.Lstat(target)
|
||||
require.NoError(t, err)
|
||||
|
||||
cc := &CopyCollector{DstDir: dst}
|
||||
require.NoError(t, cc.WriteFile("link", fi, "target", nil))
|
||||
require.NoError(t, cc.WriteFile("link", nil, "target", nil))
|
||||
|
||||
resolved, err := os.Readlink(target)
|
||||
require.NoError(t, err)
|
||||
@@ -143,7 +110,7 @@ func TestCopyCollectorWriteFileOverwritesFileWithSymlink(t *testing.T) {
|
||||
func TestFileCollectorCancellationAndWalkError(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
walk := (&FileCollector{}).CollectFiles(ctx, nil)
|
||||
walk := (&FileCollector{}).CollectFiles(ctx)
|
||||
|
||||
err := walk("file", nil, nil)
|
||||
require.EqualError(t, err, "copy cancelled")
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
@@ -25,6 +26,26 @@ var (
|
||||
findGithubRepo = git.FindGithubRepo
|
||||
)
|
||||
|
||||
type pinnedKey struct{}
|
||||
|
||||
// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves.
|
||||
func WithPinnedCheckout(ctx context.Context) context.Context {
|
||||
if ctx.Value(pinnedKey{}) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, pinnedKey{}, &sync.Map{})
|
||||
}
|
||||
|
||||
func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) {
|
||||
cache, ok := ctx.Value(pinnedKey{}).(*sync.Map)
|
||||
if !ok {
|
||||
return lookup()
|
||||
}
|
||||
cached, _ := cache.LoadOrStore(key, sync.OnceValues(lookup))
|
||||
lookupOnce, _ := cached.(func() (string, error))
|
||||
return lookupOnce()
|
||||
}
|
||||
|
||||
// SetRef resolves the ref of the context from its event payload, falling back
|
||||
// to the ref checked out in repoPath.
|
||||
func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
@@ -51,9 +72,9 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
}
|
||||
|
||||
if ghc.Ref == "" {
|
||||
ref, err := findGitRef(ctx, repoPath)
|
||||
ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) })
|
||||
if err != nil {
|
||||
logger.Warningf("unable to get git ref: %v", err)
|
||||
logger.Warnf("unable to get git ref: %v", err)
|
||||
} else {
|
||||
logger.Debugf("using github ref: %s", ref)
|
||||
ghc.Ref = ref
|
||||
@@ -95,9 +116,11 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
}
|
||||
|
||||
if ghc.Sha == "" {
|
||||
_, sha, err := findGitRevision(ctx, repoPath)
|
||||
sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) {
|
||||
return findGitRevision(ctx, repoPath)
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warningf("unable to get git revision: %v", err)
|
||||
logger.Warnf("unable to get git revision: %v", err)
|
||||
} else {
|
||||
ghc.Sha = sha
|
||||
}
|
||||
@@ -106,14 +129,14 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
|
||||
// SetRepositoryAndOwner resolves the repository of the context from the git
|
||||
// remote in repoPath when it is not set yet, and derives its owner.
|
||||
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, githubInstance, repoPath string) {
|
||||
func SetRepositoryAndOwner(ctx context.Context, ghc *model.GithubContext, repoPath string) {
|
||||
if ghc.Repository == "" {
|
||||
repo, err := findGithubRepo(ctx, repoPath, githubInstance)
|
||||
repo, err := findGithubRepo(ctx, repoPath)
|
||||
if err != nil {
|
||||
common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v, repoPath: %v): %v", githubInstance, repoPath, err)
|
||||
common.Logger(ctx).Warnf("unable to get git repo (repoPath: %v): %v", repoPath, err)
|
||||
return
|
||||
}
|
||||
ghc.Repository = repo
|
||||
}
|
||||
ghc.RepositoryOwner = strings.Split(ghc.Repository, "/")[0]
|
||||
ghc.RepositoryOwner, _, _ = strings.Cut(ghc.Repository, "/")
|
||||
}
|
||||
|
||||
@@ -10,26 +10,17 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSetRef(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
oldFindGitRef := findGitRef
|
||||
oldFindGitRevision := findGitRevision
|
||||
defer func() { findGitRef = oldFindGitRef }()
|
||||
defer func() { findGitRevision = oldFindGitRevision }()
|
||||
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "refs/heads/master", nil
|
||||
}
|
||||
|
||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
||||
return "", "1234fakesha", nil
|
||||
}
|
||||
|
||||
tables := []struct {
|
||||
eventName string
|
||||
event map[string]any
|
||||
@@ -104,7 +95,7 @@ func TestSetRef(t *testing.T) {
|
||||
Event: table.event,
|
||||
}
|
||||
|
||||
SetRef(context.Background(), ghc, "/some/dir")
|
||||
SetRef(t.Context(), ghc, "/some/dir")
|
||||
ghc.SetRefTypeAndName()
|
||||
|
||||
assert.Equal(t, table.ref, ghc.Ref)
|
||||
@@ -113,7 +104,7 @@ func TestSetRef(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Run("no-default-branch", func(t *testing.T) {
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
findGitRef = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "", errors.New("no default branch")
|
||||
}
|
||||
|
||||
@@ -122,26 +113,20 @@ func TestSetRef(t *testing.T) {
|
||||
Event: map[string]any{},
|
||||
}
|
||||
|
||||
SetRef(context.Background(), ghc, "/some/dir")
|
||||
SetRef(t.Context(), ghc, "/some/dir")
|
||||
|
||||
assert.Equal(t, "refs/heads/master", ghc.Ref)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetSha(t *testing.T) {
|
||||
log.SetLevel(log.DebugLevel)
|
||||
|
||||
oldFindGitRef := findGitRef
|
||||
oldFindGitRevision := findGitRevision
|
||||
defer func() { findGitRef = oldFindGitRef }()
|
||||
defer func() { findGitRevision = oldFindGitRevision }()
|
||||
|
||||
findGitRef = func(ctx context.Context, file string) (string, error) {
|
||||
return "refs/heads/master", nil
|
||||
}
|
||||
|
||||
findGitRevision = func(ctx context.Context, file string) (string, string, error) {
|
||||
return "", "1234fakesha", nil
|
||||
findGitRevision = func(_ctx context.Context, _file string) (string, error) {
|
||||
return "1234fakesha", nil
|
||||
}
|
||||
|
||||
tables := []struct {
|
||||
@@ -176,11 +161,6 @@ func TestSetSha(t *testing.T) {
|
||||
},
|
||||
sha: "deployment-sha",
|
||||
},
|
||||
{
|
||||
eventName: "release",
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
{
|
||||
eventName: "push",
|
||||
event: map[string]any{
|
||||
@@ -194,24 +174,31 @@ func TestSetSha(t *testing.T) {
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
{
|
||||
eventName: "no-event",
|
||||
event: map[string]any{},
|
||||
sha: "1234fakesha",
|
||||
},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
t.Run(table.eventName, func(t *testing.T) {
|
||||
ghc := &model.GithubContext{
|
||||
EventName: table.eventName,
|
||||
BaseRef: "master",
|
||||
Event: table.event,
|
||||
}
|
||||
|
||||
SetSha(context.Background(), ghc, "/some/dir")
|
||||
SetSha(t.Context(), ghc, "/some/dir")
|
||||
|
||||
assert.Equal(t, table.sha, ghc.Sha)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) {
|
||||
ctx := WithPinnedCheckout(t.Context())
|
||||
for _, head := range []string{"first", "moved"} {
|
||||
findGitRevision = func(context.Context, string) (string, error) { return head, nil }
|
||||
findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil }
|
||||
ghc := &model.GithubContext{Event: map[string]any{}}
|
||||
SetSha(ctx, ghc, "/some/dir")
|
||||
SetRef(ctx, ghc, "/some/dir")
|
||||
assert.Equal(t, "first", ghc.Sha)
|
||||
assert.Equal(t, "refs/heads/first", ghc.Ref)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,35 +13,23 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLookPath2SearchesPathAndEmptyElement(t *testing.T) {
|
||||
func TestLookPath2SearchesPathAndEmptyElementOrUsesDirectPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
exe := filepath.Join(dir, "tool")
|
||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := LookPath2("tool", map[string]string{"PATH": string(filepath.ListSeparator) + dir})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLookPath2DirectPathDoesNotSearchPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
exe := filepath.Join(dir, "tool")
|
||||
if err := os.WriteFile(exe, []byte("#!/bin/sh\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := LookPath2(exe, map[string]string{"PATH": ""})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
for file, searchPath := range map[string]string{"tool": string(filepath.ListSeparator) + dir, exe: ""} {
|
||||
t.Run(file, func(t *testing.T) {
|
||||
got, err := LookPath2(file, map[string]string{"PATH": searchPath})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != exe {
|
||||
t.Fatalf("LookPath2() = %q, want %q", got, exe)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,21 +28,15 @@ func chkStat(file string) error {
|
||||
}
|
||||
|
||||
func hasExt(file string) bool {
|
||||
i := strings.LastIndex(file, ".")
|
||||
if i < 0 {
|
||||
return false
|
||||
}
|
||||
return strings.LastIndexAny(file, `:\/`) < i
|
||||
return strings.LastIndex(file, ".") > strings.LastIndexAny(file, `:\/`)
|
||||
}
|
||||
|
||||
func findExecutable(file string, exts []string) (string, error) {
|
||||
if len(exts) == 0 {
|
||||
return file, chkStat(file)
|
||||
}
|
||||
if hasExt(file) {
|
||||
if chkStat(file) == nil {
|
||||
return file, nil
|
||||
}
|
||||
if hasExt(file) && chkStat(file) == nil {
|
||||
return file, nil
|
||||
}
|
||||
for _, e := range exts {
|
||||
if f := file + e; chkStat(f) == nil {
|
||||
|
||||
@@ -6,13 +6,14 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"embed"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"maps"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -24,6 +25,8 @@ import (
|
||||
"gitea.com/gitea/runner/act/common/git"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
_ "embed"
|
||||
|
||||
"gitea.dev/actionslib/pkg/model"
|
||||
"github.com/kballard/go-shellquote"
|
||||
)
|
||||
@@ -45,11 +48,13 @@ type fileWriter func(filename string, data []byte, perm fs.FileMode) error
|
||||
type runAction func(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor
|
||||
|
||||
//go:embed res/trampoline.js
|
||||
var trampoline embed.FS
|
||||
var trampoline string
|
||||
|
||||
var (
|
||||
ContainerImageExistsLocally = container.ImageExistsLocally
|
||||
ContainerNewDockerBuildExecutor = container.NewDockerBuildExecutor
|
||||
dockerActionNamePattern = regexp.MustCompile("[^a-zA-Z0-9]")
|
||||
inputEnvPattern = regexp.MustCompile("[^A-Z0-9-]")
|
||||
)
|
||||
|
||||
func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) {
|
||||
@@ -83,38 +88,31 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
||||
logger.Debugf("Using synthetic action %v for Dockerfile", action)
|
||||
return action, nil
|
||||
}
|
||||
if step.With != nil {
|
||||
if val, ok := step.With["args"]; ok {
|
||||
var b []byte
|
||||
if b, err = trampoline.ReadFile("res/trampoline.js"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err2 := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), b, 0o400)
|
||||
if err2 != nil {
|
||||
return nil, err2
|
||||
}
|
||||
action := &model.Action{
|
||||
Name: "(Synthetic)",
|
||||
Inputs: map[string]model.Input{
|
||||
"cwd": {
|
||||
Description: "(Actual working directory)",
|
||||
Required: false,
|
||||
Default: filepath.Join(actionDir, actionPath),
|
||||
},
|
||||
"command": {
|
||||
Description: "(Actual program)",
|
||||
Required: false,
|
||||
Default: val,
|
||||
},
|
||||
},
|
||||
Runs: model.ActionRuns{
|
||||
Using: "node12",
|
||||
Main: "trampoline.js",
|
||||
},
|
||||
}
|
||||
logger.Debugf("Using synthetic action %v", action)
|
||||
return action, nil
|
||||
if val, ok := step.With["args"]; ok {
|
||||
if err := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), []byte(trampoline), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
action := &model.Action{
|
||||
Name: "(Synthetic)",
|
||||
Inputs: map[string]model.Input{
|
||||
"cwd": {
|
||||
Description: "(Actual working directory)",
|
||||
Required: false,
|
||||
Default: filepath.Join(actionDir, actionPath),
|
||||
},
|
||||
"command": {
|
||||
Description: "(Actual program)",
|
||||
Required: false,
|
||||
Default: val,
|
||||
},
|
||||
},
|
||||
Runs: model.ActionRuns{
|
||||
Using: "node12",
|
||||
Main: "trampoline.js",
|
||||
},
|
||||
}
|
||||
logger.Debugf("Using synthetic action %v", action)
|
||||
return action, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -123,8 +121,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
||||
}
|
||||
defer closer.Close()
|
||||
|
||||
action, err := model.ReadAction(reader)
|
||||
return action, err
|
||||
return model.ReadAction(reader)
|
||||
}
|
||||
|
||||
func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actionPath, containerActionDir string) error {
|
||||
@@ -143,18 +140,18 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
|
||||
containerActionDirCopy += `/`
|
||||
}
|
||||
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
|
||||
if !rc.Config.NoActionPatch {
|
||||
// A concurrent job's prepare resets this directory, so patch under the copy's lock.
|
||||
patchActions(ctx, actionScriptPaths(filepath.Join(actionDir, actionPath), step.getActionModel()))
|
||||
}
|
||||
|
||||
if err := removeGitIgnore(ctx, actionDir); err != nil {
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", rc.Config.UseGitIgnore, true)(ctx)
|
||||
if !rc.Config.NoActionPatch {
|
||||
// Other jobs share the directory, so patch under the copy's lock.
|
||||
defer patchActions(ctx, actionScriptPaths(actionDir, filepath.Join(actionDir, actionPath), step.getActionModel()))()
|
||||
}
|
||||
|
||||
return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", false, true)(ctx) // an exported commit has no .git to apply gitignore with
|
||||
}
|
||||
|
||||
func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor {
|
||||
@@ -164,7 +161,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
return func(ctx context.Context) error {
|
||||
logger := common.Logger(ctx)
|
||||
actionPath := ""
|
||||
if remoteAction != nil && remoteAction.Path != "" {
|
||||
if remoteAction != nil {
|
||||
actionPath = remoteAction.Path
|
||||
}
|
||||
|
||||
@@ -176,8 +173,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
return err
|
||||
}
|
||||
|
||||
actionLocation := path.Join(actionDir, actionPath)
|
||||
actionName, containerActionDir := getContainerActionPaths(stepModel, actionLocation, rc)
|
||||
actionName, containerActionDir := getContainerActionPaths(step)
|
||||
|
||||
logger.Debugf("type=%v actionDir=%s actionPath=%s workdir=%s actionCacheDir=%s actionName=%s containerActionDir=%s", stepModel.Type(), actionDir, actionPath, rc.Config.Workdir, rc.ActionCacheDir(), actionName, containerActionDir)
|
||||
|
||||
@@ -194,7 +190,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
|
||||
return rc.JobContainer.Exec(containerArgs, *step.getEnv(), "", "")(ctx)
|
||||
case x.IsDocker():
|
||||
location := actionLocation
|
||||
location := path.Join(actionDir, actionPath)
|
||||
if remoteAction == nil {
|
||||
location = containerActionDir
|
||||
}
|
||||
@@ -213,12 +209,9 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Main + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Main}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Main}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return fmt.Errorf("the runs.using key must be one of: %v, got %s", []string{
|
||||
@@ -239,23 +232,6 @@ func nodeActionCommand(script string) []string {
|
||||
return []string{"node", "--preserve-symlinks-main", script}
|
||||
}
|
||||
|
||||
// https://github.com/nektos/act/issues/228#issuecomment-629709055
|
||||
// files in .gitignore are not copied in a Docker container
|
||||
// this causes issues with actions that ignore other important resources
|
||||
// such as `node_modules` for example
|
||||
func removeGitIgnore(ctx context.Context, directory string) error {
|
||||
gitIgnorePath := path.Join(directory, ".gitignore")
|
||||
if _, err := os.Stat(gitIgnorePath); err == nil {
|
||||
// .gitignore exists
|
||||
common.Logger(ctx).Debugf("Removing %s before docker cp", gitIgnorePath)
|
||||
err := os.Remove(gitIgnorePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dockerActionImageTag derives the local docker image tag used when an action
|
||||
// is built from a Dockerfile.
|
||||
//
|
||||
@@ -274,16 +250,30 @@ func dockerActionImageTag(repository, actionName string, localAction bool) strin
|
||||
name = path.Join(repository, actionName)
|
||||
}
|
||||
// The human-readable name is sanitized by collapsing every non-alphanumeric character to "-".
|
||||
sanitized := regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(name, "-")
|
||||
sanitized := dockerActionNamePattern.ReplaceAllString(name, "-")
|
||||
if localAction {
|
||||
// For local actions a short hash of the raw repository and action path is appended so the tag stays unique per repository.
|
||||
sum := sha256.Sum256([]byte(repository + "\x00" + actionName))
|
||||
sanitized += "-" + hex.EncodeToString(sum[:])[:12]
|
||||
}
|
||||
// "-dockeraction" ensures that "./", "./test " won't get converted to "act-:latest", "act-test-:latest" which are invalid docker image names
|
||||
image := fmt.Sprintf("%s-dockeraction:%s", sanitized, "latest")
|
||||
image = "act-" + strings.TrimLeft(image, "-")
|
||||
return strings.ToLower(image)
|
||||
return strings.ToLower("act-" + strings.TrimLeft(sanitized+"-dockeraction:latest", "-"))
|
||||
}
|
||||
|
||||
// One image per step and job: its stages share a build, and concurrent jobs never run each other's.
|
||||
func (rc *RunContext) dockerActionImage(ctx context.Context, step actionStep, actionName string, localAction bool) string {
|
||||
job := rc.topLevelRunContext()
|
||||
if job.dockerActionImages == nil {
|
||||
job.dockerActionImages = map[*model.Step]string{}
|
||||
}
|
||||
image, ok := job.dockerActionImages[step.getStepModel()]
|
||||
if !ok {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction) + "-" + strings.ToLower(rand.Text())
|
||||
if !common.Dryrun(ctx) { // a dry run builds nothing to remove
|
||||
job.dockerActionImages[step.getStepModel()] = image
|
||||
}
|
||||
}
|
||||
return image
|
||||
}
|
||||
|
||||
// TODO: break out parts of function to reduce complexicity
|
||||
@@ -291,6 +281,9 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
logger := common.Logger(ctx)
|
||||
rc := step.getRunContext()
|
||||
action := step.getActionModel()
|
||||
if rc.kubernetes {
|
||||
return errKubernetesDocker
|
||||
}
|
||||
|
||||
var prepImage common.Executor
|
||||
var image string
|
||||
@@ -300,7 +293,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
// Apply forcePull only for prebuild docker images
|
||||
forcePull = rc.Config.ForcePull
|
||||
} else {
|
||||
image = dockerActionImageTag(step.getGithubContext(ctx).Repository, actionName, localAction)
|
||||
image = rc.dockerActionImage(ctx, step, actionName, localAction)
|
||||
contextDir, fileName := filepath.Split(filepath.Join(basedir, action.Runs.Image))
|
||||
|
||||
anyArchExists, err := ContainerImageExistsLocally(ctx, image, "any")
|
||||
@@ -345,7 +338,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
// Held across the whole build: the daemon drains contextDir lazily.
|
||||
inner := prepImage
|
||||
prepImage = func(ctx context.Context) error {
|
||||
defer git.AcquireCloneLock(actionDir)()
|
||||
unlock, err := git.AcquireCloneLock(ctx, actionDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
return inner(ctx)
|
||||
}
|
||||
}
|
||||
@@ -353,12 +350,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
logger.Debugf("image '%s' for architecture '%s' already exists", image, rc.Config.ContainerArchitecture)
|
||||
}
|
||||
}
|
||||
eval := rc.NewActionInputsExpressionEvaluator(ctx, step)
|
||||
args, err := eval.Interpolate(ctx, step.getStepModel().With["args"])
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to interpolate with.args: %w", err)
|
||||
}
|
||||
cmd, err := shellquote.Split(args)
|
||||
cmd, err := shellquote.Split(step.getStepModel().With["args"])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -375,14 +367,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
}
|
||||
}
|
||||
}
|
||||
entrypoint, err := dockerEntrypoint(ctx, step, eval, stage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stepContainer, err := newStepContainer(ctx, step, image, cmd, entrypoint, rc.Config.ContainerOptions)
|
||||
entrypoint, err := dockerEntrypoint(step, stage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stepContainer := newStepContainer(ctx, step, image, cmd, entrypoint, rc.Config.ContainerOptions)
|
||||
return common.NewPipelineExecutor(
|
||||
prepImage,
|
||||
stepContainer.Pull(forcePull),
|
||||
@@ -394,7 +383,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
|
||||
// dockerEntrypoint returns the entrypoint the action's image runs with for the given
|
||||
// stage. Only the main stage honours the `entrypoint` input.
|
||||
func dockerEntrypoint(ctx context.Context, step actionStep, eval *expressionEvaluator, stage stepStage) ([]string, error) {
|
||||
func dockerEntrypoint(step actionStep, stage stepStage) ([]string, error) {
|
||||
runs := step.getActionModel().Runs
|
||||
|
||||
var entrypoint string
|
||||
@@ -406,11 +395,7 @@ func dockerEntrypoint(ctx context.Context, step actionStep, eval *expressionEval
|
||||
default:
|
||||
entrypoint = runs.Entrypoint
|
||||
if entrypoint == "" {
|
||||
withEntrypoint, err := eval.Interpolate(ctx, step.getStepModel().With["entrypoint"])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to interpolate with.entrypoint: %w", err)
|
||||
}
|
||||
if fields := strings.Fields(withEntrypoint); len(fields) > 0 {
|
||||
if fields := strings.Fields(step.getStepModel().With["entrypoint"]); len(fields) > 0 {
|
||||
return fields, nil
|
||||
}
|
||||
}
|
||||
@@ -425,7 +410,6 @@ func dockerEntrypoint(ctx context.Context, step actionStep, eval *expressionEval
|
||||
// evalDockerEnv returns an evaluator bound to the environment it installed.
|
||||
func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expressionEvaluator, error) {
|
||||
rc := step.getRunContext()
|
||||
stepModel := step.getStepModel()
|
||||
|
||||
var err error
|
||||
inputs := make(map[string]string)
|
||||
@@ -436,40 +420,33 @@ func evalDockerEnv(ctx context.Context, step step, action *model.Action) (*expre
|
||||
return nil, fmt.Errorf("unable to interpolate the default of input %s: %w", k, err)
|
||||
}
|
||||
}
|
||||
for k, v := range stepModel.With {
|
||||
if inputs[k], err = eval.Interpolate(ctx, v); err != nil {
|
||||
return nil, fmt.Errorf("unable to interpolate with.%s: %w", k, err)
|
||||
}
|
||||
}
|
||||
mergeIntoMap(step, step.getEnv(), inputs)
|
||||
|
||||
env := make(map[string]string, len(action.Runs.Env)+len(*step.getEnv()))
|
||||
mergeIntoMap(step, &env, action.Runs.Env, *step.getEnv())
|
||||
*step.getEnv() = env
|
||||
mergeIntoMap(step, step.getEnv(), inputs, step.getStepModel().With)
|
||||
|
||||
ee := rc.NewActionInputsExpressionEvaluator(ctx, step)
|
||||
for k, v := range *step.getEnv() {
|
||||
if (*step.getEnv())[k], err = ee.Interpolate(ctx, v); err != nil {
|
||||
runsEnv := make(map[string]string, len(action.Runs.Env))
|
||||
for k, v := range action.Runs.Env {
|
||||
if runsEnv[k], err = ee.Interpolate(ctx, v); err != nil {
|
||||
return nil, fmt.Errorf("unable to interpolate env %s: %w", k, err)
|
||||
}
|
||||
}
|
||||
mergeIntoMap(step, step.getEnv(), runsEnv, maps.Clone(*step.getEnv()))
|
||||
return ee, nil
|
||||
}
|
||||
|
||||
func newStepContainer(ctx context.Context, step step, image string, cmd, entrypoint []string, runnerOptions string) (container.Container, error) {
|
||||
func newStepContainer(ctx context.Context, step step, image string, cmd, entrypoint []string, runnerOptions string) container.Container {
|
||||
rc := step.getRunContext()
|
||||
logWriter := rc.commandLogWriter(ctx)
|
||||
envList := make([]string, 0)
|
||||
for k, v := range *step.getEnv() {
|
||||
if rc.IsHostEnv() && strings.EqualFold(k, rc.JobContainer.GetPathVariableName()) && v == rc.JobContainer.DefaultPathVariable() { // the runner's own PATH stays out, as on GitHub
|
||||
continue
|
||||
}
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
envList = append(envList, rc.runnerEnv(ctx)...)
|
||||
|
||||
binds, mounts, err := rc.GetBindsAndMounts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
binds, mounts := rc.GetBindsAndMounts(ctx)
|
||||
networkMode := "container:" + rc.jobContainerName()
|
||||
if rc.IsHostEnv() {
|
||||
networkMode = "default"
|
||||
@@ -493,24 +470,19 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
|
||||
AutoRemove: true,
|
||||
ValidVolumes: rc.validVolumes(),
|
||||
AllocatePTY: rc.Config.AllocatePTY,
|
||||
}), nil
|
||||
})
|
||||
}
|
||||
|
||||
func populateEnvsFromSavedState(env *map[string]string, step actionStep, rc *RunContext) {
|
||||
state, ok := rc.IntraActionState[step.getStepModel().ID]
|
||||
if ok {
|
||||
for name, value := range state {
|
||||
envName := "STATE_" + name
|
||||
(*env)[envName] = value
|
||||
}
|
||||
for name, value := range rc.IntraActionState[step.getStepModel().ID] {
|
||||
(*env)["STATE_"+name] = value
|
||||
}
|
||||
}
|
||||
|
||||
func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *model.Action, rc *RunContext) error {
|
||||
eval := rc.NewExpressionEvaluator(ctx)
|
||||
for inputID, input := range action.Inputs {
|
||||
envKey := regexp.MustCompile("[^A-Z0-9-]").ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
envKey = "INPUT_" + envKey
|
||||
envKey := "INPUT_" + inputEnvPattern.ReplaceAllString(strings.ToUpper(inputID), "_")
|
||||
if _, ok := (*env)[envKey]; !ok {
|
||||
var err error
|
||||
if (*env)[envKey], err = eval.Interpolate(ctx, input.Default); err != nil {
|
||||
@@ -521,25 +493,14 @@ func populateEnvsFromInput(ctx context.Context, env *map[string]string, action *
|
||||
return nil
|
||||
}
|
||||
|
||||
func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) (string, string) {
|
||||
actionName := ""
|
||||
containerActionDir := "."
|
||||
if step.Type() != model.StepTypeUsesActionRemote {
|
||||
actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir)
|
||||
containerActionDir = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName
|
||||
actionName = "./" + actionName
|
||||
} else if step.Type() == model.StepTypeUsesActionRemote {
|
||||
actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir())
|
||||
containerActionDir = rc.JobContainer.GetActPath() + "/actions/" + actionName
|
||||
func getContainerActionPaths(step actionStep) (string, string) {
|
||||
rc := step.getRunContext()
|
||||
if sar, ok := step.(*stepActionRemote); ok { // a copy per action, as its repository's directory is shared
|
||||
actionName := path.Join(sar.containerKey(), sar.remoteAction.Path)
|
||||
return actionName, rc.JobContainer.GetActPath() + "/actions/" + actionName
|
||||
}
|
||||
|
||||
if actionName == "" {
|
||||
actionName = filepath.Base(actionDir)
|
||||
if runtime.GOOS == "windows" {
|
||||
actionName = strings.ReplaceAll(actionName, "\\", "/")
|
||||
}
|
||||
}
|
||||
return actionName, containerActionDir
|
||||
actionName := getOsSafeRelativePath(filepath.Join(rc.Config.Workdir, step.getStepModel().Uses), rc.Config.Workdir)
|
||||
return "./" + actionName, rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName
|
||||
}
|
||||
|
||||
func getOsSafeRelativePath(s, prefix string) string {
|
||||
@@ -585,13 +546,13 @@ func actionStagePaths(step actionStep) (actionDir, actionPath, actionName, conta
|
||||
stepModel := step.getStepModel()
|
||||
|
||||
if sar, ok := step.(*stepActionRemote); ok {
|
||||
actionDir = sar.actionDir()
|
||||
actionDir = sar.actionDir
|
||||
actionPath = sar.remoteAction.Path
|
||||
} else {
|
||||
actionDir = filepath.Join(rc.Config.Workdir, stepModel.Uses)
|
||||
}
|
||||
|
||||
actionName, containerActionDir = getContainerActionPaths(stepModel, path.Join(actionDir, actionPath), rc)
|
||||
actionName, containerActionDir = getContainerActionPaths(step)
|
||||
return actionDir, actionPath, actionName, containerActionDir
|
||||
}
|
||||
|
||||
@@ -660,12 +621,9 @@ func runPreStep(step actionStep) common.Executor {
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Pre + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Pre}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Pre}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
default:
|
||||
return nil
|
||||
@@ -676,21 +634,21 @@ func runPreStep(step actionStep) common.Executor {
|
||||
func shouldRunPostStep(step actionStep) common.Conditional {
|
||||
return func(ctx context.Context) bool {
|
||||
log := common.Logger(ctx)
|
||||
stepResults := step.getRunContext().getStepsContext()
|
||||
stepResults := step.getRunContext().StepResults
|
||||
stepResult := stepResults[step.getStepModel().ID]
|
||||
|
||||
if stepResult == nil {
|
||||
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; step was not executed", step.getStepModel())
|
||||
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; step was not executed", step.getStepModel())
|
||||
return false
|
||||
}
|
||||
|
||||
if stepResult.Conclusion == model.StepStatusSkipped {
|
||||
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; main step was skipped", step.getStepModel())
|
||||
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s'; main step was skipped", step.getStepModel())
|
||||
return false
|
||||
}
|
||||
|
||||
if step.getActionModel() == nil {
|
||||
log.WithField("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s': no action model available", step.getStepModel())
|
||||
log.With("stepResult", model.StepStatusSkipped).Debugf("skipping post step for '%s': no action model available", step.getStepModel())
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -754,12 +712,9 @@ func runPostStep(step actionStep) common.Executor {
|
||||
rc.ApplyExtraPath(ctx, step.getEnv())
|
||||
|
||||
execFileName := action.Runs.Post + ".out"
|
||||
buildArgs := []string{"go", "build", "-o", execFileName, action.Runs.Post}
|
||||
execArgs := []string{filepath.Join(containerActionDir, execFileName)}
|
||||
|
||||
return common.NewPipelineExecutor(
|
||||
rc.JobContainer.Exec(buildArgs, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec(execArgs, *step.getEnv(), "", ""),
|
||||
rc.JobContainer.Exec([]string{"go", "build", "-o", execFileName, action.Runs.Post}, *step.getEnv(), "", containerActionDir),
|
||||
rc.JobContainer.Exec([]string{filepath.Join(containerActionDir, execFileName)}, *step.getEnv(), "", ""),
|
||||
)(ctx)
|
||||
|
||||
default:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user