From 1952e5cdb69bec152bd455d693b759539745c5c6 Mon Sep 17 00:00:00 2001 From: wmTJc9IK0Q <231276+wmtjc9ik0q@noreply.gitea.com> Date: Thu, 24 Sep 2026 15:57:18 +0000 Subject: [PATCH] feat: OpenTelemetry tracing (#1206) Adds OpenTelemetry tracing, configured through the standard `OTEL_*` environment variables and off until an OTLP endpoint is set. ![image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477) Each job becomes a trace with a span per step, so operators can see where CI time goes in their existing observability backend. Jobs receive only `TRACEPARENT`, so instrumented tools inside them join the trace without seeing the collector's credentials. Metrics stay on the Prometheus endpoint, which a collector can scrape. See [docs/telemetry.md](https://gitea.com/wmTJc9IK0Q/runner/src/branch/otel/docs/telemetry.md) for setup and security notes. Also keeps `GITHUB_SHA` and `GITHUB_REF` fixed for a whole run when HEAD moves during `exec` runs and the act tests. [image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477) [image.png](https://gitea.com/attachments/c4b49cdf-a32d-403c-94e9-30cd06ee6477) --------- Co-authored-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: silverwind Co-authored-by: bircni Reviewed-on: https://gitea.com/gitea/runner/pulls/1206 Reviewed-by: bircni Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: wmTJc9IK0Q <231276+wmtjc9ik0q@noreply.gitea.com> --- README.md | 4 + act/ghcontext/github_context.go | 28 ++- act/ghcontext/github_context_test.go | 13 ++ act/runner/cancellation_test.go | 5 +- act/runner/job_executor.go | 9 +- act/runner/runner.go | 4 +- act/runner/step.go | 14 +- act/runner/step_test.go | 29 +++ docs/telemetry.md | 46 +++++ go.mod | 10 +- go.sum | 24 ++- internal/app/cmd/daemon.go | 13 ++ internal/app/run/runner.go | 6 + internal/pkg/telemetry/telemetry.go | 240 +++++++++++++++++++++++ internal/pkg/telemetry/telemetry_test.go | 186 ++++++++++++++++++ 15 files changed, 603 insertions(+), 28 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 internal/pkg/telemetry/telemetry.go create mode 100644 internal/pkg/telemetry/telemetry_test.go diff --git a/README.md b/README.md index 42a23845..6b24473a 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,10 @@ Both hooks are synchronous and block the job while they run. Either one exiting See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes. +#### OpenTelemetry (`OTEL_*`) + +Set `OTEL_EXPORTER_OTLP_ENDPOINT` in the runner's environment to send a trace of every job to an OpenTelemetry collector. See **[docs/telemetry.md](docs/telemetry.md)**. + #### Local job logs (`log.job.dir`) Set `log.job.dir` to a path and the runner writes a copy of every task's log there as `-task-.log`: the rows exactly as Gitea received them, with the same secrets masked and the job's result on the last line. Off by default, and what Gitea shows does not change. diff --git a/act/ghcontext/github_context.go b/act/ghcontext/github_context.go index 6ade785e..cb52220c 100644 --- a/act/ghcontext/github_context.go +++ b/act/ghcontext/github_context.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "strings" + "sync" "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/common/git" @@ -25,6 +26,26 @@ var ( findGithubRepo = git.FindGithubRepo ) +type pinnedKey struct{} + +// WithPinnedCheckout keeps a run's sha and ref fixed when HEAD moves. +func WithPinnedCheckout(ctx context.Context) context.Context { + if ctx.Value(pinnedKey{}) != nil { + return ctx + } + return context.WithValue(ctx, pinnedKey{}, &sync.Map{}) +} + +func pinned(ctx context.Context, key string, lookup func() (string, error)) (string, error) { + cache, ok := ctx.Value(pinnedKey{}).(*sync.Map) + if !ok { + return lookup() + } + cached, _ := cache.LoadOrStore(key, sync.OnceValues(lookup)) + lookupOnce, _ := cached.(func() (string, error)) + return lookupOnce() +} + // SetRef resolves the ref of the context from its event payload, falling back // to the ref checked out in repoPath. func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) { @@ -51,7 +72,7 @@ func SetRef(ctx context.Context, ghc *model.GithubContext, repoPath string) { } if ghc.Ref == "" { - ref, err := findGitRef(ctx, repoPath) + ref, err := pinned(ctx, "ref:"+repoPath, func() (string, error) { return findGitRef(ctx, repoPath) }) if err != nil { logger.Warningf("unable to get git ref: %v", err) } else { @@ -95,7 +116,10 @@ func SetSha(ctx context.Context, ghc *model.GithubContext, repoPath string) { } if ghc.Sha == "" { - _, sha, err := findGitRevision(ctx, repoPath) + sha, err := pinned(ctx, "sha:"+repoPath, func() (string, error) { + _, revision, err := findGitRevision(ctx, repoPath) + return revision, err + }) if err != nil { logger.Warningf("unable to get git revision: %v", err) } else { diff --git a/act/ghcontext/github_context_test.go b/act/ghcontext/github_context_test.go index 258ca0ad..b75085f6 100644 --- a/act/ghcontext/github_context_test.go +++ b/act/ghcontext/github_context_test.go @@ -214,4 +214,17 @@ func TestSetSha(t *testing.T) { assert.Equal(t, table.sha, ghc.Sha) }) } + + t.Run("pinned checkout keeps the first sha and ref", func(t *testing.T) { + ctx := WithPinnedCheckout(context.Background()) + for _, head := range []string{"first", "moved"} { + findGitRevision = func(context.Context, string) (string, string, error) { return "", head, nil } + findGitRef = func(context.Context, string) (string, error) { return "refs/heads/" + head, nil } + ghc := &model.GithubContext{Event: map[string]any{}} + SetSha(ctx, ghc, "/some/dir") + SetRef(ctx, ghc, "/some/dir") + assert.Equal(t, "first", ghc.Sha) + assert.Equal(t, "refs/heads/first", ghc.Ref) + } + }) } diff --git a/act/runner/cancellation_test.go b/act/runner/cancellation_test.go index cc6c15bb..77dc2db7 100644 --- a/act/runner/cancellation_test.go +++ b/act/runner/cancellation_test.go @@ -14,6 +14,7 @@ import ( "gitea.dev/actionslib/pkg/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" "go.yaml.in/yaml/v4" ) @@ -249,7 +250,8 @@ func TestPreStepFailureAffectsMainStepIfStatus(t *testing.T) { // SetJobError dereferences a nil map and panics. The post context must therefore be detached // from cancellation (so the steps run) yet still carry a usable error container. func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) { - cancelled, cancel := context.WithCancel(common.WithJobErrorContainer(context.Background())) + jobSpan := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}}) + cancelled, cancel := context.WithCancel(trace.ContextWithSpanContext(common.WithJobErrorContainer(context.Background()), jobSpan)) cancel() require.ErrorIs(t, cancelled.Err(), context.Canceled) @@ -258,6 +260,7 @@ func TestPostStepsContextCancelledIsUsableForFailingStep(t *testing.T) { // Detached from cancellation, so the post steps actually run. require.NoError(t, postCtx.Err(), "post context must not be cancelled") + assert.Equal(t, jobSpan, trace.SpanContextFromContext(postCtx), "post steps must stay in the job's trace") // A failing post step records its error instead of panicking. require.NotPanics(t, func() { diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index fcc7f13f..a74e49aa 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -281,8 +281,8 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo // finished main-pipeline context. Cleanup has to run even when the run was interrupted, so the // returned context always carries a fresh bounded deadline and is never itself cancelled. // -// - context.Canceled (server cancel): detach from the cancelled context via a fresh root so -// the post steps can run. +// - context.Canceled (server cancel): detach with WithoutCancel under a fresh job-error +// container, so a failing post step cannot turn the cancellation into a failure. // - context.DeadlineExceeded (job timeout): detach the deadline with WithoutCancel, which // keeps the original values — including the job-error container — so the timeout failure and // any post-step error are preserved and the job is still reported as failed. @@ -290,10 +290,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo func postStepsContext(ctx context.Context) (context.Context, context.CancelFunc) { switch ctx.Err() { case context.Canceled: - // The cancelled context is abandoned for a fresh root, which drops the job-error - // container installed at the job root. Re-attach a fresh one so a failing post step - // records its error via SetJobError instead of panicking on a nil container. - return context.WithTimeout(common.WithJobErrorContainer(common.WithLogger(context.Background(), common.Logger(ctx))), 5*time.Minute) + return context.WithTimeout(common.WithJobErrorContainer(context.WithoutCancel(ctx)), 5*time.Minute) case context.DeadlineExceeded: return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute) default: diff --git a/act/runner/runner.go b/act/runner/runner.go index 10311ed2..11a88d8d 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -14,6 +14,7 @@ import ( "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/act/ghcontext" "gitea.dev/actionslib/pkg/model" docker_container "github.com/moby/moby/api/types/container" @@ -321,7 +322,8 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { }) } - return common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan)) + executor := common.NewPipelineExecutor(stagePipeline...).Then(handleFailure(plan)) + return func(ctx context.Context) error { return executor(ghcontext.WithPinnedCheckout(ctx)) } } func handleFailure(plan *model.Plan) common.Executor { diff --git a/act/runner/step.go b/act/runner/step.go index c459ff86..299c6f6d 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -15,6 +15,7 @@ import ( "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/internal/pkg/telemetry" "gitea.dev/actionslib/pkg/exprparser" "gitea.dev/actionslib/pkg/model" @@ -65,24 +66,27 @@ func processRunnerEnvFileCommand(ctx context.Context, fileName string, rc *RunCo } func runStepExecutor(step step, stage stepStage, executor common.Executor) common.Executor { - return func(ctx context.Context) error { + return func(ctx context.Context) (err error) { logger := common.Logger(ctx) rc := step.getRunContext() stepModel := step.getStepModel() - ifExpression := step.getIfExpression(ctx, stage) - rc.CurrentStep = stepModel.ID - stepResult := &model.StepResult{ Outcome: model.StepStatusSuccess, Conclusion: model.StepStatusSuccess, Outputs: make(map[string]string), } + ctx, endStep := telemetry.StartStep(ctx, stepModel, stage.String()) + defer func() { endStep(err, stepResult) }() + + ifExpression := step.getIfExpression(ctx, stage) + rc.CurrentStep = stepModel.ID + if stage == stepStageMain { rc.StepResults[rc.CurrentStep] = stepResult } - err := setupEnv(ctx, step) + err = setupEnv(ctx, step) var runStep bool if err == nil { runStep, err = isStepEnabled(ctx, ifExpression, step, stage) diff --git a/act/runner/step_test.go b/act/runner/step_test.go index 8c0b4dc0..915c8166 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -13,13 +13,19 @@ import ( "gitea.com/gitea/runner/act/common" "gitea.com/gitea/runner/act/container" + "gitea.com/gitea/runner/internal/pkg/telemetry" "gitea.dev/actionslib/pkg/model" + runnerv1 "gitea.dev/actionslib/runner/v1" log "github.com/sirupsen/logrus" logrustest "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" + tracenoop "go.opentelemetry.io/otel/trace/noop" yaml "go.yaml.in/yaml/v4" ) @@ -547,6 +553,29 @@ func TestRunStepExecutorParity(t *testing.T) { assert.Equal(t, "child", childRC.IntraActionState[child.Step.ID]["saved"]) }) + t.Run("stages and composite steps are traced under Gitea's step names", func(t *testing.T) { + spans := tracetest.NewSpanRecorder() + telemetry.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spans))) + t.Cleanup(func() { telemetry.SetTracerProvider(tracenoop.NewTracerProvider()) }) + ctx, _ := telemetry.StartJob(t.Context(), &runnerv1.Task{Id: 7}) + outer := newStep(t, &model.Step{ID: "outer", Uses: "actions/cache@v4", Number: 3}) + child := newStep(t, &model.Step{ID: "child", Name: "Restore", Number: 1}) + skipped := newStep(t, &model.Step{ID: "skipped", Name: "Deploy", Number: 4, If: yaml.Node{Value: "false"}}) + broken := newStep(t, &model.Step{ID: "broken", Name: "Broken", Number: 5, If: yaml.Node{Value: badExpression}}) + + require.NoError(t, runStepExecutor(outer, stepStagePost, runStepExecutor(child, stepStageMain, noopExecutor))(ctx)) + require.NoError(t, runStepExecutor(skipped, stepStageMain, noopExecutor)(ctx)) + require.Error(t, runStepExecutor(broken, stepStageMain, noopExecutor)(ctx)) + + ended := spans.Ended() + require.Len(t, ended, 4) + assert.Equal(t, []string{"Restore", "Post Run actions/cache@v4", "Deploy", "Broken"}, []string{ended[0].Name(), ended[1].Name(), ended[2].Name(), ended[3].Name()}) + assert.Equal(t, ended[1].SpanContext().SpanID(), ended[0].Parent().SpanID()) + assert.Contains(t, ended[0].Attributes(), semconv.CICDPipelineTaskRunID("7.3.post.1")) + assert.Contains(t, ended[2].Attributes(), semconv.CICDPipelineTaskRunResultSkip) + assert.Contains(t, ended[3].Attributes(), semconv.CICDPipelineTaskRunResultFailure) + }) + t.Run("timeout must be positive", func(t *testing.T) { exprEval := createRunContext(t).NewExpressionEvaluator(context.Background()) for timeout, wantDeadline := range map[string]bool{"-1": false, "0": false, "1": true} { diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 00000000..9518cf70 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,46 @@ +# OpenTelemetry tracing + +The runner can send a trace of every job it runs to an OpenTelemetry collector or any backend that accepts OTLP. Each job becomes a trace with a span per step, so you can see where CI time goes and which steps fail. Tracing stays off until you set an endpoint. + +## Setup + +Set the endpoint in the runner's environment, for example in its systemd unit, with `docker run -e` or in the Kubernetes pod spec: + +```sh +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +``` + +| Variable | Purpose | +| --- | --- | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Collector base URL, traces go to `/v1/traces` under it | +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Full traces URL, takes precedence over the base URL | +| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | Request headers such as an API key, as comma-separated `key=value` pairs with URL-encoded values | +| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes, the service name defaults to `gitea-runner` | +| `OTEL_TRACES_SAMPLER`, `OTEL_BSP_*` | Sampling and batching, see the [SDK variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) | +| `OTEL_SDK_DISABLED=true`, `OTEL_TRACES_EXPORTER=none` | Turn tracing off | + +Only the `http/protobuf` protocol is supported. + +## What is sent + +- A span per job, named `RUN {workflow} {job}`, with the repository, ref, commit, run URL, run number, attempt and result. +- A span per step, named as in Gitea's step list, including `Pre` and `Post` stages and the steps of composite actions, with the step's result. +- The runner's name, UUID and version. + +Logs, step output and secrets are never sent. Step names are taken from the workflow file as written, without expanding `${{ }}` expressions. + +## Tracing inside jobs + +Jobs receive `TRACEPARENT`, so tools in a job that use OpenTelemetry can add their spans to the job's trace. To let them export, set their `OTEL_*` variables in `runner.envs`, see [Security](#security). + +## Metrics + +Metrics stay on the Prometheus endpoint enabled by `metrics.enabled`. To get them into OpenTelemetry, scrape `/metrics` with the collector's `prometheus` receiver. + +## Security + +- OTLP has no authentication of its own. Send to a collector on the runner's host that holds the backend credentials, as the [OpenTelemetry security guide](https://opentelemetry.io/docs/security/config-best-practices/) recommends, or use `https` with a token in `OTEL_EXPORTER_OTLP_HEADERS`. Over plain `http`, headers travel unencrypted. +- TLS certificates are verified against the system CA bundle, set `SSL_CERT_FILE` to use your own. Client certificates are not supported, use a local collector for mTLS. Redirects are not followed, so headers only reach the configured endpoint. +- Jobs in containers never see the runner's `OTEL_*` variables. Jobs on the host inherit the runner's whole environment, headers included, and can read its files anyway since they run as the same user. +- Every workflow can read `runner.envs`, so never put credentials there. Point jobs at a collector that holds them instead, keeping in mind that any workflow can then send data through it. +- Traces carry the repository and step names of every job, private repositories included, so give the collector the same access controls as the runner's logs. diff --git a/go.mod b/go.mod index 5bde57f8..97f9759a 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,11 @@ require ( github.com/stretchr/testify v1.12.1 github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928 go.etcd.io/bbolt v1.5.0 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 + go.opentelemetry.io/proto/otlp v1.11.0 go.yaml.in/yaml/v4 v4.0.0-rc.6 golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 @@ -68,6 +73,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect @@ -89,9 +95,7 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect - go.opentelemetry.io/otel v1.45.0 // indirect - go.opentelemetry.io/otel/metric v1.45.0 // indirect - go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/crypto v0.57.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 84239547..440dea97 100644 --- a/go.sum +++ b/go.sum @@ -196,16 +196,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= -go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= -go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= -go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= -go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= -go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= -go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= -go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= diff --git a/internal/app/cmd/daemon.go b/internal/app/cmd/daemon.go index 787f4a63..640f5d5d 100644 --- a/internal/app/cmd/daemon.go +++ b/internal/app/cmd/daemon.go @@ -25,6 +25,7 @@ import ( "gitea.com/gitea/runner/internal/pkg/lock" "gitea.com/gitea/runner/internal/pkg/metrics" "gitea.com/gitea/runner/internal/pkg/report" + "gitea.com/gitea/runner/internal/pkg/telemetry" "gitea.com/gitea/runner/internal/pkg/ver" "connectrpc.com/connect" @@ -71,6 +72,18 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu defer func() { _ = releaseLock() }() } + shutdownTelemetry, err := telemetry.Setup(ctx, reg.UUID, reg.Name) + if err != nil { + log.WithError(err).Warn("OpenTelemetry export failed to start") + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := shutdownTelemetry(ctx); err != nil { + log.WithError(err).Warn("OpenTelemetry export failed to flush") + } + }() + lbls := resolveLabels(daemArgs.Labels, cfg.Runner.Labels, reg.Labels) ls := labels.Labels{} diff --git a/internal/app/run/runner.go b/internal/app/run/runner.go index 6f1b35bd..2d4a2d37 100644 --- a/internal/app/run/runner.go +++ b/internal/app/run/runner.go @@ -5,6 +5,7 @@ package run import ( "bytes" + "cmp" "context" "encoding/json/v2" "errors" @@ -34,6 +35,7 @@ import ( "gitea.com/gitea/runner/internal/pkg/labels" "gitea.com/gitea/runner/internal/pkg/metrics" "gitea.com/gitea/runner/internal/pkg/report" + "gitea.com/gitea/runner/internal/pkg/telemetry" "gitea.com/gitea/runner/internal/pkg/ver" "connectrpc.com/connect" @@ -298,6 +300,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error { ctx, cancel := context.WithTimeout(ctx, r.cfg.Runner.Timeout) defer cancel() + ctx, endJob := telemetry.StartJob(ctx, task) // A proxy URL may carry credentials, and every job is given it; keep them out of the log. reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords()...) var volumeCleanup []common.Executor @@ -322,6 +325,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error { metrics.JobDuration.Observe(time.Since(start).Seconds()) metrics.JobsTotal.WithLabelValues(metrics.ResultToStatusLabel(reporter.Result())).Inc() + endJob(reporter.Result()) }() reporter.RunDaemon() runErr = r.run(ctx, task, reporter) @@ -409,6 +413,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. } job := workflow.GetJob(jobID) reporter.ResetSteps(len(job.Steps)) + telemetry.SetJobName(ctx, cmp.Or(job.Name, jobID)) taskContext := task.Context.Fields envs := r.cloneEnvs() @@ -426,6 +431,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report. if v := taskContext["run_id"].GetStringValue(); v != "" { envs["GITEA_RUN_ID"] = v } + telemetry.AddJobEnv(ctx, envs) log.Infof("task %v repo is %v %v %v", task.Id, taskContext["repository"].GetStringValue(), r.getDefaultActionsURL(task), diff --git a/internal/pkg/telemetry/telemetry.go b/internal/pkg/telemetry/telemetry.go new file mode 100644 index 00000000..3af83d2d --- /dev/null +++ b/internal/pkg/telemetry/telemetry.go @@ -0,0 +1,240 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package telemetry + +import ( + "bytes" + "cmp" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "slices" + "strconv" + "strings" + + "gitea.com/gitea/runner/internal/pkg/ver" + + "gitea.dev/actionslib/pkg/model" + runnerv1 "gitea.dev/actionslib/runner/v1" + "github.com/avast/retry-go/v5" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" + "go.opentelemetry.io/otel/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" + tracepb "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" +) + +const scope = "gitea.com/gitea/runner" + +type taskRunKey struct{} + +type taskRun struct { + id, url, pipeline string + span trace.Span +} + +// tracer stays off the global provider and ctx carries only span contexts, so the Docker client's otelhttp records nothing. +var tracer trace.Tracer = tracenoop.Tracer{} + +func SetTracerProvider(provider trace.TracerProvider) { + tracer = provider.Tracer(scope) +} + +func Setup(ctx context.Context, uuid, name string) (func(context.Context) error, error) { + noop := func(context.Context) error { return nil } + endpoint := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + if base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint == "" && base != "" { + endpoint = strings.TrimSuffix(base, "/") + "/v1/traces" + } + if endpoint == "" || strings.EqualFold(os.Getenv("OTEL_SDK_DISABLED"), "true") || strings.EqualFold(os.Getenv("OTEL_TRACES_EXPORTER"), "none") { + return noop, nil + } + if protocol := cmp.Or(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"), os.Getenv("OTEL_EXPORTER_OTLP_PROTOCOL")); protocol != "" && !strings.EqualFold(protocol, "http/protobuf") { + return noop, fmt.Errorf("OTLP protocol %q is not supported, only http/protobuf is", protocol) + } + res, err := resource.New(ctx, resource.WithTelemetrySDK(), resource.WithAttributes( + semconv.ServiceName("gitea-runner"), + semconv.ServiceVersion(ver.Version()), + semconv.ServiceInstanceID(uuid), + semconv.CICDWorkerID(uuid), + semconv.CICDWorkerName(name), + ), resource.WithFromEnv()) + if err != nil { + return noop, err + } + headers := http.Header{} + for _, key := range []string{"OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"} { + for pair := range strings.SplitSeq(os.Getenv(key), ",") { + key, value, found := strings.Cut(pair, "=") + if decoded, err := url.PathUnescape(strings.TrimSpace(value)); found && err == nil { + headers.Set(strings.TrimSpace(key), decoded) + } + } + } + provider := sdktrace.NewTracerProvider(sdktrace.WithBatcher(otlptrace.NewUnstarted(&client{endpoint, headers})), sdktrace.WithResource(res)) + SetTracerProvider(provider) + return provider.Shutdown, nil +} + +// client replaces otlptracehttp, which links gRPC (+7 MB): https://github.com/open-telemetry/opentelemetry-go/issues/2579 +type client struct { + endpoint string + headers http.Header +} + +// exportClient refuses redirects, which would carry the collector's header credentials to another host. +var exportClient = &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + +func (*client) Start(context.Context) error { return nil } + +func (*client) Stop(context.Context) error { return nil } + +func (c *client) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error { + body, err := proto.Marshal(&tracepb.TracesData{ResourceSpans: spans}) + if err != nil { + return err + } + return retry.New(retry.Context(ctx), retry.Attempts(5), retry.LastErrorOnly(true)).Do(func() error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(body)) + if err != nil { + return retry.Unrecoverable(err) + } + req.Header = c.headers.Clone() + req.Header.Set("Content-Type", "application/x-protobuf") + resp, err := exportClient.Do(req) + if err != nil { + return err + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode < http.StatusMultipleChoices { + return nil + } + err = fmt.Errorf("OTLP export failed: %s", resp.Status) + if slices.Contains([]int{http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout}, resp.StatusCode) { + return err + } + return retry.Unrecoverable(err) + }) +} + +func StartJob(ctx context.Context, task *runnerv1.Task) (context.Context, func(runnerv1.Result)) { + field := func(key string) string { return task.GetContext().GetFields()[key].GetStringValue() } + pipeline, owner, repo := field("workflow"), field("repository_owner"), field("repository") + repoURL := strings.TrimSuffix(field("server_url"), "/") + "/" + repo + runURL := repoURL + "/actions/runs/" + field("run_id") + _, span := tracer.Start(ctx, "RUN "+pipeline, trace.WithSpanKind(trace.SpanKindServer), trace.WithAttributes( + semconv.CICDPipelineName(pipeline), + semconv.CICDPipelineActionNameRun, + semconv.CICDPipelineRunID(field("run_id")), + semconv.CICDPipelineRunURLFull(runURL), + semconv.VCSOwnerName(owner), + semconv.VCSRepositoryName(strings.TrimPrefix(repo, owner+"/")), + semconv.VCSRepositoryURLFull(repoURL), + semconv.VCSRefHeadName(field("ref_name")), + semconv.VCSRefHeadRevision(field("sha")), + attribute.String("gitea.job.key", field("job")), + attribute.String("gitea.run.number", field("run_number")), + attribute.String("gitea.run.attempt", field("run_attempt")), + attribute.Int64("gitea.task.id", task.Id), + )) + if !span.SpanContext().IsValid() { + return ctx, func(runnerv1.Result) {} + } + ctx = context.WithValue(trace.ContextWithSpanContext(ctx, span.SpanContext()), taskRunKey{}, taskRun{strconv.FormatInt(task.Id, 10), runURL, pipeline, span}) + return ctx, func(result runnerv1.Result) { + end(span, semconv.CICDPipelineResultKey, jobResult(ctx, result)) + } +} + +func SetJobName(ctx context.Context, job string) { + if run, ok := ctx.Value(taskRunKey{}).(taskRun); ok { + run.span.SetName("RUN " + run.pipeline + " " + job) + } +} + +func AddJobEnv(ctx context.Context, envs map[string]string) { + carrier := propagation.MapCarrier{} + propagation.TraceContext{}.Inject(ctx, carrier) + for key, value := range carrier { + envs[strings.ToUpper(key)] = value + } +} + +func StartStep(ctx context.Context, step *model.Step, stage string) (context.Context, func(error, *model.StepResult)) { + parent, ok := ctx.Value(taskRunKey{}).(taskRun) + if !ok { + return ctx, func(error, *model.StepResult) {} + } + name, id := step.Name, parent.id+"."+strconv.Itoa(step.Number) + if name == "" { + run, _, _ := strings.Cut(strings.TrimSpace(step.Run), "\n") + name = "Run " + cmp.Or(strings.TrimSpace(run), step.String()) + } + if stage != "Main" { + name, id = stage+" "+name, id+"."+strings.ToLower(stage) + } + _, span := tracer.Start(ctx, name, trace.WithAttributes( + semconv.CICDPipelineTaskName(name), + semconv.CICDPipelineTaskRunID(id), + semconv.CICDPipelineTaskRunURLFull(parent.url), + )) + ctx = context.WithValue(trace.ContextWithSpanContext(ctx, span.SpanContext()), taskRunKey{}, taskRun{id, parent.url, parent.pipeline, span}) + return ctx, func(err error, result *model.StepResult) { + end(span, semconv.CICDPipelineTaskRunResultKey, stepResult(ctx, err, result)) + } +} + +func jobResult(ctx context.Context, result runnerv1.Result) string { + switch { + case result == runnerv1.Result_RESULT_SUCCESS: + return "success" + case result == runnerv1.Result_RESULT_SKIPPED: + return "skip" + case errors.Is(ctx.Err(), context.DeadlineExceeded): + return "timeout" + case result == runnerv1.Result_RESULT_CANCELLED: + return "cancellation" + case result == runnerv1.Result_RESULT_FAILURE: + return "failure" + } + return "error" +} + +func stepResult(ctx context.Context, err error, result *model.StepResult) string { + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + return "timeout" + case errors.Is(err, context.Canceled) || ctx.Err() != nil: + return "cancellation" + case err != nil || result.Conclusion == model.StepStatusFailure: + return "failure" + case result.Conclusion == model.StepStatusSkipped: + return "skip" + } + return "success" +} + +func end(span trace.Span, key attribute.Key, result string) { + span.SetAttributes(key.String(result)) + switch result { + case "timeout": + span.SetAttributes(semconv.ErrorTypeKey.String(result)) + span.SetStatus(codes.Error, "") + case "failure", "error": + span.SetAttributes(semconv.ErrorTypeOther) + span.SetStatus(codes.Error, "") + } + span.End() +} diff --git a/internal/pkg/telemetry/telemetry_test.go b/internal/pkg/telemetry/telemetry_test.go new file mode 100644 index 00000000..556e6ab8 --- /dev/null +++ b/internal/pkg/telemetry/telemetry_test.go @@ -0,0 +1,186 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package telemetry + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "gitea.dev/actionslib/pkg/model" + runnerv1 "gitea.dev/actionslib/runner/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" + "go.opentelemetry.io/otel/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" + tracepb "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" +) + +func setEnv(t *testing.T, env map[string]string) { + t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) }) + for _, key := range []string{ + "OTEL_SDK_DISABLED", "OTEL_TRACES_EXPORTER", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + } { + t.Setenv(key, env[key]) + } +} + +func TestSetup(t *testing.T) { + for _, tc := range []struct { + env map[string]string + enabled bool + err string + }{ + {}, + {env: map[string]string{"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://collector:4318/v1/traces"}, enabled: true}, + {env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_TRACES_EXPORTER": "NONE"}}, + {env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_SDK_DISABLED": "TRUE"}}, + {env: map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": "http://collector:4318", "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc"}, err: `OTLP protocol "grpc" is not supported, only http/protobuf is`}, + } { + t.Run(fmt.Sprint(tc.env), func(t *testing.T) { + setEnv(t, tc.env) + shutdown, err := Setup(t.Context(), "uuid", "runner") + if tc.err != "" { + require.EqualError(t, err, tc.err) + } else { + require.NoError(t, err) + } + _, disabled := tracer.(tracenoop.Tracer) + assert.Equal(t, tc.enabled, !disabled) + require.NoError(t, shutdown(t.Context())) + }) + } +} + +func TestExportRetriesAndMatchesOTLP(t *testing.T) { + bodies := make(chan []byte, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/otlp/v1/traces", r.URL.Path) + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + body, _ := io.ReadAll(r.Body) + bodies <- body + if len(bodies) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + defer server.Close() + setEnv(t, map[string]string{"OTEL_EXPORTER_OTLP_ENDPOINT": server.URL + "/otlp/", "OTEL_EXPORTER_OTLP_HEADERS": "authorization=Bearer%20token"}) + + shutdown, err := Setup(t.Context(), "uuid", "runner") + require.NoError(t, err) + _, span := tracer.Start(t.Context(), "op") + span.End() + require.NoError(t, shutdown(t.Context())) + + require.Len(t, bodies, 2) + <-bodies + var decoded tracepb.TracesData + require.NoError(t, proto.Unmarshal(<-bodies, &decoded)) + assert.Equal(t, "op", decoded.GetResourceSpans()[0].GetScopeSpans()[0].GetSpans()[0].GetName()) +} + +func TestExportRefusesRedirects(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("redirect followed") })) + defer target.Close() + origin := httptest.NewServer(http.RedirectHandler(target.URL, http.StatusTemporaryRedirect)) + defer origin.Close() + require.EqualError(t, (&client{origin.URL, http.Header{}}).UploadTraces(t.Context(), nil), "OTLP export failed: 307 Temporary Redirect") +} + +func TestJob(t *testing.T) { + fields, err := structpb.NewStruct(map[string]any{ + "workflow": "ci.yml", + "repository": "owner/repo", + "repository_owner": "owner", + "server_url": "https://gitea.example/", + "run_id": "42", + "run_number": "40", + "run_attempt": "2", + "job": "build", + }) + require.NoError(t, err) + task := &runnerv1.Task{Id: 7, Context: fields} + + envs := map[string]string{} + ctx, endJob := StartJob(t.Context(), task) + AddJobEnv(ctx, envs) + stepCtx, endStep := StartStep(ctx, &model.Step{}, "Main") + endStep(nil, nil) + endJob(runnerv1.Result_RESULT_SUCCESS) + assert.Equal(t, ctx, stepCtx) + assert.Empty(t, envs) + + spans := tracetest.NewSpanRecorder() + SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spans))) + t.Cleanup(func() { SetTracerProvider(tracenoop.NewTracerProvider()) }) + + ctx, endJob = StartJob(t.Context(), task) + SetJobName(ctx, "Build and test") + AddJobEnv(ctx, envs) + stepCtx, endStep = StartStep(ctx, &model.Step{Run: "make\nmake test", Number: 1}, "Main") + assert.False(t, trace.SpanFromContext(ctx).IsRecording()) + assert.False(t, trace.SpanFromContext(stepCtx).IsRecording()) + endStep(assert.AnError, nil) + endJob(runnerv1.Result_RESULT_FAILURE) + + jobSpan := trace.SpanContextFromContext(ctx) + assert.Equal(t, "00-"+jobSpan.TraceID().String()+"-"+jobSpan.SpanID().String()+"-01", envs["TRACEPARENT"]) + + ended := spans.Ended() + require.Len(t, ended, 2) + step, job := ended[0], ended[1] + assert.Equal(t, "Run make", step.Name()) + assert.Equal(t, jobSpan.SpanID(), step.Parent().SpanID()) + assert.Equal(t, codes.Error, step.Status().Code) + assert.Subset(t, step.Attributes(), []any{ + semconv.CICDPipelineTaskRunID("7.1"), + semconv.CICDPipelineTaskRunURLFull("https://gitea.example/owner/repo/actions/runs/42"), + semconv.ErrorTypeOther, + }) + assert.Equal(t, "RUN ci.yml Build and test", job.Name()) + assert.Equal(t, trace.SpanKindServer, job.SpanKind()) + assert.Subset(t, job.Attributes(), []any{ + semconv.VCSOwnerName("owner"), + semconv.VCSRepositoryName("repo"), + semconv.CICDPipelineRunID("42"), + attribute.String("gitea.run.number", "40"), + attribute.String("gitea.run.attempt", "2"), + attribute.String("gitea.job.key", "build"), + attribute.Int64("gitea.task.id", 7), + }) +} + +func TestResults(t *testing.T) { + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + expired, expire := context.WithDeadline(t.Context(), time.Time{}) + defer expire() + for want, tc := range map[string]struct { + ctx context.Context + err error + result *model.StepResult + }{ + "timeout": {ctx: t.Context(), err: fmt.Errorf("exec: %w", context.DeadlineExceeded)}, + "cancellation": {ctx: cancelled}, + "failure": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusFailure}}, + "success": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusFailure, Conclusion: model.StepStatusSuccess}}, + "skip": {ctx: t.Context(), result: &model.StepResult{Outcome: model.StepStatusSkipped, Conclusion: model.StepStatusSkipped}}, + } { + assert.Equal(t, want, stepResult(tc.ctx, tc.err, tc.result)) + } + assert.Equal(t, "timeout", jobResult(expired, runnerv1.Result_RESULT_FAILURE)) + assert.Equal(t, "cancellation", jobResult(t.Context(), runnerv1.Result_RESULT_CANCELLED)) +}