From 329115b1487eb5860c9e677e393e7b00bfb07ab9 Mon Sep 17 00:00:00 2001 From: bircni Date: Thu, 8 Oct 2026 17:31:00 +0000 Subject: [PATCH] enhance(config): expand environment values (#1274) Config values can reference the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`, so container deployments can configure the runner without templating the file. An unset `NAME` fails the load instead of silently becoming empty, and `$${NAME}` writes a literal `${NAME}`. Keys are never expanded. Breaking: an existing value containing `${NAME}`, such as a shell command in a Kubernetes `pod_template`, now needs `$${NAME}`. Closes https://gitea.com/gitea/runner/issues/551 Co-authored-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: silverwind Reviewed-on: https://gitea.com/gitea/runner/pulls/1274 Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: bircni --- README.md | 2 +- docs/kubernetes.md | 2 +- internal/pkg/config/config.example.yaml | 1 + internal/pkg/config/config.go | 67 ++++++++++++++++++------- internal/pkg/config/config_test.go | 32 ++++++++++++ 5 files changed, 83 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 5bd1a72d..d6442c27 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo #### Environment variables -Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below. +Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below. ### Labels diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 5f7ad615..a903391d 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -45,7 +45,7 @@ kubernetes: nvidia.com/gpu.present: "true" ``` -A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. +A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. Like every config value, templates expand `${NAME}` from the runner's environment, so a shell command in a template writes `$${NAME}` for the pod's own variable. Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: `. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`. diff --git a/internal/pkg/config/config.example.yaml b/internal/pkg/config/config.example.yaml index 73fe26fe..999ec1f1 100644 --- a/internal/pkg/config/config.example.yaml +++ b/internal/pkg/config/config.example.yaml @@ -1,5 +1,6 @@ # Every option with its default value, all commented out. Read this file, do not copy it. # `./gitea-runner config init` writes a config file to copy the lines you change into. +# Values may read the runner's environment as ${NAME}, an unset NAME fails the load. $${NAME} is a literal ${NAME}. # Logging for the runner process itself (messages printed to stderr), plus the copy of # each task's log kept under log.job. Neither controls how workflow step output is streamed diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index b2845d41..59719891 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -4,12 +4,12 @@ package config import ( - "bytes" "errors" "fmt" "maps" "os" "path/filepath" + "regexp" "slices" "strings" "time" @@ -33,6 +33,8 @@ const DefaultImage = "docker.gitea.com/runner-images:ubuntu-latest" // (so a programmatically built config still gets a sane bound). const DefaultPostTaskScriptTimeout = 5 * time.Minute +var environmentReference = regexp.MustCompile(`\$?\$\{[A-Za-z_][A-Za-z0-9_]*\}`) + // Minimal is the smallest config file that runs the runner: options it does not // name keep their default, and it names none. const Minimal = `# Minimal config file. Every option it does not set keeps its default. @@ -244,14 +246,18 @@ func LoadDefault(file string) (*Config, error) { if err != nil { return nil, fmt.Errorf("open config file %q: %w", file, err) } - if err := yaml.Unmarshal(content, cfg); err != nil { + var document yaml.Node + if err := yaml.Unmarshal(content, &document); err != nil { return nil, fmt.Errorf("parse config file %q: %w", file, err) } - warnUnknownKeys(file, content) - definedRunnerKeys, err = definedRunnerConfigKeys(content) - if err != nil { - return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err) + if err := expandEnvironment(&document); err != nil { + return nil, fmt.Errorf("parse config file %q: %w", file, err) } + if err := document.Load(cfg, yaml.WithV3Defaults()); err != nil { + return nil, fmt.Errorf("parse config file %q: %w", file, err) + } + warnUnknownKeys(file, &document) + definedRunnerKeys = definedRunnerConfigKeys(&document) } if cfg.Runner.EnvFile != "" { @@ -416,30 +422,53 @@ func LoadDefault(file string) (*Config, error) { return cfg, nil } +// expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset. +func expandEnvironment(node *yaml.Node) error { + var errs []error + if node.Kind == yaml.ScalarNode { + expanded := environmentReference.ReplaceAllStringFunc(node.Value, func(match string) string { + if strings.HasPrefix(match, "$$") { + return match[1:] + } + value, ok := os.LookupEnv(strings.Trim(match, "${}")) + if !ok { + errs = append(errs, fmt.Errorf("line %d: %s is not set", node.Line, match)) + } + return value + }) + if expanded != node.Value && node.Style&yaml.TaggedStyle == 0 { + node.Tag = "" + if expanded == "" && node.Style == 0 { + node.Tag = "!!null" + } + } + node.Value = expanded + return errors.Join(errs...) + } + for index, child := range node.Content { + if node.Kind != yaml.MappingNode || index%2 == 1 { + errs = append(errs, expandEnvironment(child)) + } + } + return errors.Join(errs...) +} + // warnUnknownKeys reports keys the config does not define, which are otherwise ignored // without a trace. It only warns, so a config carrying keys from another runner version // still loads. -func warnUnknownKeys(file string, content []byte) { - decoder := yaml.NewDecoder(bytes.NewReader(content)) - decoder.KnownFields(true) - +func warnUnknownKeys(file string, document *yaml.Node) { var loadErrs *yaml.LoadErrors - if err := decoder.Decode(&Config{}); errors.As(err, &loadErrs) { + if err := document.Load(&Config{}, yaml.WithV3Defaults(), yaml.WithKnownFields()); errors.As(err, &loadErrs) { for _, loadErr := range loadErrs.Errors { log.Warnf("config file %q: %s, it will be ignored", file, loadErr) } } } -func definedRunnerConfigKeys(content []byte) (map[string]bool, error) { - var root yaml.Node - if err := yaml.Unmarshal(content, &root); err != nil { - return nil, err - } - +func definedRunnerConfigKeys(root *yaml.Node) map[string]bool { defined := map[string]bool{} if len(root.Content) == 0 { - return defined, nil + return defined } doc := root.Content[0] @@ -455,7 +484,7 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) { break } - return defined, nil + return defined } func validateExtraHeaders(headers map[string]string) error { diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index b865215f..03b244e6 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -42,6 +42,38 @@ cache: require.NoError(t, err) } +func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) { + t.Setenv("RUNNER_CAPACITY", "4") + t.Setenv("RUNNER_LABEL", "linux:host") + t.Setenv("RUNNER_KEY", "fetch_timeout") + t.Setenv("RUNNER_EMPTY", "") + t.Setenv("RUNNER_MISSING", "") + require.NoError(t, os.Unsetenv("RUNNER_MISSING")) + path := write(t, `runner: + capacity: ${RUNNER_CAPACITY} + labels: + - ${RUNNER_LABEL} + envs: + LITERAL: $${RUNNER_LABEL} + ${RUNNER_KEY}: 1s +cache: + enabled: ${RUNNER_EMPTY} +`) + + cfg, err := LoadDefault(path) + require.NoError(t, err) + assert.Equal(t, 4, cfg.Runner.Capacity) + assert.Equal(t, []string{"linux:host"}, cfg.Runner.Labels) + assert.Equal(t, map[string]string{"LITERAL": "${RUNNER_LABEL}"}, cfg.Runner.Envs) + assert.Equal(t, 5*time.Second, cfg.Runner.FetchTimeout) + assert.True(t, *cfg.Cache.Enabled) + + _, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_MISSING}\n file: ${RUNNER_MISSING}\n")) + require.ErrorContains(t, err, "line 3: ${RUNNER_MISSING} is not set\nline 4: ${RUNNER_MISSING} is not set") + _, err = LoadDefault(write(t, "runner:\n\n capacity: ${RUNNER_LABEL}\n")) + require.ErrorContains(t, err, "line 3:") +} + func TestLoadDefault_CacheS3(t *testing.T) { dir := t.TempDir() accessPath := filepath.Join(dir, "access.key")