enhance(config): expand environment values (#1274)

Config values can reference the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`, so container deployments can configure the runner without templating the file. An unset `NAME` fails the load instead of silently becoming empty, and `$${NAME}` writes a literal `${NAME}`. Keys are never expanded.

Breaking: an existing value containing `${NAME}`, such as a shell command in a Kubernetes `pod_template`, now needs `$${NAME}`.

Closes https://gitea.com/gitea/runner/issues/551
Co-authored-by: silverwind <2021+silverwind@noreply.gitea.com>
Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1274
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>
Co-authored-by: bircni <bircni@icloud.com>

README.md

@@ -169,7 +169,7 @@ Of the [runner images](https://gitea.com/gitea/runner-images), the `-full` flavo
 
 #### Environment variables
 
-Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
+Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. Config values can read the runner's environment as `${NAME}`, for example `capacity: ${RUNNER_CAPACITY}`. An unset `NAME` fails the load, and `$${NAME}` is a literal `${NAME}`. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
 
 ### Labels
 

docs/kubernetes.md

@@ -45,7 +45,7 @@ kubernetes:
           nvidia.com/gpu.present: "true"
 ```
 
-A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`.
+A job with `runs-on: [linux, gpu]` gets the `linux` and then the `gpu` entry of `pod_templates` merged over `pod_template`, while its image comes from the first of its labels the runner has. A template volume named `workspace` or `act` replaces the `emptyDir` the runner mounts at the workspace or at `/var/run/act`, for example to limit its size. Images are pulled by the cluster's `imagePullPolicy` default, a template's, or always with `container.force_pull`. Like every config value, templates expand `${NAME}` from the runner's environment, so a shell command in a template writes `$${NAME}` for the pod's own variable.
 
 Job pods mount no service account token unless the template sets `automountServiceAccountToken: true`. Job pods and their Secrets are labeled `app.kubernetes.io/managed-by: gitea-runner` and `com.gitea.runner.uuid: <runner UUID>`. The idle cleanup removes those the runner did not get to remove, once they are older than `runner.workdir_cleanup_age`.
 

internal/pkg/config/config.example.yaml

@@ -1,5 +1,6 @@
 # Every option with its default value, all commented out. Read this file, do not copy it.
 # `./gitea-runner config init` writes a config file to copy the lines you change into.
+# Values may read the runner's environment as ${NAME}, an unset NAME fails the load. $${NAME} is a literal ${NAME}.
 
 # Logging for the runner process itself (messages printed to stderr), plus the copy of
 # each task's log kept under log.job. Neither controls how workflow step output is streamed

internal/pkg/config/config.go

@@ -4,12 +4,12 @@
 package config
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"maps"
 	"os"
 	"path/filepath"
+	"regexp"
 	"slices"
 	"strings"
 	"time"
@@ -33,6 +33,8 @@ const DefaultImage = "docker.gitea.com/runner-images:ubuntu-latest"
 // (so a programmatically built config still gets a sane bound).
 const DefaultPostTaskScriptTimeout = 5 * time.Minute
 
+var environmentReference = regexp.MustCompile(`\$?\$\{[A-Za-z_][A-Za-z0-9_]*\}`)
+
 // Minimal is the smallest config file that runs the runner: options it does not
 // name keep their default, and it names none.
 const Minimal = `# Minimal config file. Every option it does not set keeps its default.
@@ -244,14 +246,18 @@ func LoadDefault(file string) (*Config, error) {
 		if err != nil {
 			return nil, fmt.Errorf("open config file %q: %w", file, err)
 		}
-		if err := yaml.Unmarshal(content, cfg); err != nil {
+		var document yaml.Node
+		if err := yaml.Unmarshal(content, &document); err != nil {
 			return nil, fmt.Errorf("parse config file %q: %w", file, err)
 		}
-		warnUnknownKeys(file, content)
-		definedRunnerKeys, err = definedRunnerConfigKeys(content)
-		if err != nil {
-			return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
+		if err := expandEnvironment(&document); err != nil {
+			return nil, fmt.Errorf("parse config file %q: %w", file, err)
 		}
+		if err := document.Load(cfg, yaml.WithV3Defaults()); err != nil {
+			return nil, fmt.Errorf("parse config file %q: %w", file, err)
+		}
+		warnUnknownKeys(file, &document)
+		definedRunnerKeys = definedRunnerConfigKeys(&document)
 	}
 
 	if cfg.Runner.EnvFile != "" {
@@ -416,30 +422,53 @@ func LoadDefault(file string) (*Config, error) {
 	return cfg, nil
 }
 
+// expandEnvironment replaces ${NAME} in values and retypes them, so ${CAPACITY} can be an int and an empty value is unset.
+func expandEnvironment(node *yaml.Node) error {
+	var errs []error
+	if node.Kind == yaml.ScalarNode {
+		expanded := environmentReference.ReplaceAllStringFunc(node.Value, func(match string) string {
+			if strings.HasPrefix(match, "$$") {
+				return match[1:]
+			}
+			value, ok := os.LookupEnv(strings.Trim(match, "${}"))
+			if !ok {
+				errs = append(errs, fmt.Errorf("line %d: %s is not set", node.Line, match))
+			}
+			return value
+		})
+		if expanded != node.Value && node.Style&yaml.TaggedStyle == 0 {
+			node.Tag = ""
+			if expanded == "" && node.Style == 0 {
+				node.Tag = "!!null"
+			}
+		}
+		node.Value = expanded
+		return errors.Join(errs...)
+	}
+	for index, child := range node.Content {
+		if node.Kind != yaml.MappingNode || index%2 == 1 {
+			errs = append(errs, expandEnvironment(child))
+		}
+	}
+	return errors.Join(errs...)
+}
+
 // warnUnknownKeys reports keys the config does not define, which are otherwise ignored
 // without a trace. It only warns, so a config carrying keys from another runner version
 // still loads.
-func warnUnknownKeys(file string, content []byte) {
-	decoder := yaml.NewDecoder(bytes.NewReader(content))
-	decoder.KnownFields(true)
-
+func warnUnknownKeys(file string, document *yaml.Node) {
 	var loadErrs *yaml.LoadErrors
-	if err := decoder.Decode(&Config{}); errors.As(err, &loadErrs) {
+	if err := document.Load(&Config{}, yaml.WithV3Defaults(), yaml.WithKnownFields()); errors.As(err, &loadErrs) {
 		for _, loadErr := range loadErrs.Errors {
 			log.Warnf("config file %q: %s, it will be ignored", file, loadErr)
 		}
 	}
 }
 
-func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
-	var root yaml.Node
-	if err := yaml.Unmarshal(content, &root); err != nil {
-		return nil, err
-	}
-
+func definedRunnerConfigKeys(root *yaml.Node) map[string]bool {
 	defined := map[string]bool{}
 	if len(root.Content) == 0 {
-		return defined, nil
+		return defined
 	}
 
 	doc := root.Content[0]
@@ -455,7 +484,7 @@ func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
 		break
 	}
 
-	return defined, nil
+	return defined
 }
 
 func validateExtraHeaders(headers map[string]string) error {

internal/pkg/config/config_test.go

@@ -42,6 +42,38 @@ cache:
 	require.NoError(t, err)
 }
 
+func TestLoadDefault_ExpandsEnvironmentValues(t *testing.T) {
+	t.Setenv("RUNNER_CAPACITY", "4")
+	t.Setenv("RUNNER_LABEL", "linux:host")
+	t.Setenv("RUNNER_KEY", "fetch_timeout")
+	t.Setenv("RUNNER_EMPTY", "")
+	t.Setenv("RUNNER_MISSING", "")
+	require.NoError(t, os.Unsetenv("RUNNER_MISSING"))
+	path := write(t, `runner:
+  capacity: ${RUNNER_CAPACITY}
+  labels:
+    - ${RUNNER_LABEL}
+  envs:
+    LITERAL: $${RUNNER_LABEL}
+  ${RUNNER_KEY}: 1s
+cache:
+  enabled: ${RUNNER_EMPTY}
+`)
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, 4, cfg.Runner.Capacity)
+	assert.Equal(t, []string{"linux:host"}, cfg.Runner.Labels)
+	assert.Equal(t, map[string]string{"LITERAL": "${RUNNER_LABEL}"}, cfg.Runner.Envs)
+	assert.Equal(t, 5*time.Second, cfg.Runner.FetchTimeout)
+	assert.True(t, *cfg.Cache.Enabled)
+
+	_, err = LoadDefault(write(t, "runner:\n\n  capacity: ${RUNNER_MISSING}\n  file: ${RUNNER_MISSING}\n"))
+	require.ErrorContains(t, err, "line 3: ${RUNNER_MISSING} is not set\nline 4: ${RUNNER_MISSING} is not set")
+	_, err = LoadDefault(write(t, "runner:\n\n  capacity: ${RUNNER_LABEL}\n"))
+	require.ErrorContains(t, err, "line 3:")
+}
+
 func TestLoadDefault_CacheS3(t *testing.T) {
 	dir := t.TempDir()
 	accessPath := filepath.Join(dir, "access.key")