From 15dd63a839b4936bbb920f9dc5a7a2e8ff5b01be Mon Sep 17 00:00:00 2001 From: Pascal Zimmermann Date: Sun, 19 Apr 2026 22:36:34 +0000 Subject: [PATCH] feat: Add support for Dynamic Matrix Strategies with Job Outputs (#149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Matrix Strategy: Support for Scalar Values and Template Expressions ## 🎯 Overview This Pull Request implements support for **scalar values and template expressions** in matrix strategies. Previously, every matrix value had to be declared as a YAML array — a bare scalar like `os: ubuntu-latest` or `version: ${{ fromJSON(...) }}` caused silent failures. Now, scalars are automatically wrapped into single-element arrays, enabling the `${{ fromJSON(...) }}` pattern that is commonly used in GitHub Actions workflows for dynamic matrix generation. ## ⚠️ Semantic Changes Two subtle behavior changes are introduced that are worth understanding: ### 1. Scalar matrix values now produce a 1-element matrix expansion **Before this PR:** A scalar value like ```yaml strategy: matrix: os: ubuntu-latest # no brackets ``` failed to decode as `map[string][]interface{}`, so `Matrix()` returned `nil`. The job ran **once, without any matrix context** (`matrix.os` was undefined). **After this PR:** The same input is wrapped to `["ubuntu-latest"]`. The job runs **one matrix iteration** with `matrix.os = "ubuntu-latest"` set. > **Impact on existing workflows:** Workflows that accidentally used bare scalars instead of arrays will now run with a matrix context where they previously did not. The functional result is usually the same, but `matrix.*` variables are now populated. This is considered an improvement, but is a semantic change. --- ### 2. Unevaluated template expressions fall back to a literal string value The actual resolution of `${{ fromJSON(needs.job1.outputs.versions) }}` into an array happens **before** `Matrix()` is called — in `EvaluateYamlNode` inside `pkg/runner/runner.go`. This PR does not change that evaluation path. If evaluation succeeds → the YAML node already contains a proper array → `Matrix()` decodes it normally. If evaluation **fails or is skipped** (e.g., expression not yet resolved) → the scalar string is now wrapped as `["${{ fromJSON(...) }}"]` → the job runs **one iteration with the literal unexpanded string** as the matrix value. > **Note:** The test `TestJobMatrix/matrix_with_template_expression` covers this fallback path, not the successful resolution path. The literal-wrap behavior is the fallback, not the primary mechanism for template expression support. --- ## 🔑 Key Changes ### 1. Flexible Matrix Decoding (`pkg/model/workflow.go`) #### New Helper Function: `normalizeMatrixValue` ```go func normalizeMatrixValue(key string, val interface{}) ([]interface{}, error) ``` - Converts a single matrix value to the expected array format - Handles three cases: 1. **Arrays**: Pass through unchanged 2. **Scalars**: Wrap in single-element array (including template expressions) 3. **Invalid types** (maps, etc.): Return error to detect misconfigurations early **Supported scalar types:** - `string` — including unevaluated template expressions - `int`, `float64` — numeric values - `bool` — boolean values - `nil` — null values #### Enhanced `Matrix()` Method **Before:** ```go func (j *Job) Matrix() map[string][]interface{} { if j.Strategy.RawMatrix.Kind == yaml.MappingNode { var val map[string][]interface{} if !decodeNode(j.Strategy.RawMatrix, &val) { return nil } return val } return nil } ``` **After:** ```go func (j *Job) Matrix() map[string][]interface{} { if j.Strategy == nil || j.Strategy.RawMatrix.Kind != yaml.MappingNode { return nil } // First decode to flexible map[string]interface{} to handle scalars and arrays var flexVal map[string]interface{} err := j.Strategy.RawMatrix.Decode(&flexVal) if err != nil { // If that fails, try the strict array format var val map[string][]interface{} if !decodeNode(j.Strategy.RawMatrix, &val) { return nil } return val } // Convert flexible format to expected format with validation val := make(map[string][]interface{}) for k, v := range flexVal { normalized, err := normalizeMatrixValue(k, v) if err != nil { log.Errorf("matrix validation error: %v", err) return nil } val[k] = normalized } return val } ``` **Key improvements:** - Handles unevaluated template expressions as scalar strings - Automatic conversion to array format for consistent processing - Validation to catch nested maps and invalid configurations - Fallback to strict array format decoding for backward compatibility - Early nil check for strategy to prevent nil pointer panics --- ## 🎁 Use Cases ### Example 1: Template Expression in Matrix (primary motivation) ```yaml jobs: setup: runs-on: ubuntu-latest outputs: versions: ${{ steps.version-map.outputs.versions }} steps: - id: version-map run: echo "versions=[\"1.17\",\"1.18\",\"1.19\"]" >> $GITHUB_OUTPUT test: needs: setup strategy: matrix: # EvaluateYamlNode resolves this to an array before Matrix() is called version: ${{ fromJSON(needs.setup.outputs.versions) }} runs-on: ubuntu-latest steps: - run: echo "Testing version ${{ matrix.version }}" ``` ### Example 2: Mixed Scalar and Array Values ```yaml jobs: build: strategy: matrix: os: [ubuntu-latest, windows-latest] # Array — unchanged go-version: 1.19 # Scalar — wrapped to [1.19] node: [14, 16, 18] # Array — unchanged runs-on: ${{ matrix.os }} ``` ### Example 3: Single Value Matrix ```yaml jobs: deploy: strategy: matrix: environment: production # Scalar — wrapped to ["production"] runs-on: ubuntu-latest ``` --- ## ✅ Benefits - ✨ **Feature Parity with GitHub Actions**: Supports template expressions in matrix strategies - 🔒 **Robust Validation**: Detects misconfigured matrices (nested maps) early - 🔄 **Backward Compatible**: Existing array-based workflows continue to work - 📝 **Clear Error Messages**: Helpful validation messages for debugging - 🛡️ **Type Safety**: Handles all YAML scalar types correctly --- ## 🚀 Breaking Changes **Technically none in a strictly backward-incompatible sense**, but two semantic changes exist (see [⚠️ Semantic Changes](#️-semantic-changes-read-before-merging) above): 1. Scalar matrix values now iterate (previously they caused a `nil` matrix / no iteration) 2. Unevaluated template strings now produce a literal iteration instead of skipping Both changes are improvements in practice, but downstream users should be aware. --- ## 📚 Related Work - Related Gitea Server PR: [go-gitea/gitea#36564](https://github.com/go-gitea/gitea/pull/36564) - Enables dynamic matrix generation from job outputs - Supports advanced CI/CD patterns used in GitHub Actions ## 🔗 References - [GitHub Actions: Matrix strategies](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) - [GitHub Actions: Expression syntax](https://docs.github.com/en/actions/learn-github-actions/expressions) - [GitHub Actions: `fromJSON` in matrix context](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrix) Reviewed-on: https://gitea.com/gitea/act/pulls/149 Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com> Co-authored-by: Pascal Zimmermann Co-committed-by: Pascal Zimmermann --- README.md | 36 +++++ pkg/model/workflow.go | 51 ++++++- pkg/model/workflow_test.go | 265 +++++++++++++++++++++++++++++++++++++ pkg/runner/runner.go | 2 + 4 files changed, 351 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 32e6a7da..7356eb78 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,42 @@ Tags: - `nektos/v0.10.1` -> `v0.1001.*`, not ~~`v0.101.*`~~ - `nektos/v0.3.100` -> not ~~`v0.3100.*`~~, I don't think it's really going to happen, if it does, we can find a way to handle it. +## Gitea-specific changes + +### Matrix strategy: scalar values and template expressions + +This fork extends the matrix strategy parser [workflow.go](pkg/model/workflow.go) to accept +bare scalar YAML values in addition to arrays, and to handle unevaluated template +expressions gracefully. + +**Scalar wrapping** + +A matrix key written without brackets is automatically promoted to a +single-element array: + +```yaml +strategy: + matrix: + go-version: 1.21 # treated as [1.21] + os: ubuntu-latest # treated as ["ubuntu-latest"] +``` + +> [!NOTE] +> Previously such a value caused the matrix decoding to fail and the job ran *without* +> a matrix context (`matrix.*` variables were undefined). Now the job runs *one* matrix iteration with the scalar as the +> value. Existing workflows that used scalars by accident may see a difference in which matrix variables are populated. + +**Template expression support (`${{ fromJSON(...) }}`)** + +Template expressions in the matrix are resolved by `EvaluateYamlNode` +(`pkg/runner/runner.go`) *before* `Matrix()` is called. When successful, the +expression is replaced by a proper YAML sequence and the matrix expands +normally. + +If the expression cannot be resolved (e.g., the necessary context is not yet +available), the literal string is wrapped as a one-element array, and the job +runs once with the unexpanded string as the matrix value (graceful degradation). + --- ![act-logo](https://raw.githubusercontent.com/wiki/nektos/act/img/logo-150.png) diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index 62326702..cbcb0f29 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -377,16 +377,61 @@ func (j *Job) Environment() map[string]string { return environment(j.Env) } -// Matrix decodes RawMatrix YAML node +// normalizeMatrixValue converts a matrix value to []interface{}. +// Arrays pass through unchanged; scalars are wrapped in a single-element array. +// Unevaluated template expressions are wrapped as a fallback — proper resolution +// happens via EvaluateYamlNode before Matrix() is called. Nested maps are rejected. +func normalizeMatrixValue(key string, val any) ([]any, error) { + switch t := val.(type) { + case []any: + // Already an array - use as-is + return t, nil + case string, int, float64, bool, nil: + // Valid scalar types that can appear in YAML + // These can be unevaluated template expressions (strings) or literal values + return []any{t}, nil + case map[string]any: + // Nested map indicates misconfiguration - likely user error + return nil, fmt.Errorf("matrix key %q has invalid nested object value - expected scalar or array, got map", key) + default: + // Unknown types might indicate parsing issues + log.Warnf("matrix key %q has unexpected type %T, wrapping as single value", key, t) + return []any{t}, nil + } +} + +// Matrix decodes the RawMatrix YAML node into a map[string][]interface{}. +// Scalar values are wrapped into single-element arrays automatically. +// Template expressions are resolved by EvaluateYamlNode before this method is +// called; if unresolved, the literal string is wrapped as a one-element fallback. func (j *Job) Matrix() map[string][]any { - if j.Strategy.RawMatrix.Kind == yaml.MappingNode { + if j.Strategy == nil || j.Strategy.RawMatrix.Kind != yaml.MappingNode { + return nil + } + + // Decode to flexible map first so that scalar values don't cause a type error. + var flexVal map[string]any + err := j.Strategy.RawMatrix.Decode(&flexVal) + if err != nil { + // Fall back to the strict array-only format for backward compatibility. var val map[string][]any if !decodeNode(j.Strategy.RawMatrix, &val) { return nil } return val } - return nil + + // Convert flexible format to expected format with validation + val := make(map[string][]any) + for k, v := range flexVal { + normalized, err := normalizeMatrixValue(k, v) + if err != nil { + log.Errorf("matrix validation error: %v", err) + return nil + } + val[k] = normalized + } + return val } // GetMatrixes returns the matrix cross product diff --git a/pkg/model/workflow_test.go b/pkg/model/workflow_test.go index ca4212c7..c57859e3 100644 --- a/pkg/model/workflow_test.go +++ b/pkg/model/workflow_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "go.yaml.in/yaml/v4" ) func TestReadWorkflow_ScheduleEvent(t *testing.T) { @@ -637,3 +638,267 @@ func TestStep_UsesHash(t *testing.T) { }) } } + +func TestNormalizeMatrixValue(t *testing.T) { + tests := []struct { + name string + key string + value interface{} + wantResult []interface{} + wantErr bool + errMsg string + }{ + { + name: "array_values_pass_through", + key: "version", + value: []interface{}{"1.0", "2.0", "3.0"}, + wantResult: []interface{}{"1.0", "2.0", "3.0"}, + wantErr: false, + }, + { + name: "string_scalar_wrapped", + key: "os", + value: "ubuntu-latest", + wantResult: []interface{}{"ubuntu-latest"}, + wantErr: false, + }, + { + name: "template_expression_wrapped", + key: "version", + value: "${{ fromJson(needs.setup.outputs.versions) }}", + wantResult: []interface{}{"${{ fromJson(needs.setup.outputs.versions) }}"}, + wantErr: false, + }, + { + name: "integer_scalar_wrapped", + key: "count", + value: 42, + wantResult: []interface{}{42}, + wantErr: false, + }, + { + name: "float_scalar_wrapped", + key: "factor", + value: 3.14, + wantResult: []interface{}{3.14}, + wantErr: false, + }, + { + name: "bool_scalar_wrapped", + key: "enabled", + value: true, + wantResult: []interface{}{true}, + wantErr: false, + }, + { + name: "nil_scalar_wrapped", + key: "optional", + value: nil, + wantResult: []interface{}{nil}, + wantErr: false, + }, + { + name: "nested_map_returns_error", + key: "config", + value: map[string]interface{}{"nested": "value"}, + wantErr: true, + errMsg: "has invalid nested object value", + }, + { + name: "empty_array_passes_through", + key: "empty", + value: []interface{}{}, + wantResult: []interface{}{}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := normalizeMatrixValue(tt.key, tt.value) + + if tt.wantErr { + assert.Error(t, err, "should return error") + if tt.errMsg != "" { + assert.Contains(t, err.Error(), tt.errMsg) + } + } else { + assert.NoError(t, err, "should not return error") + assert.Equal(t, tt.wantResult, result, "result should match expected") + } + }) + } +} + +func TestJobMatrix(t *testing.T) { + tests := []struct { + name string + yaml string + wantErr bool + wantLen int + checkFn func(*testing.T, map[string][]interface{}) + }{ + { + name: "matrix_with_arrays", + yaml: ` +name: test +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + version: [1.18, 1.19] + steps: + - run: echo test +`, + wantErr: false, + wantLen: 2, + checkFn: func(t *testing.T, m map[string][]interface{}) { + assert.Equal(t, []interface{}{"ubuntu-latest", "windows-latest"}, m["os"]) + assert.Equal(t, []interface{}{1.18, 1.19}, m["version"]) + }, + }, + { + name: "matrix_with_scalar_values", + yaml: ` +name: test +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + os: ubuntu-latest + version: 1.19 + steps: + - run: echo test +`, + wantErr: false, + wantLen: 2, + checkFn: func(t *testing.T, m map[string][]interface{}) { + assert.Equal(t, []interface{}{"ubuntu-latest"}, m["os"]) + assert.Equal(t, []interface{}{1.19}, m["version"]) + }, + }, + { + name: "matrix_with_template_expression", + yaml: ` +name: test +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + versions: ${{ fromJson(needs.setup.outputs.versions) }} + steps: + - run: echo test +`, + wantErr: false, + wantLen: 1, + checkFn: func(t *testing.T, m map[string][]interface{}) { + assert.Equal(t, []interface{}{"${{ fromJson(needs.setup.outputs.versions) }}"}, m["versions"]) + }, + }, + { + name: "matrix_mixed_arrays_and_scalars", + yaml: ` +name: test +on: push +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + version: 1.19 + node: [14, 16] + steps: + - run: echo test +`, + wantErr: false, + wantLen: 3, + checkFn: func(t *testing.T, m map[string][]interface{}) { + assert.Equal(t, []interface{}{"ubuntu-latest", "windows-latest"}, m["os"]) + assert.Equal(t, []interface{}{1.19}, m["version"]) + assert.Equal(t, []interface{}{14, 16}, m["node"]) + }, + }, + { + name: "empty_matrix", + yaml: ` +name: test +on: push +jobs: + build: + runs-on: ubuntu-latest + steps: + - run: echo test +`, + wantErr: false, + wantLen: 0, + checkFn: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workflow, err := ReadWorkflow(strings.NewReader(tt.yaml)) + assert.NoError(t, err, "reading workflow should succeed") + + job := workflow.GetJob("build") + if job == nil { + // For empty matrix test + if tt.wantLen == 0 { + return + } + t.Fatal("job not found") + } + + matrix := job.Matrix() + + if tt.wantErr { + assert.Nil(t, matrix, "matrix should be nil on error") + } else { + if tt.wantLen == 0 { + assert.Nil(t, matrix, "matrix should be nil for jobs without strategy") + } else { + assert.NotNil(t, matrix, "matrix should not be nil") + assert.Equal(t, tt.wantLen, len(matrix), "matrix should have expected number of keys") + if tt.checkFn != nil { + tt.checkFn(t, matrix) + } + } + } + }) + } +} + +func TestJobMatrixValidation(t *testing.T) { + // This test verifies that invalid nested map values are caught + t.Run("matrix_with_nested_map_fails", func(t *testing.T) { + // Manually construct a job with a problematic matrix containing a nested map + job := &Job{ + Strategy: &Strategy{ + RawMatrix: yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "config"}, + {Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "nested"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}, + }}, + }, + }, + }, + } + + // Attempt to get matrix + matrix := job.Matrix() + + // Should return nil due to validation error + assert.Nil(t, matrix, "matrix with nested map should return nil") + }) +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 53cd9c97..2a4bfd0a 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -179,6 +179,8 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { log.Debugf("Job.Strategy.RawMatrix: %v", job.Strategy.RawMatrix) strategyRc := runner.newRunContext(ctx, run, nil) + // Resolve template expressions in the matrix node before Matrix() is called. + // On failure the literal string is kept and normalizeMatrixValue wraps it as a fallback. if err := strategyRc.NewExpressionEvaluator(ctx).EvaluateYamlNode(ctx, &job.Strategy.RawMatrix); err != nil { log.Errorf("Error while evaluating matrix: %v", err) }