mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-04-27 14:30:28 +08:00
perf: reduce runner-to-server connection load with adaptive reporting and polling (#819)
## Summary Many teams self-host Gitea + Act Runner at scale. The current runner design causes excessive HTTP requests to the Gitea server, leading to high server load. This PR addresses three root causes: aggressive fixed-interval polling, per-task status reporting every 1 second regardless of activity, and unoptimized HTTP client configuration. ## Problem The original architecture has these issues: **1. Fixed 1-second reporting interval (RunDaemon)** - Every running task calls ReportLog + ReportState every 1 second (2 HTTP requests/sec/task) - These requests are sent even when there are no new log rows or state changes - With 200 runners × 3 tasks each = **1,200 req/sec just for status reporting** **2. Fixed 2-second polling interval (no backoff)** - Idle runners poll FetchTask every 2 seconds forever, even when no jobs are queued - No exponential backoff or jitter — all runners can synchronize after network recovery (thundering herd) - 200 idle runners = **100 req/sec doing nothing useful** **3. HTTP client not tuned** - Uses http.DefaultClient with MaxIdleConnsPerHost=2, causing frequent TCP/TLS reconnects - Creates two separate http.Client instances (one for Ping, one for Runner service) instead of sharing **Total: ~1,300 req/sec for 200 runners with 3 tasks each** ## Solution ### Adaptive Event-Driven Log Reporting Replace the recursive `time.AfterFunc(1s)` pattern in RunDaemon with a goroutine-based select event loop using three trigger mechanisms: | Trigger | Default | Purpose | |---------|---------|---------| | `log_report_max_latency` | 3s | Guarantee even a single log line is delivered within this time | | `log_report_interval` | 5s | Periodic sweep — steady-state cadence | | `log_report_batch_size` | 100 rows | Immediate flush during bursty output (e.g., npm install) | **Key design**: `log_report_max_latency` (3s) must be less than `log_report_interval` (5s) so the max-latency timer fires before the periodic ticker for single-line scenarios. State reporting is decoupled to its own `state_report_interval` (default 5s), with immediate flush on step transitions (start/stop) via a stateNotify channel for responsive frontend UX. Additionally: - Skip ReportLog when `len(rows) == 0` (no pending log rows) - Skip ReportState when `stateChanged == false && len(outputs) == 0` (nothing changed) - Move expensive `proto.Clone` after the early-return check to avoid deep copies on no-op paths ### Polling Backoff with Jitter Replace fixed `rate.Limiter` with adaptive exponential backoff: - Track `consecutiveEmpty` and `consecutiveErrors` counters - Interval doubles with each empty/error response: `base × 2^(n-1)`, capped at `fetch_interval_max` (default 60s) - Add ±20% random jitter to prevent thundering herd - Fetch first, sleep after ��� preserves burst=1 behavior for immediate first fetch on startup and after task completion ### HTTP Client Tuning - Configure custom `http.Transport` with `MaxIdleConnsPerHost=10` (was 2) - Share a single `http.Client` between PingService and RunnerService - Add `IdleConnTimeout=90s` for clean connection lifecycle ## Load Reduction For 200 runners × 3 tasks (70% with active log output): | Component | Before | After | Reduction | |-----------|--------|-------|-----------| | Polling (idle) | 100 req/s | ~3.4 req/s | 97% | | Log reporting | 420 req/s | ~84 req/s | 80% | | State reporting | 126 req/s | ~25 req/s | 80% | | **Total** | **~1,300 req/s** | **~113 req/s** | **~91%** | ## Frontend UX Impact | Scenario | Before | After | Notes | |----------|--------|-------|-------| | Continuous output (npm install) | ~1s | ~5s | Periodic ticker sweep | | Single line then silence | ~1s | ≤3s | maxLatencyTimer guarantee | | Bursty output (100+ lines) | ~1s | <1s | Batch size immediate flush | | Step start/stop | ~1s | <1s | stateNotify immediate flush | | Job completion | ~1s | ~1s | Close() retry unchanged | ## New Configuration Options All have safe defaults — existing config files need no changes: ```yaml runner: fetch_interval_max: 60s # Max backoff interval when idle log_report_interval: 5s # Periodic log flush interval log_report_max_latency: 3s # Max time a log row waits (must be < log_report_interval) log_report_batch_size: 100 # Immediate flush threshold state_report_interval: 5s # State flush interval (step transitions are always immediate) ``` Config validation warns on invalid combinations: - `fetch_interval_max < fetch_interval` → auto-corrected - `log_report_max_latency >= log_report_interval` → warning (timer would be redundant) ## Test Plan - [x] `go build ./...` passes - [x] `go test ./...` passes (all existing + 3 new tests) - [x] `golangci-lint run` — 0 issues - [x] TestReporter_MaxLatencyTimer — verifies single log line flushed by maxLatencyTimer before logTicker - [x] TestReporter_BatchSizeFlush — verifies batch size threshold triggers immediate flush - [x] TestReporter_StateNotifyFlush — verifies step transition triggers immediate state flush - [x] TestReporter_EphemeralRunnerDeletion — verifies Close/RunDaemon race safety - [x] TestReporter_RunDaemonClose_Race — verifies concurrent Close safety Reviewed-on: https://gitea.com/gitea/act_runner/pulls/819 Reviewed-by: Nicolas <173651+bircni@noreply.gitea.com> Co-authored-by: Bo-Yi Wu <appleboy.tw@gmail.com> Co-committed-by: Bo-Yi Wu <appleboy.tw@gmail.com>
This commit is contained in:
Bo-Yi Wu
@@ -7,13 +7,14 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
||||
"connectrpc.com/connect"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"gitea.com/gitea/act_runner/internal/app/run"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/client"
|
||||
@@ -35,6 +36,15 @@ type Poller struct {
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// workerState holds per-goroutine polling state. Backoff counters are
|
||||
// per-worker so that with Capacity > 1, N workers each seeing one empty
|
||||
// response don't combine into a "consecutive N empty" reading on a shared
|
||||
// counter and trigger an unnecessarily long backoff.
|
||||
type workerState struct {
|
||||
consecutiveEmpty int64
|
||||
consecutiveErrors int64
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, client client.Client, runner *run.Runner) *Poller {
|
||||
pollingCtx, shutdownPolling := context.WithCancel(context.Background())
|
||||
|
||||
@@ -58,11 +68,10 @@ func New(cfg *config.Config, client client.Client, runner *run.Runner) *Poller {
|
||||
}
|
||||
|
||||
func (p *Poller) Poll() {
|
||||
limiter := rate.NewLimiter(rate.Every(p.cfg.Runner.FetchInterval), 1)
|
||||
wg := &sync.WaitGroup{}
|
||||
for i := 0; i < p.cfg.Runner.Capacity; i++ {
|
||||
wg.Add(1)
|
||||
go p.poll(wg, limiter)
|
||||
go p.poll(wg)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@@ -71,9 +80,7 @@ func (p *Poller) Poll() {
|
||||
}
|
||||
|
||||
func (p *Poller) PollOnce() {
|
||||
limiter := rate.NewLimiter(rate.Every(p.cfg.Runner.FetchInterval), 1)
|
||||
|
||||
p.pollOnce(limiter)
|
||||
p.pollOnce(&workerState{})
|
||||
|
||||
// signal that we're done
|
||||
close(p.done)
|
||||
@@ -108,10 +115,11 @@ func (p *Poller) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) poll(wg *sync.WaitGroup, limiter *rate.Limiter) {
|
||||
func (p *Poller) poll(wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
s := &workerState{}
|
||||
for {
|
||||
p.pollOnce(limiter)
|
||||
p.pollOnce(s)
|
||||
|
||||
select {
|
||||
case <-p.pollingCtx.Done():
|
||||
@@ -122,19 +130,57 @@ func (p *Poller) poll(wg *sync.WaitGroup, limiter *rate.Limiter) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnce(limiter *rate.Limiter) {
|
||||
// calculateInterval returns the polling interval with exponential backoff based on
|
||||
// consecutive empty or error responses. The interval starts at FetchInterval and
|
||||
// doubles with each consecutive empty/error, capped at FetchIntervalMax.
|
||||
func (p *Poller) calculateInterval(s *workerState) time.Duration {
|
||||
base := p.cfg.Runner.FetchInterval
|
||||
maxInterval := p.cfg.Runner.FetchIntervalMax
|
||||
|
||||
n := max(s.consecutiveEmpty, s.consecutiveErrors)
|
||||
if n <= 1 {
|
||||
return base
|
||||
}
|
||||
|
||||
// Capped exponential backoff: base * 2^(n-1), max shift=5 so multiplier <= 32
|
||||
shift := min(n-1, 5)
|
||||
interval := base * time.Duration(int64(1)<<shift)
|
||||
return min(interval, maxInterval)
|
||||
}
|
||||
|
||||
// addJitter adds +/- 20% random jitter to the given duration to avoid thundering herd.
|
||||
func addJitter(d time.Duration) time.Duration {
|
||||
if d <= 0 {
|
||||
return d
|
||||
}
|
||||
// jitter range: [-20%, +20%] of d
|
||||
jitterRange := int64(d) * 2 / 5 // 40% total range
|
||||
if jitterRange <= 0 {
|
||||
return d
|
||||
}
|
||||
jitter := rand.Int64N(jitterRange) - jitterRange/2
|
||||
return d + time.Duration(jitter)
|
||||
}
|
||||
|
||||
func (p *Poller) pollOnce(s *workerState) {
|
||||
for {
|
||||
if err := limiter.Wait(p.pollingCtx); err != nil {
|
||||
if p.pollingCtx.Err() != nil {
|
||||
log.WithError(err).Debug("limiter wait failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
task, ok := p.fetchTask(p.pollingCtx)
|
||||
task, ok := p.fetchTask(p.pollingCtx, s)
|
||||
if !ok {
|
||||
interval := addJitter(p.calculateInterval(s))
|
||||
timer := time.NewTimer(interval)
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-p.pollingCtx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Got a task — reset backoff counters for fast subsequent polling.
|
||||
s.consecutiveEmpty = 0
|
||||
s.consecutiveErrors = 0
|
||||
|
||||
p.runTaskWithRecover(p.jobsCtx, task)
|
||||
return
|
||||
}
|
||||
@@ -153,7 +199,7 @@ func (p *Poller) runTaskWithRecover(ctx context.Context, task *runnerv1.Task) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Poller) fetchTask(ctx context.Context) (*runnerv1.Task, bool) {
|
||||
func (p *Poller) fetchTask(ctx context.Context, s *workerState) (*runnerv1.Task, bool) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, p.cfg.Runner.FetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -167,10 +213,15 @@ func (p *Poller) fetchTask(ctx context.Context) (*runnerv1.Task, bool) {
|
||||
}
|
||||
if err != nil {
|
||||
log.WithError(err).Error("failed to fetch task")
|
||||
s.consecutiveErrors++
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Successful response — reset error counter.
|
||||
s.consecutiveErrors = 0
|
||||
|
||||
if resp == nil || resp.Msg == nil {
|
||||
s.consecutiveEmpty++
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@@ -179,6 +230,7 @@ func (p *Poller) fetchTask(ctx context.Context) (*runnerv1.Task, bool) {
|
||||
}
|
||||
|
||||
if resp.Msg.Task == nil {
|
||||
s.consecutiveEmpty++
|
||||
return nil, false
|
||||
}
|
||||
|
||||
|
||||
108
internal/app/poll/poller_test.go
Normal file
108
internal/app/poll/poller_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package poll
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
|
||||
connect_go "connectrpc.com/connect"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"gitea.com/gitea/act_runner/internal/pkg/client/mocks"
|
||||
"gitea.com/gitea/act_runner/internal/pkg/config"
|
||||
)
|
||||
|
||||
// TestPoller_PerWorkerCounters verifies that each worker maintains its own
|
||||
// backoff counters. With a shared counter, N workers each seeing one empty
|
||||
// response would inflate the counter to N and trigger an unnecessarily long
|
||||
// backoff. With per-worker state, each worker only sees its own count.
|
||||
func TestPoller_PerWorkerCounters(t *testing.T) {
|
||||
client := mocks.NewClient(t)
|
||||
client.On("FetchTask", mock.Anything, mock.Anything).Return(
|
||||
func(_ context.Context, _ *connect_go.Request[runnerv1.FetchTaskRequest]) (*connect_go.Response[runnerv1.FetchTaskResponse], error) {
|
||||
// Always return an empty response.
|
||||
return connect_go.NewResponse(&runnerv1.FetchTaskResponse{}), nil
|
||||
},
|
||||
)
|
||||
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
p := &Poller{client: client, cfg: cfg}
|
||||
|
||||
ctx := context.Background()
|
||||
s1 := &workerState{}
|
||||
s2 := &workerState{}
|
||||
|
||||
// Each worker independently observes one empty response.
|
||||
_, ok := p.fetchTask(ctx, s1)
|
||||
require.False(t, ok)
|
||||
_, ok = p.fetchTask(ctx, s2)
|
||||
require.False(t, ok)
|
||||
|
||||
assert.Equal(t, int64(1), s1.consecutiveEmpty, "worker 1 should only count its own empty response")
|
||||
assert.Equal(t, int64(1), s2.consecutiveEmpty, "worker 2 should only count its own empty response")
|
||||
|
||||
// Worker 1 sees a second empty; worker 2 stays at 1.
|
||||
_, ok = p.fetchTask(ctx, s1)
|
||||
require.False(t, ok)
|
||||
assert.Equal(t, int64(2), s1.consecutiveEmpty)
|
||||
assert.Equal(t, int64(1), s2.consecutiveEmpty, "worker 2's counter must not be affected by worker 1's empty fetches")
|
||||
}
|
||||
|
||||
// TestPoller_FetchErrorIncrementsErrorsOnly verifies that a fetch error
|
||||
// increments only the per-worker error counter, not the empty counter.
|
||||
func TestPoller_FetchErrorIncrementsErrorsOnly(t *testing.T) {
|
||||
client := mocks.NewClient(t)
|
||||
client.On("FetchTask", mock.Anything, mock.Anything).Return(
|
||||
func(_ context.Context, _ *connect_go.Request[runnerv1.FetchTaskRequest]) (*connect_go.Response[runnerv1.FetchTaskResponse], error) {
|
||||
return nil, errors.New("network unreachable")
|
||||
},
|
||||
)
|
||||
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
p := &Poller{client: client, cfg: cfg}
|
||||
|
||||
s := &workerState{}
|
||||
_, ok := p.fetchTask(context.Background(), s)
|
||||
require.False(t, ok)
|
||||
assert.Equal(t, int64(1), s.consecutiveErrors)
|
||||
assert.Equal(t, int64(0), s.consecutiveEmpty)
|
||||
}
|
||||
|
||||
// TestPoller_CalculateInterval verifies the per-worker exponential backoff
|
||||
// math is correctly driven by the worker's own counters.
|
||||
func TestPoller_CalculateInterval(t *testing.T) {
|
||||
cfg, err := config.LoadDefault("")
|
||||
require.NoError(t, err)
|
||||
cfg.Runner.FetchInterval = 2 * time.Second
|
||||
cfg.Runner.FetchIntervalMax = 60 * time.Second
|
||||
p := &Poller{cfg: cfg}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
empty, errs int64
|
||||
wantInterval time.Duration
|
||||
}{
|
||||
{"first poll, no backoff", 0, 0, 2 * time.Second},
|
||||
{"single empty, still base", 1, 0, 2 * time.Second},
|
||||
{"two empties, doubled", 2, 0, 4 * time.Second},
|
||||
{"five empties, capped path", 5, 0, 32 * time.Second},
|
||||
{"many empties, capped at max", 20, 0, 60 * time.Second},
|
||||
{"errors drive backoff too", 0, 3, 8 * time.Second},
|
||||
{"max(empty, errors) wins", 2, 4, 16 * time.Second},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &workerState{consecutiveEmpty: tc.empty, consecutiveErrors: tc.errs}
|
||||
assert.Equal(t, tc.wantInterval, p.calculateInterval(s))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, r.cfg.Runner.Timeout)
|
||||
defer cancel()
|
||||
reporter := report.NewReporter(ctx, cancel, r.client, task)
|
||||
reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg)
|
||||
var runErr error
|
||||
defer func() {
|
||||
lastWords := ""
|
||||
|
||||
Reference in New Issue
Block a user