mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-04-24 12:50:31 +08:00
## Summary Many teams self-host Gitea + Act Runner at scale. The current runner design causes excessive HTTP requests to the Gitea server, leading to high server load. This PR addresses three root causes: aggressive fixed-interval polling, per-task status reporting every 1 second regardless of activity, and unoptimized HTTP client configuration. ## Problem The original architecture has these issues: **1. Fixed 1-second reporting interval (RunDaemon)** - Every running task calls ReportLog + ReportState every 1 second (2 HTTP requests/sec/task) - These requests are sent even when there are no new log rows or state changes - With 200 runners × 3 tasks each = **1,200 req/sec just for status reporting** **2. Fixed 2-second polling interval (no backoff)** - Idle runners poll FetchTask every 2 seconds forever, even when no jobs are queued - No exponential backoff or jitter — all runners can synchronize after network recovery (thundering herd) - 200 idle runners = **100 req/sec doing nothing useful** **3. HTTP client not tuned** - Uses http.DefaultClient with MaxIdleConnsPerHost=2, causing frequent TCP/TLS reconnects - Creates two separate http.Client instances (one for Ping, one for Runner service) instead of sharing **Total: ~1,300 req/sec for 200 runners with 3 tasks each** ## Solution ### Adaptive Event-Driven Log Reporting Replace the recursive `time.AfterFunc(1s)` pattern in RunDaemon with a goroutine-based select event loop using three trigger mechanisms: | Trigger | Default | Purpose | |---------|---------|---------| | `log_report_max_latency` | 3s | Guarantee even a single log line is delivered within this time | | `log_report_interval` | 5s | Periodic sweep — steady-state cadence | | `log_report_batch_size` | 100 rows | Immediate flush during bursty output (e.g., npm install) | **Key design**: `log_report_max_latency` (3s) must be less than `log_report_interval` (5s) so the max-latency timer fires before the periodic ticker for single-line scenarios. State reporting is decoupled to its own `state_report_interval` (default 5s), with immediate flush on step transitions (start/stop) via a stateNotify channel for responsive frontend UX. Additionally: - Skip ReportLog when `len(rows) == 0` (no pending log rows) - Skip ReportState when `stateChanged == false && len(outputs) == 0` (nothing changed) - Move expensive `proto.Clone` after the early-return check to avoid deep copies on no-op paths ### Polling Backoff with Jitter Replace fixed `rate.Limiter` with adaptive exponential backoff: - Track `consecutiveEmpty` and `consecutiveErrors` counters - Interval doubles with each empty/error response: `base × 2^(n-1)`, capped at `fetch_interval_max` (default 60s) - Add ±20% random jitter to prevent thundering herd - Fetch first, sleep after ��� preserves burst=1 behavior for immediate first fetch on startup and after task completion ### HTTP Client Tuning - Configure custom `http.Transport` with `MaxIdleConnsPerHost=10` (was 2) - Share a single `http.Client` between PingService and RunnerService - Add `IdleConnTimeout=90s` for clean connection lifecycle ## Load Reduction For 200 runners × 3 tasks (70% with active log output): | Component | Before | After | Reduction | |-----------|--------|-------|-----------| | Polling (idle) | 100 req/s | ~3.4 req/s | 97% | | Log reporting | 420 req/s | ~84 req/s | 80% | | State reporting | 126 req/s | ~25 req/s | 80% | | **Total** | **~1,300 req/s** | **~113 req/s** | **~91%** | ## Frontend UX Impact | Scenario | Before | After | Notes | |----------|--------|-------|-------| | Continuous output (npm install) | ~1s | ~5s | Periodic ticker sweep | | Single line then silence | ~1s | ≤3s | maxLatencyTimer guarantee | | Bursty output (100+ lines) | ~1s | <1s | Batch size immediate flush | | Step start/stop | ~1s | <1s | stateNotify immediate flush | | Job completion | ~1s | ~1s | Close() retry unchanged | ## New Configuration Options All have safe defaults — existing config files need no changes: ```yaml runner: fetch_interval_max: 60s # Max backoff interval when idle log_report_interval: 5s # Periodic log flush interval log_report_max_latency: 3s # Max time a log row waits (must be < log_report_interval) log_report_batch_size: 100 # Immediate flush threshold state_report_interval: 5s # State flush interval (step transitions are always immediate) ``` Config validation warns on invalid combinations: - `fetch_interval_max < fetch_interval` → auto-corrected - `log_report_max_latency >= log_report_interval` → warning (timer would be redundant) ## Test Plan - [x] `go build ./...` passes - [x] `go test ./...` passes (all existing + 3 new tests) - [x] `golangci-lint run` — 0 issues - [x] TestReporter_MaxLatencyTimer — verifies single log line flushed by maxLatencyTimer before logTicker - [x] TestReporter_BatchSizeFlush — verifies batch size threshold triggers immediate flush - [x] TestReporter_StateNotifyFlush — verifies step transition triggers immediate state flush - [x] TestReporter_EphemeralRunnerDeletion — verifies Close/RunDaemon race safety - [x] TestReporter_RunDaemonClose_Race — verifies concurrent Close safety Reviewed-on: https://gitea.com/gitea/act_runner/pulls/819 Reviewed-by: Nicolas <173651+bircni@noreply.gitea.com> Co-authored-by: Bo-Yi Wu <appleboy.tw@gmail.com> Co-committed-by: Bo-Yi Wu <appleboy.tw@gmail.com>
187 lines
9.3 KiB
Go
187 lines
9.3 KiB
Go
// Copyright 2022 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/joho/godotenv"
|
|
log "github.com/sirupsen/logrus"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// Log represents the configuration for logging.
|
|
type Log struct {
|
|
Level string `yaml:"level"` // Level indicates the logging level.
|
|
}
|
|
|
|
// Runner represents the configuration for the runner.
|
|
type Runner struct {
|
|
File string `yaml:"file"` // File specifies the file path for the runner.
|
|
Capacity int `yaml:"capacity"` // Capacity specifies the capacity of the runner.
|
|
Envs map[string]string `yaml:"envs"` // Envs stores environment variables for the runner.
|
|
EnvFile string `yaml:"env_file"` // EnvFile specifies the path to the file containing environment variables for the runner.
|
|
Timeout time.Duration `yaml:"timeout"` // Timeout specifies the duration for runner timeout.
|
|
ShutdownTimeout time.Duration `yaml:"shutdown_timeout"` // ShutdownTimeout specifies the duration to wait for running jobs to complete during a shutdown of the runner.
|
|
Insecure bool `yaml:"insecure"` // Insecure indicates whether the runner operates in an insecure mode.
|
|
FetchTimeout time.Duration `yaml:"fetch_timeout"` // FetchTimeout specifies the timeout duration for fetching resources.
|
|
FetchInterval time.Duration `yaml:"fetch_interval"` // FetchInterval specifies the interval duration for fetching resources.
|
|
FetchIntervalMax time.Duration `yaml:"fetch_interval_max"` // FetchIntervalMax specifies the maximum backoff interval when idle.
|
|
LogReportInterval time.Duration `yaml:"log_report_interval"` // LogReportInterval specifies the base interval for periodic log flush.
|
|
LogReportMaxLatency time.Duration `yaml:"log_report_max_latency"` // LogReportMaxLatency specifies the max time a log row can wait before being sent.
|
|
LogReportBatchSize int `yaml:"log_report_batch_size"` // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
|
|
StateReportInterval time.Duration `yaml:"state_report_interval"` // StateReportInterval specifies the interval for state reporting.
|
|
Labels []string `yaml:"labels"` // Labels specify the labels of the runner. Labels are declared on each startup
|
|
GithubMirror string `yaml:"github_mirror"` // GithubMirror defines what mirrors should be used when using github
|
|
}
|
|
|
|
// Cache represents the configuration for caching.
|
|
type Cache struct {
|
|
Enabled *bool `yaml:"enabled"` // Enabled indicates whether caching is enabled. It is a pointer to distinguish between false and not set. If not set, it will be true.
|
|
Dir string `yaml:"dir"` // Dir specifies the directory path for caching.
|
|
Host string `yaml:"host"` // Host specifies the caching host.
|
|
Port uint16 `yaml:"port"` // Port specifies the caching port.
|
|
ExternalServer string `yaml:"external_server"` // ExternalServer specifies the URL of external cache server
|
|
}
|
|
|
|
// Container represents the configuration for the container.
|
|
type Container struct {
|
|
Network string `yaml:"network"` // Network specifies the network for the container.
|
|
NetworkMode string `yaml:"network_mode"` // Deprecated: use Network instead. Could be removed after Gitea 1.20
|
|
Privileged bool `yaml:"privileged"` // Privileged indicates whether the container runs in privileged mode.
|
|
Options string `yaml:"options"` // Options specifies additional options for the container.
|
|
WorkdirParent string `yaml:"workdir_parent"` // WorkdirParent specifies the parent directory for the container's working directory.
|
|
ValidVolumes []string `yaml:"valid_volumes"` // ValidVolumes specifies the volumes (including bind mounts) can be mounted to containers.
|
|
DockerHost string `yaml:"docker_host"` // DockerHost specifies the Docker host. It overrides the value specified in environment variable DOCKER_HOST.
|
|
ForcePull bool `yaml:"force_pull"` // Pull docker image(s) even if already present
|
|
ForceRebuild bool `yaml:"force_rebuild"` // Rebuild docker image(s) even if already present
|
|
RequireDocker bool `yaml:"require_docker"` // Always require a reachable docker daemon, even if not required by act_runner
|
|
DockerTimeout time.Duration `yaml:"docker_timeout"` // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or act_runner
|
|
BindWorkdir bool `yaml:"bind_workdir"` // BindWorkdir binds the workspace to the host filesystem instead of using Docker volumes. Required for DinD when jobs use docker compose with bind mounts.
|
|
}
|
|
|
|
// Host represents the configuration for the host.
|
|
type Host struct {
|
|
WorkdirParent string `yaml:"workdir_parent"` // WorkdirParent specifies the parent directory for the host's working directory.
|
|
}
|
|
|
|
// Config represents the overall configuration.
|
|
type Config struct {
|
|
Log Log `yaml:"log"` // Log represents the configuration for logging.
|
|
Runner Runner `yaml:"runner"` // Runner represents the configuration for the runner.
|
|
Cache Cache `yaml:"cache"` // Cache represents the configuration for caching.
|
|
Container Container `yaml:"container"` // Container represents the configuration for the container.
|
|
Host Host `yaml:"host"` // Host represents the configuration for the host.
|
|
}
|
|
|
|
// LoadDefault returns the default configuration.
|
|
// If file is not empty, it will be used to load the configuration.
|
|
func LoadDefault(file string) (*Config, error) {
|
|
cfg := &Config{}
|
|
if file != "" {
|
|
content, err := os.ReadFile(file)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("open config file %q: %w", file, err)
|
|
}
|
|
if err := yaml.Unmarshal(content, cfg); err != nil {
|
|
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
|
}
|
|
}
|
|
compatibleWithOldEnvs(file != "", cfg)
|
|
|
|
if cfg.Runner.EnvFile != "" {
|
|
if stat, err := os.Stat(cfg.Runner.EnvFile); err == nil && !stat.IsDir() {
|
|
envs, err := godotenv.Read(cfg.Runner.EnvFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read env file %q: %w", cfg.Runner.EnvFile, err)
|
|
}
|
|
if cfg.Runner.Envs == nil {
|
|
cfg.Runner.Envs = map[string]string{}
|
|
}
|
|
maps.Copy(cfg.Runner.Envs, envs)
|
|
}
|
|
}
|
|
|
|
if cfg.Log.Level == "" {
|
|
cfg.Log.Level = "info"
|
|
}
|
|
if cfg.Runner.File == "" {
|
|
cfg.Runner.File = ".runner"
|
|
}
|
|
if cfg.Runner.Capacity <= 0 {
|
|
cfg.Runner.Capacity = 1
|
|
}
|
|
if cfg.Runner.Timeout <= 0 {
|
|
cfg.Runner.Timeout = 3 * time.Hour
|
|
}
|
|
if cfg.Cache.Enabled == nil {
|
|
b := true
|
|
cfg.Cache.Enabled = &b
|
|
}
|
|
if *cfg.Cache.Enabled {
|
|
if cfg.Cache.Dir == "" {
|
|
home, _ := os.UserHomeDir()
|
|
cfg.Cache.Dir = filepath.Join(home, ".cache", "actcache")
|
|
}
|
|
}
|
|
if cfg.Container.WorkdirParent == "" {
|
|
cfg.Container.WorkdirParent = "workspace"
|
|
}
|
|
if cfg.Host.WorkdirParent == "" {
|
|
home, _ := os.UserHomeDir()
|
|
cfg.Host.WorkdirParent = filepath.Join(home, ".cache", "act")
|
|
}
|
|
if cfg.Runner.FetchTimeout <= 0 {
|
|
cfg.Runner.FetchTimeout = 5 * time.Second
|
|
}
|
|
if cfg.Runner.FetchInterval <= 0 {
|
|
cfg.Runner.FetchInterval = 2 * time.Second
|
|
}
|
|
if cfg.Runner.FetchIntervalMax <= 0 {
|
|
cfg.Runner.FetchIntervalMax = 60 * time.Second
|
|
}
|
|
if cfg.Runner.LogReportInterval <= 0 {
|
|
cfg.Runner.LogReportInterval = 5 * time.Second
|
|
}
|
|
if cfg.Runner.LogReportMaxLatency <= 0 {
|
|
cfg.Runner.LogReportMaxLatency = 3 * time.Second
|
|
}
|
|
if cfg.Runner.LogReportBatchSize <= 0 {
|
|
cfg.Runner.LogReportBatchSize = 100
|
|
}
|
|
if cfg.Runner.StateReportInterval <= 0 {
|
|
cfg.Runner.StateReportInterval = 5 * time.Second
|
|
}
|
|
|
|
// Validate and fix invalid config combinations to prevent confusing behavior.
|
|
if cfg.Runner.FetchIntervalMax < cfg.Runner.FetchInterval {
|
|
log.Warnf("fetch_interval_max (%v) is less than fetch_interval (%v), setting fetch_interval_max to fetch_interval",
|
|
cfg.Runner.FetchIntervalMax, cfg.Runner.FetchInterval)
|
|
cfg.Runner.FetchIntervalMax = cfg.Runner.FetchInterval
|
|
}
|
|
if cfg.Runner.LogReportMaxLatency >= cfg.Runner.LogReportInterval {
|
|
log.Warnf("log_report_max_latency (%v) >= log_report_interval (%v), the max-latency timer will never fire before the periodic ticker; consider lowering log_report_max_latency",
|
|
cfg.Runner.LogReportMaxLatency, cfg.Runner.LogReportInterval)
|
|
}
|
|
|
|
// although `container.network_mode` will be deprecated, but we have to be compatible with it for now.
|
|
if cfg.Container.NetworkMode != "" && cfg.Container.Network == "" {
|
|
log.Warn("You are trying to use deprecated configuration item of `container.network_mode`, please use `container.network` instead.")
|
|
if cfg.Container.NetworkMode == "bridge" {
|
|
// Previously, if the value of `container.network_mode` is `bridge`, we will create a new network for job.
|
|
// But “bridge” is easily confused with the bridge network created by Docker by default.
|
|
// So we set the value of `container.network` to empty string to make `act_runner` automatically create a new network for job.
|
|
cfg.Container.Network = ""
|
|
} else {
|
|
cfg.Container.Network = cfg.Container.NetworkMode
|
|
}
|
|
}
|
|
|
|
return cfg, nil
|
|
}
|