From 3c16cbcd19d1397a14d46a4bbc995c4aa2c54a33 Mon Sep 17 00:00:00 2001 From: David Kirwan Date: Wed, 5 Nov 2025 16:43:23 +0000 Subject: [PATCH] forgejo: templates for deploying valkey cluster. Signed-off-by: David Kirwan --- .../templates/valkey-configmap.yaml.j2 | 78 ++++++++++++++++--- .../forgejo/templates/valkey-secret.yaml.j2 | 8 ++ .../forgejo/templates/valkey-service.yaml.j2 | 11 +-- .../templates/valkey-statefulset.yaml.j2 | 71 +++++++++++++++-- 4 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 roles/openshift-apps/forgejo/templates/valkey-secret.yaml.j2 diff --git a/roles/openshift-apps/forgejo/templates/valkey-configmap.yaml.j2 b/roles/openshift-apps/forgejo/templates/valkey-configmap.yaml.j2 index f416c6568c..5b24a19940 100644 --- a/roles/openshift-apps/forgejo/templates/valkey-configmap.yaml.j2 +++ b/roles/openshift-apps/forgejo/templates/valkey-configmap.yaml.j2 @@ -2,19 +2,73 @@ apiVersion: v1 kind: ConfigMap metadata: - name: valkey-config + name: valkey namespace: forgejo data: valkey.conf: | - bind 0.0.0.0 - port 6379 - cluster-enabled yes - requirepass "{{ (env == 'production') | ternary(forgejo_valkey_password, forgejo_stg_valkey_password) }}" - masterauth "{{ (env == 'production') | ternary(forgejo_valkey_masterauth, forgejo_stg_valkey_masterauth) }}" protected-mode no - dir /data - dbfilename dump.rdb - save 900 1 - save 300 10 - save 60 10000 - logfile /data/valkey.log + cluster-enabled yes + cluster-config-file nodes.conf + cluster-node-timeout 5000 + appendonly yes + port 6379 + + init-config.sh: | + #!/bin/sh + cp /etc/valkey/valkey.conf /tmp/valkey.conf + + echo "requirepass ${VALKEY_PASSWORD}" >> /tmp/valkey.conf + echo "masterauth ${VALKEY_PASSWORD}" >> /tmp/valkey.conf + echo "cluster-announce-ip ${POD_IP}" >> /tmp/valkey.conf + cp /tmp/valkey.conf /config/valkey.conf + + mkdir -p /data + chown -R 1000:1000 /data + + init-cluster.sh: | + #!/bin/sh + set -e + + until valkey-cli -h localhost -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do + echo "Waiting for local Valkey..." + sleep 2 + done + + ORDINAL=$(hostname | rev | cut -d'-' -f1 | rev) + PRIMARIES=$(( ({{ replicas }} + 1) / 2 )) + + if [ "$ORDINAL" -eq 0 ]; then + # ---- Primary-0 creates the cluster --------------------------------- + if ! valkey-cli -a "${VALKEY_PASSWORD}" cluster info | grep -q 'cluster_state:ok'; then + NODES="" + for i in $(seq 0 $(({{ replicas }}-1))); do + if [ "$i" -eq 0 ]; then + NODES="${NODES} ${POD_IP}:6379" + else + NODES="${NODES} valkey-${i}.valkey.{{ namespace | default('default') }}.svc.cluster.local:6379" + fi + done + REPLICAS_PER_PRIMARY=$(( ({{ replicas }} - ${PRIMARIES}) / ${PRIMARIES} )) + echo "yes" | valkey-cli -a "${VALKEY_PASSWORD}" \ + --cluster create ${NODES} --cluster-replicas ${REPLICAS_PER_PRIMARY} + fi + elif [ "$ORDINAL" -ge "$PRIMARIES" ]; then + # ---- Replica node --------------------------------------------------- + PRIMARY_IDX=$(( ORDINAL % PRIMARIES )) + PRIMARY_HOST="valkey-${PRIMARY_IDX}.valkey.{{ namespace | default('default') }}.svc.cluster.local" + until valkey-cli -h "${PRIMARY_HOST}" -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do + echo "Waiting for primary ${PRIMARY_HOST}..." + sleep 5 + done + valkey-cli -a "${VALKEY_PASSWORD}" \ + --cluster add-node "${POD_IP}:6379" "${PRIMARY_HOST}:6379" --cluster-slave + else + # ---- Additional primary -------------------------------------------- + until valkey-cli -h valkey-0.valkey.{{ namespace | default('default') }}.svc.cluster.local \ + -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do + echo "Waiting for valkey-0..." + sleep 5 + done + valkey-cli -a "${VALKEY_PASSWORD}" \ + --cluster add-node "${POD_IP}:6379" valkey-0.valkey.{{ namespace | default('default') }}.svc.cluster.local:6379 + fi diff --git a/roles/openshift-apps/forgejo/templates/valkey-secret.yaml.j2 b/roles/openshift-apps/forgejo/templates/valkey-secret.yaml.j2 new file mode 100644 index 0000000000..1ddf07a1d1 --- /dev/null +++ b/roles/openshift-apps/forgejo/templates/valkey-secret.yaml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: valkey-auth + namespace: forgejo +type: Opaque +stringData: + password: "{{ (env == 'production') | ternary(forgejo_valkey_password, forgejo_stg_valkey_password) }}" diff --git a/roles/openshift-apps/forgejo/templates/valkey-service.yaml.j2 b/roles/openshift-apps/forgejo/templates/valkey-service.yaml.j2 index 279e4b516e..a2054249e7 100644 --- a/roles/openshift-apps/forgejo/templates/valkey-service.yaml.j2 +++ b/roles/openshift-apps/forgejo/templates/valkey-service.yaml.j2 @@ -2,13 +2,14 @@ apiVersion: v1 kind: Service metadata: - name: valkey-headless + name: valkey namespace: forgejo spec: - clusterIP: None # Headless Service + clusterIP: None selector: app: valkey ports: - - name: valkey - port: 6379 - targetPort: 6379 + - name: client + port: 6379 + - name: cluster-bus + port: 16379 diff --git a/roles/openshift-apps/forgejo/templates/valkey-statefulset.yaml.j2 b/roles/openshift-apps/forgejo/templates/valkey-statefulset.yaml.j2 index 22b6faf0e8..74bd9c0b7c 100644 --- a/roles/openshift-apps/forgejo/templates/valkey-statefulset.yaml.j2 +++ b/roles/openshift-apps/forgejo/templates/valkey-statefulset.yaml.j2 @@ -5,8 +5,9 @@ metadata: name: valkey namespace: forgejo spec: - serviceName: valkey-headless + serviceName: valkey replicas: 3 + podManagementPolicy: Parallel selector: matchLabels: app: valkey @@ -15,23 +16,79 @@ spec: labels: app: valkey spec: + initContainers: + - name: init-config + image: busybox:1.36 + command: ["/scripts/init-config.sh"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-auth + key: password + volumeMounts: + - name: config + mountPath: /etc/valkey + readOnly: true + - name: scripts + mountPath: /scripts + - name: workdir + mountPath: /config + - name: data + mountPath: /data containers: - name: valkey image: valkey/valkey:7.2.5 - args: ["/etc/valkey/valkey.conf"] + command: ["/bin/sh", "-c"] + args: + - | + /scripts/init-cluster.sh & + exec valkey-server /config/valkey.conf + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: valkey-auth + key: password ports: - containerPort: 6379 - name: valkey + name: client + - containerPort: 16379 + name: cluster-bus volumeMounts: - - name: config - mountPath: /etc/valkey/valkey.conf - subPath: valkey.conf - name: data mountPath: /data + - name: workdir + mountPath: /config + - name: scripts + mountPath: /scripts + readinessProbe: + exec: + command: ["valkey-cli", "-a", "x", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 volumes: - name: config configMap: - name: valkey-config + name: valkey + - name: scripts + configMap: + name: valkey + defaultMode: 0755 + - name: workdir + emptyDir: {} volumeClaimTemplates: - metadata: name: data