forgejo: templates for deploying valkey cluster.

Signed-off-by: David Kirwan <davidkirwanirl@gmail.com>

roles/openshift-apps/forgejo/templates/valkey-configmap.yaml.j2

@@ -2,19 +2,73 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: valkey-config
+  name: valkey
   namespace: forgejo
 data:
   valkey.conf: |
-    bind 0.0.0.0
-    port 6379
-    cluster-enabled yes
-    requirepass "{{ (env == 'production') | ternary(forgejo_valkey_password, forgejo_stg_valkey_password) }}"
-    masterauth "{{ (env == 'production') | ternary(forgejo_valkey_masterauth, forgejo_stg_valkey_masterauth) }}"
     protected-mode no
-    dir /data
-    dbfilename dump.rdb
-    save 900 1
-    save 300 10
-    save 60 10000
-    logfile /data/valkey.log
+    cluster-enabled yes
+    cluster-config-file nodes.conf
+    cluster-node-timeout 5000
+    appendonly yes
+    port 6379
+
+  init-config.sh: |
+    #!/bin/sh
+    cp /etc/valkey/valkey.conf /tmp/valkey.conf
+
+    echo "requirepass ${VALKEY_PASSWORD}" >> /tmp/valkey.conf
+    echo "masterauth ${VALKEY_PASSWORD}" >> /tmp/valkey.conf
+    echo "cluster-announce-ip ${POD_IP}" >> /tmp/valkey.conf
+    cp /tmp/valkey.conf /config/valkey.conf
+
+    mkdir -p /data
+    chown -R 1000:1000 /data
+
+  init-cluster.sh: |
+    #!/bin/sh
+    set -e
+
+    until valkey-cli -h localhost -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do
+      echo "Waiting for local Valkey..."
+      sleep 2
+    done
+
+    ORDINAL=$(hostname | rev | cut -d'-' -f1 | rev)
+    PRIMARIES=$(( ({{ replicas }} + 1) / 2 ))
+
+    if [ "$ORDINAL" -eq 0 ]; then
+      # ---- Primary-0 creates the cluster ---------------------------------
+      if ! valkey-cli -a "${VALKEY_PASSWORD}" cluster info | grep -q 'cluster_state:ok'; then
+        NODES=""
+        for i in $(seq 0 $(({{ replicas }}-1))); do
+          if [ "$i" -eq 0 ]; then
+            NODES="${NODES} ${POD_IP}:6379"
+          else
+            NODES="${NODES} valkey-${i}.valkey.{{ namespace | default('default') }}.svc.cluster.local:6379"
+          fi
+        done
+        REPLICAS_PER_PRIMARY=$(( ({{ replicas }} - ${PRIMARIES}) / ${PRIMARIES} ))
+        echo "yes" | valkey-cli -a "${VALKEY_PASSWORD}" \
+          --cluster create ${NODES} --cluster-replicas ${REPLICAS_PER_PRIMARY}
+      fi
+    elif [ "$ORDINAL" -ge "$PRIMARIES" ]; then
+      # ---- Replica node ---------------------------------------------------
+      PRIMARY_IDX=$(( ORDINAL % PRIMARIES ))
+      PRIMARY_HOST="valkey-${PRIMARY_IDX}.valkey.{{ namespace | default('default') }}.svc.cluster.local"
+      until valkey-cli -h "${PRIMARY_HOST}" -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do
+        echo "Waiting for primary ${PRIMARY_HOST}..."
+        sleep 5
+      done
+      valkey-cli -a "${VALKEY_PASSWORD}" \
+        --cluster add-node "${POD_IP}:6379" "${PRIMARY_HOST}:6379" --cluster-slave
+    else
+      # ---- Additional primary --------------------------------------------
+      until valkey-cli -h valkey-0.valkey.{{ namespace | default('default') }}.svc.cluster.local \
+            -p 6379 -a "${VALKEY_PASSWORD}" ping >/dev/null 2>&1; do
+        echo "Waiting for valkey-0..."
+        sleep 5
+      done
+      valkey-cli -a "${VALKEY_PASSWORD}" \
+        --cluster add-node "${POD_IP}:6379" valkey-0.valkey.{{ namespace | default('default') }}.svc.cluster.local:6379
+    fi

roles/openshift-apps/forgejo/templates/valkey-secret.yaml.j2

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: valkey-auth
+  namespace: forgejo
+type: Opaque
+stringData:
+  password: "{{ (env == 'production') | ternary(forgejo_valkey_password, forgejo_stg_valkey_password) }}"

roles/openshift-apps/forgejo/templates/valkey-service.yaml.j2

@@ -2,13 +2,14 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: valkey-headless
+  name: valkey
   namespace: forgejo
 spec:
-  clusterIP: None # Headless Service
+  clusterIP: None
   selector:
     app: valkey
   ports:
-    - name: valkey
-      port: 6379
-      targetPort: 6379
+  - name: client
+    port: 6379
+  - name: cluster-bus
+    port: 16379

roles/openshift-apps/forgejo/templates/valkey-statefulset.yaml.j2

@@ -5,8 +5,9 @@ metadata:
   name: valkey
   namespace: forgejo
 spec:
-  serviceName: valkey-headless
+  serviceName: valkey
   replicas: 3
+  podManagementPolicy: Parallel
   selector:
     matchLabels:
       app: valkey
@@ -15,23 +16,79 @@ spec:
       labels:
         app: valkey
     spec:
+      initContainers:
+        - name: init-config
+          image: busybox:1.36
+          command: ["/scripts/init-config.sh"]
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: VALKEY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: valkey-auth
+                  key: password
+          volumeMounts:
+            - name: config
+              mountPath: /etc/valkey
+              readOnly: true
+            - name: scripts
+              mountPath: /scripts
+            - name: workdir
+              mountPath: /config
+            - name: data
+              mountPath: /data
       containers:
         - name: valkey
           image: valkey/valkey:7.2.5
-          args: ["/etc/valkey/valkey.conf"]
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              /scripts/init-cluster.sh &
+              exec valkey-server /config/valkey.conf
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: VALKEY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: valkey-auth
+                  key: password
           ports:
             - containerPort: 6379
-              name: valkey
+              name: client
+            - containerPort: 16379
+              name: cluster-bus
           volumeMounts:
-            - name: config
-              mountPath: /etc/valkey/valkey.conf
-              subPath: valkey.conf
             - name: data
               mountPath: /data
+            - name: workdir
+              mountPath: /config
+            - name: scripts
+              mountPath: /scripts
+          readinessProbe:
+            exec:
+              command: ["valkey-cli", "-a", "x", "ping"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
       volumes:
         - name: config
           configMap:
-            name: valkey-config
+            name: valkey
+        - name: scripts
+          configMap:
+            name: valkey
+            defaultMode: 0755
+        - name: workdir
+          emptyDir: {}
   volumeClaimTemplates:
     - metadata:
         name: data