Merge pull request #823 from stavros-k/minio-logsearchapi

NAS-118859 / Add LogSearchApi to Minio App
This commit is contained in:
Stavros Kois
2022-12-21 13:34:10 +02:00
committed by GitHub
32 changed files with 1313 additions and 0 deletions

View File

@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
# OWNERS file for Kubernetes
OWNERS

View File

@@ -0,0 +1,8 @@
# Change Log
This contains all the notable changes to the MinIO application.
## [1.5.0]
1. On fresh installation, minIO data directory's ownership will be updated to minio:minio.
2. For existing installations that are exhibiting the upgrade, the minIO data directory's ownership will be migrated to minio:minio.

View File

@@ -0,0 +1,19 @@
apiVersion: v1
appVersion: '2022-10-24'
dependencies:
- name: common
repository: file://../../../library/common/2207.0.0
version: 2207.0.0
description: High Performance, Kubernetes Native Object Storage
home: https://min.io
icon: https://min.io/resources/img/logo/MINIO_wordmark.png
keywords:
- storage
- object-storage
- S3
name: minio
sources:
- https://github.com/minio/minio
- https://github.com/minio/charts
upstream_version: 8.0.5
version: 1.7.0

52
test/minio/1.7.0/README.md Executable file
View File

@@ -0,0 +1,52 @@
MinIO
=====
[MinIO](https://min.io) is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
MinIO supports [distributed mode](https://docs.minio.io/docs/distributed-minio-quickstart-guide). In distributed mode, you can pool multiple drives (even on different machines) into a single object storage server.
For more detailed documentation please visit [here](https://docs.minio.io/)
Introduction
------------
This chart bootstraps MinIO deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager.
Configuration
-------------
The following table lists the configurable parameters of the MinIO chart and their default values.
| Parameter | Description | Default |
|:-------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------|
| `image.repository` | Image repository | `minio/minio` |
| `image.tag` | MinIO image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/). | `RELEASE.2020-11-06T23-17-07Z` |
| `image.pullPolicy` | Image pull policy | `IfNotPresent` |
| `extraArgs` | Additional command line arguments to pass to the MinIO server | `[]` |
| `accessKey` | Default access key (5 to 20 characters) | random 20 chars |
| `secretKey` | Default secret key (8 to 40 characters) | random 40 chars |
| `persistence.enabled` | Use persistent volume to store data | `true` |
| `persistence.size` | Size of persistent volume claim | `500Gi` |
| `persistence.existingClaim` | Use an existing PVC to persist data | `nil` |
| `persistence.storageClass` | Storage class name of PVC | `nil` |
| `persistence.accessMode` | ReadWriteOnce or ReadOnly | `ReadWriteOnce` |
| `persistence.subPath` | Mount a sub directory of the persistent volume if set | `""` |
| `environment` | Set MinIO server relevant environment variables in `values.yaml` file. MinIO containers will be passed these variables when they start. | `MINIO_STORAGE_CLASS_STANDARD: EC:4"` |
Some parameters above map to the env variables defined in the [MinIO DockerHub image](https://hub.docker.com/r/minio/minio/).
Pass environment variables to MinIO containers
----------------------------------------------
To pass environment variables to MinIO containers when deploying via Helm chart, use the below command line format
```bash
$ helm install --set environment.MINIO_BROWSER=on,environment.MINIO_DOMAIN=domain-name minio/minio
```
You can add as many environment variables as required, using the above format. Just add `environment.<VARIABLE_NAME>=<value>` under `set` flag.
**NOTE**
- On fresh installation, minIO data directory's ownership will be updated to minio:minio.
- For existing installations that are exhibiting the upgrade to >=1.5.0, the minIO data directory's ownership will be migrated to minio:minio.

View File

@@ -0,0 +1,5 @@
[MinIO](https://min.io) is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
**NOTE**
- On installation, minIO data directory's ownership will be updated to minio:minio.
- For existing installations that are exhibiting the upgrade to >=1.5.0, the minIO data directory's ownership will be migrated to minio:minio.

Binary file not shown.

View File

@@ -0,0 +1,13 @@
image:
pullPolicy: IfNotPresent
repository: minio/minio
tag: RELEASE.2022-11-10T18-20-21Z
logSearchImage:
pullPolicy: IfNotPresent
repository: minio/operator
tag: v4.5.4
logsearchapi:
enabled: false
diskCapacityGB: 5
runAsGroup: 473
runAsUser: 473

View File

@@ -0,0 +1,120 @@
#!/usr/bin/python3
import json
import os
import sys
import subprocess
from pathlib import Path
from middlewared.client import Client
from middlewared.service import ValidationErrors, CallError
def path_in_locked_datasets(path: str) -> bool:
with Client() as c:
return c.call('pool.dataset.path_in_locked_datasets', path)
def get_configured_user_group(path: str) -> dict:
with Client() as c:
return c.call('filesystem.stat', path)
def get_host_path_attachments(path: str) -> set:
with Client() as c:
return {
attachment['type']
for attachment in c.call('pool.dataset.attachments_with_path', path)
if attachment['type'].lower() not in ['kubernetes', 'chart releases']
}
def get_kubernetes_config() -> dict:
with Client() as c:
return c.call('kubernetes.config')
def validate_host_path(path: str, schema_name: str, verrors: ValidationErrors) -> None:
"""
These validations are taken from `FilesystemService._common_perm_path_validate`.
Including an additional validation that makes sure all the children under
a path are on same device.
"""
schema_name += ".migration.chown"
p = Path(path)
if not p.is_absolute():
verrors.add(schema_name, f"Must be an absolute path: {path}")
if p.is_file():
verrors.add(schema_name, f"Recursive operations on a file are invalid: {path}")
if not p.absolute().as_posix().startswith("/mnt/"):
verrors.add(
schema_name,
f"Changes to permissions on paths that are not beneath the directory /mnt are not permitted: {path}"
)
elif len(p.resolve().parents) == 2:
verrors.add(schema_name, f"The specified path is a ZFS pool mountpoint: {path}")
# Make sure that dataset is not locked
if path_in_locked_datasets(path):
verrors.add(schema_name, f"Dataset is locked at path: {path}.")
# Validate attachments
if attachments := get_host_path_attachments(path):
verrors.add(schema_name, f"The path '{path}' is already attached to service(s): {', '.join(attachments)}.")
# Make sure all the minio's data directory children are on same device.
device_id = os.stat(path).st_dev
for root, dirs, files in os.walk(path):
for child in dirs + files:
abs_path = os.path.join(root, child)
if os.stat(abs_path).st_dev != device_id:
verrors.add(
schema_name,
(f"All the children of MinIO data directory should be on "
f"same device as root: path={abs_path} device={os.stat(abs_path).st_dev}")
)
break
def migrate(values: dict) -> dict:
# minio user / group ID
uid = gid = 473
verrors = ValidationErrors()
k8s_config = get_kubernetes_config()
if values["appVolumeMounts"]["export"]["hostPathEnabled"]:
host_path = values["appVolumeMounts"]["export"]["hostPath"]
else:
app_dataset = values["appVolumeMounts"]["export"]["datasetName"]
host_path = os.path.join(
"/mnt", k8s_config['dataset'], "releases", values["release_name"], "volumes/ix_volumes", app_dataset
)
current_config = get_configured_user_group(host_path)
if current_config["uid"] == uid and current_config["gid"] == gid:
return values
validate_host_path(host_path, values['release_name'], verrors)
verrors.check()
# chown the host path
acltool = subprocess.run([
"/usr/bin/nfs4xdr_winacl",
"-a", "chown",
"-O", str(uid), "-G", str(gid),
"-r",
"-c", host_path,
"-p", host_path], check=False, capture_output=True
)
if acltool.returncode != 0:
raise CallError(f"acltool [chown] on path {host_path} failed with error: [{acltool.stderr.decode().strip()}]")
return values
if __name__ == "__main__":
if len(sys.argv) != 2:
exit(1)
if os.path.exists(sys.argv[1]):
with open(sys.argv[1], "r") as f:
print(json.dumps(migrate(json.loads(f.read()))))

View File

@@ -0,0 +1,28 @@
#!/usr/bin/python3
import json
import os
import sys
def migrate(values):
if values.get('appVolumeMounts'):
return values
values.update({
'appVolumeMounts': {
'export': {
'hostPathEnabled': values['minioHostPathEnabled'],
**({'hostPath': values['minioHostPath']} if values.get('minioHostPath') else {})
},
},
})
return values
if __name__ == '__main__':
if len(sys.argv) != 2:
exit(1)
if os.path.exists(sys.argv[1]):
with open(sys.argv[1], 'r') as f:
print(json.dumps(migrate(json.loads(f.read()))))

View File

@@ -0,0 +1,378 @@
groups:
- name: "Container Images"
description: "Image to be used for container"
- name: "Workload Configuration"
description: "Configure workload deployment"
- name: "Minio Configuration"
description: "Configure Minio credentials"
- name: "Storage"
description: "Configure Storage for Nextcloud"
- name: "Advanced DNS Settings"
description: "Configure DNS settings"
- name: "Resource Limits"
description: "Set CPU/memory limits for Kubernetes Pod"
portals:
web_portal:
protocols:
- "$kubernetes-resource_configmap_minio-config_protocol"
host:
- "$node_ip"
ports:
- "$variable-service.consolePort"
questions:
- variable: dnsConfig
label: "DNS Configuration"
group: "Advanced DNS Settings"
schema:
type: dict
attrs:
- variable: options
label: "DNS Options"
schema:
type: list
items:
- variable: optionsEntry
label: "Option Entry Configuration"
schema:
type: dict
attrs:
- variable: name
label: "Option Name"
schema:
type: string
required: true
- variable: value
label: "Option Value"
schema:
type: string
required: true
- variable: updateStrategy
label: "Minio update strategy"
group: "Workload Configuration"
schema:
type: string
default: "RollingUpdate"
enum:
- value: "RollingUpdate"
description: "Create new pods and then kill old ones"
- value: "Recreate"
description: "Kill existing pods before creating new ones"
- variable: distributedMode
label: "Enable Distributed Mode"
description: "Run Minio instance to connect to a distributed minio cluster"
group: "Minio Configuration"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: distributedIps
label: "Distributed Minio Instance URI(s)"
schema:
type: list
required: true
default: []
items:
- variable: ip
label: "Distributed Minio Instance URI"
schema:
type: string
empty: false
- variable: extraArgs
label: "Minio Extra Arguments"
group: "Minio Configuration"
schema:
type: list
default: []
items:
- variable: arg
label: "Argument"
schema:
type: string
- variable: accessKey
label: "Root User"
group: "Minio Configuration"
description: "Enter the S3 Root User"
schema:
type: string
private: true
required: true
min_length: 5
max_length: 20
- variable: secretKey
label: "Root Password"
group: "Minio Configuration"
description: "Enter the S3 Root Password"
schema:
type: string
private: true
required: true
min_length: 8
max_length: 40
- variable: environmentVariables
label: "Minio image environment"
group: "Minio Configuration"
schema:
type: list
default: []
items:
- variable: environmentVariable
label: "Environment Variable"
schema:
type: dict
attrs:
- variable: name
label: "Name"
schema:
type: string
- variable: value
label: "Value"
schema:
type: string
- variable: service
description: "Minio Service Configuration"
label: "Minio Service Configuration"
group: "Minio Configuration"
schema:
type: dict
required: true
attrs:
- variable: nodePort
label: "Node Port to use for Minio API"
schema:
type: int
min: 9000
max: 65535
default: 9000
required: true
- variable: consolePort
label: "Node Port to use for Minio UI Access"
schema:
type: int
min: 9000
max: 65535
default: 9002
required: true
- variable: minioDomain
label: "Minio Domain Name"
description: "This is only required if TLS is configured for Minio"
group: "Minio Configuration"
schema:
type: string
default: null
"null": true
show_if: [["certificate", "!=", null]]
- variable: certificate
description: "Minio Certificate"
label: "Minio Certificate"
group: "Minio Configuration"
schema:
type: int
$ref:
- "definitions/certificate"
- variable: logsearchapi
label: Log Search API Configuration
description: Log Search API Configuration
group: Minio Configuration
schema:
type: dict
attrs:
- variable: enabled
label: Enable Log Search API
description: Enables Log Search API and configures MinIO to use it. It also deploys a postgres DB for it.
schema:
type: boolean
default: false
- variable: diskCapacityGB
label: Disk Capacity in GB
description: Capacity in GB that logs are allowed to occupy.
schema:
type: int
show_if: [["enabled", "=", true]]
default: 5
- variable: appVolumeMounts
label: "Minio Storage"
group: "Storage"
schema:
type: dict
show_if: [["distributedMode", "=", false]]
attrs:
- variable: export
label: "Data Volume"
schema:
type: dict
attrs:
- variable: datasetName
label: "Minio Data Volume Name"
schema:
type: string
hidden: true
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-minio"
editable: false
- variable: mountPath
label: "Minio Data Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
default: "/export"
immutable: true
- variable: hostPathEnabled
label: "Enable Host Path for Minio Data Volume"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: "Host Path for Minio Data Volume"
schema:
type: hostpath
required: true
immutable: true
- variable: extraAppVolumeMounts
label: "Extra Host Path Volumes"
group: "Storage"
schema:
type: list
items:
- variable: extraAppVolume
label: "Host Path Volume"
description: "Add an extra host path volume for Minio application"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: postgresAppVolumeMounts
label: Postgres Storage
group: Storage
schema:
type: dict
attrs:
- variable: postgres-data
label: Postgres Data Volume
schema:
type: dict
attrs:
- variable: datasetName
label: Postgres Data Volume Name
schema:
type: string
hidden: true
$ref:
- normalize/ixVolume
default: ix-postgres_data
show_if: [["hostPathEnabled", "=", false]]
editable: false
- variable: mountPath
label: Postgresql Data Mount Path
description: Path where the volume will be mounted inside the pod
schema:
type: path
hidden: true
editable: false
default: /var/lib/postgresql/data
- variable: hostPathEnabled
label: Enable Host Path for Postgres Data Volume
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: Host Path for Postgres Data Volume
schema:
type: hostpath
required: true
immutable: true
- variable: postgres-backup
label: Postgres Backup Volume
schema:
type: dict
attrs:
- variable: datasetName
label: Postgres Backup Volume Name
schema:
type: string
hidden: true
$ref:
- normalize/ixVolume
default: ix-postgres_backups
show_if: [["hostPathEnabled", "=", false]]
editable: false
- variable: mountPath
label: Postgresql Backup Mount Path
description: Path where the volume will be mounted inside the pod
schema:
type: path
hidden: true
editable: false
default: /postgres_backups
- variable: hostPathEnabled
label: Enable Host Path for Postgres Backup Volume
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: Host Path for Postgres Backup Volume
schema:
type: hostpath
required: true
- variable: enableResourceLimits
label: "Enable Pod resource limits"
group: "Resource Limits"
schema:
type: boolean
default: false
- variable: cpuLimit
label: "CPU Limresource limitsit"
description: "CPU resource limit allow plain integer values with suffix m(milli) e.g 1000m, 100."
group: "Resource Limits"
schema:
type: string
show_if: [["enableResourceLimits", "=", true]]
valid_chars: "^\\d+(?:\\.\\d+(?!.*m$)|m?$)"
default: "4000m"
- variable: memLimit
label: "Memory Limit"
group: "Resource Limits"
description: "Memory limits is specified by number of bytes. Followed by quantity suffix like E,P,T,G,M,k and Ei,Pi,Ti,Mi,Gi,Ki can also be used. e.g 129e6, 129M, 128974848000m, 123Mi"
schema:
type: string
show_if: [["enableResourceLimits", "=", true]]
valid_chars: "^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$"
default: "8Gi"

View File

@@ -0,0 +1,6 @@
dependencies:
- name: common
repository: file://../../../library/common/2207.0.0
version: 2207.0.0
digest: sha256:f17f3d458ca0210a52e39da0dce35034e900b36f2040d4b19bed46a7aae91506
generated: "2022-07-25T00:20:47.987169596+05:00"

View File

@@ -0,0 +1,2 @@
Minio can be accessed from the following URL:
http://$node_ip:{{ .Values.service.nodePort }}/

View File

@@ -0,0 +1,33 @@
{{/*
Formats volumeMount for Minio tls keys and trusted certs
*/}}
{{- define "minio.tlsKeysVolumeMount" -}}
{{- if eq (include "minio.certAvailable" .) "true" -}}
- name: cert-secret-volume
mountPath: "/etc/minio/certs"
- name: trusted-cert-secret-volume
mountPath: "/etc/minio/certs/CAs"
{{- end }}
{{- end -}}
{{/*
Formats volume for Minio tls keys and trusted certs
*/}}
{{- define "minio.tlsKeysVolume" -}}
{{- if eq (include "minio.certAvailable" .) "true" -}}
- name: cert-secret-volume
secret:
secretName: {{ include "minio.secretName" . }}
items:
- key: certPublicKey
path: public.crt
- key: certPrivateKey
path: private.key
- name: trusted-cert-secret-volume
secret:
secretName: {{ include "minio.secretName" . }}
items:
- key: certPublicKey
path: public.crt
{{- end }}
{{- end -}}

View File

@@ -0,0 +1,77 @@
{{/*
Determine secret name.
*/}}
{{- define "minio.secretName" -}}
{{- include "common.names.fullname" . -}}
{{- end -}}
{{/*
Retrieve true/false if minio certificate is configured
*/}}
{{- define "minio.certAvailable" -}}
{{- if .Values.certificate -}}
{{- $values := (. | mustDeepCopy) -}}
{{- $_ := set $values "commonCertOptions" (dict "certKeyName" $values.Values.certificate) -}}
{{- template "common.resources.cert_present" $values -}}
{{- else -}}
{{- false -}}
{{- end -}}
{{- end -}}
{{/*
Retrieve public key of minio certificate
*/}}
{{- define "minio.cert.publicKey" -}}
{{- $values := (. | mustDeepCopy) -}}
{{- $_ := set $values "commonCertOptions" (dict "certKeyName" $values.Values.certificate "publicKey" true) -}}
{{ include "common.resources.cert" $values }}
{{- end -}}
{{/*
Retrieve private key of minio certificate
*/}}
{{- define "minio.cert.privateKey" -}}
{{- $values := (. | mustDeepCopy) -}}
{{- $_ := set $values "commonCertOptions" (dict "certKeyName" $values.Values.certificate) -}}
{{ include "common.resources.cert" $values }}
{{- end -}}
{{/*
Retrieve scheme/protocol for minio
*/}}
{{- define "minio.scheme" -}}
{{- if eq (include "minio.certAvailable" .) "true" -}}
{{- print "https" -}}
{{- else -}}
{{- print "http" -}}
{{- end -}}
{{- end -}}
{{/*
Retrieve command for minio application
*/}}
{{- define "minio.commandArgs" -}}
{{- $arg := (printf "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs server --console-address=':%d'" (.Values.service.consolePort | int)) -}}
{{- if .Values.distributedMode -}}
{{- cat $arg (join " " (concat (.Values.distributedIps | default list) (.Values.extraArgs | default list))) -}}
{{- else -}}
{{- cat $arg ((concat (list (printf "--address ':%d'" (.Values.service.nodePort | int))) (list (printf "%s" .Values.appVolumeMounts.export.mountPath)) (.Values.extraArgs | default list)) | join " ") -}}
{{- end -}}
{{- end -}}
{{/*
Enable host networking
*/}}
{{- define "minio.hostNetworking" -}}
{{- if .Values.distributedMode -}}
{{- print "true" -}}
{{- else -}}
{{- print "false" -}}
{{- end -}}
{{- end -}}

View File

@@ -0,0 +1,34 @@
{{- define "logsearchapi.imageName" -}}
{{- printf "%s:%s" .Values.logSearchImage.repository .Values.logSearchImage.tag -}}
{{- end -}}
{{- define "logsearchapi.nameSuffix" -}}
{{- print "logsearchapi" -}}
{{- end -}}
{{- define "logsearchapi.command" -}}
{{- print "/logsearchapi" -}}
{{- end -}}
{{- define "logsearchapi.secretName" -}}
{{- print "logsearchapi-details" -}}
{{- end -}}
{{- define "logsearchapi.envVariableConfiguration" -}}
{{- $envList := list -}}
{{- $secretName := (include "logsearchapi.secretName" .) -}}
{{- $postgresSecretName := (include "postgres.secretName" .) -}}
{{- $envList = mustAppend $envList (dict "name" "MINIO_LOG_QUERY_AUTH_TOKEN" "valueFromSecret" true "secretName" $secretName "secretKey" "queryToken") -}}
{{- $envList = mustAppend $envList (dict "name" "LOGSEARCH_AUDIT_AUTH_TOKEN" "valueFromSecret" true "secretName" $secretName "secretKey" "auditToken") -}}
{{- $envList = mustAppend $envList (dict "name" "LOGSEARCH_PG_CONN_STR" "valueFromSecret" true "secretName" $postgresSecretName "secretKey" "postgresURL") -}}
{{- $envList = mustAppend $envList (dict "name" "LOGSEARCH_DISK_CAPACITY_GB" "value" .Values.logsearchapi.diskCapacityGB) -}}
{{- include "common.containers.environmentVariables" (dict "environmentVariables" $envList) -}}
{{- end -}}
{{/* Used in the minio init container (checks that logsearchapi is available) */}}
{{- define "logsearchapiInit.envVariableConfiguration" -}}
{{- $envList := list -}}
{{- $secretName := (include "logsearchapi.secretName" .) -}}
{{- $envList = mustAppend $envList (dict "name" "apiURL" "valueFromSecret" true "secretName" $secretName "secretKey" "logQueryURL") -}}
{{- include "common.containers.environmentVariables" (dict "environmentVariables" $envList) -}}
{{- end -}}

View File

@@ -0,0 +1,71 @@
{{- define "postgres.imageName" -}}
{{- print "postgres:14.5" -}}
{{- end -}}
{{- define "postgres.nameSuffix" -}}
{{- print "postgres" -}}
{{- end -}}
{{- define "postgres.secretName" -}}
{{- print "postgres-details" -}}
{{- end -}}
{{- define "postgres.dbName" -}}
{{- print "logsearchapi" -}}
{{- end -}}
{{- define "postgres.dbUser" -}}
{{- print "logsearchapi" -}}
{{- end -}}
{{/*
Retrieve postgres backup name
This will return a unique name based on revision and chart numbers specified.
*/}}
{{- define "postgres.backupName" -}}
{{- $upgradeDict := .Values.ixChartContext.upgradeMetadata -}}
{{- printf "postgres-backup-from-%s-to-%s-revision-%d" $upgradeDict.oldChartVersion $upgradeDict.newChartVersion (int64 $upgradeDict.preUpgradeRevision) -}}
{{- end -}}
{{- define "postgres.envVariableConfiguration" -}}
{{- $envList := list -}}
{{- $secretName := (include "postgres.secretName" .) -}}
{{- $envList = mustAppend $envList (dict "name" "POSTGRES_USER" "valueFromSecret" true "secretName" $secretName "secretKey" "db_user") -}}
{{- $envList = mustAppend $envList (dict "name" "POSTGRES_DB" "valueFromSecret" true "secretName" $secretName "secretKey" "db_name") -}}
{{- $envList = mustAppend $envList (dict "name" "POSTGRES_PASSWORD" "valueFromSecret" true "secretName" $secretName "secretKey" "db_password") -}}
{{- include "common.containers.environmentVariables" (dict "environmentVariables" $envList) -}}
{{- end -}}
{{- define "postgresBackup.envVariableConfiguration" -}}
{{- $envList := list -}}
{{- $secretName := (include "postgres.secretName" .) -}}
{{- $envList = mustAppend $envList (dict "name" "POSTGRES_USER" "valueFromSecret" true "secretName" $secretName "secretKey" "db_user") -}}
{{- $envList = mustAppend $envList (dict "name" "POSTGRES_DB" "valueFromSecret" true "secretName" $secretName "secretKey" "db_name") -}}
{{/* PGPASSWORD is used by pg_dump */}}
{{- $envList = mustAppend $envList (dict "name" "PGPASSWORD" "valueFromSecret" true "secretName" $secretName "secretKey" "db_password") -}}
{{- $envList = mustAppend $envList (dict "name" "pgHost" "valueFromSecret" true "secretName" $secretName "secretKey" "postgresHost") -}}
{{- include "common.containers.environmentVariables" (dict "environmentVariables" $envList) -}}
{{- end -}}
{{/* Used in the logsearchapi init container (checks that postgres is available) */}}
{{- define "postgresInit.envVariableConfiguration" -}}
{{- $envList := list -}}
{{- $secretName := (include "postgres.secretName" .) -}}
{{- $envList = mustAppend $envList (dict "name" "pgHost" "valueFromSecret" true "secretName" $secretName "secretKey" "postgresHost") -}}
{{- $envList = mustAppend $envList (dict "name" "pguser" "valueFromSecret" true "secretName" $secretName "secretKey" "db_user") -}}
{{- include "common.containers.environmentVariables" (dict "environmentVariables" $envList) -}}
{{- end -}}
{{/*
Retrieve postgres volume configuration
*/}}
{{- define "postgres.volumeConfiguration" -}}
{{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.postgresAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 0 }}
{{- end -}}
{{/*
Retrieve postgres volume mounts configuration
*/}}
{{- define "postgres.volumeMountsConfiguration" -}}
{{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.postgresAppVolumeMounts ) | nindent 0 }}
{{- end -}}

View File

@@ -0,0 +1,15 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: postgres-backup-hook-config-map
annotations:
rollme: {{ randAlphaNum 5 | quote }}
data:
entrypoint.sh: |-
#!/bin/sh
cmd="/docker-entrypoint.sh postgres"
eval "${cmd}" & disown;
until pg_isready -U "$POSTGRES_USER" -h "$pgHost"; do
sleep 5;
done;
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -h "$pgHost" -f /postgres_backups/$BACKUP_NAME.sql;

View File

@@ -0,0 +1,40 @@
{{- if .Values.logsearchapi.enabled -}}
{{- if .Values.ixChartContext.isUpgrade -}}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" (include "postgres.nameSuffix" .)) }}
apiVersion: batch/v1
kind: Job
metadata:
name: pre-upgrade-hook
annotations:
"helm.sh/hook": pre-upgrade
"helm.sh/hook-weight": "1"
"helm.sh/hook-delete-policy": hook-succeeded
rollme: {{ randAlphaNum 5 | quote }}
spec:
template:
metadata:
name: pre-upgrade-hook
spec:
restartPolicy: Never
containers:
- name: {{ .Chart.Name }}-postgres-backup
image: {{ include "postgres.imageName" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: {{ include "postgresBackup.envVariableConfiguration" $values | nindent 10 }}
- name: BACKUP_NAME
value: {{ include "postgres.backupName" . }}
volumeMounts: {{ include "postgres.volumeMountsConfiguration" $values | nindent 10 }}
- name: backup-script-configmap
mountPath: /bin/backup_entrypoint.sh
readOnly: true
subPath: entrypoint.sh
command:
- /bin/backup_entrypoint.sh
volumes: {{ include "postgres.volumeConfiguration" $values | nindent 8 }}
- name: backup-script-configmap
configMap:
defaultMode: 0700
name: postgres-backup-hook-config-map
{{- end -}}
{{- end -}}

View File

@@ -0,0 +1,6 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: minio-config
data:
protocol: {{ include "minio.scheme" . }}

View File

@@ -0,0 +1,96 @@
{{ include "common.storage.hostPathValidate" .Values }}
{{ $logsearchValues := (. | mustDeepCopy) }}
{{ $_ := set $logsearchValues "common" (dict "nameSuffix" (include "logsearchapi.nameSuffix" .)) }}
apiVersion: {{ template "common.capabilities.deployment.apiVersion" . }}
kind: Deployment
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
chart: {{ template "common.names.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
replicas: {{ (default 1 .Values.replicas) }}
strategy:
type: {{ (default "Recreate" .Values.updateStrategy ) }}
selector:
matchLabels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
template:
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
{{- include "common.labels.selectorLabels" . | nindent 8 }}
annotations: {{ include "common.annotations" . | nindent 8 }}
spec:
securityContext:
runAsUser: {{ .Values.runAsUser }}
runAsGroup: {{ .Values.runAsGroup }}
serviceAccountName: {{ include "common.names.serviceAccountName" . | quote }}
hostNetwork: {{ include "minio.hostNetworking" . }}
{{ if .Values.logsearchapi.enabled }}
initContainers:
- name: {{ .Chart.Name }}-waitapi
image: alpine/curl
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: {{ include "logsearchapiInit.envVariableConfiguration" $logsearchValues | nindent 10 }}
command:
- sh
- -c
- "until curl $apiURL/status; do sleep 2; done"
{{ end }}
containers:
- name: {{ .Chart.Name }}
{{ include "common.resources.limitation" . | nindent 10 }}
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
{{- include "minio.tlsKeysVolumeMount" . | nindent 12 }}
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
command:
- "/bin/sh"
- "-ce"
- {{ include "minio.commandArgs" . }}
ports:
- name: api
containerPort: {{ .Values.service.nodePort | int }}
- name: console
containerPort: {{ .Values.service.consolePort | int }}
env:
{{ $secretName := (include "minio.secretName" .) }}
{{ $envList := (default list .Values.environmentVariables) }}
{{ if and (eq (include "minio.certAvailable" .) "true") .Values.minioDomain }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_BROWSER_REDIRECT_URL" "value" (printf "%s://%s:%d" (include "minio.scheme" .) .Values.minioDomain (.Values.service.consolePort | int))) }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_SERVER_URL" "value" (printf "%s://%s:%d" (include "minio.scheme" .) .Values.minioDomain (.Values.service.nodePort | int))) }}
{{ end }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_ROOT_USER" "valueFromSecret" true "secretName" $secretName "secretKey" "accesskey") }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_ROOT_PASSWORD" "valueFromSecret" true "secretName" $secretName "secretKey" "secretkey") }}
{{ if .Values.logsearchapi.enabled }}
{{/*
We can put any ID we want here. Just make sure it's unique
It can be rolled on each startup without problems, or se can set a static one.
*/}}
{{ $webhookID := (printf "ix-%v" (randAlphaNum 5)) }}
{{ $logsearchSecretName := (include "logsearchapi.secretName" .) }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_LOG_QUERY_AUTH_TOKEN" "valueFromSecret" true "secretName" $logsearchSecretName "secretKey" "queryToken") }}
{{ $envList = mustAppend $envList (dict "name" "MINIO_LOG_QUERY_URL" "valueFromSecret" true "secretName" $logsearchSecretName "secretKey" "logQueryURL") }}
{{ $envList = mustAppend $envList (dict "name" (printf "MINIO_AUDIT_WEBHOOK_ENDPOINT_%v" $webhookID) "valueFromSecret" true "secretName" $logsearchSecretName "secretKey" "webhookURL") }}
{{ $envList = mustAppend $envList (dict "name" (printf "MINIO_AUDIT_WEBHOOK_ENABLE_%v" $webhookID) "value" "on") }}
{{ end }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" .Values | nindent 8 }}
{{- include "minio.tlsKeysVolume" . | nindent 8 }}
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}

View File

@@ -0,0 +1,51 @@
{{ if .Values.logsearchapi.enabled }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" (include "logsearchapi.nameSuffix" .)) }}
{{ $pg_values := (. | mustDeepCopy) }}
{{ $_ := set $pg_values "common" (dict "nameSuffix" (include "postgres.nameSuffix" .)) }}
{{ include "common.deployment.common_config" $values | nindent 0 }}
spec: {{ include "common.deployment.common_spec" $values | nindent 2 }}
template: {{ include "common.deployment.pod.metadata" $values | nindent 4 }}
spec:
initContainers:
- name: {{ .Chart.Name }}-{{ include "logsearchapi.nameSuffix" . }}-waitdb
image: {{ include "postgres.imageName" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: {{ include "postgresInit.envVariableConfiguration" $pg_values | nindent 10 }}
command:
- sh
- -c
- 'until pg_isready -U "$pgUser" -h "$pgHost"; do sleep 2; done'
containers:
- name: {{ .Chart.Name }}-{{ include "logsearchapi.nameSuffix" . }}
image: {{ include "logsearchapi.imageName" . }}
imagePullPolicy: {{ .Values.logSearchImage.pullPolicy }}
env: {{ include "logsearchapi.envVariableConfiguration" $values | nindent 10 }}
command:
- {{ include "logsearchapi.command" . }}
ports:
- name: logsearchapi
containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /status
port: 8080
failureThreshold: 5
periodSeconds: 15
initialDelaySeconds: 15
livenessProbe:
httpGet:
path: /status
port: 8080
failureThreshold: 5
periodSeconds: 15
initialDelaySeconds: 15
startupProbe:
httpGet:
path: /status
port: 8080
failureThreshold: 5
periodSeconds: 15
initialDelaySeconds: 15
{{ end }}

View File

@@ -0,0 +1,25 @@
{{ if .Values.logsearchapi.enabled }}
{{ $logSearchValues := (. | mustDeepCopy) }}
{{ $_ := set $logSearchValues "common" (dict "nameSuffix" (include "logsearchapi.nameSuffix" .)) }}
{{ $auditToken := randAlphaNum 32 | b64enc }}
{{ $queryToken := randAlphaNum 32 | b64enc }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "logsearchapi.secretName" . }}
data:
{{ with (lookup "v1" "Secret" .Release.Namespace (include "logsearchapi.secretName" .)) }}
{{ $auditToken = (index .data "auditToken") }}
{{ end }}
auditToken: {{ $auditToken }}
{{ with (lookup "v1" "Secret" .Release.Namespace (include "logsearchapi.secretName" .)) }}
{{ $queryToken = (index .data "queryToken") }}
{{ end }}
queryToken: {{ $queryToken }}
logQueryURL: {{ (printf "http://%v:8080" (include "common.names.fullname" $logSearchValues)) | b64enc }}
webhookURL: {{ (printf "http://%v:8080/api/ingest?token=%v" (include "common.names.fullname" $logSearchValues) ($auditToken | b64dec)) | b64enc }}
{{ end }}

View File

@@ -0,0 +1,6 @@
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "logsearchapi-tcp" "port" 8080 "targetPort" 8080) }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" (include "logsearchapi.nameSuffix" .)) }}
{{ $_1 := set $values "commonService" (dict "type" "ClusterIP" "ports" $ports ) }}
{{ include "common.classes.service" $values }}

View File

@@ -0,0 +1,56 @@
{{ if .Values.logsearchapi.enabled }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" (include "postgres.nameSuffix" .)) }}
{{ include "common.deployment.common_config" $values | nindent 0 }}
spec:
replicas: {{ (default 1 .Values.replicas) }}
strategy:
type: Recreate
selector:
matchLabels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
template: {{ include "common.deployment.pod.metadata" $values | nindent 4 }}
spec:
containers:
- name: {{ .Chart.Name }}-{{ include "postgres.nameSuffix" .}}
image: {{ template "postgres.imageName" . }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
env: {{ include "postgres.envVariableConfiguration" $values | nindent 10 }}
volumeMounts: {{ include "postgres.volumeMountsConfiguration" $values | nindent 10 }}
ports:
- name: postgres-tcp
containerPort: 5432
protocol: TCP
readinessProbe:
exec:
command:
- sh
- -c
- "until pg_isready -U${POSTGRES_USER} -h localhost; do sleep 2; done"
initialDelaySeconds: 15
failureThreshold: 5
periodSeconds: 15
timeoutSeconds: 2
livenessProbe:
exec:
command:
- sh
- -c
- "until pg_isready -U${POSTGRES_USER} -h localhost; do sleep 2; done"
initialDelaySeconds: 15
failureThreshold: 5
periodSeconds: 15
timeoutSeconds: 2
startupProbe:
exec:
command:
- sh
- -c
- "until pg_isready -U${POSTGRES_USER} -h localhost; do sleep 2; done"
initialDelaySeconds: 15
failureThreshold: 20
periodSeconds: 15
timeoutSeconds: 2
volumes: {{ include "postgres.volumeConfiguration" $values | nindent 8 }}
{{ end }}

View File

@@ -0,0 +1,22 @@
{{ if .Values.logsearchapi.enabled }}
{{ $pgValues := (. | mustDeepCopy) }}
{{ $_ := set $pgValues "common" (dict "nameSuffix" (include "postgres.nameSuffix" .)) }}
{{ $dbPass := randAlphaNum 32 | b64enc }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "postgres.secretName" . }}
data:
{{ with (lookup "v1" "Secret" .Release.Namespace (include "postgres.secretName" .)) }}
{{ $dbPass = (index .data "db_password") }}
{{ end }}
db_password: {{ $dbPass }}
db_user: {{ include "postgres.dbUser" . | b64enc }}
db_name: {{ include "postgres.dbName" . | b64enc }}
postgresURL: {{ printf "postgres://%v:%v@%v:5432/%v?sslmode=disable" (include "postgres.dbUser" .) ($dbPass | b64dec) (include "common.names.fullname" $pgValues) (include "postgres.dbName" .) | b64enc }}
postgresHost: {{ printf "%v" (include "common.names.fullname" $pgValues) | b64enc }}
{{ end }}

View File

@@ -0,0 +1,6 @@
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "postgres-tcp" "port" 5432 "targetPort" 5432) }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" (include "postgres.nameSuffix" .)) }}
{{ $_1 := set $values "commonService" (dict "type" "ClusterIP" "ports" $ports ) }}
{{ include "common.classes.service" $values }}

View File

@@ -0,0 +1,28 @@
apiVersion: batch/v1
kind: Job
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-job"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion }}
helm.sh/chart: {{ template "common.names.chart" . }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-hook"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
spec:
restartPolicy: Never
containers:
- name: pre-install-job
image: "alpine:latest"
command: ["chown", "-R", "{{ .Values.runAsUser }}:{{ .Values.runAsGroup }}", "{{ .Values.appVolumeMounts.export.mountPath }}"]
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
volumes: {{ include "common.storage.configureAppVolumes" .Values | nindent 8 }}

View File

@@ -0,0 +1,13 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ template "minio.secretName" . }}
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
accesskey: {{ if .Values.accessKey }}{{ .Values.accessKey | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 20 | b64enc | quote }}{{ end }}
secretkey: {{ if .Values.secretKey }}{{ .Values.secretKey | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 40 | b64enc | quote }}{{ end }}
{{ if eq (include "minio.certAvailable" .) "true" }}
certPublicKey: {{ (include "minio.cert.publicKey" .) | toString | b64enc | quote }}
certPrivateKey: {{ (include "minio.cert.privateKey" .) | toString | b64enc | quote }}
{{ end }}

View File

@@ -0,0 +1,13 @@
{{ if eq (include "minio.hostNetworking" .) "false" }}
{{ $svc := .Values.service }}
{{ $selectors := list }}
{{ $selectors = mustAppend $selectors (dict "key" "app" "value" (include "common.names.name" .) ) }}
{{ $selectors = mustAppend $selectors (dict "key" "release" "value" .Release.Name ) }}
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "api" "port" $svc.nodePort "nodePort" $svc.nodePort "targetPort" (.Values.service.nodePort | int)) }}
{{ $ports = mustAppend $ports (dict "name" "console" "port" $svc.consolePort "nodePort" $svc.consolePort "targetPort" (.Values.service.consolePort | int)) }}
{{ $params := . }}
{{ $_ := set $params "commonService" (dict "type" "NodePort" "ports" $ports ) }}
{{ $_1 := set .Values "extraSelectorLabels" $selectors }}
{{ include "common.classes.service" $params }}
{{ end }}

View File

@@ -0,0 +1 @@
{{ include "common.serviceaccount" . }}

View File

@@ -0,0 +1,21 @@
{{- $serviceName := (include "common.names.fullname" .) -}}
apiVersion: v1
kind: Pod
metadata:
name: "{{ .Release.Name }}-minio-test"
labels:
app: {{ .Release.Name }}
release: {{ .Release.Name }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test-curl
image: alpine/curl
imagePullPolicy: "IfNotPresent"
command:
- /bin/sh
- -ec
- |
curl --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 15 --retry-max-time 90 --retry-all-errors -ksf http://{{ $serviceName }}:{{ .Values.service.consolePort }}/
restartPolicy: Never

View File

@@ -0,0 +1,45 @@
appVolumeMounts:
export:
emptyDir: true
mountPath: /export
postgresAppVolumeMounts:
postgres-backup:
emptyDir: true
mountPath: /postgres_backups
postgres-data:
emptyDir: true
mountPath: /var/lib/postgresql/data
postgresql:
backupVolume:
datasetName: ix-postgres_backups
mountPath: /postgres_backups
dataVolume:
datasetName: ix-postgres_data
mountPath: /var/lib/postgresql/data
ixChartContext: {}
distributedIps: []
distributedMode: false
dnsConfig:
options: []
emptyDirVolumes: true
environmentVariables: []
extraAppVolumeMounts: []
extraArgs: []
image:
pullPolicy: IfNotPresent
repository: minio/minio
tag: RELEASE.2022-10-24T18-35-07Z
logSearchImage:
pullPolicy: IfNotPresent
repository: minio/operator
tag: v4.5.4
logsearchapi:
enabled: true
diskCapacityGB: 5
minioDomain: null
runAsGroup: 473
runAsUser: 473
service:
consolePort: 32325
nodePort: 32324
updateStrategy: RollingUpdate