From 7b4077922fcf4640c65320a3e4ffb10633f5ea1e Mon Sep 17 00:00:00 2001 From: Stavros Kois <47820033+stavros-k@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:45:06 +0200 Subject: [PATCH] elastic search - migrate storage section (#1823) * use pvc in ci * adapt values * Update version in Chart.yaml * adapt template * update readme * adapt questions.yaml * add migration and fix typo in values * fix description --- .../ix-dev/charts/elastic-search/Chart.yaml | 2 +- .../ix-dev/charts/elastic-search/README.md | 4 - .../charts/elastic-search/app-readme.md | 4 - .../elastic-search/ci/basic-values.yaml | 3 +- .../elastic-search/ci/https-values.yaml | 3 +- .../charts/elastic-search/migrations/migrate | 40 +++- .../charts/elastic-search/questions.yaml | 209 +++++++++++++++++- .../charts/elastic-search/templates/_es.tpl | 34 ++- .../ix-dev/charts/elastic-search/values.yaml | 4 +- 9 files changed, 260 insertions(+), 43 deletions(-) diff --git a/library/ix-dev/charts/elastic-search/Chart.yaml b/library/ix-dev/charts/elastic-search/Chart.yaml index c010b07120..a51f51b134 100644 --- a/library/ix-dev/charts/elastic-search/Chart.yaml +++ b/library/ix-dev/charts/elastic-search/Chart.yaml @@ -3,7 +3,7 @@ description: Elasticsearch is the distributed, RESTful search and analytics engi annotations: title: Elastic Search type: application -version: 1.0.32 +version: 1.1.0 apiVersion: v2 appVersion: 8.11.1 kubeVersion: '>=1.16.0-0' diff --git a/library/ix-dev/charts/elastic-search/README.md b/library/ix-dev/charts/elastic-search/README.md index beec2c1bc3..bbf6cdb439 100644 --- a/library/ix-dev/charts/elastic-search/README.md +++ b/library/ix-dev/charts/elastic-search/README.md @@ -1,9 +1,5 @@ # Elastic Search -> During the installation process, a container will be launched with **root** privileges. This is required -> in order to apply the correct permissions to the `Elastic Search` data directory. Afterward, the `Elastic Search` container -> will run as a **non**-root user (default `568`). - If you want to apply additional configuration you can by using additional environment variables. See the [Elastic Search documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/docker.html#docker-configuration-methods) diff --git a/library/ix-dev/charts/elastic-search/app-readme.md b/library/ix-dev/charts/elastic-search/app-readme.md index beec2c1bc3..bbf6cdb439 100644 --- a/library/ix-dev/charts/elastic-search/app-readme.md +++ b/library/ix-dev/charts/elastic-search/app-readme.md @@ -1,9 +1,5 @@ # Elastic Search -> During the installation process, a container will be launched with **root** privileges. This is required -> in order to apply the correct permissions to the `Elastic Search` data directory. Afterward, the `Elastic Search` container -> will run as a **non**-root user (default `568`). - If you want to apply additional configuration you can by using additional environment variables. See the [Elastic Search documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/docker.html#docker-configuration-methods) diff --git a/library/ix-dev/charts/elastic-search/ci/basic-values.yaml b/library/ix-dev/charts/elastic-search/ci/basic-values.yaml index 5e6c993ef6..0a32d54d64 100644 --- a/library/ix-dev/charts/elastic-search/ci/basic-values.yaml +++ b/library/ix-dev/charts/elastic-search/ci/basic-values.yaml @@ -1,7 +1,6 @@ esStorage: data: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/data + type: pvc esConfig: password: some-secret diff --git a/library/ix-dev/charts/elastic-search/ci/https-values.yaml b/library/ix-dev/charts/elastic-search/ci/https-values.yaml index ce7d041923..7e98262336 100644 --- a/library/ix-dev/charts/elastic-search/ci/https-values.yaml +++ b/library/ix-dev/charts/elastic-search/ci/https-values.yaml @@ -1,7 +1,6 @@ esStorage: data: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/data + type: pvc esConfig: password: some-https-secret diff --git a/library/ix-dev/charts/elastic-search/migrations/migrate b/library/ix-dev/charts/elastic-search/migrations/migrate index a578a218c2..00c21112ee 100755 --- a/library/ix-dev/charts/elastic-search/migrations/migrate +++ b/library/ix-dev/charts/elastic-search/migrations/migrate @@ -4,18 +4,44 @@ import os import sys +def storage_migrate(storage): + delete_keys = [] + if storage['type'] == 'hostPath': + # Check if the key exists, if not we have already migrated + if not storage.get('hostPath'): + return storage + + storage['hostPathConfig'] = {'hostPath': storage['hostPath']} + delete_keys.append('hostPath') + + elif storage['type'] == 'ixVolume': + # Check if the key exists, if not we have already migrated + if not storage.get('datasetName'): + return storage + + storage['ixVolumeConfig'] = {'datasetName': storage['datasetName']} + delete_keys.append('datasetName') + + # Clean up for some older versions. + if storage.get('hostPath'): + delete_keys.append('hostPath') + + for key in delete_keys: + storage.pop(key, None) + + return storage + + def migrate(values): - storageKey = 'esStorage' + storage_key = 'esStorage' storages = ['data'] - for storage in storages: - check_val = values.get(storageKey, {}).get(storage, {}) - if not isinstance(check_val, dict) or not check_val or check_val.get('type', 'hostPath') == 'hostPath': - continue - - values[storageKey][storage] = {key: value for key, value in check_val.items() if key != 'hostPath'} + check_val = values.get(storage_key, {}).get(storage, {}) + if not isinstance(check_val, dict) or not check_val: + raise Exception(f'Storage section {storage} is malformed') + values[storage_key][storage] = storage_migrate(check_val) return values diff --git a/library/ix-dev/charts/elastic-search/questions.yaml b/library/ix-dev/charts/elastic-search/questions.yaml index f84311560b..f868e92739 100644 --- a/library/ix-dev/charts/elastic-search/questions.yaml +++ b/library/ix-dev/charts/elastic-search/questions.yaml @@ -131,24 +131,209 @@ questions: description: Host Path (Path that already exists on the system) - value: "ixVolume" description: ixVolume (Dataset created automatically by the system) - - variable: datasetName - label: Dataset Name + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - hidden: true - immutable: true - default: "data" $ref: - "normalize/ixVolume" - - variable: hostPath - label: Host Path + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "data" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration schema: - type: hostpath + type: dict show_if: [["type", "=", "hostPath"]] - immutable: true - required: true + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + immutable: true + required: true + + - variable: additionalStorages + label: Additional Storage + description: Additional storage for Elastic Search. + schema: + type: list + default: [] + items: + - variable: storageEntry + label: Storage Entry + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system.
+ SMB Share: Is a SMB share that is mounted to a persistent volume claim. + schema: + type: string + required: true + default: "ixVolume" + immutable: true + enum: + - value: "hostPath" + description: Host Path (Path that already exists on the system) + - value: "ixVolume" + description: ixVolume (Dataset created automatically by the system) + - value: "smb-pv-pvc" + description: SMB Share (Mounts a persistent volume claim to a SMB share) + - variable: readOnly + label: Read Only + description: Mount the volume as read only. + schema: + type: boolean + default: false + - variable: mountPath + label: Mount Path + description: The path inside the container to mount the storage. + schema: + type: path + required: true + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "hostPath"]] + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + immutable: true + required: true + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ixVolume"]] + $ref: + - "normalize/ixVolume" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + default: "storage_entry" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: smbConfig + label: SMB Share Configuration + description: The configuration for the SMB Share. + schema: + type: dict + show_if: [["type", "=", "smb-pv-pvc"]] + attrs: + - variable: server + label: Server + description: The server for the SMB share. + schema: + type: string + required: true + - variable: share + label: Share + description: The share name for the SMB share. + schema: + type: string + required: true + - variable: domain + label: Domain (Optional) + description: The domain for the SMB share. + schema: + type: string + - variable: username + label: Username + description: The username for the SMB share. + schema: + type: string + required: true + - variable: password + label: Password + description: The password for the SMB share. + schema: + type: string + required: true + private: true + - variable: size + label: Size (in Gi) + description: The size of the volume quota. + schema: + type: int + required: true + min: 1 + default: 1 - variable: resources label: "" diff --git a/library/ix-dev/charts/elastic-search/templates/_es.tpl b/library/ix-dev/charts/elastic-search/templates/_es.tpl index e6ff70aed6..8aa13c8133 100644 --- a/library/ix-dev/charts/elastic-search/templates/_es.tpl +++ b/library/ix-dev/charts/elastic-search/templates/_es.tpl @@ -60,11 +60,6 @@ workload: port: {{ .Values.esNetwork.httpPort }} httpHeaders: Authorization: Basic {{ printf "elastic:%s" .Values.esConfig.password | b64enc }} - initContainers: - {{- include "ix.v1.common.app.permissions" (dict "containerName" "01-permissions" - "UID" .Values.esRunAs.user - "GID" .Values.esRunAs.group - "type" "install") | nindent 8 }} {{/* Service */}} service: es: @@ -84,15 +79,24 @@ service: persistence: data: enabled: true - type: {{ .Values.esStorage.data.type }} - datasetName: {{ .Values.esStorage.data.datasetName | default "" }} - hostPath: {{ .Values.esStorage.data.hostPath | default "" }} + {{- include "es.storage.ci.migration" (dict "storage" .Values.esStorage.data) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.esStorage.data) | nindent 4 }} targetSelector: es: es: mountPath: /usr/share/elasticsearch/data - 01-permissions: - mountPath: /mnt/directories/data + + {{- range $idx, $storage := .Values.esStorage.additionalStorages }} + {{ printf "es-%v:" (int $idx) }} + enabled: true + {{- include "es.storage.ci.migration" (dict "storage" $storage) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }} + targetSelector: + es: + es: + mountPath: {{ $storage.mountPath }} + {{- end }} + {{- if .Values.esNetwork.certificateID }} certs: enabled: true @@ -118,3 +122,13 @@ scaleCertificate: id: {{ .Values.esNetwork.certificateID }} {{- end -}} {{- end -}} + +{{/* TODO: Remove on the next version bump, eg 1.1.0+ */}} +{{- define "es.storage.ci.migration" -}} + {{- $storage := .storage -}} + + {{- if $storage.hostPath -}} + {{- $_ := set $storage "hostPathConfig" dict -}} + {{- $_ := set $storage.hostPathConfig "hostPath" $storage.hostPath -}} + {{- end -}} +{{- end -}} diff --git a/library/ix-dev/charts/elastic-search/values.yaml b/library/ix-dev/charts/elastic-search/values.yaml index 6cab91d3ad..d8dc7fb3a0 100644 --- a/library/ix-dev/charts/elastic-search/values.yaml +++ b/library/ix-dev/charts/elastic-search/values.yaml @@ -27,4 +27,6 @@ esNetwork: esStorage: data: type: ixVolume - datasetName: data + ixVolumeConfig: + datasetName: data + additionalStorages: []