From aeb7f5af3676f2f6866af981d5b3d1a581af110b Mon Sep 17 00:00:00 2001 From: Stavros Kois <47820033+stavros-k@users.noreply.github.com> Date: Fri, 5 Jan 2024 14:42:28 +0200 Subject: [PATCH] Tdarr - migrate storage section (adds acl) (#2000) --- library/ix-dev/community/tdarr/Chart.yaml | 2 +- .../community/tdarr/ci/basic-values.yaml | 14 +- .../tdarr/ci/no-internal-node-values.yaml | 13 +- .../ix-dev/community/tdarr/migrations/migrate | 76 ++- library/ix-dev/community/tdarr/questions.yaml | 456 +++++++++++++----- .../community/tdarr/templates/_tdarr.tpl | 58 +-- library/ix-dev/community/tdarr/values.yaml | 14 +- 7 files changed, 452 insertions(+), 181 deletions(-) diff --git a/library/ix-dev/community/tdarr/Chart.yaml b/library/ix-dev/community/tdarr/Chart.yaml index 7b7f3b32b8..fbc0c09e3a 100644 --- a/library/ix-dev/community/tdarr/Chart.yaml +++ b/library/ix-dev/community/tdarr/Chart.yaml @@ -3,7 +3,7 @@ description: Tdarr is a Distributed Transcoding System annotations: title: Tdarr type: application -version: 1.1.7 +version: 1.2.0 apiVersion: v2 appVersion: '2.17.01' kubeVersion: '>=1.16.0-0' diff --git a/library/ix-dev/community/tdarr/ci/basic-values.yaml b/library/ix-dev/community/tdarr/ci/basic-values.yaml index c3756cd60b..96db80fc31 100644 --- a/library/ix-dev/community/tdarr/ci/basic-values.yaml +++ b/library/ix-dev/community/tdarr/ci/basic-values.yaml @@ -5,18 +5,16 @@ tdarrID: group: 1001 tdarrStorage: server: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/server + type: pvc configs: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/configs + type: pvc logs: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/logs + type: pvc transcodes: type: emptyDir - medium: Memory - size: 1Gi + emptyDirConfig: + medium: Memory + size: 1 # Make sure installation don't fail when 0 GPU are assigned tdarrGPU: nvidia.com/gpu: 0 diff --git a/library/ix-dev/community/tdarr/ci/no-internal-node-values.yaml b/library/ix-dev/community/tdarr/ci/no-internal-node-values.yaml index bd5432f14d..e61a631ed0 100644 --- a/library/ix-dev/community/tdarr/ci/no-internal-node-values.yaml +++ b/library/ix-dev/community/tdarr/ci/no-internal-node-values.yaml @@ -3,14 +3,13 @@ tdarrConfig: internalNode: false tdarrStorage: server: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/server + type: pvc configs: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/configs + type: pvc logs: - type: hostPath - hostPath: /mnt/{{ .Release.Name }}/logs + type: pvc transcodes: type: emptyDir - size: 1Gi + emptyDirConfig: + size: 1 + medium: "" diff --git a/library/ix-dev/community/tdarr/migrations/migrate b/library/ix-dev/community/tdarr/migrations/migrate index 6fe555ffd2..4756f7150a 100755 --- a/library/ix-dev/community/tdarr/migrations/migrate +++ b/library/ix-dev/community/tdarr/migrations/migrate @@ -4,18 +4,82 @@ import os import sys +def storage_migrate(storage): + delete_keys = [] + if storage['type'] == 'hostPath': + # Check if the key exists, if not we have already migrated + if not storage.get('hostPath'): + return storage + + storage['hostPathConfig'] = {'hostPath': storage['hostPath']} + delete_keys.append('hostPath') + + elif storage['type'] == 'ixVolume': + # Check if the key exists, if not we have already migrated + if not storage.get('datasetName'): + return storage + + storage['ixVolumeConfig'] = {'datasetName': storage['datasetName']} + delete_keys.append('datasetName') + + # Clean up for some older versions. + if storage.get('hostPath'): + delete_keys.append('hostPath') + + elif storage['type'] == 'emptyDir': + # Check if the key exists, if not we have already migrated + if not 'medium' in storage.keys(): + return storage + + if storage['size'] and (not storage['size'].endswith('Gi')): + raise Exception('In order to proceed with the update, ' \ + 'please change the size for the storage using ' \ + '"emptyDir" in the format xGi. eg. "2Gi"') + + storage['emptyDirConfig'] = { + 'medium': storage['medium'], + 'size': int(storage['size'].replace('Gi', '')) if storage['size'] else None, + } + delete_keys.extend(['medium', 'size']) + + elif storage['type'] == 'smb-pv-pvc': + # Check if the key exists, if not we have already migrated + if not storage.get('server'): + return storage + + storage['smbConfig'] = { + 'server': storage['server'], + 'share': storage['share'], + 'domain': storage['domain'], + 'username': storage['username'], + 'password': storage['password'], + 'size': storage['size'], + } + delete_keys.extend(['server', 'share', 'domain', 'username', 'password', 'size']) + + for key in delete_keys: + storage.pop(key, None) + + return storage + + def migrate(values): - storageKey = 'tdarrStorage' + storage_key = 'tdarrStorage' storages = ['server', 'configs', 'logs', 'transcodes'] - for storage in storages: - check_val = values.get(storageKey, {}).get(storage, {}) - if not isinstance(check_val, dict) or not check_val or check_val.get('type', 'hostPath') == 'hostPath': - continue + check_val = values.get(storage_key, {}).get(storage, {}) + if not isinstance(check_val, dict) or not check_val: + raise Exception(f'Storage section {storage} is malformed') - values[storageKey][storage] = {key: value for key, value in check_val.items() if key != 'hostPath'} + values[storage_key][storage] = storage_migrate(check_val) + additionalStorages = values.get(storage_key, {}).get('additionalStorages', []) + for idx, storage in enumerate(additionalStorages): + if not isinstance(storage, dict) or not storage: + raise Exception(f'Item {idx} in additionalStorages is malformed') + + values[storage_key]['additionalStorages'][idx] = storage_migrate(storage) return values diff --git a/library/ix-dev/community/tdarr/questions.yaml b/library/ix-dev/community/tdarr/questions.yaml index 6cfe1f7b61..194eed4d17 100644 --- a/library/ix-dev/community/tdarr/questions.yaml +++ b/library/ix-dev/community/tdarr/questions.yaml @@ -151,23 +151,64 @@ questions: description: Host Path (Path that already exists on the system) - value: "ixVolume" description: ixVolume (Dataset created automatically by the system) - - variable: datasetName - label: Dataset Name + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - hidden: true - immutable: true - default: "server" $ref: - "normalize/ixVolume" - - variable: hostPath - label: Host Path + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "server" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration schema: - type: hostpath + type: dict show_if: [["type", "=", "hostPath"]] - required: true + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: configs label: Tdarr Config Storage description: The path to store Tdarr Configuration. @@ -189,23 +230,64 @@ questions: description: Host Path (Path that already exists on the system) - value: "ixVolume" description: ixVolume (Dataset created automatically by the system) - - variable: datasetName - label: Dataset Name + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - hidden: true - immutable: true - default: "configs" $ref: - "normalize/ixVolume" - - variable: hostPath - label: Host Path + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "configs" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration schema: - type: hostpath + type: dict show_if: [["type", "=", "hostPath"]] - required: true + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: logs label: Tdarr Logs Storage description: The path to store Tdarr Logs. @@ -227,23 +309,64 @@ questions: description: Host Path (Path that already exists on the system) - value: "ixVolume" description: ixVolume (Dataset created automatically by the system) - - variable: datasetName - label: Dataset Name + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - hidden: true - immutable: true - default: "logs" $ref: - "normalize/ixVolume" - - variable: hostPath - label: Host Path + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "logs" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration schema: - type: hostpath + type: dict show_if: [["type", "=", "hostPath"]] - required: true + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: transcodes label: Tdarr Transcodes Storage description: The path to store Tdarr Transcodes. @@ -268,46 +391,92 @@ questions: description: ixVolume (Dataset created automatically by the system) - value: "emptyDir" description: emptyDir (Temporary directory created on the disk or in memory) - - variable: datasetName - label: Dataset Name + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - hidden: true - immutable: true - default: "transcodes" $ref: - "normalize/ixVolume" - - variable: hostPath - label: Host Path + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "transcodes" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: emptyDirConfig + label: emptyDir Configuration schema: - type: hostpath + type: dict + show_if: [["type", "=", "emptyDir"]] + attrs: + - variable: medium + label: emptyDir Medium + description: | + Disk: Creates a temporary directory on the disk.
+ Memory: Creates a temporary directory in memory. + schema: + type: string + default: "" + enum: + - value: "" + description: Disk (Temporary directory created on the disk) + - value: "Memory" + description: Memory (Temporary directory created in memory) + - variable: size + label: emptyDir Size Limit (in Gi) + description: | + The maximum size (in Gi) of the temporary directory.
+ For example: 2 + schema: + type: int + "null": true + default: 2 + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict show_if: [["type", "=", "hostPath"]] - required: true - - variable: medium - label: emptyDir Medium - description: | - Disk: Creates a temporary directory on the disk.
- Memory: Creates a temporary directory in memory. - schema: - type: string - show_if: [["type", "=", "emptyDir"]] - default: "" - enum: - - value: "" - description: Disk (Temporary directory created on the disk) - - value: "Memory" - description: Memory (Temporary directory created in memory) - - variable: size - label: emptyDir Size Limit - description: | - The maximum size of the temporary directory.
- For example: 2Gi - schema: - type: string - show_if: [["type", "=", "emptyDir"]] - default: "2Gi" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: additionalStorages label: Additional Storage description: Additional storage for Tdarr. @@ -338,74 +507,119 @@ questions: description: ixVolume (Dataset created automatically by the system) - value: "smb-pv-pvc" description: SMB Share (Mounts a persistent volume claim to a SMB share) + - variable: readOnly + label: Read Only + description: Mount the volume as read only. + schema: + type: boolean + default: false - variable: mountPath label: Mount Path description: The path inside the container to mount the storage. schema: type: path required: true - - variable: hostPath - label: Host Path - description: The host path to use for storage. + - variable: hostPathConfig + label: Host Path Configuration schema: - type: hostpath + type: dict show_if: [["type", "=", "hostPath"]] - required: true - - variable: datasetName - label: Dataset Name - description: The name of the dataset to use for storage. + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. schema: - type: string + type: dict show_if: [["type", "=", "ixVolume"]] - required: true - immutable: true - default: "storage_entry" $ref: - "normalize/ixVolume" - - variable: server - label: Server - description: The server for the SMB share. + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + default: "storage_entry" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: smbConfig + label: SMB Share Configuration + description: The configuration for the SMB Share. schema: - type: string + type: dict show_if: [["type", "=", "smb-pv-pvc"]] - required: true - - variable: share - label: Share - description: The share name for the SMB share. - schema: - type: string - show_if: [["type", "=", "smb-pv-pvc"]] - required: true - - variable: domain - label: Domain (Optional) - description: The domain for the SMB share. - schema: - type: string - show_if: [["type", "=", "smb-pv-pvc"]] - - variable: username - label: Username - description: The username for the SMB share. - schema: - type: string - show_if: [["type", "=", "smb-pv-pvc"]] - required: true - - variable: password - label: Password - description: The password for the SMB share. - schema: - type: string - show_if: [["type", "=", "smb-pv-pvc"]] - required: true - private: true - - variable: size - label: Size (in Gi) - description: The size of the volume quota. - schema: - type: int - show_if: [["type", "=", "smb-pv-pvc"]] - required: true - min: 1 - default: 1 + attrs: + - variable: server + label: Server + description: The server for the SMB share. + schema: + type: string + required: true + - variable: share + label: Share + description: The share name for the SMB share. + schema: + type: string + required: true + - variable: domain + label: Domain (Optional) + description: The domain for the SMB share. + schema: + type: string + - variable: username + label: Username + description: The username for the SMB share. + schema: + type: string + required: true + - variable: password + label: Password + description: The password for the SMB share. + schema: + type: string + required: true + private: true + - variable: size + label: Size (in Gi) + description: The size of the volume quota. + schema: + type: int + required: true + min: 1 + default: 1 - variable: resources group: Resources Configuration diff --git a/library/ix-dev/community/tdarr/templates/_tdarr.tpl b/library/ix-dev/community/tdarr/templates/_tdarr.tpl index de3154a181..8ace96c0bd 100644 --- a/library/ix-dev/community/tdarr/templates/_tdarr.tpl +++ b/library/ix-dev/community/tdarr/templates/_tdarr.tpl @@ -81,63 +81,41 @@ service: persistence: server: enabled: true - type: {{ .Values.tdarrStorage.server.type }} - datasetName: {{ .Values.tdarrStorage.server.datasetName | default "" }} - hostPath: {{ .Values.tdarrStorage.server.hostPath | default "" }} + {{- include "tdarr.storage.ci.migration" (dict "storage" .Values.tdarrStorage.server) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.tdarrStorage.server) | nindent 4 }} targetSelector: tdarr: tdarr: mountPath: /app/server configs: enabled: true - type: {{ .Values.tdarrStorage.configs.type }} - datasetName: {{ .Values.tdarrStorage.configs.datasetName | default "" }} - hostPath: {{ .Values.tdarrStorage.configs.hostPath | default "" }} + {{- include "tdarr.storage.ci.migration" (dict "storage" .Values.tdarrStorage.configs) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.tdarrStorage.configs) | nindent 4 }} targetSelector: tdarr: tdarr: mountPath: /app/configs logs: enabled: true - type: {{ .Values.tdarrStorage.logs.type }} - datasetName: {{ .Values.tdarrStorage.logs.datasetName | default "" }} - hostPath: {{ .Values.tdarrStorage.logs.hostPath | default "" }} + {{- include "tdarr.storage.ci.migration" (dict "storage" .Values.tdarrStorage.logs) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.tdarrStorage.logs) | nindent 4 }} targetSelector: tdarr: tdarr: mountPath: /app/logs transcode: enabled: true - type: {{ .Values.tdarrStorage.transcodes.type }} - datasetName: {{ .Values.tdarrStorage.transcodes.datasetName | default "" }} - hostPath: {{ .Values.tdarrStorage.transcodes.hostPath | default "" }} - medium: {{ .Values.tdarrStorage.transcodes.medium | default "" }} - {{/* Size of the emptyDir */}} - size: {{ .Values.tdarrStorage.transcodes.size | default "" }} + {{- include "tdarr.storage.ci.migration" (dict "storage" .Values.tdarrStorage.transcodes) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.tdarrStorage.transcodes) | nindent 4 }} targetSelector: tdarr: tdarr: mountPath: /temp {{- range $idx, $storage := .Values.tdarrStorage.additionalStorages }} {{ printf "tdarr-%v" (int $idx) }}: - {{- $size := "" -}} - {{- if $storage.size -}} - {{- $size = (printf "%vGi" $storage.size) -}} - {{- end }} enabled: true - type: {{ $storage.type }} - datasetName: {{ $storage.datasetName | default "" }} - hostPath: {{ $storage.hostPath | default "" }} - server: {{ $storage.server | default "" }} - share: {{ $storage.share | default "" }} - domain: {{ $storage.domain | default "" }} - username: {{ $storage.username | default "" }} - password: {{ $storage.password | default "" }} - size: {{ $size }} - {{- if eq $storage.type "smb-pv-pvc" }} - mountOptions: - - key: noperm - {{- end }} + {{- include "tdarr.storage.ci.migration" (dict "storage" $storage) }} + {{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }} targetSelector: tdarr: tdarr: @@ -155,3 +133,19 @@ scaleGPU: {{ end }} {{ end }} {{- end -}} + +{{/* TODO: Remove on the next version bump, eg 1.2.0+ */}} +{{- define "tdarr.storage.ci.migration" -}} + {{- $storage := .storage -}} + + {{- if $storage.hostPath -}} + {{- $_ := set $storage "hostPathConfig" dict -}} + {{- $_ := set $storage.hostPathConfig "hostPath" $storage.hostPath -}} + {{- end -}} + + {{- if eq $storage.type "emptyDir" -}} + {{- if not $storage.emptyDirConfig -}} + {{- $_ := set $storage "emptyDirConfig" (dict "medium" "" "size" "") -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/library/ix-dev/community/tdarr/values.yaml b/library/ix-dev/community/tdarr/values.yaml index 5b306f80ec..545e66af54 100644 --- a/library/ix-dev/community/tdarr/values.yaml +++ b/library/ix-dev/community/tdarr/values.yaml @@ -22,16 +22,18 @@ tdarrID: tdarrStorage: server: type: ixVolume - datasetName: server + ixVolumeConfig: + datasetName: server configs: type: ixVolume - datasetName: configs + ixVolumeConfig: + datasetName: configs logs: type: ixVolume - datasetName: logs + ixVolumeConfig: + datasetName: logs transcodes: type: ixVolume - datasetName: transcodes - medium: '' - size: '' + ixVolumeConfig: + datasetName: transcodes additionalStorages: []