truenas/chart
1
0
Fork 0
mirror of https://github.com/truenas/charts.git synced 2026-04-05 03:39:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

diskoverdata - migration (#2230)

Browse Source 
* init commit

* fixes

* more fixes

* fix

* fiox

* fix pre-flight

* cron

* update questions

* add migration

* add to_keep_versions

* fix typos

* remove comment

* clean
This commit is contained in:
Stavros Kois
2024-03-01 14:55:59 +02:00
committed by GitHub
parent 69fd9de5cf
commit 71d0cd2ac6
31 changed files with 1213 additions and 900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
library/ix-dev/charts/diskoverdata/Chart.lock
View File

@@ -1,6 +1,6 @@
dependencies:
- name: common
repository: file://../../../common/2304.0.1
version: 2304.0.1
digest: sha256:1ed155c6760e1166e2cb75b52bc5e81c6bdf0252c16ff5ede001157077c41670
generated: "2023-04-24T13:39:12.281094547+03:00"
repository: file://../../../common
version: 1.2.9
digest: sha256:af1a9a1f87e3e48453c9f25f909f5ebcd7fa6e25162b7b425448ba752bcdbc5c
generated: "2024-02-27T20:26:48.651992686+02:00"

8
library/ix-dev/charts/diskoverdata/Chart.yaml
View File

@@ -3,18 +3,18 @@ description: Diskover is used to monitor size/volumes of distributed dataset.
annotations:
title: Diskover Data
type: application
version: 1.0.15
version: 2.0.0
apiVersion: v2
appVersion: "2.0.1"
kubeVersion: '>=1.16.0-0'
kubeVersion: ">=1.16.0-0"
maintainers:
- name: truenas
url: https://www.truenas.com/
email: dev@ixsystems.com
dependencies:
- name: common
repository: file://../../../common/2304.0.1
version: 2304.0.1
repository: file://../../../common
version: 1.2.9
home: https://github.com/diskoverdata/diskover-community
icon: https://media.sys.truenas.net/apps/diskoverdata/icons/icon.png
sources:

15
library/ix-dev/charts/diskoverdata/README.md
View File

@@ -1,14 +1,3 @@
# DiskOverData
# Diskover Data
DiskOver App for TrueNAS SCALE
[Diskover](https://www.diskoverdata.com/) is a sustainable file management solution for your distributed data.
# Introduction
This chart is based on [diskoverdata](https://hub.docker.com/r/linuxserver/diskover) and
deployed on kubernetes via helm chart
## Configuration
Please refer to questions.yaml for a detailed overview on supported configurable values.
[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset.

4
library/ix-dev/charts/diskoverdata/app-readme.md
View File

@@ -1 +1,3 @@
DiskOver App for TrueNAS SCALE
# Diskover Data
[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset.

BIN
library/ix-dev/charts/diskoverdata/charts/common-1.2.9.tgz Normal file
View File

Binary file not shown.

BIN
library/ix-dev/charts/diskoverdata/charts/common-2304.0.1.tgz
View File

Binary file not shown.

19
library/ix-dev/charts/diskoverdata/ci/basic-values.yaml Normal file
View File

@@ -0,0 +1,19 @@
diskoverConfig:
cronSchedule: '* * * * *'
username: someuser
password: somepass
diskoverStorage:
config:
type: pvc
data:
type: pvc
esdata:
type: pvc
additionalStorages:
- type: pvc
diskoverDataIndex: true
mountPath: /data1
- type: pvc
diskoverDataIndex: true
mountPath: /data2

30
library/ix-dev/charts/diskoverdata/ci/test-values.yaml
View File

@@ -1,30 +0,0 @@
environmentVariables: []
extraAppVolumeMounts: []
extraDataVolumeMounts: []
web_port: 32000
dnsConfig:
options: []
emptyDirVolumes: true
appVolumeMounts:
config:
emptyDir: true
mountPath: /config
data:
emptyDir: true
mountPath: /data
elasticSearchAppVolumeMounts:
esdata:
emptyDir: true
mountPath: /usr/share/elasticsearch/data
ownerUID: 568
ownerGID: 568
username: "admin"
password: "admin"
host: "192.169.0.156"
hostNetwork: false
timezone: "America/Los_Angeles"
diskoverCredentials:
username: admin
password: admin
cronjobSchedule: "0 3 * * *"
es_user: elasticsearch

106
library/ix-dev/charts/diskoverdata/migrations/migrate Executable file
View File

@@ -0,0 +1,106 @@
#!/usr/bin/python3
import json
import os
import sys
def migrate_volume(volume):
return {
'type': 'hostPath',
'hostPathConfig': {
'hostPath': volume['hostPath']
},
} if volume.get('hostPathEnabled', False) else {
'type': 'ixVolume',
'ixVolumeConfig': {
'datasetName': volume['datasetName'],
},
}
def migrate_common_lib(values):
delete_keys = [
'web_port', 'timezone', 'dnsConfig', 'environmentVariables', 'ownerUID',
'cronjobSchedule', 'diskoverCredentials', 'appVolumeMounts', 'ownerGID',
'extraAppVolumeMounts', 'extraDataVolumeMounts', 'elasticSearchAppVolumeMounts',
'elasticsearch', 'enableResourceLimits', 'es_user', 'python',
]
additionalVols = [
{
'type': 'hostPath',
'hostPathConfig': {'hostPath': e['hostPath']},
'mountPath': e['mountPath'],
} for e in values.get('extraAppVolumeMounts', [])
] + [
{
'type': 'hostPath',
'hostPathConfig': {'hostPath': e['hostPath']},
'mountPath': e['mountPath'],
'diskoverDataIndex': True
} for e in values.get('extraDataVolumeMounts', [])
]
values.update({
'TZ': values['timezone'],
# Migrate Network
'diskoverNetwork': {
'webPort': values['web_port'],
},
# Migrate DNS
'podOptions': {
'dnsConfig': {
'options': [
{'name': opt['name'], 'value': opt['value']}
for opt in values.get('dnsConfig', {}).get('options', [])
]
}
},
# Migrate Resources
'resources': {
'limits': {
'cpu': values.get('cpuLimit', '4000m'),
'memory': values.get('memLimit', '8Gi'),
}
},
# Migrate ID
'diskoverID': {
'user': values['ownerUID'],
'group': values['ownerGID'],
},
# Migrate Config
'diskoverConfig': {
'cronSchedule': values['cronjobSchedule'],
'username': values['diskoverCredentials']['username'],
'password': values['diskoverCredentials']['password'],
'additionalEnvs': values.get('environmentVariables', []),
},
# Migrate Storage
'diskoverStorage': {
'config': migrate_volume(values['appVolumeMounts']['config']),
'data': migrate_volume(values['appVolumeMounts']['data']),
'esdata': migrate_volume(values['elasticSearchAppVolumeMounts']['esdata']),
'additionalStorages': additionalVols
},
})
for k in delete_keys:
values.pop(k, None)
return values
def migrate(values):
# If this missing, we have already migrated
if not 'appVolumeMounts' in values.keys():
return values
return migrate_common_lib(values)
if __name__ == '__main__':
if len(sys.argv) != 2:
exit(1)
if os.path.exists(sys.argv[1]):
with open(sys.argv[1], 'r') as f:
print(json.dumps(migrate(json.loads(f.read()))))

745
library/ix-dev/charts/diskoverdata/questions.yaml
View File

@@ -1,322 +1,585 @@
groups:
- name: "Configuration"
description: "Diskover application configuration"
- name: "Storage"
description: "Configure storage for Diskover"
- name: "Networking"
description: "Networking Configuration for Diskover"
- name: "Advanced DNS Settings"
description: "Configure DNS settings"
- name: "Resource Limits"
description: "Set CPU/memory limits for Kubernetes Pod"
- name: Diskover Data Configuration
description: Configure Diskover Data
- name: User and Group Configuration
description: Configure User and Group for Diskover Data
- name: Advanced Pod Configuration
description: Configure Advanced Pod Options for Diskover Data
- name: Network Configuration
description: Configure Network for Diskover Data
- name: Storage Configuration
description: Configure Storage for Diskover Data
- name: Resources Configuration
description: Configure Resources for Diskover Data
portals:
web_portal:
protocols:
- "http"
- "$kubernetes-resource_configmap_portal_protocol"
host:
- "$node_ip"
- "$kubernetes-resource_configmap_portal_host"
ports:
- "$variable-web_port"
path: "/"
- "$kubernetes-resource_configmap_portal_port"
path: "$kubernetes-resource_configmap_portal_path"
questions:
- variable: web_port
label: "Web Port for Diskover"
group: Networking
schema:
type: int
min: 8000
max: 65535
default: 22510
required: true
- variable: timezone
label: "Configure timezone"
group: "Configuration"
description: "Configure timezone for Diskover"
- variable: TZ
group: Diskover Data Configuration
label: Timezone
schema:
type: string
default: Etc/UTC
required: true
$ref:
- "definitions/timezone"
- definitions/timezone
- variable: dnsConfig
label: "DNS Configuration"
group: "Advanced DNS Settings"
- variable: discoverConfig
label: ""
group: Diskover Data Configuration
schema:
type: dict
attrs:
- variable: options
label: "DNS Options"
- variable: cronSchedule
label: Cron Schedule
description: The cron schedule for Diskover Data.
schema:
type: string
default: "0 3 * * *"
- variable: username
label: Username
description: The username for Diskover Data.
schema:
type: string
default: ""
required: true
- variable: password
label: Password
description: The password for Diskover Data.
schema:
type: string
default: ""
required: true
private: true
- variable: additionalEnvs
label: Additional Environment Variables
description: Additional environment variables for Diskover Data.
schema:
type: list
items:
- variable: optionsEntry
label: "Option Entry Configuration"
- variable: env
label: Environment Variable
schema:
type: dict
attrs:
- variable: name
label: "Option Name"
label: Name
schema:
type: string
required: true
- variable: value
label: "Option Value"
label: Value
schema:
type: string
required: true
- variable: ownerUID
label: "Config folder's user id"
description: "Linuxserver uses this user id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: ownerGID
label: "Config folder's group id"
description: "Linuxserver uses this group id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: diskoverCredentials
description: "Configure Diskover Initial Password"
label: "Configure Diskover Initial Username and password"
group: "Configuration"
- variable: podOptions
label: ""
group: Advanced Pod Configuration
schema:
type: dict
required: true
additional_attrs: true
attrs:
- variable: username
label: "Username"
description: "UserName for Diskover User"
schema:
type: string
default: "admin"
required: true
- variable: password
label: "Password"
description: "Initial Password for Diskover User"
schema:
type: string
private: true
default: "changeme"
required: true
- variable: environmentVariables
label: "Diskover Extra Environment"
group: "Configuration"
schema:
type: list
default: []
items:
- variable: environmentVariable
label: "Environment Variable"
- variable: dnsConfig
label: Advanced DNS Configuration
schema:
type: dict
attrs:
- variable: name
label: "Name"
- variable: options
label: DNS Options
schema:
type: string
- variable: value
label: "Value"
schema:
type: string
type: list
items:
- variable: optionsEntry
label: DNS Option Entry
schema:
type: dict
attrs:
- variable: name
label: Option Name
schema:
type: string
required: true
- variable: value
label: Option Value
schema:
type: string
required: true
- variable: cronjobSchedule
description: "Cronjobs Consist on 5 values in this specific format 'Minute Hour Day Month Week'"
label: "Define cronjob schedule for diskover"
group: "Configuration"
- variable: diskoverID
label: ""
group: User and Group Configuration
schema:
type: string
default: "0 3 * * *"
type: dict
attrs:
- variable: user
label: User ID
description: The user id that Diskover Data files will be owned by.
schema:
type: int
min: 568
default: 568
required: true
- variable: group
label: Group ID
description: The group id that Diskover Data files will be owned by.
schema:
type: int
min: 568
default: 568
required: true
- variable: appVolumeMounts
label: "Diskover Storage"
group: "Storage"
- variable: discoverNetwork
label: ""
group: Network Configuration
schema:
type: dict
attrs:
- variable: webPort
label: Web Port
description: The port for the Diskover Data Web UI.
schema:
type: int
default: 22510
min: 9000
max: 65535
required: true
- variable: discoverStorage
label: ""
group: Storage Configuration
schema:
type: dict
attrs:
- variable: config
label: "Storage Volume for Configuration"
label: Diskover Data Config Storage
description: The path to store Diskover Data Configuration.
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
hidden: true
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-config"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "config"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: path
hidden: true
editable: true
default: "/config"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Configuration Storage Volume"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: "Host Path for Diskover Configuration Storage Volume"
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: data
label: "Storage Volume for Data"
label: Diskover Data Data Storage
description: The path to store Diskover Data Data.
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
hidden: true
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "data"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: path
hidden: true
editable: true
default: "/data"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Data folder to monitor"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: "Host Path for Diskover Data folder to monitor"
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: extraDataVolumeMounts
label: "Add Extra Host Paths For Diskover To Monitor"
group: "Storage"
schema:
type: list
items:
- variable: dataAppVolume
label: "Host Path Volume"
description: "Add extra Data Volumes for diskover to monitor"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: extraAppVolumeMounts
label: "Extra Host Path Volumes"
group: "Storage"
schema:
type: list
items:
- variable: extraAppVolume
label: "Host Path Volume"
description: "Add an extra host path volume for Diskover application"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: elasticSearchAppVolumeMounts
label: "elastic search Storage"
group: "Storage"
schema:
type: dict
hidden: true
attrs:
- variable: esdata
label: "Storage Volume for Configuration"
label: Elastic Search Data Storage
description: The path to store Elastic Search Data.
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
default: "ix-elasticsearch-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "esdata"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: path
editable: false
default: "/usr/share/elasticsearch/data"
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: enableResourceLimits
label: "Enable Pod resource limits"
group: "Resource Limits"
- variable: additionalStorages
label: Additional Storage
description: Additional storage for Diskover Data.
schema:
type: list
default: []
items:
- variable: storageEntry
label: Storage Entry
schema:
type: dict
attrs:
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.</br>
SMB Share: Is a SMB share that is mounted to a persistent volume claim.
schema:
type: string
required: true
default: "ixVolume"
immutable: true
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- value: "smb-pv-pvc"
description: SMB Share (Mounts a persistent volume claim to a SMB share)
- variable: diskoverDataIndex
label: Enable Diskover Data Indexing
description: Enable Diskover Data Indexing
schema:
type: boolean
- variable: readOnly
label: Read Only
description: Mount the volume as read only.
schema:
type: boolean
default: false
- variable: mountPath
label: Mount Path
description: The path inside the container to mount the storage.
schema:
type: path
required: true
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
default: "storage_entry"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: smbConfig
label: SMB Share Configuration
description: The configuration for the SMB Share.
schema:
type: dict
show_if: [["type", "=", "smb-pv-pvc"]]
attrs:
- variable: server
label: Server
description: The server for the SMB share.
schema:
type: string
required: true
- variable: share
label: Share
description: The share name for the SMB share.
schema:
type: string
required: true
- variable: domain
label: Domain (Optional)
description: The domain for the SMB share.
schema:
type: string
- variable: username
label: Username
description: The username for the SMB share.
schema:
type: string
required: true
- variable: password
label: Password
description: The password for the SMB share.
schema:
type: string
required: true
private: true
- variable: size
label: Size (in Gi)
description: The size of the volume quota.
schema:
type: int
required: true
min: 1
default: 1
- variable: resources
group: Resources Configuration
label: ""
schema:
type: boolean
default: false
- variable: cpuLimit
label: "CPU Limit"
description: "CPU resource limit allow plain integer values with suffix m(milli) e.g 1000m, 100."
group: "Resource Limits"
schema:
type: string
show_if: [["enableResourceLimits", "=", true]]
valid_chars: "^\\d+(?:\\.\\d+(?!.*m$)|m?$)"
default: "4000m"
- variable: memLimit
label: "Memory Limit"
group: "Resource Limits"
description: "Memory limits is specified by number of bytes. Followed by quantity suffix like E,P,T,G,M,k and Ei,Pi,Ti,Mi,Gi,Ki can also be used. e.g 129e6, 129M, 128974848000m, 123Mi"
schema:
type: string
show_if: [["enableResourceLimits", "=", true]]
valid_chars: "^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$"
default: "8Gi"
type: dict
attrs:
- variable: limits
label: Limits
schema:
type: dict
attrs:
- variable: cpu
label: CPU
description: CPU limit for WG-Easy.
schema:
type: string
max_length: 6
valid_chars: '^(0\.[1-9]|[1-9][0-9]*)(\.[0-9]|m?)$'
valid_chars_error: |
Valid CPU limit formats are</br>
- Plain Integer - eg. 1</br>
- Float - eg. 0.5</br>
- Milicpu - eg. 500m
default: "4000m"
required: true
- variable: memory
label: Memory
description: Memory limit for WG-Easy.
schema:
type: string
max_length: 12
valid_chars: '^[1-9][0-9]*([EPTGMK]i?|e[0-9]+)?$'
valid_chars_error: |
Valid Memory limit formats are</br>
- Suffixed with E/P/T/G/M/K - eg. 1G</br>
- Suffixed with Ei/Pi/Ti/Gi/Mi/Ki - eg. 1Gi</br>
- Plain Integer in bytes - eg. 1024</br>
- Exponent - eg. 134e6
default: "8Gi"
required: true

1
library/ix-dev/charts/diskoverdata/templates/NOTES.txt Normal file
View File

@@ -0,0 +1 @@
{{ include "ix.v1.common.lib.chart.notes" $ }}

212
library/ix-dev/charts/diskoverdata/templates/_configuration.tpl Normal file
View File

@@ -0,0 +1,212 @@
{{- define "diskover.configuration" -}}
{{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) }}
{{- $elasticsearch := printf "%s-elasticsearch" $fullname }}
{{- $esPassword := randAlphaNum 32 }}
secret:
diskover-secret:
enabled: true
data:
es-password: {{ $esPassword }}
configmap:
diskover-config:
enabled: true
data:
.default_crawler.sh: |-
#!/bin/sh
while true; do
# this condition wait for the script to copy into the container .
if test -f "$1"; then
# Empty folders don't generate indices . if folder is empty a default file is generated
if ! [ "$(ls -A $2)" ]; then
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
fi
python3 $1 $2/;
break;
fi
sleep 5
done
Constants.php: |
<?php
namespace diskover;
class Constants {
const TIMEZONE = '{{ .Values.TZ }}';
const ES_HOST = '{{ $elasticsearch }}';
const ES_PORT = 9200;
const ES_USER = 'elastic';
const ES_PASS = '{{ $esPassword }}';
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
// override with env var ES_HTTPS
const ES_HTTPS = FALSE;
// login auth for diskover-web
const LOGIN_REQUIRED = TRUE;
// default username and password to login
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
const USER = '{{ .Values.diskoverConfig.username }}';
const PASS = '{{ .Values.diskoverConfig.password }}';
// default results per search page
const SEARCH_RESULTS = 50;
// default size field (size, size_du) to use for sizes on file tree and charts
const SIZE_FIELD = 'size';
// default file types, used by quick search (file type) and dashboard file type usage chart
// additional extensions can be added/removed from each file types list
const FILE_TYPES = [
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
];
// extra fields for search results and view file/dir info pages
// key is description for field and value is ES field name
// Example:
//const EXTRA_FIELDS = [
// 'Date Changed' => 'ctime'
//];
const EXTRA_FIELDS = [];
// Maximum number of indices to load by default, indices are loaded in order by creation date
// setting this too high can cause slow logins and other timeout issues
// This setting can bo overridden on indices page per user and stored in maxindex cookie
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
const MAX_INDEX = 250;
// time in seconds for index info to be cached, clicking reload indices forces update
const INDEXINFO_CACHETIME = 600;
// time in seconds to check Elasticsearch for new index info
const NEWINDEX_CHECKTIME = 10;
// sqlite database file path
const DATABASE = '../diskoverdb.sqlite3';
}
config.yaml: |
# diskover default/sample config file
#
# default search paths for config
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
# Other Unix: ~/.config/diskover and /etc/diskover
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
#
appName: diskover
logLevel: INFO
logToFile: False
logDirectory: /tmp/
diskover:
# max number of crawl threads
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
# set to a number or leave blank to auto set based on number of cpus
#maxthreads: 20
maxthreads:
# block size used for du size
blocksize: 512
excludes:
# directory names and absolute paths you want to exclude from crawl
# directory excludes uses python re.search for string search (regex)
# directory excludes are case-sensitive
# Examples: .* or .backup or .backup* or /dir/dirname
# to exclude none use empty list []
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
#dirs: []
# files you want to exclude from crawl
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
# file names are case-sensitive, extensions are not
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
#files: []
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
emptyfiles: True
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
emptydirs: True
# exclude files smaller than min size in bytes
minfilesize: 1
#minfilesize: 512
# exclude files modified less than x days ago
minmtime: 0
#minmtime: 30
# exclude files modified more than x days ago
maxmtime: 36500
# exclude files changed less than x days ago
minctime: 0
# exclude files changed more than x days ago
maxctime: 36500
# exclude files accessed less than x days ago
minatime: 0
# exclude files accessed more than x days ago
maxatime: 36500
includes:
# directory names and absolute paths you want to include (whitelist), case-sensitive,
# to include none use empty list []
#dirs: [".recycle"]
dirs: []
# files you want to include (whitelist), case-sensitive
files: []
ownersgroups:
# control how owner (username) and group fields are stored for file and directory docs
# store uid and gid's instead of trying to get owner and group names
uidgidonly: False
# owner/group names contain domain name set to True
domain: False
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
domainsep: \
# if domain name comes first before character separator, set this to True, otherwise False
domainfirst: True
# when indexing owner and group fields, keep the domain name
keepdomain: False
replacepaths:
# translate path names set to True to enable or False to disable.
# Set to True if crawling in Windows to replace drive letters and \ with /
replace: False
#from: /mnt/
#to: /vols/
from:
to:
plugins:
# set to True to enable all plugins or False to disable all plugins
enable: False
# list of plugins (by name) to use for directories
dirs: ['unixperms']
# list of plugins (by name) to use for files
files: ['unixperms']
other:
# restore atime/mtime for files and dirs during crawl
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
restoretimes: False
databases:
elasticsearch:
host: '{{ $elasticsearch }}'
port: 9200
user: 'elastic'
password: '{{ $esPassword }}'
# set https to True if using HTTP TLS/SSL or False if using http
# for AWS ES, you will most likely want to set this to True
# override with env var ES_HTTPS
https: False
# compress http data
# for AWS ES, you will most likely want to set this to True
httpcompress: False
# timeout for connection to ES (default is 10)
timeout: 30
# number of connections kept open to ES when crawling (default is 10)
maxsize: 20
# max retries for ES operations (default is 0)
maxretries: 10
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait: False
# chunk size for ES bulk operations (default is 500)
chunksize: 1000
# the below settings are to optimize ES for crawling
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh: 30s
# transaction log flush threshold size (default 512mb)
translogsize: 1gb
# transaction log sync interval time (default 5s)
translogsyncint: 30s
# search scroll size (default 100 docs)
scrollsize: 1000
{{- end -}}

138
library/ix-dev/charts/diskoverdata/templates/_diskover.tpl
View File

@@ -1,35 +1,105 @@
{{- define "add.user" -}}
{{- $user := .Values.es_user -}}
{{- printf "adduser %s -D;" $user -}}
{{- end -}}
{{- define "change.user.permissions" -}}
{{- $user := .Values.es_user -}}
{{- $mountPath := .Values.elasticSearchAppVolumeMounts.esdata.mountPath -}}
{{- printf "chown -R %s:%s %s;" $user $user $mountPath -}}
{{- end -}}
{{- define "elasticsearch.IP" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_HOST" "value" (printf "%s-es" (include "common.names.fullname" .))) }}
{{ $envList = mustAppend $envList (dict "name" "ES_PORT" "value" "9200") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "elasticsearch.credentials" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_USER" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-username") }}
{{ $envList = mustAppend $envList (dict "name" "ES_PASS" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "config.file.path" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "DEST" "value" .mountPath) }}
{{ $envList = mustAppend $envList (dict "name" "FILE" "value" .configFile) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- define "diskover.workload" -}}
{{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) -}}
{{- $elasticsearch := printf "http://%s-elasticsearch:%v/_cluster/health?local=true" $fullname 9200 }}
workload:
diskover:
enabled: true
primary: true
type: Deployment
podSpec:
hostNetwork: false
securityContext:
fsGroup: {{ .Values.diskoverID.group }}
containers:
diskover:
enabled: true
primary: true
imageSelector: image
securityContext:
runAsUser: 0
runAsGroup: 0
runAsNonRoot: false
readOnlyRootFilesystem: false
capabilities:
add:
- CHOWN
- DAC_OVERRIDE
- FOWNER
- SETGID
- SETUID
- KILL
fixedEnv:
PUID: {{ .Values.diskoverID.user }}
{{ with .Values.diskoverConfig.additionalEnvs }}
envList:
{{ range $env := . }}
- name: {{ $env.name }}
value: {{ $env.value }}
{{ end }}
{{ end }}
probes:
liveness:
enabled: true
type: http
path: /login.php
port: 80
readiness:
enabled: true
type: http
path: /login.php
port: 80
startup:
enabled: true
type: http
path: /login.php
port: 80
lifecycle:
{{- $sched := .Values.diskoverConfig.cronSchedule }}
postStart:
type: exec
command:
- /bin/sh
- -c
- |
/scripts/.default_crawler.sh /app/diskover/diskover.py /data;
{{- $cron := printf "%s python3 /app/diskover/diskover.py /data" $sched }}
if ! cat /config/crontab | grep -q "{{ $cron }}"; then
echo "{{ $cron }}" >> /config/crontab;
fi
{{- range $item := .Values.diskoverStorage.additionalStorages }}
/scripts/.default_crawler.sh /app/diskover/diskover.py {{ $item.mountPath }};
{{- end -}}
{{- range $item := .Values.diskoverStorage.additionalStorages }}
{{- if $item.diskoverDataIndex }}
{{- $cron := printf "%s python3 /app/diskover/diskover.py %s" $sched $item.mountPath }}
if ! cat /config/crontab | grep -q "{{ $cron }}"; then
echo "{{ $cron }}" >> /config/crontab;
fi
{{- end }}
{{- end }}
crontab /config/crontab;
initContainers:
01-wait-for-elasticsearch:
enabled: true
type: init
imageSelector: bashImage
env:
ELASTIC_PASSWORD:
secretKeyRef:
name: diskover-secret
key: es-password
command:
- bash
- -c
args:
- |
echo "Pinging [{{ $elasticsearch }}] until it is ready..."
head="--header=Authorization: Basic x$(base64 <<< "elastic:$ELASTIC_PASSWORD")"
time="--timeout=3"
until wget "$head" "$time" --spider -qO- "{{ $elasticsearch }}"; do
echo "Waiting for [{{ $elasticsearch }}] to be ready..."
sleep 2
done
echo "URL [{{ $elasticsearch }}] is ready!"
{{- end -}}

53
library/ix-dev/charts/diskoverdata/templates/_es.tpl Normal file
View File

@@ -0,0 +1,53 @@
{{- define "es.workload" -}}
workload:
elasticsearch:
enabled: true
type: Deployment
podSpec:
hostNetwork: false
containers:
elasticsearch:
enabled: true
primary: true
imageSelector: elasticSearchImage
securityContext:
runAsUser: 1000
runAsGroup: 1000
readOnlyRootFilesystem: false
env:
ELASTIC_PASSWORD:
secretKeyRef:
name: diskover-secret
key: es-password
http.port: 9200
discovery.type: single-node
node.name: diskoverdata
probes:
liveness:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
readiness:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
startup:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
{{- end -}}

35
library/ix-dev/charts/diskoverdata/templates/_migration.tpl Normal file
View File

@@ -0,0 +1,35 @@
{{- define "diskover.get-versions" -}}
{{- $oldChartVersion := "" -}}
{{- $newChartVersion := "" -}}
{{/* Safely access the context, so it wont block CI */}}
{{- if hasKey .Values.global "ixChartContext" -}}
{{- if .Values.global.ixChartContext.upgradeMetadata -}}
{{- $oldChartVersion = .Values.global.ixChartContext.upgradeMetadata.oldChartVersion -}}
{{- $newChartVersion = .Values.global.ixChartContext.upgradeMetadata.newChartVersion -}}
{{- if and (not $oldChartVersion) (not $newChartVersion) -}}
{{- fail "Upgrade Metadata is missing. Cannot proceed" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- toYaml (dict "old" $oldChartVersion "new" $newChartVersion) -}}
{{- end -}}
{{- define "diskover.migration" -}}
{{- $versions := (fromYaml (include "diskover.get-versions" $)) -}}
{{- if and $versions.old $versions.new -}}
{{- $oldV := semver $versions.old -}}
{{- $newV := semver $versions.new -}}
{{/* If new is v2.x.x */}}
{{- if eq ($newV.Major | int) 2 -}}
{{/* And old is v1.x.x, but lower than 0.15 */}}
{{- if and (eq $oldV.Major 1) (lt ($oldV.Patch | int) 15) -}}
{{/* Block the upgrade */}}
{{- fail "Migration to 2.x.x is only allowed from 1.0.15 or higher" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}

68
library/ix-dev/charts/diskoverdata/templates/_persistence.tpl Normal file
View File

@@ -0,0 +1,68 @@
{{- define "diskover.persistence" -}}
persistence:
config:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.config) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: /config
data:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.data) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: /data
esdata:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.esdata) | nindent 4 }}
targetSelector:
elasticsearch:
elasticsearch:
mountPath: /usr/share/elasticsearch/data
defaultcrawler:
enabled: true
type: configmap
objectName: diskover-config
defaultMode: "0755"
targetSelector:
diskover:
diskover:
mountPath: /scripts/default_crawler.sh
subPath: .default_crawler.sh
phpfile:
enabled: true
type: configmap
objectName: diskover-config
targetSelector:
diskover:
diskover:
mountPath: /config/diskover-web.conf.d/Constants.php
subPath: Constants.php
yamlfile:
enabled: true
type: configmap
objectName: diskover-config
targetSelector:
diskover:
diskover:
mountPath: /config/diskover.conf.d/diskover/config.yaml
subPath: config.yaml
tmp:
enabled: true
type: emptyDir
targetSelector:
diskover:
01-wait-for-elasticsearch:
mountPath: /tmp
{{- range $idx, $storage := .Values.diskoverStorage.additionalStorages }}
{{ printf "diskover-%v:" (int $idx) }}
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: {{ $storage.mountPath }}
{{- end }}
{{- end -}}

12
library/ix-dev/charts/diskoverdata/templates/_portal.tpl Normal file
View File

@@ -0,0 +1,12 @@
{{- define "diskover.portal" -}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: portal
data:
path: "/"
port: {{ .Values.diskoverNetwork.webPort | quote }}
protocol: "http"
host: "$node_ip"
{{- end -}}

27
library/ix-dev/charts/diskoverdata/templates/_service.tpl Normal file
View File

@@ -0,0 +1,27 @@
{{- define "diskover.service" -}}
service:
diskover:
enabled: true
primary: true
type: NodePort
targetSelector: diskover
ports:
webui:
enabled: true
primary: true
port: {{ .Values.diskoverNetwork.webPort }}
nodePort: {{ .Values.diskoverNetwork.webPort }}
targetPort: 80
targetSelector: diskover
elasticsearch:
enabled: true
type: ClusterIP
targetSelector: elasticsearch
ports:
elasticsearch:
enabled: true
primary: true
port: 9200
targetPort: 9200
targetSelector: elasticsearch
{{- end -}}

15
library/ix-dev/charts/diskoverdata/templates/common.yaml Normal file
View File

@@ -0,0 +1,15 @@
{{- include "ix.v1.common.loader.init" . -}}
{{- include "diskover.migration" $ -}}
{{/* Merge the templates with Values */}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.configuration" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.workload" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "es.workload" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.service" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.persistence" $ | fromYaml) -}}
{{/* Create the configmap for portal manually */}}
{{- include "diskover.portal" $ -}}
{{- include "ix.v1.common.loader.apply" . -}}

160
library/ix-dev/charts/diskoverdata/templates/deployment.yaml
View File

@@ -1,160 +0,0 @@
{{ include "common.storage.hostPathValidate" .Values }}
{{ $elastic_search := (. | mustDeepCopy) }}
{{ $_ := set $elastic_search "common" (dict "nameSuffix" "elasticsearch") }}
apiVersion: {{ template "common.capabilities.deployment.apiVersion" . }}
kind: Deployment
metadata:
name: {{ template "common.names.fullname" . }}-diskover
labels:
app: {{ template "common.names.name" . }}
chart: {{ template "common.names.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
replicas: {{ (default 1 .Values.replicas) }}
strategy:
type: "Recreate"
selector:
matchLabels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
template:
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
{{- include "common.labels.selectorLabels" . | nindent 8 }}
annotations: {{ include "common.annotations" . | nindent 8 }}
spec:
initContainers:
- name: init-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/init_config.py"]
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "TZ" "value" .Values.timezone) }}
{{ $envList = mustAppend $envList (dict "name" "DS_USER" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "username") }}
{{ $envList = mustAppend $envList (dict "name" "DS_PASS" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPath := (dict "mountPath" (printf "%s/diskover-web.conf.d/" .Values.appVolumeMounts.config.mountPath) "configFile" "Constants.php") }}
{{ include "config.file.path" $configPath | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: wait-es-search
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
env:
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
command: ["python3", "/init_scripts/wait_for_elastic_search.py"]
volumeMounts:
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: init-es-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/initial_es_config.py"]
env:
{{ $envListConfig := (default list .Values.environmentVariables) }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPathES := (dict "mountPath" (printf "%s/diskover.conf.d/diskover/" .Values.appVolumeMounts.config.mountPath) "configFile" "config.yaml") }}
{{ include "config.file.path" $configPathES | nindent 12 }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListConfig) | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
containers:
- name: {{ .Chart.Name }}
{{ include "common.resources.limitation" . | nindent 10 }}
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
ports:
- name: web
containerPort: 80
{{ $cronjobSchedule := .Values.cronjobSchedule }}
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- |
./init_scripts/.default_crawler.sh /app/diskover/diskover.py /data;
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
./init_scripts/.default_crawler.sh /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }};
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
echo "{{$cronjobSchedule}} python3 /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }}" >> /config/crontab;
{{ end }}
echo "{{.Values.cronjobSchedule}} python3 /app/diskover/diskover.py /data" >> /config/crontab;
crontab /config/crontab;
env:
{{ $envListDiskover := (default list .Values.environmentVariables) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PUID" "value" .Values.ownerUID) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PGID" "value" .Values.ownerGID) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListDiskover) | nindent 12 }}
livenessProbe:
httpGet:
path: /login.php
port: 80
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 5
successThreshold: 1
readinessProbe:
httpGet:
path: /login.php
port: 80
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 5
successThreshold: 2
startupProbe:
httpGet:
path: /login.php
port: 80
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 60
successThreshold: 1
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" .Values | nindent 8 }}
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
- name: diskover-initial-scripts
configMap:
defaultMode: 0700
name: "diskover-initial-scripts"

9
library/ix-dev/charts/diskoverdata/templates/diskover-secrets.yaml
View File

@@ -1,9 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: diskover-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
username: {{ .Values.diskoverCredentials.username | b64enc | quote }}
password: {{ .Values.diskoverCredentials.password | b64enc | quote }}

46
library/ix-dev/charts/diskoverdata/templates/elastic_search_deployment.yaml
View File

@@ -1,46 +0,0 @@
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch-es") }}
{{ include "common.deployment.common_config" $values | nindent 0 }}
spec: {{ include "common.deployment.common_spec" $values | nindent 2 }}
template: {{ include "common.deployment.pod.metadata" $values | nindent 4 }}
spec:
containers:
- name: {{ .Chart.Name }}
{{ include "common.containers.imageConfig" .Values.elasticsearch.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
ports:
- name: es-port
containerPort: 9200
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "discovery.type" "value" "single-node") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
livenessProbe:
httpGet:
path: /
port: 9200
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 5
successThreshold: 1
readinessProbe:
httpGet:
path: /
port: 9200
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 5
successThreshold: 2
startupProbe:
httpGet:
path: /
port: 9200
initialDelaySeconds: 10
periodSeconds: 5
timeoutSeconds: 2
failureThreshold: 60
successThreshold: 1
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

6
library/ix-dev/charts/diskoverdata/templates/elastic_search_service.yaml
View File

@@ -1,6 +0,0 @@
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "es-port" "port" 9200 "targetPort" 9200) }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch-es") }}
{{ $_1 := set $values "commonService" (dict "type" "ClusterIP" "ports" $ports ) }}
{{ include "common.classes.service" $values }}

9
library/ix-dev/charts/diskoverdata/templates/elasticsearch-secret.yaml
View File

@@ -1,9 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: elastic-search-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
es-username: {{ "elastic" | b64enc | quote }}
es-password: {{ "changeme" | b64enc | quote }}

270
library/ix-dev/charts/diskoverdata/templates/initial_scripts.yaml
View File

@@ -1,270 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "diskover-initial-scripts"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
data:
wait_for_elastic_search.py: |-
# This Script Wait for elastic search to setup completely
import requests
import os
import time
timeout = 100
while True:
try:
if timeout < 0:
print("timeout")
raise requests.exceptions.ConnectTimeout("Elasticsearch is not responding")
timeout -= 1
response = requests.get(f"http://{os.environ['ES_HOST']}:{os.environ['ES_PORT']}")
if response.status_code == 200:
break
except requests.exceptions.ConnectTimeout as e:
print(e)
break
except requests.exceptions.ConnectionError:
print("Trying to connect to elastic search")
time.sleep(3)
.default_crawler.sh: |-
#!/bin/sh
while :
do
# this condition wait for the script to copy into the container .
if test -f "$1"; then
# Empty folders don't generate indices . if folder is empty a default file is generated
if ! [ "$(ls -A $2)" ]; then
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
fi
python3 $1 $2/;
break;
fi
sleep 5
done
init_config.py: |-
import os
Config = f"""<?php
namespace diskover;
class Constants {{`{{
const TIMEZONE = '{os.environ['TZ']}';
const ES_HOST = '{os.environ['ES_HOST']}';
const ES_PORT = {os.environ['ES_PORT']};
const ES_USER = '{os.environ['ES_USER']}';
const ES_PASS = '{os.environ['ES_PASS']}';
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
// override with env var ES_HTTPS
const ES_HTTPS = FALSE;
// login auth for diskover-web
const LOGIN_REQUIRED = TRUE;
// default username and password to login
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
const USER = '{os.environ['DS_USER']}';
const PASS = '{os.environ['DS_PASS']}';
// default results per search page
const SEARCH_RESULTS = 50;
// default size field (size, size_du) to use for sizes on file tree and charts
const SIZE_FIELD = 'size';
// default file types, used by quick search (file type) and dashboard file type usage chart
// additional extensions can be added/removed from each file types list
const FILE_TYPES = [
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
];
// extra fields for search results and view file/dir info pages
// key is description for field and value is ES field name
// Example:
//const EXTRA_FIELDS = [
// 'Date Changed' => 'ctime'
//];
const EXTRA_FIELDS = [];
// Maximum number of indices to load by default, indices are loaded in order by creation date
// setting this too high can cause slow logins and other timeout issues
// This setting can bo overridden on indices page per user and stored in maxindex cookie
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
const MAX_INDEX = 250;
// time in seconds for index info to be cached, clicking reload indices forces update
const INDEXINFO_CACHETIME = 600;
// time in seconds to check Elasticsearch for new index info
const NEWINDEX_CHECKTIME = 10;
// sqlite database file path
const DATABASE = '../diskoverdb.sqlite3';
}}`}}
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)
initial_es_config.py: |-
import os
Config = f"""# diskover default/sample config file
#
# default search paths for config
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
# Other Unix: ~/.config/diskover and /etc/diskover
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
#
appName: diskover
#logLevel: WARN
#logLevel: DEBUG
logLevel: INFO
logToFile: False
#logToFile: True
logDirectory: /tmp/
diskover:
# max number of crawl threads
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
# set to a number or leave blank to auto set based on number of cpus
#maxthreads: 20
maxthreads:
# block size used for du size
blocksize: 512
excludes:
# directory names and absolute paths you want to exclude from crawl
# directory excludes uses python re.search for string search (regex)
# directory excludes are case-sensitive
# Examples: .* or .backup or .backup* or /dir/dirname
# to exclude none use empty list []
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
#dirs: []
# files you want to exclude from crawl
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
# file names are case-sensitive, extensions are not
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
#files: []
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
emptyfiles: True
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
emptydirs: True
# exclude files smaller than min size in bytes
minfilesize: 1
#minfilesize: 512
# exclude files modified less than x days ago
minmtime: 0
#minmtime: 30
# exclude files modified more than x days ago
maxmtime: 36500
# exclude files changed less than x days ago
minctime: 0
# exclude files changed more than x days ago
maxctime: 36500
# exclude files accessed less than x days ago
minatime: 0
# exclude files accessed more than x days ago
maxatime: 36500
includes:
# directory names and absolute paths you want to include (whitelist), case-sensitive,
# to include none use empty list []
#dirs: [".recycle"]
dirs: []
# files you want to include (whitelist), case-sensitive
files: []
ownersgroups:
# control how owner (username) and group fields are stored for file and directory docs
# store uid and gid's instead of trying to get owner and group names
uidgidonly: False
# owner/group names contain domain name set to True
domain: False
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
domainsep: \
# if domain name comes first before character separator, set this to True, otherwise False
domainfirst: True
# when indexing owner and group fields, keep the domain name
keepdomain: False
replacepaths:
# translate path names set to True to enable or False to disable.
# Set to True if crawling in Windows to replace drive letters and \ with /
replace: False
#from: /mnt/
#to: /vols/
from:
to:
plugins:
# set to True to enable all plugins or False to disable all plugins
enable: False
# list of plugins (by name) to use for directories
dirs: ['unixperms']
# list of plugins (by name) to use for files
files: ['unixperms']
other:
# restore atime/mtime for files and dirs during crawl
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
restoretimes: False
databases:
elasticsearch:
host: '{os.environ['ES_HOST']}'
port: {os.environ['ES_PORT']}
#user: elastic
#password: changeme
user: '{os.environ['ES_USER']}'
password: '{os.environ['ES_PASS']}'
# set https to True if using HTTP TLS/SSL or False if using http
# for AWS ES, you will most likely want to set this to True
# override with env var ES_HTTPS
https: False
# compress http data
# for AWS ES, you will most likely want to set this to True
httpcompress: False
# timeout for connection to ES (default is 10)
timeout: 30
# number of connections kept open to ES when crawling (default is 10)
maxsize: 20
# max retries for ES operations (default is 0)
maxretries: 10
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait: False
# chunk size for ES bulk operations (default is 500)
chunksize: 1000
# the below settings are to optimize ES for crawling
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh: 30s
# transaction log flush threshold size (default 512mb)
translogsize: 1gb
# transaction log sync interval time (default 5s)
translogsyncint: 30s
# search scroll size (default 100 docs)
scrollsize: 1000
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)

31
library/ix-dev/charts/diskoverdata/templates/pre-install-job.yaml
View File

@@ -1,31 +0,0 @@
apiVersion: batch/v1
kind: Job
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-job"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-hook"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
spec:
restartPolicy: Never
containers:
- name: pre-install-job
image: "alpine:latest"
command: ["/bin/sh", "-c"]
args:
- {{ include "add.user" . }}
{{ include "change.user.permissions" . }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

10
library/ix-dev/charts/diskoverdata/templates/service.yaml
View File

@@ -1,10 +0,0 @@
{{ $svc := .Values.service }}
{{ $selectors := list }}
{{ $selectors = mustAppend $selectors (dict "key" "app" "value" (include "common.names.name" .) ) }}
{{ $selectors = mustAppend $selectors (dict "key" "release" "value" .Release.Name ) }}
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "web" "port" .Values.web_port "nodePort" .Values.web_port "targetPort" 80) }}
{{ $params := . }}
{{ $_ := set $params "commonService" (dict "type" "NodePort" "ports" $ports ) }}
{{ $_1 := set .Values "extraSelectorLabels" $selectors }}
{{ include "common.classes.service" $params }}

21
library/ix-dev/charts/diskoverdata/templates/tests/deployment-check.yaml
View File

@@ -1,21 +0,0 @@
{{- $serviceName := (include "common.names.fullname" .) -}}
apiVersion: v1
kind: Pod
metadata:
name: {{ .Release.Name }}-test-pod
labels:
app: {{ .Release.Name }}
release: {{ .Release.Name }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test-curl
image: alpine/curl
imagePullPolicy: "IfNotPresent"
command:
- /bin/sh
- -ec
- |
curl --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 15 --retry-max-time 90 --retry-all-errors -ksf http://{{ $serviceName }}:{{.Values.web_port}}/
restartPolicy: Never

4
library/ix-dev/charts/diskoverdata/to_keep_versions.md Normal file
View File

@@ -0,0 +1,4 @@
# 1.0.15
This version is kept because it contains a fix that is needed for migration to v2.x.x
It should be safe to remove few months after v2.x.x is released.

1
library/ix-dev/charts/diskoverdata/to_keep_versions.yaml Normal file
View File

@@ -0,0 +1 @@
- 1.0.15

50
library/ix-dev/charts/diskoverdata/values.yaml
View File

@@ -3,16 +3,44 @@ image:
repository: linuxserver/diskover
tag: "2.0.1"
elasticsearch:
image:
pullPolicy: IfNotPresent
repository: elasticsearch
tag: "7.5.2"
elasticSearchImage:
pullPolicy: IfNotPresent
repository: elasticsearch
tag: "7.5.2"
python:
image:
pullPolicy: IfNotPresent
repository: python
tag: "3.10"
resources:
limits:
cpu: 4000m
memory: 8Gi
es_user: elasticsearch
podOptions:
dnsConfig:
options: []
diskoverConfig:
cronSchedule: ''
username: ''
password: ''
additionalEnvs: []
diskoverID:
user: 568
group: 568
diskoverNetwork:
webPort: 32000
diskoverStorage:
config:
type: ixVolume
ixVolumeConfig:
datasetName: config
data:
type: ixVolume
ixVolumeConfig:
datasetName: data
esdata:
type: ixVolume
ixVolumeConfig:
datasetName: esdata
additionalStorages: []