diff --git a/charts/diskoverdata/2.0.0/Chart.lock b/charts/diskoverdata/2.0.0/Chart.lock new file mode 100644 index 0000000000..00bcc6e479 --- /dev/null +++ b/charts/diskoverdata/2.0.0/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../../../common + version: 1.2.9 +digest: sha256:af1a9a1f87e3e48453c9f25f909f5ebcd7fa6e25162b7b425448ba752bcdbc5c +generated: "2024-02-27T20:26:48.651992686+02:00" diff --git a/charts/diskoverdata/2.0.0/Chart.yaml b/charts/diskoverdata/2.0.0/Chart.yaml new file mode 100644 index 0000000000..ea71cb8ed7 --- /dev/null +++ b/charts/diskoverdata/2.0.0/Chart.yaml @@ -0,0 +1,26 @@ +name: diskoverdata +description: Diskover is used to monitor size/volumes of distributed dataset. +annotations: + title: Diskover Data +type: application +version: 2.0.0 +apiVersion: v2 +appVersion: "2.0.1" +kubeVersion: ">=1.16.0-0" +maintainers: + - name: truenas + url: https://www.truenas.com/ + email: dev@ixsystems.com +dependencies: + - name: common + repository: file://../../../common + version: 1.2.9 +home: https://github.com/diskoverdata/diskover-community +icon: https://media.sys.truenas.net/apps/diskoverdata/icons/icon.png +sources: + - https://github.com/diskoverdata/diskover-community + - https://github.com/truenas/charts/tree/master/charts/diskoverdata +keywords: + - storage + - monitoring + - management diff --git a/charts/diskoverdata/2.0.0/README.md b/charts/diskoverdata/2.0.0/README.md new file mode 100644 index 0000000000..817612c364 --- /dev/null +++ b/charts/diskoverdata/2.0.0/README.md @@ -0,0 +1,3 @@ +# Diskover Data + +[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset. diff --git a/charts/diskoverdata/2.0.0/app-readme.md b/charts/diskoverdata/2.0.0/app-readme.md new file mode 100644 index 0000000000..817612c364 --- /dev/null +++ b/charts/diskoverdata/2.0.0/app-readme.md @@ -0,0 +1,3 @@ +# Diskover Data + +[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset. diff --git a/charts/diskoverdata/2.0.0/charts/common-1.2.9.tgz b/charts/diskoverdata/2.0.0/charts/common-1.2.9.tgz new file mode 100644 index 0000000000..797faff0d7 Binary files /dev/null and b/charts/diskoverdata/2.0.0/charts/common-1.2.9.tgz differ diff --git a/charts/diskoverdata/2.0.0/ci/basic-values.yaml b/charts/diskoverdata/2.0.0/ci/basic-values.yaml new file mode 100644 index 0000000000..910fbce8f8 --- /dev/null +++ b/charts/diskoverdata/2.0.0/ci/basic-values.yaml @@ -0,0 +1,19 @@ +diskoverConfig: + cronSchedule: '* * * * *' + username: someuser + password: somepass + +diskoverStorage: + config: + type: pvc + data: + type: pvc + esdata: + type: pvc + additionalStorages: + - type: pvc + diskoverDataIndex: true + mountPath: /data1 + - type: pvc + diskoverDataIndex: true + mountPath: /data2 diff --git a/charts/diskoverdata/2.0.0/ix_values.yaml b/charts/diskoverdata/2.0.0/ix_values.yaml new file mode 100644 index 0000000000..db702b5582 --- /dev/null +++ b/charts/diskoverdata/2.0.0/ix_values.yaml @@ -0,0 +1,46 @@ +image: + pullPolicy: IfNotPresent + repository: linuxserver/diskover + tag: "2.0.1" + +elasticSearchImage: + pullPolicy: IfNotPresent + repository: elasticsearch + tag: "7.5.2" + +resources: + limits: + cpu: 4000m + memory: 8Gi + +podOptions: + dnsConfig: + options: [] + +diskoverConfig: + cronSchedule: '' + username: '' + password: '' + additionalEnvs: [] + +diskoverID: + user: 568 + group: 568 + +diskoverNetwork: + webPort: 32000 + +diskoverStorage: + config: + type: ixVolume + ixVolumeConfig: + datasetName: config + data: + type: ixVolume + ixVolumeConfig: + datasetName: data + esdata: + type: ixVolume + ixVolumeConfig: + datasetName: esdata + additionalStorages: [] diff --git a/charts/diskoverdata/2.0.0/metadata.yaml b/charts/diskoverdata/2.0.0/metadata.yaml new file mode 100644 index 0000000000..fe06b896fe --- /dev/null +++ b/charts/diskoverdata/2.0.0/metadata.yaml @@ -0,0 +1,41 @@ +runAsContext: + - userName: root + groupName: root + gid: 0 + uid: 0 + description: Diskover runs as root user. + - userName: root + groupName: root + gid: 0 + uid: 0 + description: Elastic Search runs as root user. +capabilities: + - name: CHOWN + description: Diskover and Elastic Search are able to chown files. + - name: FOWNER + description: Diskover and Elastic Search are able to bypass permission checks for it's sub-processes. + - name: SYS_CHROOT + description: Diskover and Elastic Search are able to use chroot. + - name: MKNOD + description: Diskover and Elastic Search are able to create device nodes. + - name: DAC_OVERRIDE + description: Diskover and Elastic Search are able to bypass permission checks. + - name: FSETID + description: Diskover and Elastic Search are able to set file capabilities. + - name: KILL + description: Diskover and Elastic Search are able to kill processes. + - name: SETGID + description: Diskover and Elastic Search are able to set group ID for it's sub-processes. + - name: SETUID + description: Diskover and Elastic Search are able to set user ID for it's sub-processes. + - name: SETPCAP + description: Diskover and Elastic Search are able to set process capabilities. + - name: NET_BIND_SERVICE + description: Diskover and Elastic Search are able to bind to privileged ports. + - name: SETFCAP + description: Diskover and Elastic Search are able to set file capabilities. + - name: NET_RAW + description: Diskover and Elastic Search are able to use raw sockets. + - name: AUDIT_WRITE + description: Diskover and Elastic Search are able to write to audit log. +hostMounts: [] diff --git a/charts/diskoverdata/2.0.0/migrations/migrate b/charts/diskoverdata/2.0.0/migrations/migrate new file mode 100755 index 0000000000..9eea742117 --- /dev/null +++ b/charts/diskoverdata/2.0.0/migrations/migrate @@ -0,0 +1,106 @@ +#!/usr/bin/python3 +import json +import os +import sys + + +def migrate_volume(volume): + return { + 'type': 'hostPath', + 'hostPathConfig': { + 'hostPath': volume['hostPath'] + }, + } if volume.get('hostPathEnabled', False) else { + 'type': 'ixVolume', + 'ixVolumeConfig': { + 'datasetName': volume['datasetName'], + }, + } + + +def migrate_common_lib(values): + delete_keys = [ + 'web_port', 'timezone', 'dnsConfig', 'environmentVariables', 'ownerUID', + 'cronjobSchedule', 'diskoverCredentials', 'appVolumeMounts', 'ownerGID', + 'extraAppVolumeMounts', 'extraDataVolumeMounts', 'elasticSearchAppVolumeMounts', + 'elasticsearch', 'enableResourceLimits', 'es_user', 'python', + ] + + additionalVols = [ + { + 'type': 'hostPath', + 'hostPathConfig': {'hostPath': e['hostPath']}, + 'mountPath': e['mountPath'], + } for e in values.get('extraAppVolumeMounts', []) + ] + [ + { + 'type': 'hostPath', + 'hostPathConfig': {'hostPath': e['hostPath']}, + 'mountPath': e['mountPath'], + 'diskoverDataIndex': True + } for e in values.get('extraDataVolumeMounts', []) + ] + + values.update({ + 'TZ': values['timezone'], + # Migrate Network + 'diskoverNetwork': { + 'webPort': values['web_port'], + }, + # Migrate DNS + 'podOptions': { + 'dnsConfig': { + 'options': [ + {'name': opt['name'], 'value': opt['value']} + for opt in values.get('dnsConfig', {}).get('options', []) + ] + } + }, + # Migrate Resources + 'resources': { + 'limits': { + 'cpu': values.get('cpuLimit', '4000m'), + 'memory': values.get('memLimit', '8Gi'), + } + }, + # Migrate ID + 'diskoverID': { + 'user': values['ownerUID'], + 'group': values['ownerGID'], + }, + # Migrate Config + 'diskoverConfig': { + 'cronSchedule': values['cronjobSchedule'], + 'username': values['diskoverCredentials']['username'], + 'password': values['diskoverCredentials']['password'], + 'additionalEnvs': values.get('environmentVariables', []), + }, + # Migrate Storage + 'diskoverStorage': { + 'config': migrate_volume(values['appVolumeMounts']['config']), + 'data': migrate_volume(values['appVolumeMounts']['data']), + 'esdata': migrate_volume(values['elasticSearchAppVolumeMounts']['esdata']), + 'additionalStorages': additionalVols + }, + }) + + for k in delete_keys: + values.pop(k, None) + + return values + +def migrate(values): + # If this missing, we have already migrated + if not 'appVolumeMounts' in values.keys(): + return values + + return migrate_common_lib(values) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + exit(1) + + if os.path.exists(sys.argv[1]): + with open(sys.argv[1], 'r') as f: + print(json.dumps(migrate(json.loads(f.read())))) diff --git a/charts/diskoverdata/2.0.0/questions.yaml b/charts/diskoverdata/2.0.0/questions.yaml new file mode 100644 index 0000000000..c29e2361a6 --- /dev/null +++ b/charts/diskoverdata/2.0.0/questions.yaml @@ -0,0 +1,585 @@ +groups: + - name: Diskover Data Configuration + description: Configure Diskover Data + - name: User and Group Configuration + description: Configure User and Group for Diskover Data + - name: Advanced Pod Configuration + description: Configure Advanced Pod Options for Diskover Data + - name: Network Configuration + description: Configure Network for Diskover Data + - name: Storage Configuration + description: Configure Storage for Diskover Data + - name: Resources Configuration + description: Configure Resources for Diskover Data + +portals: + web_portal: + protocols: + - "$kubernetes-resource_configmap_portal_protocol" + host: + - "$kubernetes-resource_configmap_portal_host" + ports: + - "$kubernetes-resource_configmap_portal_port" + path: "$kubernetes-resource_configmap_portal_path" + +questions: + - variable: TZ + group: Diskover Data Configuration + label: Timezone + schema: + type: string + default: Etc/UTC + required: true + $ref: + - definitions/timezone + + - variable: discoverConfig + label: "" + group: Diskover Data Configuration + schema: + type: dict + attrs: + - variable: cronSchedule + label: Cron Schedule + description: The cron schedule for Diskover Data. + schema: + type: string + default: "0 3 * * *" + - variable: username + label: Username + description: The username for Diskover Data. + schema: + type: string + default: "" + required: true + - variable: password + label: Password + description: The password for Diskover Data. + schema: + type: string + default: "" + required: true + private: true + - variable: additionalEnvs + label: Additional Environment Variables + description: Additional environment variables for Diskover Data. + schema: + type: list + items: + - variable: env + label: Environment Variable + schema: + type: dict + attrs: + - variable: name + label: Name + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + + - variable: podOptions + label: "" + group: Advanced Pod Configuration + schema: + type: dict + attrs: + - variable: dnsConfig + label: Advanced DNS Configuration + schema: + type: dict + attrs: + - variable: options + label: DNS Options + schema: + type: list + items: + - variable: optionsEntry + label: DNS Option Entry + schema: + type: dict + attrs: + - variable: name + label: Option Name + schema: + type: string + required: true + - variable: value + label: Option Value + schema: + type: string + required: true + + - variable: diskoverID + label: "" + group: User and Group Configuration + schema: + type: dict + attrs: + - variable: user + label: User ID + description: The user id that Diskover Data files will be owned by. + schema: + type: int + min: 568 + default: 568 + required: true + - variable: group + label: Group ID + description: The group id that Diskover Data files will be owned by. + schema: + type: int + min: 568 + default: 568 + required: true + + - variable: discoverNetwork + label: "" + group: Network Configuration + schema: + type: dict + attrs: + - variable: webPort + label: Web Port + description: The port for the Diskover Data Web UI. + schema: + type: int + default: 22510 + min: 9000 + max: 65535 + required: true + + - variable: discoverStorage + label: "" + group: Storage Configuration + schema: + type: dict + attrs: + - variable: config + label: Diskover Data Config Storage + description: The path to store Diskover Data Configuration. + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system. + schema: + type: string + required: true + immutable: true + default: "ixVolume" + enum: + - value: "hostPath" + description: Host Path (Path that already exists on the system) + - value: "ixVolume" + description: ixVolume (Dataset created automatically by the system) + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ixVolume"]] + $ref: + - "normalize/ixVolume" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "config" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "hostPath"]] + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: data + label: Diskover Data Data Storage + description: The path to store Diskover Data Data. + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system. + schema: + type: string + required: true + immutable: true + default: "ixVolume" + enum: + - value: "hostPath" + description: Host Path (Path that already exists on the system) + - value: "ixVolume" + description: ixVolume (Dataset created automatically by the system) + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ixVolume"]] + $ref: + - "normalize/ixVolume" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "data" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "hostPath"]] + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: esdata + label: Elastic Search Data Storage + description: The path to store Elastic Search Data. + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system. + schema: + type: string + required: true + immutable: true + default: "ixVolume" + enum: + - value: "hostPath" + description: Host Path (Path that already exists on the system) + - value: "ixVolume" + description: ixVolume (Dataset created automatically by the system) + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ixVolume"]] + $ref: + - "normalize/ixVolume" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + hidden: true + default: "esdata" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "hostPath"]] + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + + - variable: additionalStorages + label: Additional Storage + description: Additional storage for Diskover Data. + schema: + type: list + default: [] + items: + - variable: storageEntry + label: Storage Entry + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system.
+ SMB Share: Is a SMB share that is mounted to a persistent volume claim. + schema: + type: string + required: true + default: "ixVolume" + immutable: true + enum: + - value: "hostPath" + description: Host Path (Path that already exists on the system) + - value: "ixVolume" + description: ixVolume (Dataset created automatically by the system) + - value: "smb-pv-pvc" + description: SMB Share (Mounts a persistent volume claim to a SMB share) + - variable: diskoverDataIndex + label: Enable Diskover Data Indexing + description: Enable Diskover Data Indexing + schema: + type: boolean + - variable: readOnly + label: Read Only + description: Mount the volume as read only. + schema: + type: boolean + default: false + - variable: mountPath + label: Mount Path + description: The path inside the container to mount the storage. + schema: + type: path + required: true + - variable: hostPathConfig + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "hostPath"]] + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: hostPath + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["aclEnable", "=", false]] + required: true + - variable: ixVolumeConfig + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ixVolume"]] + $ref: + - "normalize/ixVolume" + attrs: + - variable: aclEnable + label: Enable ACL + description: Enable ACL for the dataset. + schema: + type: boolean + default: false + - variable: datasetName + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + immutable: true + default: "storage_entry" + - variable: aclEntries + label: ACL Configuration + schema: + type: dict + show_if: [["aclEnable", "=", true]] + attrs: [] + - variable: smbConfig + label: SMB Share Configuration + description: The configuration for the SMB Share. + schema: + type: dict + show_if: [["type", "=", "smb-pv-pvc"]] + attrs: + - variable: server + label: Server + description: The server for the SMB share. + schema: + type: string + required: true + - variable: share + label: Share + description: The share name for the SMB share. + schema: + type: string + required: true + - variable: domain + label: Domain (Optional) + description: The domain for the SMB share. + schema: + type: string + - variable: username + label: Username + description: The username for the SMB share. + schema: + type: string + required: true + - variable: password + label: Password + description: The password for the SMB share. + schema: + type: string + required: true + private: true + - variable: size + label: Size (in Gi) + description: The size of the volume quota. + schema: + type: int + required: true + min: 1 + default: 1 + + - variable: resources + group: Resources Configuration + label: "" + schema: + type: dict + attrs: + - variable: limits + label: Limits + schema: + type: dict + attrs: + - variable: cpu + label: CPU + description: CPU limit for WG-Easy. + schema: + type: string + max_length: 6 + valid_chars: '^(0\.[1-9]|[1-9][0-9]*)(\.[0-9]|m?)$' + valid_chars_error: | + Valid CPU limit formats are
+ - Plain Integer - eg. 1
+ - Float - eg. 0.5
+ - Milicpu - eg. 500m + default: "4000m" + required: true + - variable: memory + label: Memory + description: Memory limit for WG-Easy. + schema: + type: string + max_length: 12 + valid_chars: '^[1-9][0-9]*([EPTGMK]i?|e[0-9]+)?$' + valid_chars_error: | + Valid Memory limit formats are
+ - Suffixed with E/P/T/G/M/K - eg. 1G
+ - Suffixed with Ei/Pi/Ti/Gi/Mi/Ki - eg. 1Gi
+ - Plain Integer in bytes - eg. 1024
+ - Exponent - eg. 134e6 + default: "8Gi" + required: true diff --git a/charts/diskoverdata/2.0.0/templates/NOTES.txt b/charts/diskoverdata/2.0.0/templates/NOTES.txt new file mode 100644 index 0000000000..ba4e01146c --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/NOTES.txt @@ -0,0 +1 @@ +{{ include "ix.v1.common.lib.chart.notes" $ }} diff --git a/charts/diskoverdata/2.0.0/templates/_configuration.tpl b/charts/diskoverdata/2.0.0/templates/_configuration.tpl new file mode 100644 index 0000000000..ceb6dcd7bd --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_configuration.tpl @@ -0,0 +1,212 @@ +{{- define "diskover.configuration" -}} + {{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) }} + {{- $elasticsearch := printf "%s-elasticsearch" $fullname }} + +{{- $esPassword := randAlphaNum 32 }} +secret: + diskover-secret: + enabled: true + data: + es-password: {{ $esPassword }} +configmap: + diskover-config: + enabled: true + data: + .default_crawler.sh: |- + #!/bin/sh + while true; do + # this condition wait for the script to copy into the container . + if test -f "$1"; then + # Empty folders don't generate indices . if folder is empty a default file is generated + if ! [ "$(ls -A $2)" ]; then + echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt; + fi + python3 $1 $2/; + break; + fi + sleep 5 + done + Constants.php: | + ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'], + 'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'], + 'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'], + 'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'], + 'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'], + 'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'], + 'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'], + 'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'], + 'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'], + 'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'], + 'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'], + 'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove'] + ]; + // extra fields for search results and view file/dir info pages + // key is description for field and value is ES field name + // Example: + //const EXTRA_FIELDS = [ + // 'Date Changed' => 'ctime' + //]; + const EXTRA_FIELDS = []; + // Maximum number of indices to load by default, indices are loaded in order by creation date + // setting this too high can cause slow logins and other timeout issues + // This setting can bo overridden on indices page per user and stored in maxindex cookie + // If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value + const MAX_INDEX = 250; + // time in seconds for index info to be cached, clicking reload indices forces update + const INDEXINFO_CACHETIME = 600; + // time in seconds to check Elasticsearch for new index info + const NEWINDEX_CHECKTIME = 10; + // sqlite database file path + const DATABASE = '../diskoverdb.sqlite3'; + } + config.yaml: | + # diskover default/sample config file + # + # default search paths for config + # macOS: ~/.config/diskover and ~/Library/Application Support/diskover + # Other Unix: ~/.config/diskover and /etc/diskover + # Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined + # + appName: diskover + logLevel: INFO + logToFile: False + logDirectory: /tmp/ + + diskover: + # max number of crawl threads + # a thread is created up to maxthreads for each directory at level 1 of tree dir arg + # set to a number or leave blank to auto set based on number of cpus + #maxthreads: 20 + maxthreads: + # block size used for du size + blocksize: 512 + excludes: + # directory names and absolute paths you want to exclude from crawl + # directory excludes uses python re.search for string search (regex) + # directory excludes are case-sensitive + # Examples: .* or .backup or .backup* or /dir/dirname + # to exclude none use empty list [] + dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"] + #dirs: [] + # files you want to exclude from crawl + # can include wildcards (.*, *.doc or NULLEXT for files with no extension) + # file names are case-sensitive, extensions are not + files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"] + #files: [] + # exclude empty 0 byte files, set to True to exclude empty files or False to not exclude + emptyfiles: True + # exclude empty dirs, set to True to exclude empty dirs or False to not exclude + emptydirs: True + # exclude files smaller than min size in bytes + minfilesize: 1 + #minfilesize: 512 + # exclude files modified less than x days ago + minmtime: 0 + #minmtime: 30 + # exclude files modified more than x days ago + maxmtime: 36500 + # exclude files changed less than x days ago + minctime: 0 + # exclude files changed more than x days ago + maxctime: 36500 + # exclude files accessed less than x days ago + minatime: 0 + # exclude files accessed more than x days ago + maxatime: 36500 + includes: + # directory names and absolute paths you want to include (whitelist), case-sensitive, + # to include none use empty list [] + #dirs: [".recycle"] + dirs: [] + # files you want to include (whitelist), case-sensitive + files: [] + ownersgroups: + # control how owner (username) and group fields are stored for file and directory docs + # store uid and gid's instead of trying to get owner and group names + uidgidonly: False + # owner/group names contain domain name set to True + domain: False + # character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @ + domainsep: \ + # if domain name comes first before character separator, set this to True, otherwise False + domainfirst: True + # when indexing owner and group fields, keep the domain name + keepdomain: False + replacepaths: + # translate path names set to True to enable or False to disable. + # Set to True if crawling in Windows to replace drive letters and \ with / + replace: False + #from: /mnt/ + #to: /vols/ + from: + to: + plugins: + # set to True to enable all plugins or False to disable all plugins + enable: False + # list of plugins (by name) to use for directories + dirs: ['unixperms'] + # list of plugins (by name) to use for files + files: ['unixperms'] + other: + # restore atime/mtime for files and dirs during crawl + # set to True or False, default False (useful for cifs which does not work with noatime mount option) + # for nfs, it's preferable to use mount options ro,noatime,nodiratime + restoretimes: False + + databases: + elasticsearch: + host: '{{ $elasticsearch }}' + port: 9200 + user: 'elastic' + password: '{{ $esPassword }}' + # set https to True if using HTTP TLS/SSL or False if using http + # for AWS ES, you will most likely want to set this to True + # override with env var ES_HTTPS + https: False + # compress http data + # for AWS ES, you will most likely want to set this to True + httpcompress: False + # timeout for connection to ES (default is 10) + timeout: 30 + # number of connections kept open to ES when crawling (default is 10) + maxsize: 20 + # max retries for ES operations (default is 0) + maxretries: 10 + # wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait + wait: False + # chunk size for ES bulk operations (default is 500) + chunksize: 1000 + # the below settings are to optimize ES for crawling + # index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s + indexrefresh: 30s + # transaction log flush threshold size (default 512mb) + translogsize: 1gb + # transaction log sync interval time (default 5s) + translogsyncint: 30s + # search scroll size (default 100 docs) + scrollsize: 1000 +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_diskover.tpl b/charts/diskoverdata/2.0.0/templates/_diskover.tpl new file mode 100644 index 0000000000..bc482679d1 --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_diskover.tpl @@ -0,0 +1,105 @@ +{{- define "diskover.workload" -}} + {{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) -}} + {{- $elasticsearch := printf "http://%s-elasticsearch:%v/_cluster/health?local=true" $fullname 9200 }} +workload: + diskover: + enabled: true + primary: true + type: Deployment + podSpec: + hostNetwork: false + securityContext: + fsGroup: {{ .Values.diskoverID.group }} + containers: + diskover: + enabled: true + primary: true + imageSelector: image + securityContext: + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + readOnlyRootFilesystem: false + capabilities: + add: + - CHOWN + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + - KILL + fixedEnv: + PUID: {{ .Values.diskoverID.user }} + {{ with .Values.diskoverConfig.additionalEnvs }} + envList: + {{ range $env := . }} + - name: {{ $env.name }} + value: {{ $env.value }} + {{ end }} + {{ end }} + probes: + liveness: + enabled: true + type: http + path: /login.php + port: 80 + readiness: + enabled: true + type: http + path: /login.php + port: 80 + startup: + enabled: true + type: http + path: /login.php + port: 80 + lifecycle: + {{- $sched := .Values.diskoverConfig.cronSchedule }} + postStart: + type: exec + command: + - /bin/sh + - -c + - | + /scripts/.default_crawler.sh /app/diskover/diskover.py /data; + {{- $cron := printf "%s python3 /app/diskover/diskover.py /data" $sched }} + if ! cat /config/crontab | grep -q "{{ $cron }}"; then + echo "{{ $cron }}" >> /config/crontab; + fi + {{- range $item := .Values.diskoverStorage.additionalStorages }} + /scripts/.default_crawler.sh /app/diskover/diskover.py {{ $item.mountPath }}; + {{- end -}} + {{- range $item := .Values.diskoverStorage.additionalStorages }} + {{- if $item.diskoverDataIndex }} + {{- $cron := printf "%s python3 /app/diskover/diskover.py %s" $sched $item.mountPath }} + if ! cat /config/crontab | grep -q "{{ $cron }}"; then + echo "{{ $cron }}" >> /config/crontab; + fi + {{- end }} + {{- end }} + crontab /config/crontab; + + initContainers: + 01-wait-for-elasticsearch: + enabled: true + type: init + imageSelector: bashImage + env: + ELASTIC_PASSWORD: + secretKeyRef: + name: diskover-secret + key: es-password + command: + - bash + - -c + args: + - | + echo "Pinging [{{ $elasticsearch }}] until it is ready..." + head="--header=Authorization: Basic x$(base64 <<< "elastic:$ELASTIC_PASSWORD")" + time="--timeout=3" + until wget "$head" "$time" --spider -qO- "{{ $elasticsearch }}"; do + echo "Waiting for [{{ $elasticsearch }}] to be ready..." + sleep 2 + done + echo "URL [{{ $elasticsearch }}] is ready!" +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_es.tpl b/charts/diskoverdata/2.0.0/templates/_es.tpl new file mode 100644 index 0000000000..fc4f51393f --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_es.tpl @@ -0,0 +1,53 @@ +{{- define "es.workload" -}} +workload: + elasticsearch: + enabled: true + type: Deployment + podSpec: + hostNetwork: false + containers: + elasticsearch: + enabled: true + primary: true + imageSelector: elasticSearchImage + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + env: + ELASTIC_PASSWORD: + secretKeyRef: + name: diskover-secret + key: es-password + http.port: 9200 + discovery.type: single-node + node.name: diskoverdata + probes: + liveness: + enabled: true + type: exec + command: + - /bin/bash + - -c + - | + curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \ + http://localhost:9200/_cluster/health?local=true + readiness: + enabled: true + type: exec + command: + - /bin/bash + - -c + - | + curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \ + http://localhost:9200/_cluster/health?local=true + startup: + enabled: true + type: exec + command: + - /bin/bash + - -c + - | + curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \ + http://localhost:9200/_cluster/health?local=true +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_migration.tpl b/charts/diskoverdata/2.0.0/templates/_migration.tpl new file mode 100644 index 0000000000..7427253ba0 --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_migration.tpl @@ -0,0 +1,35 @@ +{{- define "diskover.get-versions" -}} + {{- $oldChartVersion := "" -}} + {{- $newChartVersion := "" -}} + + {{/* Safely access the context, so it wont block CI */}} + {{- if hasKey .Values.global "ixChartContext" -}} + {{- if .Values.global.ixChartContext.upgradeMetadata -}} + + {{- $oldChartVersion = .Values.global.ixChartContext.upgradeMetadata.oldChartVersion -}} + {{- $newChartVersion = .Values.global.ixChartContext.upgradeMetadata.newChartVersion -}} + {{- if and (not $oldChartVersion) (not $newChartVersion) -}} + {{- fail "Upgrade Metadata is missing. Cannot proceed" -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- toYaml (dict "old" $oldChartVersion "new" $newChartVersion) -}} +{{- end -}} + +{{- define "diskover.migration" -}} + {{- $versions := (fromYaml (include "diskover.get-versions" $)) -}} + {{- if and $versions.old $versions.new -}} + {{- $oldV := semver $versions.old -}} + {{- $newV := semver $versions.new -}} + + {{/* If new is v2.x.x */}} + {{- if eq ($newV.Major | int) 2 -}} + {{/* And old is v1.x.x, but lower than 0.15 */}} + {{- if and (eq $oldV.Major 1) (lt ($oldV.Patch | int) 15) -}} + {{/* Block the upgrade */}} + {{- fail "Migration to 2.x.x is only allowed from 1.0.15 or higher" -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_persistence.tpl b/charts/diskoverdata/2.0.0/templates/_persistence.tpl new file mode 100644 index 0000000000..d623b24ba8 --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_persistence.tpl @@ -0,0 +1,68 @@ +{{- define "diskover.persistence" -}} +persistence: + config: + enabled: true + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.config) | nindent 4 }} + targetSelector: + diskover: + diskover: + mountPath: /config + data: + enabled: true + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.data) | nindent 4 }} + targetSelector: + diskover: + diskover: + mountPath: /data + esdata: + enabled: true + {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.esdata) | nindent 4 }} + targetSelector: + elasticsearch: + elasticsearch: + mountPath: /usr/share/elasticsearch/data + defaultcrawler: + enabled: true + type: configmap + objectName: diskover-config + defaultMode: "0755" + targetSelector: + diskover: + diskover: + mountPath: /scripts/default_crawler.sh + subPath: .default_crawler.sh + phpfile: + enabled: true + type: configmap + objectName: diskover-config + targetSelector: + diskover: + diskover: + mountPath: /config/diskover-web.conf.d/Constants.php + subPath: Constants.php + yamlfile: + enabled: true + type: configmap + objectName: diskover-config + targetSelector: + diskover: + diskover: + mountPath: /config/diskover.conf.d/diskover/config.yaml + subPath: config.yaml + tmp: + enabled: true + type: emptyDir + targetSelector: + diskover: + 01-wait-for-elasticsearch: + mountPath: /tmp + {{- range $idx, $storage := .Values.diskoverStorage.additionalStorages }} + {{ printf "diskover-%v:" (int $idx) }} + enabled: true + {{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }} + targetSelector: + diskover: + diskover: + mountPath: {{ $storage.mountPath }} + {{- end }} +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_portal.tpl b/charts/diskoverdata/2.0.0/templates/_portal.tpl new file mode 100644 index 0000000000..61dd8bf69e --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_portal.tpl @@ -0,0 +1,12 @@ +{{- define "diskover.portal" -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: portal +data: + path: "/" + port: {{ .Values.diskoverNetwork.webPort | quote }} + protocol: "http" + host: "$node_ip" +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/_service.tpl b/charts/diskoverdata/2.0.0/templates/_service.tpl new file mode 100644 index 0000000000..950488ca83 --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/_service.tpl @@ -0,0 +1,27 @@ +{{- define "diskover.service" -}} +service: + diskover: + enabled: true + primary: true + type: NodePort + targetSelector: diskover + ports: + webui: + enabled: true + primary: true + port: {{ .Values.diskoverNetwork.webPort }} + nodePort: {{ .Values.diskoverNetwork.webPort }} + targetPort: 80 + targetSelector: diskover + elasticsearch: + enabled: true + type: ClusterIP + targetSelector: elasticsearch + ports: + elasticsearch: + enabled: true + primary: true + port: 9200 + targetPort: 9200 + targetSelector: elasticsearch +{{- end -}} diff --git a/charts/diskoverdata/2.0.0/templates/common.yaml b/charts/diskoverdata/2.0.0/templates/common.yaml new file mode 100644 index 0000000000..28349243dd --- /dev/null +++ b/charts/diskoverdata/2.0.0/templates/common.yaml @@ -0,0 +1,15 @@ +{{- include "ix.v1.common.loader.init" . -}} + +{{- include "diskover.migration" $ -}} + +{{/* Merge the templates with Values */}} +{{- $_ := mustMergeOverwrite .Values (include "diskover.configuration" $ | fromYaml) -}} +{{- $_ := mustMergeOverwrite .Values (include "diskover.workload" $ | fromYaml) -}} +{{- $_ := mustMergeOverwrite .Values (include "es.workload" $ | fromYaml) -}} +{{- $_ := mustMergeOverwrite .Values (include "diskover.service" $ | fromYaml) -}} +{{- $_ := mustMergeOverwrite .Values (include "diskover.persistence" $ | fromYaml) -}} + +{{/* Create the configmap for portal manually */}} +{{- include "diskover.portal" $ -}} + +{{- include "ix.v1.common.loader.apply" . -}} diff --git a/charts/diskoverdata/2.0.0/to_keep_versions.md b/charts/diskoverdata/2.0.0/to_keep_versions.md new file mode 100644 index 0000000000..a3d09442c3 --- /dev/null +++ b/charts/diskoverdata/2.0.0/to_keep_versions.md @@ -0,0 +1,4 @@ +# 1.0.15 + +This version is kept because it contains a fix that is needed for migration to v2.x.x +It should be safe to remove few months after v2.x.x is released.