Publish new changes in catalog

This commit is contained in:
sonicaj
2024-03-01 12:57:21 +00:00
parent 71d0cd2ac6
commit 2e20502723
20 changed files with 1367 additions and 0 deletions

View File

@@ -0,0 +1,6 @@
dependencies:
- name: common
repository: file://../../../common
version: 1.2.9
digest: sha256:af1a9a1f87e3e48453c9f25f909f5ebcd7fa6e25162b7b425448ba752bcdbc5c
generated: "2024-02-27T20:26:48.651992686+02:00"

View File

@@ -0,0 +1,26 @@
name: diskoverdata
description: Diskover is used to monitor size/volumes of distributed dataset.
annotations:
title: Diskover Data
type: application
version: 2.0.0
apiVersion: v2
appVersion: "2.0.1"
kubeVersion: ">=1.16.0-0"
maintainers:
- name: truenas
url: https://www.truenas.com/
email: dev@ixsystems.com
dependencies:
- name: common
repository: file://../../../common
version: 1.2.9
home: https://github.com/diskoverdata/diskover-community
icon: https://media.sys.truenas.net/apps/diskoverdata/icons/icon.png
sources:
- https://github.com/diskoverdata/diskover-community
- https://github.com/truenas/charts/tree/master/charts/diskoverdata
keywords:
- storage
- monitoring
- management

View File

@@ -0,0 +1,3 @@
# Diskover Data
[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset.

View File

@@ -0,0 +1,3 @@
# Diskover Data
[Diskover Data](https://github.com/diskoverdata/diskover-community) is used to monitor size/volumes of distributed dataset.

Binary file not shown.

View File

@@ -0,0 +1,19 @@
diskoverConfig:
cronSchedule: '* * * * *'
username: someuser
password: somepass
diskoverStorage:
config:
type: pvc
data:
type: pvc
esdata:
type: pvc
additionalStorages:
- type: pvc
diskoverDataIndex: true
mountPath: /data1
- type: pvc
diskoverDataIndex: true
mountPath: /data2

View File

@@ -0,0 +1,46 @@
image:
pullPolicy: IfNotPresent
repository: linuxserver/diskover
tag: "2.0.1"
elasticSearchImage:
pullPolicy: IfNotPresent
repository: elasticsearch
tag: "7.5.2"
resources:
limits:
cpu: 4000m
memory: 8Gi
podOptions:
dnsConfig:
options: []
diskoverConfig:
cronSchedule: ''
username: ''
password: ''
additionalEnvs: []
diskoverID:
user: 568
group: 568
diskoverNetwork:
webPort: 32000
diskoverStorage:
config:
type: ixVolume
ixVolumeConfig:
datasetName: config
data:
type: ixVolume
ixVolumeConfig:
datasetName: data
esdata:
type: ixVolume
ixVolumeConfig:
datasetName: esdata
additionalStorages: []

View File

@@ -0,0 +1,41 @@
runAsContext:
- userName: root
groupName: root
gid: 0
uid: 0
description: Diskover runs as root user.
- userName: root
groupName: root
gid: 0
uid: 0
description: Elastic Search runs as root user.
capabilities:
- name: CHOWN
description: Diskover and Elastic Search are able to chown files.
- name: FOWNER
description: Diskover and Elastic Search are able to bypass permission checks for it's sub-processes.
- name: SYS_CHROOT
description: Diskover and Elastic Search are able to use chroot.
- name: MKNOD
description: Diskover and Elastic Search are able to create device nodes.
- name: DAC_OVERRIDE
description: Diskover and Elastic Search are able to bypass permission checks.
- name: FSETID
description: Diskover and Elastic Search are able to set file capabilities.
- name: KILL
description: Diskover and Elastic Search are able to kill processes.
- name: SETGID
description: Diskover and Elastic Search are able to set group ID for it's sub-processes.
- name: SETUID
description: Diskover and Elastic Search are able to set user ID for it's sub-processes.
- name: SETPCAP
description: Diskover and Elastic Search are able to set process capabilities.
- name: NET_BIND_SERVICE
description: Diskover and Elastic Search are able to bind to privileged ports.
- name: SETFCAP
description: Diskover and Elastic Search are able to set file capabilities.
- name: NET_RAW
description: Diskover and Elastic Search are able to use raw sockets.
- name: AUDIT_WRITE
description: Diskover and Elastic Search are able to write to audit log.
hostMounts: []

View File

@@ -0,0 +1,106 @@
#!/usr/bin/python3
import json
import os
import sys
def migrate_volume(volume):
return {
'type': 'hostPath',
'hostPathConfig': {
'hostPath': volume['hostPath']
},
} if volume.get('hostPathEnabled', False) else {
'type': 'ixVolume',
'ixVolumeConfig': {
'datasetName': volume['datasetName'],
},
}
def migrate_common_lib(values):
delete_keys = [
'web_port', 'timezone', 'dnsConfig', 'environmentVariables', 'ownerUID',
'cronjobSchedule', 'diskoverCredentials', 'appVolumeMounts', 'ownerGID',
'extraAppVolumeMounts', 'extraDataVolumeMounts', 'elasticSearchAppVolumeMounts',
'elasticsearch', 'enableResourceLimits', 'es_user', 'python',
]
additionalVols = [
{
'type': 'hostPath',
'hostPathConfig': {'hostPath': e['hostPath']},
'mountPath': e['mountPath'],
} for e in values.get('extraAppVolumeMounts', [])
] + [
{
'type': 'hostPath',
'hostPathConfig': {'hostPath': e['hostPath']},
'mountPath': e['mountPath'],
'diskoverDataIndex': True
} for e in values.get('extraDataVolumeMounts', [])
]
values.update({
'TZ': values['timezone'],
# Migrate Network
'diskoverNetwork': {
'webPort': values['web_port'],
},
# Migrate DNS
'podOptions': {
'dnsConfig': {
'options': [
{'name': opt['name'], 'value': opt['value']}
for opt in values.get('dnsConfig', {}).get('options', [])
]
}
},
# Migrate Resources
'resources': {
'limits': {
'cpu': values.get('cpuLimit', '4000m'),
'memory': values.get('memLimit', '8Gi'),
}
},
# Migrate ID
'diskoverID': {
'user': values['ownerUID'],
'group': values['ownerGID'],
},
# Migrate Config
'diskoverConfig': {
'cronSchedule': values['cronjobSchedule'],
'username': values['diskoverCredentials']['username'],
'password': values['diskoverCredentials']['password'],
'additionalEnvs': values.get('environmentVariables', []),
},
# Migrate Storage
'diskoverStorage': {
'config': migrate_volume(values['appVolumeMounts']['config']),
'data': migrate_volume(values['appVolumeMounts']['data']),
'esdata': migrate_volume(values['elasticSearchAppVolumeMounts']['esdata']),
'additionalStorages': additionalVols
},
})
for k in delete_keys:
values.pop(k, None)
return values
def migrate(values):
# If this missing, we have already migrated
if not 'appVolumeMounts' in values.keys():
return values
return migrate_common_lib(values)
if __name__ == '__main__':
if len(sys.argv) != 2:
exit(1)
if os.path.exists(sys.argv[1]):
with open(sys.argv[1], 'r') as f:
print(json.dumps(migrate(json.loads(f.read()))))

View File

@@ -0,0 +1,585 @@
groups:
- name: Diskover Data Configuration
description: Configure Diskover Data
- name: User and Group Configuration
description: Configure User and Group for Diskover Data
- name: Advanced Pod Configuration
description: Configure Advanced Pod Options for Diskover Data
- name: Network Configuration
description: Configure Network for Diskover Data
- name: Storage Configuration
description: Configure Storage for Diskover Data
- name: Resources Configuration
description: Configure Resources for Diskover Data
portals:
web_portal:
protocols:
- "$kubernetes-resource_configmap_portal_protocol"
host:
- "$kubernetes-resource_configmap_portal_host"
ports:
- "$kubernetes-resource_configmap_portal_port"
path: "$kubernetes-resource_configmap_portal_path"
questions:
- variable: TZ
group: Diskover Data Configuration
label: Timezone
schema:
type: string
default: Etc/UTC
required: true
$ref:
- definitions/timezone
- variable: discoverConfig
label: ""
group: Diskover Data Configuration
schema:
type: dict
attrs:
- variable: cronSchedule
label: Cron Schedule
description: The cron schedule for Diskover Data.
schema:
type: string
default: "0 3 * * *"
- variable: username
label: Username
description: The username for Diskover Data.
schema:
type: string
default: ""
required: true
- variable: password
label: Password
description: The password for Diskover Data.
schema:
type: string
default: ""
required: true
private: true
- variable: additionalEnvs
label: Additional Environment Variables
description: Additional environment variables for Diskover Data.
schema:
type: list
items:
- variable: env
label: Environment Variable
schema:
type: dict
attrs:
- variable: name
label: Name
schema:
type: string
required: true
- variable: value
label: Value
schema:
type: string
required: true
- variable: podOptions
label: ""
group: Advanced Pod Configuration
schema:
type: dict
attrs:
- variable: dnsConfig
label: Advanced DNS Configuration
schema:
type: dict
attrs:
- variable: options
label: DNS Options
schema:
type: list
items:
- variable: optionsEntry
label: DNS Option Entry
schema:
type: dict
attrs:
- variable: name
label: Option Name
schema:
type: string
required: true
- variable: value
label: Option Value
schema:
type: string
required: true
- variable: diskoverID
label: ""
group: User and Group Configuration
schema:
type: dict
attrs:
- variable: user
label: User ID
description: The user id that Diskover Data files will be owned by.
schema:
type: int
min: 568
default: 568
required: true
- variable: group
label: Group ID
description: The group id that Diskover Data files will be owned by.
schema:
type: int
min: 568
default: 568
required: true
- variable: discoverNetwork
label: ""
group: Network Configuration
schema:
type: dict
attrs:
- variable: webPort
label: Web Port
description: The port for the Diskover Data Web UI.
schema:
type: int
default: 22510
min: 9000
max: 65535
required: true
- variable: discoverStorage
label: ""
group: Storage Configuration
schema:
type: dict
attrs:
- variable: config
label: Diskover Data Config Storage
description: The path to store Diskover Data Configuration.
schema:
type: dict
attrs:
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "config"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: data
label: Diskover Data Data Storage
description: The path to store Diskover Data Data.
schema:
type: dict
attrs:
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "data"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: esdata
label: Elastic Search Data Storage
description: The path to store Elastic Search Data.
schema:
type: dict
attrs:
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.
schema:
type: string
required: true
immutable: true
default: "ixVolume"
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
hidden: true
default: "esdata"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: additionalStorages
label: Additional Storage
description: Additional storage for Diskover Data.
schema:
type: list
default: []
items:
- variable: storageEntry
label: Storage Entry
schema:
type: dict
attrs:
- variable: type
label: Type
description: |
ixVolume: Is dataset created automatically by the system.</br>
Host Path: Is a path that already exists on the system.</br>
SMB Share: Is a SMB share that is mounted to a persistent volume claim.
schema:
type: string
required: true
default: "ixVolume"
immutable: true
enum:
- value: "hostPath"
description: Host Path (Path that already exists on the system)
- value: "ixVolume"
description: ixVolume (Dataset created automatically by the system)
- value: "smb-pv-pvc"
description: SMB Share (Mounts a persistent volume claim to a SMB share)
- variable: diskoverDataIndex
label: Enable Diskover Data Indexing
description: Enable Diskover Data Indexing
schema:
type: boolean
- variable: readOnly
label: Read Only
description: Mount the volume as read only.
schema:
type: boolean
default: false
- variable: mountPath
label: Mount Path
description: The path inside the container to mount the storage.
schema:
type: path
required: true
- variable: hostPathConfig
label: Host Path Configuration
schema:
type: dict
show_if: [["type", "=", "hostPath"]]
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: acl
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
$ref:
- "normalize/acl"
- variable: hostPath
label: Host Path
description: The host path to use for storage.
schema:
type: hostpath
show_if: [["aclEnable", "=", false]]
required: true
- variable: ixVolumeConfig
label: ixVolume Configuration
description: The configuration for the ixVolume dataset.
schema:
type: dict
show_if: [["type", "=", "ixVolume"]]
$ref:
- "normalize/ixVolume"
attrs:
- variable: aclEnable
label: Enable ACL
description: Enable ACL for the dataset.
schema:
type: boolean
default: false
- variable: datasetName
label: Dataset Name
description: The name of the dataset to use for storage.
schema:
type: string
required: true
immutable: true
default: "storage_entry"
- variable: aclEntries
label: ACL Configuration
schema:
type: dict
show_if: [["aclEnable", "=", true]]
attrs: []
- variable: smbConfig
label: SMB Share Configuration
description: The configuration for the SMB Share.
schema:
type: dict
show_if: [["type", "=", "smb-pv-pvc"]]
attrs:
- variable: server
label: Server
description: The server for the SMB share.
schema:
type: string
required: true
- variable: share
label: Share
description: The share name for the SMB share.
schema:
type: string
required: true
- variable: domain
label: Domain (Optional)
description: The domain for the SMB share.
schema:
type: string
- variable: username
label: Username
description: The username for the SMB share.
schema:
type: string
required: true
- variable: password
label: Password
description: The password for the SMB share.
schema:
type: string
required: true
private: true
- variable: size
label: Size (in Gi)
description: The size of the volume quota.
schema:
type: int
required: true
min: 1
default: 1
- variable: resources
group: Resources Configuration
label: ""
schema:
type: dict
attrs:
- variable: limits
label: Limits
schema:
type: dict
attrs:
- variable: cpu
label: CPU
description: CPU limit for WG-Easy.
schema:
type: string
max_length: 6
valid_chars: '^(0\.[1-9]|[1-9][0-9]*)(\.[0-9]|m?)$'
valid_chars_error: |
Valid CPU limit formats are</br>
- Plain Integer - eg. 1</br>
- Float - eg. 0.5</br>
- Milicpu - eg. 500m
default: "4000m"
required: true
- variable: memory
label: Memory
description: Memory limit for WG-Easy.
schema:
type: string
max_length: 12
valid_chars: '^[1-9][0-9]*([EPTGMK]i?|e[0-9]+)?$'
valid_chars_error: |
Valid Memory limit formats are</br>
- Suffixed with E/P/T/G/M/K - eg. 1G</br>
- Suffixed with Ei/Pi/Ti/Gi/Mi/Ki - eg. 1Gi</br>
- Plain Integer in bytes - eg. 1024</br>
- Exponent - eg. 134e6
default: "8Gi"
required: true

View File

@@ -0,0 +1 @@
{{ include "ix.v1.common.lib.chart.notes" $ }}

View File

@@ -0,0 +1,212 @@
{{- define "diskover.configuration" -}}
{{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) }}
{{- $elasticsearch := printf "%s-elasticsearch" $fullname }}
{{- $esPassword := randAlphaNum 32 }}
secret:
diskover-secret:
enabled: true
data:
es-password: {{ $esPassword }}
configmap:
diskover-config:
enabled: true
data:
.default_crawler.sh: |-
#!/bin/sh
while true; do
# this condition wait for the script to copy into the container .
if test -f "$1"; then
# Empty folders don't generate indices . if folder is empty a default file is generated
if ! [ "$(ls -A $2)" ]; then
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
fi
python3 $1 $2/;
break;
fi
sleep 5
done
Constants.php: |
<?php
namespace diskover;
class Constants {
const TIMEZONE = '{{ .Values.TZ }}';
const ES_HOST = '{{ $elasticsearch }}';
const ES_PORT = 9200;
const ES_USER = 'elastic';
const ES_PASS = '{{ $esPassword }}';
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
// override with env var ES_HTTPS
const ES_HTTPS = FALSE;
// login auth for diskover-web
const LOGIN_REQUIRED = TRUE;
// default username and password to login
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
const USER = '{{ .Values.diskoverConfig.username }}';
const PASS = '{{ .Values.diskoverConfig.password }}';
// default results per search page
const SEARCH_RESULTS = 50;
// default size field (size, size_du) to use for sizes on file tree and charts
const SIZE_FIELD = 'size';
// default file types, used by quick search (file type) and dashboard file type usage chart
// additional extensions can be added/removed from each file types list
const FILE_TYPES = [
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
];
// extra fields for search results and view file/dir info pages
// key is description for field and value is ES field name
// Example:
//const EXTRA_FIELDS = [
// 'Date Changed' => 'ctime'
//];
const EXTRA_FIELDS = [];
// Maximum number of indices to load by default, indices are loaded in order by creation date
// setting this too high can cause slow logins and other timeout issues
// This setting can bo overridden on indices page per user and stored in maxindex cookie
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
const MAX_INDEX = 250;
// time in seconds for index info to be cached, clicking reload indices forces update
const INDEXINFO_CACHETIME = 600;
// time in seconds to check Elasticsearch for new index info
const NEWINDEX_CHECKTIME = 10;
// sqlite database file path
const DATABASE = '../diskoverdb.sqlite3';
}
config.yaml: |
# diskover default/sample config file
#
# default search paths for config
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
# Other Unix: ~/.config/diskover and /etc/diskover
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
#
appName: diskover
logLevel: INFO
logToFile: False
logDirectory: /tmp/
diskover:
# max number of crawl threads
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
# set to a number or leave blank to auto set based on number of cpus
#maxthreads: 20
maxthreads:
# block size used for du size
blocksize: 512
excludes:
# directory names and absolute paths you want to exclude from crawl
# directory excludes uses python re.search for string search (regex)
# directory excludes are case-sensitive
# Examples: .* or .backup or .backup* or /dir/dirname
# to exclude none use empty list []
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
#dirs: []
# files you want to exclude from crawl
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
# file names are case-sensitive, extensions are not
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
#files: []
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
emptyfiles: True
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
emptydirs: True
# exclude files smaller than min size in bytes
minfilesize: 1
#minfilesize: 512
# exclude files modified less than x days ago
minmtime: 0
#minmtime: 30
# exclude files modified more than x days ago
maxmtime: 36500
# exclude files changed less than x days ago
minctime: 0
# exclude files changed more than x days ago
maxctime: 36500
# exclude files accessed less than x days ago
minatime: 0
# exclude files accessed more than x days ago
maxatime: 36500
includes:
# directory names and absolute paths you want to include (whitelist), case-sensitive,
# to include none use empty list []
#dirs: [".recycle"]
dirs: []
# files you want to include (whitelist), case-sensitive
files: []
ownersgroups:
# control how owner (username) and group fields are stored for file and directory docs
# store uid and gid's instead of trying to get owner and group names
uidgidonly: False
# owner/group names contain domain name set to True
domain: False
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
domainsep: \
# if domain name comes first before character separator, set this to True, otherwise False
domainfirst: True
# when indexing owner and group fields, keep the domain name
keepdomain: False
replacepaths:
# translate path names set to True to enable or False to disable.
# Set to True if crawling in Windows to replace drive letters and \ with /
replace: False
#from: /mnt/
#to: /vols/
from:
to:
plugins:
# set to True to enable all plugins or False to disable all plugins
enable: False
# list of plugins (by name) to use for directories
dirs: ['unixperms']
# list of plugins (by name) to use for files
files: ['unixperms']
other:
# restore atime/mtime for files and dirs during crawl
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
restoretimes: False
databases:
elasticsearch:
host: '{{ $elasticsearch }}'
port: 9200
user: 'elastic'
password: '{{ $esPassword }}'
# set https to True if using HTTP TLS/SSL or False if using http
# for AWS ES, you will most likely want to set this to True
# override with env var ES_HTTPS
https: False
# compress http data
# for AWS ES, you will most likely want to set this to True
httpcompress: False
# timeout for connection to ES (default is 10)
timeout: 30
# number of connections kept open to ES when crawling (default is 10)
maxsize: 20
# max retries for ES operations (default is 0)
maxretries: 10
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait: False
# chunk size for ES bulk operations (default is 500)
chunksize: 1000
# the below settings are to optimize ES for crawling
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh: 30s
# transaction log flush threshold size (default 512mb)
translogsize: 1gb
# transaction log sync interval time (default 5s)
translogsyncint: 30s
# search scroll size (default 100 docs)
scrollsize: 1000
{{- end -}}

View File

@@ -0,0 +1,105 @@
{{- define "diskover.workload" -}}
{{- $fullname := (include "ix.v1.common.lib.chart.names.fullname" $) -}}
{{- $elasticsearch := printf "http://%s-elasticsearch:%v/_cluster/health?local=true" $fullname 9200 }}
workload:
diskover:
enabled: true
primary: true
type: Deployment
podSpec:
hostNetwork: false
securityContext:
fsGroup: {{ .Values.diskoverID.group }}
containers:
diskover:
enabled: true
primary: true
imageSelector: image
securityContext:
runAsUser: 0
runAsGroup: 0
runAsNonRoot: false
readOnlyRootFilesystem: false
capabilities:
add:
- CHOWN
- DAC_OVERRIDE
- FOWNER
- SETGID
- SETUID
- KILL
fixedEnv:
PUID: {{ .Values.diskoverID.user }}
{{ with .Values.diskoverConfig.additionalEnvs }}
envList:
{{ range $env := . }}
- name: {{ $env.name }}
value: {{ $env.value }}
{{ end }}
{{ end }}
probes:
liveness:
enabled: true
type: http
path: /login.php
port: 80
readiness:
enabled: true
type: http
path: /login.php
port: 80
startup:
enabled: true
type: http
path: /login.php
port: 80
lifecycle:
{{- $sched := .Values.diskoverConfig.cronSchedule }}
postStart:
type: exec
command:
- /bin/sh
- -c
- |
/scripts/.default_crawler.sh /app/diskover/diskover.py /data;
{{- $cron := printf "%s python3 /app/diskover/diskover.py /data" $sched }}
if ! cat /config/crontab | grep -q "{{ $cron }}"; then
echo "{{ $cron }}" >> /config/crontab;
fi
{{- range $item := .Values.diskoverStorage.additionalStorages }}
/scripts/.default_crawler.sh /app/diskover/diskover.py {{ $item.mountPath }};
{{- end -}}
{{- range $item := .Values.diskoverStorage.additionalStorages }}
{{- if $item.diskoverDataIndex }}
{{- $cron := printf "%s python3 /app/diskover/diskover.py %s" $sched $item.mountPath }}
if ! cat /config/crontab | grep -q "{{ $cron }}"; then
echo "{{ $cron }}" >> /config/crontab;
fi
{{- end }}
{{- end }}
crontab /config/crontab;
initContainers:
01-wait-for-elasticsearch:
enabled: true
type: init
imageSelector: bashImage
env:
ELASTIC_PASSWORD:
secretKeyRef:
name: diskover-secret
key: es-password
command:
- bash
- -c
args:
- |
echo "Pinging [{{ $elasticsearch }}] until it is ready..."
head="--header=Authorization: Basic x$(base64 <<< "elastic:$ELASTIC_PASSWORD")"
time="--timeout=3"
until wget "$head" "$time" --spider -qO- "{{ $elasticsearch }}"; do
echo "Waiting for [{{ $elasticsearch }}] to be ready..."
sleep 2
done
echo "URL [{{ $elasticsearch }}] is ready!"
{{- end -}}

View File

@@ -0,0 +1,53 @@
{{- define "es.workload" -}}
workload:
elasticsearch:
enabled: true
type: Deployment
podSpec:
hostNetwork: false
containers:
elasticsearch:
enabled: true
primary: true
imageSelector: elasticSearchImage
securityContext:
runAsUser: 1000
runAsGroup: 1000
readOnlyRootFilesystem: false
env:
ELASTIC_PASSWORD:
secretKeyRef:
name: diskover-secret
key: es-password
http.port: 9200
discovery.type: single-node
node.name: diskoverdata
probes:
liveness:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
readiness:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
startup:
enabled: true
type: exec
command:
- /bin/bash
- -c
- |
curl -s -H "Authorization: Basic $(base64 <<< "elastic:$ELASTIC_PASSWORD")" \
http://localhost:9200/_cluster/health?local=true
{{- end -}}

View File

@@ -0,0 +1,35 @@
{{- define "diskover.get-versions" -}}
{{- $oldChartVersion := "" -}}
{{- $newChartVersion := "" -}}
{{/* Safely access the context, so it wont block CI */}}
{{- if hasKey .Values.global "ixChartContext" -}}
{{- if .Values.global.ixChartContext.upgradeMetadata -}}
{{- $oldChartVersion = .Values.global.ixChartContext.upgradeMetadata.oldChartVersion -}}
{{- $newChartVersion = .Values.global.ixChartContext.upgradeMetadata.newChartVersion -}}
{{- if and (not $oldChartVersion) (not $newChartVersion) -}}
{{- fail "Upgrade Metadata is missing. Cannot proceed" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- toYaml (dict "old" $oldChartVersion "new" $newChartVersion) -}}
{{- end -}}
{{- define "diskover.migration" -}}
{{- $versions := (fromYaml (include "diskover.get-versions" $)) -}}
{{- if and $versions.old $versions.new -}}
{{- $oldV := semver $versions.old -}}
{{- $newV := semver $versions.new -}}
{{/* If new is v2.x.x */}}
{{- if eq ($newV.Major | int) 2 -}}
{{/* And old is v1.x.x, but lower than 0.15 */}}
{{- if and (eq $oldV.Major 1) (lt ($oldV.Patch | int) 15) -}}
{{/* Block the upgrade */}}
{{- fail "Migration to 2.x.x is only allowed from 1.0.15 or higher" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{- end -}}

View File

@@ -0,0 +1,68 @@
{{- define "diskover.persistence" -}}
persistence:
config:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.config) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: /config
data:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.data) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: /data
esdata:
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.diskoverStorage.esdata) | nindent 4 }}
targetSelector:
elasticsearch:
elasticsearch:
mountPath: /usr/share/elasticsearch/data
defaultcrawler:
enabled: true
type: configmap
objectName: diskover-config
defaultMode: "0755"
targetSelector:
diskover:
diskover:
mountPath: /scripts/default_crawler.sh
subPath: .default_crawler.sh
phpfile:
enabled: true
type: configmap
objectName: diskover-config
targetSelector:
diskover:
diskover:
mountPath: /config/diskover-web.conf.d/Constants.php
subPath: Constants.php
yamlfile:
enabled: true
type: configmap
objectName: diskover-config
targetSelector:
diskover:
diskover:
mountPath: /config/diskover.conf.d/diskover/config.yaml
subPath: config.yaml
tmp:
enabled: true
type: emptyDir
targetSelector:
diskover:
01-wait-for-elasticsearch:
mountPath: /tmp
{{- range $idx, $storage := .Values.diskoverStorage.additionalStorages }}
{{ printf "diskover-%v:" (int $idx) }}
enabled: true
{{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }}
targetSelector:
diskover:
diskover:
mountPath: {{ $storage.mountPath }}
{{- end }}
{{- end -}}

View File

@@ -0,0 +1,12 @@
{{- define "diskover.portal" -}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: portal
data:
path: "/"
port: {{ .Values.diskoverNetwork.webPort | quote }}
protocol: "http"
host: "$node_ip"
{{- end -}}

View File

@@ -0,0 +1,27 @@
{{- define "diskover.service" -}}
service:
diskover:
enabled: true
primary: true
type: NodePort
targetSelector: diskover
ports:
webui:
enabled: true
primary: true
port: {{ .Values.diskoverNetwork.webPort }}
nodePort: {{ .Values.diskoverNetwork.webPort }}
targetPort: 80
targetSelector: diskover
elasticsearch:
enabled: true
type: ClusterIP
targetSelector: elasticsearch
ports:
elasticsearch:
enabled: true
primary: true
port: 9200
targetPort: 9200
targetSelector: elasticsearch
{{- end -}}

View File

@@ -0,0 +1,15 @@
{{- include "ix.v1.common.loader.init" . -}}
{{- include "diskover.migration" $ -}}
{{/* Merge the templates with Values */}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.configuration" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.workload" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "es.workload" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.service" $ | fromYaml) -}}
{{- $_ := mustMergeOverwrite .Values (include "diskover.persistence" $ | fromYaml) -}}
{{/* Create the configmap for portal manually */}}
{{- include "diskover.portal" $ -}}
{{- include "ix.v1.common.loader.apply" . -}}

View File

@@ -0,0 +1,4 @@
# 1.0.15
This version is kept because it contains a fix that is needed for migration to v2.x.x
It should be safe to remove few months after v2.x.x is released.