Add diskover-community application

This commit is contained in:
M. Rehan
2022-05-29 04:49:53 +05:00
parent cc10ba912d
commit 247c504ace
36 changed files with 1884 additions and 0 deletions

View File

@@ -0,0 +1,17 @@
apiVersion: v2
appVersion: "2.0.1"
icon: http://www.diskoverdata.com/wp-content/uploads/2019/09/diskover.png
description: Diskover is used to monitor size/volumes of distributed dataset.
name: diskoverdata
version: 1.0.0
dependencies:
- name: common
repository: file://../../../library/common/2112.0.0
version: 2112.0.0
home: https://github.com/diskoverdata/diskover-community/
keywords:
- storage
- monitoring
- management
sources:
- https://github.com/diskoverdata/diskover-community/

View File

@@ -0,0 +1,14 @@
# DiskOverData
DiskOver App for TrueNAS SCALE
[Diskover](https://www.diskoverdata.com/) is a sustainable file management solution for your distributed data.
# Introduction
This chart is based on [diskoverdata](https://hub.docker.com/r/linuxserver/diskover) and
deployed on kubernetes via helm chart
## Configuration
Please refer to questions.yaml for a detailed overview on supported configurable values.

View File

@@ -0,0 +1 @@
DiskOver App for TrueNAS SCALE

Binary file not shown.

View File

@@ -0,0 +1,18 @@
image:
pullPolicy: IfNotPresent
repository: linuxserver/diskover
tag: "2.0.1"
elasticsearch:
image:
pullPolicy: IfNotPresent
repository: docker.elastic.co/elasticsearch/elasticsearch
tag: "7.5.2"
python:
image:
pullPolicy: IfNotPresent
repository: python
tag: "3.10"
es_user: elasticsearch

View File

@@ -0,0 +1,295 @@
groups:
- name: "Configuration"
description: "Diskover application configuration"
- name: "Storage"
description: "Configure storage for Diskover"
- name: "Networking"
description: "Networking Configuration for Diskover"
- name: "Advanced DNS Settings"
description: "Configure DNS settings"
portals:
web_portal:
protocols:
- "http"
host:
- "$node_ip"
ports:
- "$variable-web_port"
path: "/"
questions:
- variable: web_port
label: "Web Port for Diskover"
group: Networking
schema:
type: int
min: 8000
max: 65535
default: 22510
required: true
- variable: timezone
label: "Configure timezone"
group: "Configuration"
description: "Configure timezone for Diskover"
schema:
type: string
$ref:
- "definitions/timezone"
- variable: dnsConfig
label: "DNS Configuration"
group: "Advanced DNS Settings"
schema:
type: dict
attrs:
- variable: options
label: "DNS Options"
schema:
type: list
items:
- variable: optionsEntry
label: "Option Entry Configuration"
schema:
type: dict
attrs:
- variable: name
label: "Option Name"
schema:
type: string
required: true
- variable: value
label: "Option Value"
schema:
type: string
required: true
- variable: ownerUID
label: "Config folder's user id"
description: "Linuxserver uses this user id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: ownerGID
label: "Config folder's group id"
description: "Linuxserver uses this group id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: diskoverCredentials
description: "Configure Diskover Initial Password"
label: "Configure Diskover Initial Username and password"
group: "Configuration"
schema:
type: dict
required: true
additional_attrs: true
attrs:
- variable: username
label: "Username"
description: "UserName for Diskover User"
schema:
type: string
default: "admin"
required: true
- variable: password
label: "Password"
description: "Initial Password for Diskover User"
schema:
type: string
private: true
default: "changeme"
required: true
- variable: environmentVariables
label: "Diskover Extra Environment"
group: "Configuration"
schema:
type: list
default: []
items:
- variable: environmentVariable
label: "Environment Variable"
schema:
type: dict
attrs:
- variable: name
label: "Name"
schema:
type: string
- variable: value
label: "Value"
schema:
type: string
- variable: cronjobSchedule
description: "Cronjobs Consist on 5 values in this specific format 'Minute Hour Day Month Week'"
label: "Define cronjob schedule for diskover"
group: "Configuration"
schema:
type: string
default: "0 3 * * *"
- variable: appVolumeMounts
label: "Diskover Storage"
group: "Storage"
schema:
type: dict
attrs:
- variable: config
label: "Storage Volume for Configuration"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
hidden: true
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-config"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
hidden: true
editable: true
default: "/config"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Configuration Storage Volume"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: "Host Path for Diskover Configuration Storage Volume"
schema:
type: hostpath
required: true
- variable: data
label: "Storage Volume for Data"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
hidden: true
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
hidden: true
editable: true
default: "/data"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Data folder to monitor"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: "Host Path for Diskover Data folder to monitor"
schema:
type: hostpath
required: true
- variable: extraDataVolumeMounts
label: "Add Extra Host Paths For Diskover To Monitor"
group: "Storage"
schema:
type: list
items:
- variable: dataAppVolume
label: "Host Path Volume"
description: "Add extra Data Volumes for diskover to monitor"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: extraAppVolumeMounts
label: "Extra Host Path Volumes"
group: "Storage"
schema:
type: list
items:
- variable: extraAppVolume
label: "Host Path Volume"
description: "Add an extra host path volume for Diskover application"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: elasticSearchAppVolumeMounts
label: "elastic search Storage"
group: "Storage"
schema:
type: dict
hidden: true
attrs:
- variable: esdata
label: "Storage Volume for Configuration"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
$ref:
- "normalize/ixVolume"
default: "ix-elasticsearch-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
editable: false
default: "/usr/share/elasticsearch/data"

View File

@@ -0,0 +1,35 @@
{{- define "add.user" -}}
{{- $user := .Values.es_user -}}
{{- printf "adduser %s -D;" $user -}}
{{- end -}}
{{- define "change.user.permissions" -}}
{{- $user := .Values.es_user -}}
{{- $mountPath := .Values.elasticSearchAppVolumeMounts.esdata.mountPath -}}
{{- printf "chown -R %s:%s %s;" $user $user $mountPath -}}
{{- end -}}
{{- define "elasticsearch.IP" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_HOST" "value" (printf "%s" (include "common.names.fullname" .))) }}
{{ $envList = mustAppend $envList (dict "name" "ES_PORT" "value" "9200") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "elasticsearch.credentials" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_USER" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-username") }}
{{ $envList = mustAppend $envList (dict "name" "ES_PASS" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "config.file.path" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "DEST" "value" .mountPath) }}
{{ $envList = mustAppend $envList (dict "name" "FILE" "value" .configFile) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}

View File

@@ -0,0 +1,131 @@
{{ $elastic_search := (. | mustDeepCopy) }}
{{ $_ := set $elastic_search "common" (dict "nameSuffix" "elasticsearch") }}
apiVersion: {{ template "common.capabilities.deployment.apiVersion" . }}
kind: Deployment
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
chart: {{ template "common.names.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
replicas: {{ (default 1 .Values.replicas) }}
strategy:
type: "Recreate"
selector:
matchLabels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
template:
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
{{- include "common.labels.selectorLabels" . | nindent 8 }}
annotations: {{ include "common.annotations" . | nindent 8 }}
spec:
initContainers:
- name: init-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/init_config.py"]
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "TZ" "value" .Values.timezone) }}
{{ $envList = mustAppend $envList (dict "name" "DS_USER" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "username") }}
{{ $envList = mustAppend $envList (dict "name" "DS_PASS" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPath := (dict "mountPath" (printf "%s/diskover-web.conf.d/" .Values.appVolumeMounts.config.mountPath) "configFile" "Constants.php") }}
{{ include "config.file.path" $configPath | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: wait-es-search
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
env:
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
command: ["python3", "/init_scripts/wait_for_elastic_search.py"]
volumeMounts:
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: init-es-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/initial_es_config.py"]
env:
{{ $envListConfig := (default list .Values.environmentVariables) }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPathES := (dict "mountPath" (printf "%s/diskover.conf.d/diskover/" .Values.appVolumeMounts.config.mountPath) "configFile" "config.yaml") }}
{{ include "config.file.path" $configPathES | nindent 12 }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListConfig) | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
containers:
- name: {{ .Chart.Name }}
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
ports:
- name: web
containerPort: 80
{{ $cronjobSchedule := .Values.cronjobSchedule }}
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- |
./init_scripts/.default_crawler.sh /app/diskover/diskover.py /data;
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
./init_scripts/.default_crawler.sh /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }};
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
echo "{{$cronjobSchedule}} python3 /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }}" >> /config/crontab;
{{ end }}
echo "{{.Values.cronjobSchedule}} python3 /app/diskover/diskover.py /data" >> /config/crontab;
crontab /config/crontab;
env:
{{ $envListDiskover := (default list .Values.environmentVariables) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PUID" "value" .Values.ownerUID) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PGID" "value" .Values.ownerGID) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListDiskover) | nindent 12 }}
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" .Values | nindent 8 }}
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
- name: diskover-initial-scripts
configMap:
defaultMode: 0700
name: "diskover-initial-scripts"

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: diskover-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
username: {{ .Values.diskoverCredentials.username | b64enc | quote }}
password: {{ .Values.diskoverCredentials.password | b64enc | quote }}

View File

@@ -0,0 +1,20 @@
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch") }}
{{ include "common.deployment.common_config" $values | nindent 0 }}
spec: {{ include "common.deployment.common_spec" $values | nindent 2 }}
template: {{ include "common.deployment.pod.metadata" $values | nindent 4 }}
spec:
containers:
- name: {{ .Chart.Name }}
{{ include "common.containers.imageConfig" .Values.elasticsearch.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
ports:
- name: es-port
containerPort: 9200
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "discovery.type" "value" "single-node") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

View File

@@ -0,0 +1,6 @@
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "es-port" "port" 9200 "targetPort" 9200) }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch") }}
{{ $_1 := set $values "commonService" (dict "type" "ClusterIP" "ports" $ports ) }}
{{ include "common.classes.service" $values }}

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: elastic-search-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
es-username: {{ "elastic" | b64enc | quote }}
es-password: {{ "changeme" | b64enc | quote }}

View File

@@ -0,0 +1,270 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "diskover-initial-scripts"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
data:
wait_for_elastic_search.py: |-
# This Script Wait for elastic search to setup completely
import requests
import os
import time
timeout = 100
while True:
try:
if timeout < 0:
print("timeout")
raise requests.exceptions.ConnectTimeout("Elasticsearch is not responding")
timeout -= 1
response = requests.get(f"http://{os.environ['ES_HOST']}:{os.environ['ES_PORT']}")
if response.status_code == 200:
break
except requests.exceptions.ConnectTimeout as e:
print(e)
break
except requests.exceptions.ConnectionError:
print("Trying to connect to elastic search")
time.sleep(3)
.default_crawler.sh: |-
#!/bin/sh
while :
do
# this condition wait for the script to copy into the container .
if test -f "$1"; then
# Empty folders don't generate indices . if folder is empty a default file is generated
if ! [ "$(ls -A $2)" ]; then
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
fi
python3 $1 $2/;
break;
fi
sleep 5
done
init_config.py: |-
import os
Config = f"""<?php
namespace diskover;
class Constants {{`{{
const TIMEZONE = '{os.environ['TZ']}';
const ES_HOST = '{os.environ['ES_HOST']}';
const ES_PORT = {os.environ['ES_PORT']};
const ES_USER = '{os.environ['ES_USER']}';
const ES_PASS = '{os.environ['ES_PASS']}';
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
// override with env var ES_HTTPS
const ES_HTTPS = FALSE;
// login auth for diskover-web
const LOGIN_REQUIRED = TRUE;
// default username and password to login
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
const USER = '{os.environ['DS_USER']}';
const PASS = '{os.environ['DS_PASS']}';
// default results per search page
const SEARCH_RESULTS = 50;
// default size field (size, size_du) to use for sizes on file tree and charts
const SIZE_FIELD = 'size';
// default file types, used by quick search (file type) and dashboard file type usage chart
// additional extensions can be added/removed from each file types list
const FILE_TYPES = [
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
];
// extra fields for search results and view file/dir info pages
// key is description for field and value is ES field name
// Example:
//const EXTRA_FIELDS = [
// 'Date Changed' => 'ctime'
//];
const EXTRA_FIELDS = [];
// Maximum number of indices to load by default, indices are loaded in order by creation date
// setting this too high can cause slow logins and other timeout issues
// This setting can bo overridden on indices page per user and stored in maxindex cookie
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
const MAX_INDEX = 250;
// time in seconds for index info to be cached, clicking reload indices forces update
const INDEXINFO_CACHETIME = 600;
// time in seconds to check Elasticsearch for new index info
const NEWINDEX_CHECKTIME = 10;
// sqlite database file path
const DATABASE = '../diskoverdb.sqlite3';
}}`}}
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)
initial_es_config.py: |-
import os
Config = f"""# diskover default/sample config file
#
# default search paths for config
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
# Other Unix: ~/.config/diskover and /etc/diskover
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
#
appName: diskover
#logLevel: WARN
#logLevel: DEBUG
logLevel: INFO
logToFile: False
#logToFile: True
logDirectory: /tmp/
diskover:
# max number of crawl threads
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
# set to a number or leave blank to auto set based on number of cpus
#maxthreads: 20
maxthreads:
# block size used for du size
blocksize: 512
excludes:
# directory names and absolute paths you want to exclude from crawl
# directory excludes uses python re.search for string search (regex)
# directory excludes are case-sensitive
# Examples: .* or .backup or .backup* or /dir/dirname
# to exclude none use empty list []
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
#dirs: []
# files you want to exclude from crawl
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
# file names are case-sensitive, extensions are not
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
#files: []
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
emptyfiles: True
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
emptydirs: True
# exclude files smaller than min size in bytes
minfilesize: 1
#minfilesize: 512
# exclude files modified less than x days ago
minmtime: 0
#minmtime: 30
# exclude files modified more than x days ago
maxmtime: 36500
# exclude files changed less than x days ago
minctime: 0
# exclude files changed more than x days ago
maxctime: 36500
# exclude files accessed less than x days ago
minatime: 0
# exclude files accessed more than x days ago
maxatime: 36500
includes:
# directory names and absolute paths you want to include (whitelist), case-sensitive,
# to include none use empty list []
#dirs: [".recycle"]
dirs: []
# files you want to include (whitelist), case-sensitive
files: []
ownersgroups:
# control how owner (username) and group fields are stored for file and directory docs
# store uid and gid's instead of trying to get owner and group names
uidgidonly: False
# owner/group names contain domain name set to True
domain: False
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
domainsep: \
# if domain name comes first before character separator, set this to True, otherwise False
domainfirst: True
# when indexing owner and group fields, keep the domain name
keepdomain: False
replacepaths:
# translate path names set to True to enable or False to disable.
# Set to True if crawling in Windows to replace drive letters and \ with /
replace: False
#from: /mnt/
#to: /vols/
from:
to:
plugins:
# set to True to enable all plugins or False to disable all plugins
enable: False
# list of plugins (by name) to use for directories
dirs: ['unixperms']
# list of plugins (by name) to use for files
files: ['unixperms']
other:
# restore atime/mtime for files and dirs during crawl
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
restoretimes: False
databases:
elasticsearch:
host: '{os.environ['ES_HOST']}'
port: {os.environ['ES_PORT']}
#user: elastic
#password: changeme
user: '{os.environ['ES_USER']}'
password: '{os.environ['ES_PASS']}'
# set https to True if using HTTP TLS/SSL or False if using http
# for AWS ES, you will most likely want to set this to True
# override with env var ES_HTTPS
https: False
# compress http data
# for AWS ES, you will most likely want to set this to True
httpcompress: False
# timeout for connection to ES (default is 10)
timeout: 30
# number of connections kept open to ES when crawling (default is 10)
maxsize: 20
# max retries for ES operations (default is 0)
maxretries: 10
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait: False
# chunk size for ES bulk operations (default is 500)
chunksize: 1000
# the below settings are to optimize ES for crawling
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh: 30s
# transaction log flush threshold size (default 512mb)
translogsize: 1gb
# transaction log sync interval time (default 5s)
translogsyncint: 30s
# search scroll size (default 100 docs)
scrollsize: 1000
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)

View File

@@ -0,0 +1,31 @@
apiVersion: batch/v1
kind: Job
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-job"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-hook"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
spec:
restartPolicy: Never
containers:
- name: pre-install-job
image: "alpine:latest"
command: ["/bin/sh", "-c"]
args:
- {{ include "add.user" . }}
{{ include "change.user.permissions" . }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

View File

@@ -0,0 +1,10 @@
{{ $svc := .Values.service }}
{{ $selectors := list }}
{{ $selectors = mustAppend $selectors (dict "key" "app" "value" (include "common.names.name" .) ) }}
{{ $selectors = mustAppend $selectors (dict "key" "release" "value" .Release.Name ) }}
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "web" "port" .Values.web_port "nodePort" .Values.web_port "targetPort" 80) }}
{{ $params := . }}
{{ $_ := set $params "commonService" (dict "type" "NodePort" "ports" $ports ) }}
{{ $_1 := set .Values "extraSelectorLabels" $selectors }}
{{ include "common.classes.service" $params }}

View File

@@ -0,0 +1,21 @@
{{- $serviceName := (include "common.names.fullname" .) -}}
apiVersion: v1
kind: Pod
metadata:
name: {{ .Release.Name }}-test-pod
labels:
app: {{ .Release.Name }}
release: {{ .Release.Name }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test-curl
image: alpine/curl
imagePullPolicy: "IfNotPresent"
command:
- /bin/sh
- -ec
- |
curl --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 15 --retry-max-time 90 --retry-all-errors -ksf http://{{ $serviceName }}:{{.Values.web_port}}/
restartPolicy: Never

View File

@@ -0,0 +1,50 @@
image:
pullPolicy: IfNotPresent
repository: linuxserver/diskover
tag: "2.0.1"
elasticsearch:
image:
pullPolicy: IfNotPresent
repository: docker.elastic.co/elasticsearch/elasticsearch
tag: "7.5.2"
python:
image:
pullPolicy: IfNotPresent
repository: python
tag: "3.10"
environmentVariables: []
extraAppVolumeMounts: []
extraDataVolumeMounts: []
web_port: 32000
dnsConfig:
options: []
emptyDirVolumes: true
appVolumeMounts:
config:
emptyDir: true
mountPath: /config
data:
emptyDir: true
mountPath: /data
elasticSearchAppVolumeMounts:
esdata:
emptyDir: true
mountPath: /usr/share/elasticsearch/data
ownerUID: 568
ownerGID: 568
username: "admin"
password: "admin"
host: "192.169.0.156"
hostNetwork: false
timezone: "America/Los_Angeles"
diskoverCredentials:
username: admin
password: admin
cronjobSchedule: "0 3 * * *"
es_user: elasticsearch

View File

@@ -0,0 +1,5 @@
icon_url: http://www.diskoverdata.com/wp-content/uploads/2019/09/diskover.png
categories:
- storage
- monitoring
- management

View File

@@ -0,0 +1,17 @@
apiVersion: v2
appVersion: "2.0.1"
icon: http://www.diskoverdata.com/wp-content/uploads/2019/09/diskover.png
description: Diskover is used to monitor size/volumes of distributed dataset.
name: diskoverdata
version: 1.0.0
dependencies:
- name: common
repository: file://../../../library/common/2112.0.0
version: 2112.0.0
home: https://github.com/diskoverdata/diskover-community/
keywords:
- storage
- monitoring
- management
sources:
- https://github.com/diskoverdata/diskover-community/

View File

@@ -0,0 +1,14 @@
# DiskOverData
DiskOver App for TrueNAS SCALE
[Diskover](https://www.diskoverdata.com/) is a sustainable file management solution for your distributed data.
# Introduction
This chart is based on [diskoverdata](https://hub.docker.com/r/linuxserver/diskover) and
deployed on kubernetes via helm chart
## Configuration
Please refer to questions.yaml for a detailed overview on supported configurable values.

View File

@@ -0,0 +1 @@
DiskOver App for TrueNAS SCALE

Binary file not shown.

View File

@@ -0,0 +1,18 @@
image:
pullPolicy: IfNotPresent
repository: linuxserver/diskover
tag: "2.0.1"
elasticsearch:
image:
pullPolicy: IfNotPresent
repository: docker.elastic.co/elasticsearch/elasticsearch
tag: "7.5.2"
python:
image:
pullPolicy: IfNotPresent
repository: python
tag: "3.10"
es_user: elasticsearch

View File

@@ -0,0 +1,295 @@
groups:
- name: "Configuration"
description: "Diskover application configuration"
- name: "Storage"
description: "Configure storage for Diskover"
- name: "Networking"
description: "Networking Configuration for Diskover"
- name: "Advanced DNS Settings"
description: "Configure DNS settings"
portals:
web_portal:
protocols:
- "http"
host:
- "$node_ip"
ports:
- "$variable-web_port"
path: "/"
questions:
- variable: web_port
label: "Web Port for Diskover"
group: Networking
schema:
type: int
min: 8000
max: 65535
default: 22510
required: true
- variable: timezone
label: "Configure timezone"
group: "Configuration"
description: "Configure timezone for Diskover"
schema:
type: string
$ref:
- "definitions/timezone"
- variable: dnsConfig
label: "DNS Configuration"
group: "Advanced DNS Settings"
schema:
type: dict
attrs:
- variable: options
label: "DNS Options"
schema:
type: list
items:
- variable: optionsEntry
label: "Option Entry Configuration"
schema:
type: dict
attrs:
- variable: name
label: "Option Name"
schema:
type: string
required: true
- variable: value
label: "Option Value"
schema:
type: string
required: true
- variable: ownerUID
label: "Config folder's user id"
description: "Linuxserver uses this user id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: ownerGID
label: "Config folder's group id"
description: "Linuxserver uses this group id to configure config's folders permissions"
group: Configuration
schema:
type: int
default: 568
min: 1
max: 65535
- variable: diskoverCredentials
description: "Configure Diskover Initial Password"
label: "Configure Diskover Initial Username and password"
group: "Configuration"
schema:
type: dict
required: true
additional_attrs: true
attrs:
- variable: username
label: "Username"
description: "UserName for Diskover User"
schema:
type: string
default: "admin"
required: true
- variable: password
label: "Password"
description: "Initial Password for Diskover User"
schema:
type: string
private: true
default: "changeme"
required: true
- variable: environmentVariables
label: "Diskover Extra Environment"
group: "Configuration"
schema:
type: list
default: []
items:
- variable: environmentVariable
label: "Environment Variable"
schema:
type: dict
attrs:
- variable: name
label: "Name"
schema:
type: string
- variable: value
label: "Value"
schema:
type: string
- variable: cronjobSchedule
description: "Cronjobs Consist on 5 values in this specific format 'Minute Hour Day Month Week'"
label: "Define cronjob schedule for diskover"
group: "Configuration"
schema:
type: string
default: "0 3 * * *"
- variable: appVolumeMounts
label: "Diskover Storage"
group: "Storage"
schema:
type: dict
attrs:
- variable: config
label: "Storage Volume for Configuration"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
hidden: true
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-config"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
hidden: true
editable: true
default: "/config"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Configuration Storage Volume"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: "Host Path for Diskover Configuration Storage Volume"
schema:
type: hostpath
required: true
- variable: data
label: "Storage Volume for Data"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
hidden: true
$ref:
- "normalize/ixVolume"
show_if: [["hostPathEnabled", "=", false]]
default: "ix-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
hidden: true
editable: true
default: "/data"
- variable: hostPathEnabled
label: "Enable Custom Host Path for Diskover Data folder to monitor"
schema:
type: boolean
default: false
show_subquestions_if: true
subquestions:
- variable: hostPath
label: "Host Path for Diskover Data folder to monitor"
schema:
type: hostpath
required: true
- variable: extraDataVolumeMounts
label: "Add Extra Host Paths For Diskover To Monitor"
group: "Storage"
schema:
type: list
items:
- variable: dataAppVolume
label: "Host Path Volume"
description: "Add extra Data Volumes for diskover to monitor"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: extraAppVolumeMounts
label: "Extra Host Path Volumes"
group: "Storage"
schema:
type: list
items:
- variable: extraAppVolume
label: "Host Path Volume"
description: "Add an extra host path volume for Diskover application"
schema:
type: dict
attrs:
- variable: mountPath
label: "Mount Path in Pod"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
required: true
- variable: hostPath
label: "Host Path"
description: "Host path"
schema:
type: hostpath
required: true
- variable: elasticSearchAppVolumeMounts
label: "elastic search Storage"
group: "Storage"
schema:
type: dict
hidden: true
attrs:
- variable: esdata
label: "Storage Volume for Configuration"
schema:
type: dict
attrs:
- variable: datasetName
label: "Configuration Storage Volume Dataset Name"
schema:
type: string
$ref:
- "normalize/ixVolume"
default: "ix-elasticsearch-data"
editable: false
- variable: mountPath
label: "Configuration Storage Mount Path"
description: "Path where the volume will be mounted inside the pod"
schema:
type: path
editable: false
default: "/usr/share/elasticsearch/data"

View File

@@ -0,0 +1,35 @@
{{- define "add.user" -}}
{{- $user := .Values.es_user -}}
{{- printf "adduser %s -D;" $user -}}
{{- end -}}
{{- define "change.user.permissions" -}}
{{- $user := .Values.es_user -}}
{{- $mountPath := .Values.elasticSearchAppVolumeMounts.esdata.mountPath -}}
{{- printf "chown -R %s:%s %s;" $user $user $mountPath -}}
{{- end -}}
{{- define "elasticsearch.IP" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_HOST" "value" (printf "%s" (include "common.names.fullname" .))) }}
{{ $envList = mustAppend $envList (dict "name" "ES_PORT" "value" "9200") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "elasticsearch.credentials" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "ES_USER" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-username") }}
{{ $envList = mustAppend $envList (dict "name" "ES_PASS" "valueFromSecret" true "secretName" "elastic-search-credentials" "secretKey" "es-password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}
{{- define "config.file.path" -}}
{{ $envList := (default list) }}
{{ $envList = mustAppend $envList (dict "name" "DEST" "value" .mountPath) }}
{{ $envList = mustAppend $envList (dict "name" "FILE" "value" .configFile) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) }}
{{- end -}}

View File

@@ -0,0 +1,131 @@
{{ $elastic_search := (. | mustDeepCopy) }}
{{ $_ := set $elastic_search "common" (dict "nameSuffix" "elasticsearch") }}
apiVersion: {{ template "common.capabilities.deployment.apiVersion" . }}
kind: Deployment
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
chart: {{ template "common.names.chart" . }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
replicas: {{ (default 1 .Values.replicas) }}
strategy:
type: "Recreate"
selector:
matchLabels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
template:
metadata:
name: {{ template "common.names.fullname" . }}
labels:
app: {{ template "common.names.name" . }}
release: {{ .Release.Name }}
{{- include "common.labels.selectorLabels" . | nindent 8 }}
annotations: {{ include "common.annotations" . | nindent 8 }}
spec:
initContainers:
- name: init-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/init_config.py"]
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "TZ" "value" .Values.timezone) }}
{{ $envList = mustAppend $envList (dict "name" "DS_USER" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "username") }}
{{ $envList = mustAppend $envList (dict "name" "DS_PASS" "valueFromSecret" true "secretName" "diskover-credentials" "secretKey" "password") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPath := (dict "mountPath" (printf "%s/diskover-web.conf.d/" .Values.appVolumeMounts.config.mountPath) "configFile" "Constants.php") }}
{{ include "config.file.path" $configPath | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: wait-es-search
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
env:
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
command: ["python3", "/init_scripts/wait_for_elastic_search.py"]
volumeMounts:
- name: diskover-initial-scripts
mountPath: /init_scripts/
- name: init-es-config
{{ include "common.containers.imageConfig" .Values.python.image | nindent 10 }}
command: ["python3", "/init_scripts/initial_es_config.py"]
env:
{{ $envListConfig := (default list .Values.environmentVariables) }}
{{ include "elasticsearch.IP" $elastic_search | nindent 12 }}
{{ include "elasticsearch.credentials" . | nindent 12 }}
{{ $configPathES := (dict "mountPath" (printf "%s/diskover.conf.d/diskover/" .Values.appVolumeMounts.config.mountPath) "configFile" "config.yaml") }}
{{ include "config.file.path" $configPathES | nindent 12 }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListConfig) | nindent 12 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
containers:
- name: {{ .Chart.Name }}
{{ include "common.containers.imageConfig" .Values.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" .Values | nindent 12 }}
- name: diskover-initial-scripts
mountPath: /init_scripts/
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
mountPath: {{ $hostPathConfiguration.mountPath }}
{{ end }}
ports:
- name: web
containerPort: 80
{{ $cronjobSchedule := .Values.cronjobSchedule }}
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -c
- |
./init_scripts/.default_crawler.sh /app/diskover/diskover.py /data;
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
./init_scripts/.default_crawler.sh /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }};
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
echo "{{$cronjobSchedule}} python3 /app/diskover/diskover.py {{ $hostPathConfiguration.mountPath }}" >> /config/crontab;
{{ end }}
echo "{{.Values.cronjobSchedule}} python3 /app/diskover/diskover.py /data" >> /config/crontab;
crontab /config/crontab;
env:
{{ $envListDiskover := (default list .Values.environmentVariables) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PUID" "value" .Values.ownerUID) }}
{{ $envListDiskover = mustAppend $envListDiskover (dict "name" "PGID" "value" .Values.ownerGID) }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envListDiskover) | nindent 12 }}
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" .Values | nindent 8 }}
{{ range $index, $hostPathConfiguration := .Values.extraAppVolumeMounts }}
- name: extrappvolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
{{ range $index, $hostPathConfiguration := .Values.extraDataVolumeMounts }}
- name: extradatavolume-{{ $index }}
hostPath:
path: {{ $hostPathConfiguration.hostPath }}
{{ end }}
- name: diskover-initial-scripts
configMap:
defaultMode: 0700
name: "diskover-initial-scripts"

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: diskover-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
username: {{ .Values.diskoverCredentials.username | b64enc | quote }}
password: {{ .Values.diskoverCredentials.password | b64enc | quote }}

View File

@@ -0,0 +1,20 @@
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch") }}
{{ include "common.deployment.common_config" $values | nindent 0 }}
spec: {{ include "common.deployment.common_spec" $values | nindent 2 }}
template: {{ include "common.deployment.pod.metadata" $values | nindent 4 }}
spec:
containers:
- name: {{ .Chart.Name }}
{{ include "common.containers.imageConfig" .Values.elasticsearch.image | nindent 10 }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
ports:
- name: es-port
containerPort: 9200
env:
{{ $envList := (default list .Values.environmentVariables) }}
{{ $envList = mustAppend $envList (dict "name" "discovery.type" "value" "single-node") }}
{{ include "common.containers.environmentVariables" (dict "environmentVariables" $envList) | nindent 12 }}
{{ include "common.networking.dnsConfiguration" .Values | nindent 6 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

View File

@@ -0,0 +1,6 @@
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "es-port" "port" 9200 "targetPort" 9200) }}
{{ $values := (. | mustDeepCopy) }}
{{ $_ := set $values "common" (dict "nameSuffix" "elasticsearch") }}
{{ $_1 := set $values "commonService" (dict "type" "ClusterIP" "ports" $ports ) }}
{{ include "common.classes.service" $values }}

View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: elastic-search-credentials
labels: {{ include "common.labels" . | nindent 4 }}
type: Opaque
data:
es-username: {{ "elastic" | b64enc | quote }}
es-password: {{ "changeme" | b64enc | quote }}

View File

@@ -0,0 +1,270 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: "diskover-initial-scripts"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
data:
wait_for_elastic_search.py: |-
# This Script Wait for elastic search to setup completely
import requests
import os
import time
timeout = 100
while True:
try:
if timeout < 0:
print("timeout")
raise requests.exceptions.ConnectTimeout("Elasticsearch is not responding")
timeout -= 1
response = requests.get(f"http://{os.environ['ES_HOST']}:{os.environ['ES_PORT']}")
if response.status_code == 200:
break
except requests.exceptions.ConnectTimeout as e:
print(e)
break
except requests.exceptions.ConnectionError:
print("Trying to connect to elastic search")
time.sleep(3)
.default_crawler.sh: |-
#!/bin/sh
while :
do
# this condition wait for the script to copy into the container .
if test -f "$1"; then
# Empty folders don't generate indices . if folder is empty a default file is generated
if ! [ "$(ls -A $2)" ]; then
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
fi
python3 $1 $2/;
break;
fi
sleep 5
done
init_config.py: |-
import os
Config = f"""<?php
namespace diskover;
class Constants {{`{{
const TIMEZONE = '{os.environ['TZ']}';
const ES_HOST = '{os.environ['ES_HOST']}';
const ES_PORT = {os.environ['ES_PORT']};
const ES_USER = '{os.environ['ES_USER']}';
const ES_PASS = '{os.environ['ES_PASS']}';
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
// override with env var ES_HTTPS
const ES_HTTPS = FALSE;
// login auth for diskover-web
const LOGIN_REQUIRED = TRUE;
// default username and password to login
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
const USER = '{os.environ['DS_USER']}';
const PASS = '{os.environ['DS_PASS']}';
// default results per search page
const SEARCH_RESULTS = 50;
// default size field (size, size_du) to use for sizes on file tree and charts
const SIZE_FIELD = 'size';
// default file types, used by quick search (file type) and dashboard file type usage chart
// additional extensions can be added/removed from each file types list
const FILE_TYPES = [
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
];
// extra fields for search results and view file/dir info pages
// key is description for field and value is ES field name
// Example:
//const EXTRA_FIELDS = [
// 'Date Changed' => 'ctime'
//];
const EXTRA_FIELDS = [];
// Maximum number of indices to load by default, indices are loaded in order by creation date
// setting this too high can cause slow logins and other timeout issues
// This setting can bo overridden on indices page per user and stored in maxindex cookie
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
const MAX_INDEX = 250;
// time in seconds for index info to be cached, clicking reload indices forces update
const INDEXINFO_CACHETIME = 600;
// time in seconds to check Elasticsearch for new index info
const NEWINDEX_CHECKTIME = 10;
// sqlite database file path
const DATABASE = '../diskoverdb.sqlite3';
}}`}}
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)
initial_es_config.py: |-
import os
Config = f"""# diskover default/sample config file
#
# default search paths for config
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
# Other Unix: ~/.config/diskover and /etc/diskover
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
#
appName: diskover
#logLevel: WARN
#logLevel: DEBUG
logLevel: INFO
logToFile: False
#logToFile: True
logDirectory: /tmp/
diskover:
# max number of crawl threads
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
# set to a number or leave blank to auto set based on number of cpus
#maxthreads: 20
maxthreads:
# block size used for du size
blocksize: 512
excludes:
# directory names and absolute paths you want to exclude from crawl
# directory excludes uses python re.search for string search (regex)
# directory excludes are case-sensitive
# Examples: .* or .backup or .backup* or /dir/dirname
# to exclude none use empty list []
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
#dirs: []
# files you want to exclude from crawl
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
# file names are case-sensitive, extensions are not
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
#files: []
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
emptyfiles: True
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
emptydirs: True
# exclude files smaller than min size in bytes
minfilesize: 1
#minfilesize: 512
# exclude files modified less than x days ago
minmtime: 0
#minmtime: 30
# exclude files modified more than x days ago
maxmtime: 36500
# exclude files changed less than x days ago
minctime: 0
# exclude files changed more than x days ago
maxctime: 36500
# exclude files accessed less than x days ago
minatime: 0
# exclude files accessed more than x days ago
maxatime: 36500
includes:
# directory names and absolute paths you want to include (whitelist), case-sensitive,
# to include none use empty list []
#dirs: [".recycle"]
dirs: []
# files you want to include (whitelist), case-sensitive
files: []
ownersgroups:
# control how owner (username) and group fields are stored for file and directory docs
# store uid and gid's instead of trying to get owner and group names
uidgidonly: False
# owner/group names contain domain name set to True
domain: False
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
domainsep: \
# if domain name comes first before character separator, set this to True, otherwise False
domainfirst: True
# when indexing owner and group fields, keep the domain name
keepdomain: False
replacepaths:
# translate path names set to True to enable or False to disable.
# Set to True if crawling in Windows to replace drive letters and \ with /
replace: False
#from: /mnt/
#to: /vols/
from:
to:
plugins:
# set to True to enable all plugins or False to disable all plugins
enable: False
# list of plugins (by name) to use for directories
dirs: ['unixperms']
# list of plugins (by name) to use for files
files: ['unixperms']
other:
# restore atime/mtime for files and dirs during crawl
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
restoretimes: False
databases:
elasticsearch:
host: '{os.environ['ES_HOST']}'
port: {os.environ['ES_PORT']}
#user: elastic
#password: changeme
user: '{os.environ['ES_USER']}'
password: '{os.environ['ES_PASS']}'
# set https to True if using HTTP TLS/SSL or False if using http
# for AWS ES, you will most likely want to set this to True
# override with env var ES_HTTPS
https: False
# compress http data
# for AWS ES, you will most likely want to set this to True
httpcompress: False
# timeout for connection to ES (default is 10)
timeout: 30
# number of connections kept open to ES when crawling (default is 10)
maxsize: 20
# max retries for ES operations (default is 0)
maxretries: 10
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
wait: False
# chunk size for ES bulk operations (default is 500)
chunksize: 1000
# the below settings are to optimize ES for crawling
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
indexrefresh: 30s
# transaction log flush threshold size (default 512mb)
translogsize: 1gb
# transaction log sync interval time (default 5s)
translogsyncint: 30s
# search scroll size (default 100 docs)
scrollsize: 1000
"""
os.makedirs(os.environ['DEST'], exist_ok=True)
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
with open(path, 'w') as w:
w.write(Config)

View File

@@ -0,0 +1,31 @@
apiVersion: batch/v1
kind: Job
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-job"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
name: "{{ template "common.names.fullname" . }}-preinstall-hook"
labels:
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
app.kubernetes.io/instance: {{ .Release.Name | quote }}
helm.sh/chart: {{ template "common.names.chart" . }}
spec:
restartPolicy: Never
containers:
- name: pre-install-job
image: "alpine:latest"
command: ["/bin/sh", "-c"]
args:
- {{ include "add.user" . }}
{{ include "change.user.permissions" . }}
volumeMounts: {{ include "common.storage.configureAppVolumeMountsInContainer" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts ) | nindent 12 }}
volumes: {{ include "common.storage.configureAppVolumes" (dict "appVolumeMounts" .Values.elasticSearchAppVolumeMounts "emptyDirVolumes" .Values.emptyDirVolumes "ixVolumes" .Values.ixVolumes) | nindent 8 }}

View File

@@ -0,0 +1,10 @@
{{ $svc := .Values.service }}
{{ $selectors := list }}
{{ $selectors = mustAppend $selectors (dict "key" "app" "value" (include "common.names.name" .) ) }}
{{ $selectors = mustAppend $selectors (dict "key" "release" "value" .Release.Name ) }}
{{ $ports := list }}
{{ $ports = mustAppend $ports (dict "name" "web" "port" .Values.web_port "nodePort" .Values.web_port "targetPort" 80) }}
{{ $params := . }}
{{ $_ := set $params "commonService" (dict "type" "NodePort" "ports" $ports ) }}
{{ $_1 := set .Values "extraSelectorLabels" $selectors }}
{{ include "common.classes.service" $params }}

View File

@@ -0,0 +1,21 @@
{{- $serviceName := (include "common.names.fullname" .) -}}
apiVersion: v1
kind: Pod
metadata:
name: {{ .Release.Name }}-test-pod
labels:
app: {{ .Release.Name }}
release: {{ .Release.Name }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test-curl
image: alpine/curl
imagePullPolicy: "IfNotPresent"
command:
- /bin/sh
- -ec
- |
curl --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 15 --retry-max-time 90 --retry-all-errors -ksf http://{{ $serviceName }}:{{.Values.web_port}}/
restartPolicy: Never

View File

@@ -0,0 +1,50 @@
image:
pullPolicy: IfNotPresent
repository: linuxserver/diskover
tag: "2.0.1"
elasticsearch:
image:
pullPolicy: IfNotPresent
repository: docker.elastic.co/elasticsearch/elasticsearch
tag: "7.5.2"
python:
image:
pullPolicy: IfNotPresent
repository: python
tag: "3.10"
environmentVariables: []
extraAppVolumeMounts: []
extraDataVolumeMounts: []
web_port: 32000
dnsConfig:
options: []
emptyDirVolumes: true
appVolumeMounts:
config:
emptyDir: true
mountPath: /config
data:
emptyDir: true
mountPath: /data
elasticSearchAppVolumeMounts:
esdata:
emptyDir: true
mountPath: /usr/share/elasticsearch/data
ownerUID: 568
ownerGID: 568
username: "admin"
password: "admin"
host: "192.169.0.156"
hostNetwork: false
timezone: "America/Los_Angeles"
diskoverCredentials:
username: admin
password: admin
cronjobSchedule: "0 3 * * *"
es_user: elasticsearch

View File

@@ -0,0 +1,5 @@
icon_url: http://www.diskoverdata.com/wp-content/uploads/2019/09/diskover.png
categories:
- storage
- monitoring
- management