mirror of
https://github.com/truenas/charts.git
synced 2026-02-11 14:26:39 +08:00
271 lines
11 KiB
YAML
271 lines
11 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: "diskover-initial-scripts"
|
|
annotations:
|
|
rollme: {{ randAlphaNum 5 | quote }}
|
|
|
|
data:
|
|
wait_for_elastic_search.py: |-
|
|
# This Script Wait for elastic search to setup completely
|
|
import requests
|
|
import os
|
|
import time
|
|
timeout = 100
|
|
while True:
|
|
try:
|
|
if timeout < 0:
|
|
print("timeout")
|
|
raise requests.exceptions.ConnectTimeout("Elasticsearch is not responding")
|
|
timeout -= 1
|
|
response = requests.get(f"http://{os.environ['ES_HOST']}:{os.environ['ES_PORT']}")
|
|
if response.status_code == 200:
|
|
break
|
|
except requests.exceptions.ConnectTimeout as e:
|
|
print(e)
|
|
break
|
|
except requests.exceptions.ConnectionError:
|
|
print("Trying to connect to elastic search")
|
|
time.sleep(3)
|
|
|
|
|
|
.default_crawler.sh: |-
|
|
#!/bin/sh
|
|
while :
|
|
do
|
|
# this condition wait for the script to copy into the container .
|
|
if test -f "$1"; then
|
|
# Empty folders don't generate indices . if folder is empty a default file is generated
|
|
if ! [ "$(ls -A $2)" ]; then
|
|
echo "Dummy file created as empty dirs are rejected" > $2/diskover_test.txt;
|
|
fi
|
|
python3 $1 $2/;
|
|
break;
|
|
fi
|
|
sleep 5
|
|
|
|
done
|
|
|
|
init_config.py: |-
|
|
import os
|
|
Config = f"""<?php
|
|
namespace diskover;
|
|
|
|
class Constants {{`{{
|
|
|
|
const TIMEZONE = '{os.environ['TZ']}';
|
|
|
|
const ES_HOST = '{os.environ['ES_HOST']}';
|
|
|
|
const ES_PORT = {os.environ['ES_PORT']};
|
|
|
|
const ES_USER = '{os.environ['ES_USER']}';
|
|
const ES_PASS = '{os.environ['ES_PASS']}';
|
|
|
|
// if your Elasticsearch cluster uses HTTP TLS/SSL, set ES_HTTPS to TRUE
|
|
// override with env var ES_HTTPS
|
|
const ES_HTTPS = FALSE;
|
|
|
|
// login auth for diskover-web
|
|
const LOGIN_REQUIRED = TRUE;
|
|
|
|
// default username and password to login
|
|
// the password is no longer used after first login, a hashed password gets stored in separate sqlite db
|
|
const USER = '{os.environ['DS_USER']}';
|
|
const PASS = '{os.environ['DS_PASS']}';
|
|
|
|
// default results per search page
|
|
const SEARCH_RESULTS = 50;
|
|
|
|
// default size field (size, size_du) to use for sizes on file tree and charts
|
|
const SIZE_FIELD = 'size';
|
|
|
|
// default file types, used by quick search (file type) and dashboard file type usage chart
|
|
// additional extensions can be added/removed from each file types list
|
|
const FILE_TYPES = [
|
|
'docs' => ['doc', 'docx', 'odt', 'pdf', 'tex', 'wpd', 'wks', 'txt', 'rtf', 'key', 'odp', 'pps', 'ppt', 'pptx', 'ods', 'xls', 'xlsm', 'xlsx'],
|
|
'images' => ['ai', 'bmp', 'gif', 'ico', 'jpeg', 'jpg', 'png', 'ps', 'psd', 'psp', 'svg', 'tif', 'tiff', 'exr', 'tga'],
|
|
'video' => ['3g2', '3gp', 'avi', 'flv', 'h264', 'm4v', 'mkv', 'qt', 'mov', 'mp4', 'mpg', 'mpeg', 'rm', 'swf', 'vob', 'wmv', 'ogg', 'ogv', 'webm'],
|
|
'audio' => ['au', 'aif', 'aiff', 'cda', 'mid', 'midi', 'mp3', 'm4a', 'mpa', 'ogg', 'wav', 'wma', 'wpl'],
|
|
'apps' => ['apk', 'exe', 'bat', 'bin', 'cgi', 'pl', 'gadget', 'com', 'jar', 'msi', 'py', 'wsf'],
|
|
'programming' => ['c', 'cgi', 'pl', 'class', 'cpp', 'cs', 'h', 'java', 'php', 'py', 'sh', 'swift', 'vb'],
|
|
'internet' => ['asp', 'aspx', 'cer', 'cfm', 'cgi', 'pl', 'css', 'htm', 'html', 'js', 'jsp', 'part', 'php', 'py', 'rss', 'xhtml'],
|
|
'system' => ['bak', 'cab', 'cfg', 'cpl', 'cur', 'dll', 'dmp', 'drv', 'icns', 'ico', 'ini', 'lnk', 'msi', 'sys', 'tmp', 'vdi', 'raw'],
|
|
'data' => ['csv', 'dat', 'db', 'dbf', 'log', 'mdb', 'sav', 'sql', 'tar', 'xml'],
|
|
'disc' => ['bin', 'dmg', 'iso', 'toast', 'vcd', 'img'],
|
|
'compressed' => ['7z', 'arj', 'deb', 'pkg', 'rar', 'rpm', 'tar', 'gz', 'z', 'zip'],
|
|
'trash' => ['old', 'trash', 'tmp', 'temp', 'junk', 'recycle', 'delete', 'deleteme', 'clean', 'remove']
|
|
];
|
|
|
|
// extra fields for search results and view file/dir info pages
|
|
// key is description for field and value is ES field name
|
|
// Example:
|
|
//const EXTRA_FIELDS = [
|
|
// 'Date Changed' => 'ctime'
|
|
//];
|
|
const EXTRA_FIELDS = [];
|
|
|
|
// Maximum number of indices to load by default, indices are loaded in order by creation date
|
|
// setting this too high can cause slow logins and other timeout issues
|
|
// This setting can bo overridden on indices page per user and stored in maxindex cookie
|
|
// If MAX_INDEX is set higher than maxindex browser cookie, the cookie will be set to this value
|
|
const MAX_INDEX = 250;
|
|
|
|
// time in seconds for index info to be cached, clicking reload indices forces update
|
|
const INDEXINFO_CACHETIME = 600;
|
|
|
|
// time in seconds to check Elasticsearch for new index info
|
|
const NEWINDEX_CHECKTIME = 10;
|
|
|
|
// sqlite database file path
|
|
const DATABASE = '../diskoverdb.sqlite3';
|
|
}}`}}
|
|
|
|
"""
|
|
|
|
os.makedirs(os.environ['DEST'], exist_ok=True)
|
|
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
|
|
with open(path, 'w') as w:
|
|
w.write(Config)
|
|
|
|
|
|
initial_es_config.py: |-
|
|
import os
|
|
Config = f"""# diskover default/sample config file
|
|
#
|
|
# default search paths for config
|
|
# macOS: ~/.config/diskover and ~/Library/Application Support/diskover
|
|
# Other Unix: ~/.config/diskover and /etc/diskover
|
|
# Windows: %APPDATA%\diskover where the APPDATA environment variable falls back to %HOME%\AppData\Roaming if undefined
|
|
#
|
|
appName: diskover
|
|
#logLevel: WARN
|
|
#logLevel: DEBUG
|
|
logLevel: INFO
|
|
logToFile: False
|
|
#logToFile: True
|
|
logDirectory: /tmp/
|
|
|
|
diskover:
|
|
# max number of crawl threads
|
|
# a thread is created up to maxthreads for each directory at level 1 of tree dir arg
|
|
# set to a number or leave blank to auto set based on number of cpus
|
|
#maxthreads: 20
|
|
maxthreads:
|
|
# block size used for du size
|
|
blocksize: 512
|
|
excludes:
|
|
# directory names and absolute paths you want to exclude from crawl
|
|
# directory excludes uses python re.search for string search (regex)
|
|
# directory excludes are case-sensitive
|
|
# Examples: .* or .backup or .backup* or /dir/dirname
|
|
# to exclude none use empty list []
|
|
dirs: [".*", ".snapshot", ".Snapshot", "~snapshot", "~Snapshot", ".zfs"]
|
|
#dirs: []
|
|
# files you want to exclude from crawl
|
|
# can include wildcards (.*, *.doc or NULLEXT for files with no extension)
|
|
# file names are case-sensitive, extensions are not
|
|
files: [".*", "Thumbs.db", ".DS_Store", "._.DS_Store", ".localized", "desktop.ini"]
|
|
#files: []
|
|
# exclude empty 0 byte files, set to True to exclude empty files or False to not exclude
|
|
emptyfiles: True
|
|
# exclude empty dirs, set to True to exclude empty dirs or False to not exclude
|
|
emptydirs: True
|
|
# exclude files smaller than min size in bytes
|
|
minfilesize: 1
|
|
#minfilesize: 512
|
|
# exclude files modified less than x days ago
|
|
minmtime: 0
|
|
#minmtime: 30
|
|
# exclude files modified more than x days ago
|
|
maxmtime: 36500
|
|
# exclude files changed less than x days ago
|
|
minctime: 0
|
|
# exclude files changed more than x days ago
|
|
maxctime: 36500
|
|
# exclude files accessed less than x days ago
|
|
minatime: 0
|
|
# exclude files accessed more than x days ago
|
|
maxatime: 36500
|
|
includes:
|
|
# directory names and absolute paths you want to include (whitelist), case-sensitive,
|
|
# to include none use empty list []
|
|
#dirs: [".recycle"]
|
|
dirs: []
|
|
# files you want to include (whitelist), case-sensitive
|
|
files: []
|
|
ownersgroups:
|
|
# control how owner (username) and group fields are stored for file and directory docs
|
|
# store uid and gid's instead of trying to get owner and group names
|
|
uidgidonly: False
|
|
# owner/group names contain domain name set to True
|
|
domain: False
|
|
# character separator used on cifs/nfs mounts to separte user/group and domain name, usually \ or @
|
|
domainsep: \
|
|
# if domain name comes first before character separator, set this to True, otherwise False
|
|
domainfirst: True
|
|
# when indexing owner and group fields, keep the domain name
|
|
keepdomain: False
|
|
replacepaths:
|
|
# translate path names set to True to enable or False to disable.
|
|
# Set to True if crawling in Windows to replace drive letters and \ with /
|
|
replace: False
|
|
#from: /mnt/
|
|
#to: /vols/
|
|
from:
|
|
to:
|
|
plugins:
|
|
# set to True to enable all plugins or False to disable all plugins
|
|
enable: False
|
|
# list of plugins (by name) to use for directories
|
|
dirs: ['unixperms']
|
|
# list of plugins (by name) to use for files
|
|
files: ['unixperms']
|
|
other:
|
|
# restore atime/mtime for files and dirs during crawl
|
|
# set to True or False, default False (useful for cifs which does not work with noatime mount option)
|
|
# for nfs, it's preferable to use mount options ro,noatime,nodiratime
|
|
restoretimes: False
|
|
|
|
databases:
|
|
elasticsearch:
|
|
host: '{os.environ['ES_HOST']}'
|
|
port: {os.environ['ES_PORT']}
|
|
#user: elastic
|
|
#password: changeme
|
|
user: '{os.environ['ES_USER']}'
|
|
password: '{os.environ['ES_PASS']}'
|
|
# set https to True if using HTTP TLS/SSL or False if using http
|
|
# for AWS ES, you will most likely want to set this to True
|
|
# override with env var ES_HTTPS
|
|
https: False
|
|
# compress http data
|
|
# for AWS ES, you will most likely want to set this to True
|
|
httpcompress: False
|
|
# timeout for connection to ES (default is 10)
|
|
timeout: 30
|
|
# number of connections kept open to ES when crawling (default is 10)
|
|
maxsize: 20
|
|
# max retries for ES operations (default is 0)
|
|
maxretries: 10
|
|
# wait for at least yellow status before bulk uploading (default is False), set to True if you want to wait
|
|
wait: False
|
|
# chunk size for ES bulk operations (default is 500)
|
|
chunksize: 1000
|
|
# the below settings are to optimize ES for crawling
|
|
# index refresh interval (default is 1s), set to -1 to disable refresh during crawl (fastest performance but no index searches), after crawl is set back to 1s
|
|
indexrefresh: 30s
|
|
# transaction log flush threshold size (default 512mb)
|
|
translogsize: 1gb
|
|
# transaction log sync interval time (default 5s)
|
|
translogsyncint: 30s
|
|
# search scroll size (default 100 docs)
|
|
scrollsize: 1000
|
|
"""
|
|
|
|
os.makedirs(os.environ['DEST'], exist_ok=True)
|
|
path = os.path.join(os.environ['DEST'], os.environ['FILE'])
|
|
|
|
with open(path, 'w') as w:
|
|
w.write(Config)
|