NAS-120628 / 23.10 / Add IPFS app to community train (#1023)

* Add ipfs to community train

* run chown before config and only once

* fix selector

* update readme(s)

* add update strategy

library/ix-dev/community/ipfs/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: file://../../../common
+  version: 1.0.0
+digest: sha256:f90dc95623b075f1d9ec4c4ba12464e2e789176ce7282a3aede37222201763b6
+generated: "2023-03-09T22:30:24.240799718+02:00"

library/ix-dev/community/ipfs/Chart.yaml

@@ -0,0 +1,26 @@
+name: ipfs
+description: Interplanetary Filesystem - the Web3 standard for content-addressing, interoperable with HTTP
+annotations:
+  title: IPFS
+type: application
+version: 1.0.0
+apiVersion: v2
+appVersion: '0.18.1'
+kubeVersion: '>=1.16.0-0'
+maintainers:
+  - name: truenas
+    url: https://www.truenas.com/
+dependencies:
+- name: common
+  repository: file://../../../common
+  version: 1.0.0
+home: https://ipfs.tech/
+icon: https://avatars.githubusercontent.com/u/10536621
+sources:
+- https://github.com/ipfs/kubo
+- https://github.com/truenas/charts/tree/master/community/ipfs
+- https://ipfs.tech/
+keywords:
+- storage
+- ipfs
+- kubo

library/ix-dev/community/ipfs/README.md

@@ -0,0 +1,7 @@
+# IPFS
+
+[Interplanetary Filesystem](https://ipfs.tech) - the Web3 standard for content-addressing, interoperable with HTTP
+
+> When application is installed, a container will be launched with **root** privileges.
+> This is required in order to apply the correct permissions to the ipfs directories.
+> Afterward, the `ipfs` container will run as a **non**-root user (Default: `568`).

library/ix-dev/community/ipfs/app-readme.md

@@ -0,0 +1,7 @@
+# IPFS
+
+[Interplanetary Filesystem](https://ipfs.tech) - the Web3 standard for content-addressing, interoperable with HTTP
+
+> When application is installed, a container will be launched with **root** privileges.
+> This is required in order to apply the correct permissions to the ipfs directories.
+> Afterward, the `ipfs` container will run as a **non**-root user (Default: `568`).

library/ix-dev/community/ipfs/ci/basic-values.yaml

@@ -0,0 +1,9 @@
+ipfsStorage:
+  data:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/data
+    datasetName: data
+  staging:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/staging
+    datasetName: staging

library/ix-dev/community/ipfs/ci/extra-env-values.yaml

@@ -0,0 +1,16 @@
+ipfsConfig:
+  additionalEnvs:
+    - name: GOLOG_LOG_LEVEL
+      value: debug
+    - name: GOLOG_LOG_FMT
+      value: json
+
+ipfsStorage:
+  data:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/data
+    datasetName: data
+  staging:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/staging
+    datasetName: staging

library/ix-dev/community/ipfs/ci/hostNet-values.yaml

@@ -0,0 +1,11 @@
+ipfsStorage:
+  data:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/data
+    datasetName: data
+  staging:
+    type: hostPath
+    hostPath: /mnt/{{ .Release.Namespace }}/staging
+    datasetName: staging
+ipfsNetwork:
+  hostNetwork: true

library/ix-dev/community/ipfs/item.yaml

@@ -0,0 +1,5 @@
+icon_url: https://avatars.githubusercontent.com/u/10536621
+categories:
+  - storage
+  - ipfs
+  - kubo

library/ix-dev/community/ipfs/questions.yaml

@@ -0,0 +1,219 @@
+groups:
+  - name: IPFS Configuration
+    description: Configure IPFS
+  - name: User and Group Configuration
+    description: Configure User and Group for IPFS
+  - name: Network Configuration
+    description: Configure Network for IPFS
+  - name: Storage Configuration
+    description: Configure Storage for IPFS
+  - name: Resources Configuration
+    description: Configure Resources for IPFS
+
+portals:
+  web_portal:
+    protocols:
+      - "$kubernetes-resource_configmap_portal_protocol"
+    host:
+      - "$kubernetes-resource_configmap_portal_host"
+    ports:
+      - "$kubernetes-resource_configmap_portal_port"
+    path: "$kubernetes-resource_configmap_portal_path"
+
+questions:
+
+  - variable: ipfsConfig
+    label: ""
+    group: IPFS Configuration
+    schema:
+      type: dict
+      attrs:
+        - variable: additionalEnvs
+          label: Additional Environment Variables
+          description: Configure additional environment variables for IPFS.
+          schema:
+            type: list
+            default: []
+            items:
+              - variable: env
+                label: Environment Variable
+                schema:
+                  type: dict
+                  attrs:
+                    - variable: name
+                      label: Name
+                      schema:
+                        type: string
+                        required: true
+                    - variable: value
+                      label: Value
+                      schema:
+                        type: string
+                        required: true
+
+  - variable: ipfsRunAs
+    label: ""
+    group: User and Group Configuration
+    schema:
+      type: dict
+      attrs:
+        - variable: user
+          label: User ID
+          description: The user id that IPFS will run as.
+          schema:
+            type: int
+            min: 1
+            default: 568
+            required: true
+        - variable: group
+          label: Group ID
+          description: The group id that IPFS will run as.
+          schema:
+            type: int
+            min: 1
+            default: 568
+            required: true
+
+  - variable: ipfsNetwork
+    label: ""
+    group: Network Configuration
+    schema:
+      type: dict
+      attrs:
+        - variable: apiPort
+          label: API Port
+          description: The port for the IPFS API (And WebUI).
+          schema:
+            type: int
+            default: 30000
+            min: 9000
+            max: 65535
+            required: true
+        - variable: swarmPort
+          label: Swarm Port
+          description: The port for the IPFS Swarm. Both TCP and UDP
+          schema:
+            type: int
+            default: 30001
+            min: 9000
+            max: 65535
+            required: true
+        - variable: gatewayPort
+          label: Gateway Port
+          description: The port for the IPFS Gateway.
+          schema:
+            type: int
+            default: 30002
+            min: 9000
+            max: 65535
+            required: true
+        - variable: hostNetwork
+          label: Host Network
+          description: |
+            Bind to the host network. It's recommended to keep this disabled.
+          schema:
+            type: boolean
+            default: false
+
+  - variable: ipfsStorage
+    label: ""
+    group: Storage Configuration
+    schema:
+      type: dict
+      attrs:
+        - variable: data
+          label: IPFS Data Storage
+          description: The path to store IPFS data.
+          schema:
+            type: dict
+            attrs:
+              - variable: type
+                label: Type
+                schema:
+                  type: string
+                  required: true
+                  default: ixVolume
+                  enum:
+                    - value: hostPath
+                      description: Host Path
+                    - value: ixVolume
+                      description: ixVolume
+              - variable: datasetName
+                label: Dataset Name
+                schema:
+                  type: string
+                  show_if: [["type", "=", "ixVolume"]]
+                  required: true
+                  hidden: true
+                  immutable: true
+                  default: data
+                  $ref:
+                    - "normalize/ixVolume"
+              - variable: hostPath
+                label: Host Path
+                schema:
+                  type: hostpath
+                  show_if: [["type", "=", "hostPath"]]
+                  immutable: true
+                  required: true
+        - variable: staging
+          label: IPFS Staging Storage
+          description: The path to store IPFS staging storage.
+          schema:
+            type: dict
+            attrs:
+              - variable: type
+                label: Type
+                schema:
+                  type: string
+                  required: true
+                  default: ixVolume
+                  enum:
+                    - value: hostPath
+                      description: Host Path
+                    - value: ixVolume
+                      description: ixVolume
+              - variable: datasetName
+                label: Dataset Name
+                schema:
+                  type: string
+                  show_if: [["type", "=", "ixVolume"]]
+                  required: true
+                  hidden: true
+                  immutable: true
+                  default: staging
+                  $ref:
+                    - "normalize/ixVolume"
+              - variable: hostPath
+                label: Host Path
+                schema:
+                  type: hostpath
+                  show_if: [["type", "=", "hostPath"]]
+                  immutable: true
+                  required: true
+
+  - variable: resources
+    label: ""
+    group: Resources Configuration
+    schema:
+      type: dict
+      attrs:
+        - variable: limits
+          label: Limits
+          schema:
+            type: dict
+            attrs:
+              - variable: cpu
+                label: CPU
+                description: CPU limit for qBittorrent.
+                schema:
+                  type: string
+                  default: 4000m
+                  required: true
+              - variable: memory
+                label: Memory
+                description: Memory limit for qBittorrent.
+                schema:
+                  type: string
+                  default: 8Gi
+                  required: true

library/ix-dev/community/ipfs/templates/NOTES.txt

@@ -0,0 +1 @@
+{{ include "ix.v1.common.lib.chart.notes" $ }}

library/ix-dev/community/ipfs/templates/_configuration.tpl

@@ -0,0 +1,60 @@
+{{- define "ipfs.configuration" -}}
+{{/* Default Swarm Addresses https://github.com/ipfs/kubo/blob/master/docs/config.md#addressesswarm */}}
+{{ $swarmAddressesList := (list
+      (printf "/ip4/0.0.0.0/tcp/%v" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip6/::/tcp/%v" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip4/0.0.0.0/udp/%v/quic" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip4/0.0.0.0/udp/%v/quic-v1" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip4/0.0.0.0/udp/%v/quic-v1/webtransport" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip6/::/udp/%v/quic" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip6/::/udp/%v/quic-v1" .Values.ipfsNetwork.swarmPort)
+      (printf "/ip6/::/udp/%v/quic-v1/webtransport" .Values.ipfsNetwork.swarmPort)
+) }}
+
+{{ $swarmAddresses := printf "[ \"%s\" ]" (join "\", \"" $swarmAddressesList) }}
+
+{{/* Default API Address https://github.com/ipfs/kubo/blob/master/docs/config.md#addressesapi */}}
+{{ $apiAddresses := printf "/ip4/0.0.0.0/tcp/%v" .Values.ipfsNetwork.apiPort }}
+{{/* Default Gateway Address https://github.com/ipfs/kubo/blob/master/docs/config.md#addressesgateway */}}
+{{ $gatewayAddresses := printf "/ip4/0.0.0.0/tcp/%v" .Values.ipfsNetwork.gatewayPort }}
+{{ $allowOrigins := "[ \"*\" ]" }}
+{{ $allowMethods := "[ \"PUT\", \"POST\" ]" }}
+
+
+{{/* Configmaps */}}
+configmap:
+  config-script:
+    enabled: true
+    data:
+      init-config.sh: |
+        #!/bin/sh
+        set -e
+
+        if [ ! -f /data/ipfs/config ]; then
+          # Create the IPFS config file
+          echo "Initializing IPFS"
+          ipfs init
+        fi
+
+        # Configure the Addresses.API
+        echo "Configuring the Addresses.API to {{ $apiAddresses }}"
+        ipfs config Addresses.API {{ $apiAddresses }}
+
+        # Configure the Addresses.Gateway
+        echo "Configuring the Addresses.Gateway to {{ $gatewayAddresses }}"
+        ipfs config Addresses.Gateway {{ $gatewayAddresses }}
+
+        # Configure the Addresses.Swarm
+        echo "Configuring the Addresses.Swarm to {{ $swarmAddresses | squote }}"
+        ipfs config Addresses.Swarm --json {{ $swarmAddresses | squote }}
+
+        # Configure the API.HTTPHeaders.Access-Control-Allow-Origin
+        echo "Configuring the API.HTTPHeaders.Access-Control-Allow-Origin to {{ $allowOrigins | squote }}"
+        ipfs config API.HTTPHeaders.Access-Control-Allow-Origin --json {{ $allowOrigins | squote }}
+
+        # Configure the API.HTTPHeaders.Access-Control-Allow-Methods
+        echo "Configuring the API.HTTPHeaders.Access-Control-Allow-Methods to {{ $allowMethods | squote }}"
+        ipfs config API.HTTPHeaders.Access-Control-Allow-Methods --json {{ $allowMethods | squote }}
+
+        echo "Finished configuring IPFS"
+{{- end -}}

library/ix-dev/community/ipfs/templates/_ipfs.tpl

@@ -0,0 +1,149 @@
+{{- define "ipfs.workload" -}}
+workload:
+  ipfs:
+    enabled: true
+    primary: true
+    type: Deployment
+    podSpec:
+      hostNetwork: {{ .Values.ipfsNetwork.hostNetwork }}
+      containers:
+        ipfs:
+          enabled: true
+          primary: true
+          imageSelector: image
+          securityContext:
+            runAsUser: {{ .Values.ipfsRunAs.user }}
+            runAsGroup: {{ .Values.ipfsRunAs.group }}
+          {{ with .Values.ipfsConfig.additionalEnvs }}
+          env:
+            {{ range $env := . }}
+            {{ $env.name }}: {{ $env.value }}
+            {{ end }}
+          {{ end }}
+          probes:
+            liveness:
+              enabled: true
+              type: exec
+              command:
+                - ipfs
+                - dag
+                - stat
+                # https://github.com/ipfs/kubo/blob/8f638dcbcd875ecff92021e4b62d0af8848022ce/Dockerfile#L116
+                - /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn
+            readiness:
+              enabled: true
+              type: exec
+              command:
+                - ipfs
+                - dag
+                - stat
+                - /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn
+            startup:
+              enabled: true
+              type: exec
+              command:
+                - ipfs
+                - dag
+                - stat
+                - /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn
+      initContainers:
+      {{- include "ix.v1.common.app.permissions" (dict "containerName" "01-permissions"
+                                                        "UID" .Values.ipfsRunAs.user
+                                                        "GID" .Values.ipfsRunAs.group
+                                                        "type" "install") | nindent 8 }}
+        # "02" prefix is used to ensure this container runs after the permissions container
+        02-init-config:
+          enabled: true
+          type: init
+          imageSelector: image
+          securityContext:
+            runAsUser: {{ .Values.ipfsRunAs.user }}
+            runAsGroup: {{ .Values.ipfsRunAs.group }}
+          command: /init-config.sh
+          resources:
+            limits:
+              memory: 512Mi
+              cpu: 1000m
+{{/* Service */}}
+service:
+  ipfs:
+    enabled: true
+    primary: true
+    type: NodePort
+    targetSelector: ipfs
+    ports:
+      api:
+        enabled: true
+        primary: true
+        port: {{ .Values.ipfsNetwork.apiPort }}
+        nodePort: {{ .Values.ipfsNetwork.apiPort }}
+        targetSelector: ipfs
+  ipfs-swarm:
+    enabled: true
+    type: NodePort
+    targetSelector: ipfs
+    ports:
+      swarm-tcp:
+        enabled: true
+        primary: true
+        port: {{ .Values.ipfsNetwork.swarmPort }}
+        nodePort: {{ .Values.ipfsNetwork.swarmPort }}
+        targetSelector: ipfs
+      swarm-udp:
+        enabled: true
+        primary: true
+        port: {{ .Values.ipfsNetwork.swarmPort }}
+        nodePort: {{ .Values.ipfsNetwork.swarmPort }}
+        protocol: udp
+        targetSelector: ipfs
+  ipfs-gateway:
+    enabled: true
+    type: NodePort
+    targetSelector: ipfs
+    ports:
+      ipfs-gateway:
+        enabled: true
+        primary: true
+        port: {{ .Values.ipfsNetwork.gatewayPort }}
+        nodePort: {{ .Values.ipfsNetwork.gatewayPort }}
+        targetSelector: ipfs
+
+{{/* Persistence */}}
+persistence:
+  data:
+    enabled: true
+    type: {{ .Values.ipfsStorage.data.type }}
+    datasetName: {{ .Values.ipfsStorage.data.datasetName | default "" }}
+    hostPath: {{ .Values.ipfsStorage.data.hostPath | default "" }}
+    targetSelector:
+      ipfs:
+        ipfs:
+          mountPath: /data/ipfs
+        01-permissions:
+          mountPath: /mnt/directories/data
+        02-init-config:
+          mountPath: /data/ipfs
+  staging:
+    enabled: true
+    type: {{ .Values.ipfsStorage.staging.type }}
+    datasetName: {{ .Values.ipfsStorage.staging.datasetName | default "" }}
+    hostPath: {{ .Values.ipfsStorage.staging.hostPath | default "" }}
+    targetSelector:
+      ipfs:
+        ipfs:
+          mountPath: /export
+        01-permissions:
+          mountPath: /mnt/directories/export
+  config-script:
+    enabled: true
+    type: configmap
+    objectName: config-script
+    defaultMode: "0755"
+    targetSelector:
+      ipfs:
+        02-init-config:
+          mountPath: /init-config.sh
+          readOnly: true
+          subPath: init-config.sh
+
+{{- end -}}

library/ix-dev/community/ipfs/templates/_portal.tpl

@@ -0,0 +1,12 @@
+{{- define "ipfs.portal" -}}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: portal
+data:
+  path: /webui
+  port: {{ .Values.ipfsNetwork.apiPort | quote }}
+  protocol: http
+  host: $node_ip
+{{- end -}}

library/ix-dev/community/ipfs/templates/common.yaml

@@ -0,0 +1,10 @@
+{{- include "ix.v1.common.loader.init" . -}}
+
+{{/* Merge the templates with Values */}}
+{{- $_ := mustMergeOverwrite .Values (include "ipfs.workload" $ | fromYaml) -}}
+{{- $_ := mustMergeOverwrite .Values (include "ipfs.configuration" $ | fromYaml) -}}
+
+{{/* Create the configmap for portal manually*/}}
+{{- include "ipfs.portal" $ -}}
+
+{{- include "ix.v1.common.loader.apply" . -}}

library/ix-dev/community/ipfs/upgrade_strategy

@@ -0,0 +1,26 @@
+#!/usr/bin/python3
+import json
+import sys
+
+from catalog_update.upgrade_strategy import semantic_versioning
+
+
+def newer_mapping(image_tags):
+    key = list(image_tags.keys())[0]
+    version = semantic_versioning(image_tags[key])
+    if not version:
+        return {}
+
+    return {
+        'tags': {key: f'v{version}'},
+        'app_version': f'v{version}',
+    }
+
+
+if __name__ == '__main__':
+    try:
+        versions_json = json.loads(sys.stdin.read())
+    except ValueError:
+        raise ValueError('Invalid json specified')
+
+    print(json.dumps(newer_mapping(versions_json)))

library/ix-dev/community/ipfs/values.yaml

@@ -0,0 +1,29 @@
+image:
+  repository: ipfs/kubo
+  pullPolicy: IfNotPresent
+  tag: v0.18.1
+
+resources:
+  limits:
+    cpu: 4000m
+    memory: 8Gi
+
+ipfsConfig:
+  additionalEnvs: []
+ipfsNetwork:
+  apiPort: 30000
+  swarmPort: 30001
+  gatewayPort: 30002
+  hostNetwork: false
+ipfsRunAs:
+  user: 568
+  group: 568
+ipfsStorage:
+  data:
+    type: ixVolume
+    hostPath: ""
+    datasetName: data
+  staging:
+    type: ixVolume
+    hostPath: ""
+    datasetName: staging