Integrate Container and Website Repo's + docs restructure

This commit is contained in:
kjeld Schouten-Lebbing
2021-03-17 10:35:56 +01:00
parent e8f9ef02d4
commit c9a3f01580
364 changed files with 7580 additions and 234 deletions

View File

@@ -1,96 +0,0 @@
name: "Charts: Release"
on:
push:
branches:
- master
tags-ignore:
- '**'
paths:
- 'library/**'
- '!library/**/*.md'
- '!library/**/README.md'
- '!library/**/README.md.gotmpl'
- '!library/**/app-readme.md'
- '!library/**/app-readme.md.gotmpl'
- '!library/**/docs/*'
jobs:
pre-release:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
release:
needs: pre-release
runs-on: ubuntu-latest
steps:
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "TrueCharts-Bot"
git config user.email "TrueCharts Bot"
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.5.2
- name: Run chart-releaser
uses: helm/chart-releaser-action@v1.2.0
with:
charts_dir: library
charts_repo_url: https://charts.truecharts.org/
env:
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
# Update the generated timestamp in the index.yaml
# needed until https://github.com/helm/chart-releaser/issues/90
# or helm/chart-releaser-action supports this
post-release:
needs: release
runs-on: ubuntu-latest
steps:
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout
uses: actions/checkout@v2
with:
ref: "gh-pages"
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
- name: Commit and push timestamp updates
run: |
if [[ -f index.yaml ]]; then
git pull
export generated_date=$(date --utc +%FT%T.%9NZ)
sed -i -e "s/^generated:.*/generated: \"$generated_date\"/" index.yaml
git add index.yaml
git commit -sm "Update generated timestamp [ci-skip]" || exit 0
git push
fi

111
.github/workflows/charts.release.yaml vendored Normal file
View File

@@ -0,0 +1,111 @@
name: "Charts: Release"
on:
push:
branches:
- master
- staging
tags-ignore:
- '**'
paths:
- 'library/**'
- '!library/**/*.md'
- '!library/**/README.md'
- '!library/**/README.md.gotmpl'
- '!library/**/app-readme.md'
- '!library/**/app-readme.md.gotmpl'
- '!library/**/docs/*'
- '.github/workflows/charts.release.yaml'
- '.github/cr.yaml'
jobs:
pre-release:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
release:
needs: pre-release
runs-on: ubuntu-latest
steps:
- name: Block concurrent jobs
uses: softprops/turnstyle@v1
with:
continue-after-seconds: 180
env:
GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
token: ${{ secrets.BOT_TOKEN }}
- name: Configure Git
run: |
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
- name: Install Helm
uses: azure/setup-helm@v1
with:
version: v3.5.2
- name: Run chart-releaser
uses: helm/chart-releaser-action@v1.2.0
with:
charts_dir: library
charts_repo_url: ""
config: .github/cr.yaml
env:
CR_TOKEN: "${{ secrets.BOT_TOKEN }}"
- name: Commit and Push new index
run: |
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
git add --all
git commit -sm "Commit chart-index updates" || exit 0
git push
# Update the generated timestamp in the index.yaml
# needed until https://github.com/helm/chart-releaser/issues/90
# or helm/chart-releaser-action supports this
# post-release:
# needs: release
# runs-on: ubuntu-latest
# steps:
# - name: Block concurrent jobs
# uses: softprops/turnstyle@v1
# with:
# continue-after-seconds: 180
# env:
# GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
#
# - name: Checkout
# uses: actions/checkout@v2
# with:
# ref: "gh-pages"
# fetch-depth: 0
#
# - name: Configure Git
# run: |
# git config user.name "TrueCharts-Bot"
# git config user.email "bot@truecharts.org"
#
# - name: Commit and push timestamp updates
# run: |
# if [[ -f index.yaml ]]; then
# git pull
# export generated_date=$(date --utc +%FT%T.%9NZ)
# sed -i -e "s/^generated:.*/generated: \"$generated_date\"/" index.yaml
# git add index.yaml
# git commit -sm "Update generated timestamp [ci-skip]" || exit 0
# git push
# fi

View File

@@ -1,4 +1,4 @@
name: "Charts: Tests"
name: "Charts: Test"
on:
pull_request:
@@ -6,6 +6,22 @@ on:
- '**'
tags-ignore:
- '**'
paths:
- 'library/**'
- '!library/**/*.md'
- '!library/**/README.md'
- '!library/**/README.md.gotmpl'
- '!library/**/app-readme.md'
- '!library/**/app-readme.md.gotmpl'
- '!library/**/docs/*'
- 'charts/**'
- '!charts/**/*.md'
- '!charts/**/README.md'
- '!charts/**/README.md.gotmpl'
- '!charts/**/app-readme.md'
- '!charts/**/app-readme.md.gotmpl'
- '!charts/**/docs/*'
- '.github/workflows/charts.test.yaml'
jobs:
catalog-tests:

View File

@@ -0,0 +1,21 @@
---
name: 'Community: Invalid Template'
on:
issues:
types: [labeled, unlabeled, reopened]
jobs:
support:
runs-on: ubuntu-20.04
steps:
- uses: dessant/support-requests@v2
with:
github-token: ${{ secrets.BOT_TOKEN }}
support-label: 'invalid:template-incomplete'
issue-comment: >
:wave: @{issue-author}, please follow the template provided.
close-issue: true
lock-issue: true
issue-lock-reason: 'resolved'

189
.github/workflows/containers.build.yaml vendored Normal file
View File

@@ -0,0 +1,189 @@
name: "Containers: Test-and-Build"
on:
push:
branches:
- main
- staging
paths:
- '.containers/apps/**'
- '.containers/base/**'
- ".github/workflows/containers.build.yaml"
pull_request:
paths:
- '.containers/apps/**'
- '.containers/base/**'
- ".github/workflows/containers.build.yaml"
env:
# How long to sleep before running the tests (gives the application time to start)
GOSS_SLEEP: 30
# Detect which folders in project-root (which contain the containers) contain changes
jobs:
changes:
name: Get changes
runs-on: ubuntu-20.04
outputs:
matrix: "{\"container\": ${{ steps.reduce.outputs.containers }} }"
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: dorny/paths-filter@v2
id: filter
with:
list-files: json
filters: |
changed:
- '.containers/apps/**'
- '.containers/base/**'
- run: echo '${{ toJson(steps.filter.outputs) }}' > changes.json
- id: reduce
run: |
CONTAINERS=$(jq --raw-output '.changed_files | fromjson | .[] |= sub("(?<filepath>(?<first_directory>(?<root1>[\/]?)[^\/]+\/)(?<second_directory>(?<root2>[\/]?)[^\/]+\/(?<third_directory>(?<root3>[\/]?)[^\/]+)(?<extra_paths>.+)))"; "\(.third_directory)") | unique' changes.json)
echo ::set-output name=containers::${CONTAINERS}
hadolint:
name: Run hadolint
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
- name: hadolint
uses: reviewdog/action-hadolint@v1
with:
github_token: ${{ secrets.BOT_TOKEN }}
reporter: github-pr-review
filter_mode: diff_context
fail_on_error: true
build:
name: Build
runs-on: ubuntu-20.04
needs:
- hadolint
- changes
strategy:
matrix: ${{ fromJson(needs.changes.outputs.matrix) }}
fail-fast: false
if: "!contains(github.event.head_commit.message, '[ci-skip]')"
steps:
- name: Checkout
uses: actions/checkout@v2
# Define if tests and push should be run against which versions/platforms
- name: Prepare
id: prep
run: |
if test -f "./.containers/apps/${{ matrix.container }}/Dockerfile"; then
CATEGORY="apps"
else
CATEGORY="base"
fi
echo ::set-output name=category::${CATEGORY}
VERSION=$(cat ./.containers/${CATEGORY}/${{ matrix.container }}/VERSION)
echo ::set-output name=version::${VERSION}
PLATFORM=$(cat ./.containers/${CATEGORY}/${{ matrix.container }}/PLATFORM)
echo ::set-output name=platform::${PLATFORM}
if test -f "./.containers/${CATEGORY}/${{ matrix.container }}/goss.yaml"; then
echo ::set-output name=goss::true
else
echo ::set-output name=goss::false
fi
if [ "${{github.event_name}}" == "pull_request" ]; then
echo ::set-output name=push::false
else
echo ::set-output name=push::true
fi
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
with:
platforms: all
- name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
# Install and configure Buildx
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
with:
install: true
version: latest
driver-opts: image=moby/buildkit:master
# Make sure we can store buildx cache in-between builds
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
# Install the GOSS testing framework
- name: Set up goss/dgoss
uses: e1himself/goss-installation-action@v1.0.3
if: ${{ steps.prep.outputs.goss == 'true' }}
with:
version: 'v0.3.16'
# Creates a local build to run tests on
- name: Build and Load local test-container
if: ${{ steps.prep.outputs.goss == 'true' }}
uses: docker/build-push-action@v2
with:
build-args: VERSION=${{ steps.prep.outputs.version }}
context: .
file: ./.containers/${{ steps.prep.outputs.category }}/${{ matrix.container }}/Dockerfile
load: true
tags: |
ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}:test
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
# Temporary fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move cache
if: ${{ steps.prep.outputs.goss == 'true' }}
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache | exit 0
# Run GOSS tests if included with the container
- name: Run GOSS tests
if: ${{ steps.prep.outputs.goss == 'true' }}
env:
GOSS_FILE: ./.containers/${{ steps.prep.outputs.category }}/${{ matrix.container }}/goss.yaml
run: |
dgoss run ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}:test
# Push if not a PR, otherwise just test the build process for all requested platforms
- name: Build and Push
uses: docker/build-push-action@v2
with:
build-args: VERSION=${{ steps.prep.outputs.version }}
context: .
platforms: ${{ steps.prep.outputs.platform }}
file: ./.containers/${{ steps.prep.outputs.category }}/${{ matrix.container }}/Dockerfile
push: ${{ steps.prep.outputs.push }}
tags: |
ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}:latest
ghcr.io/${{ github.repository_owner }}/${{ matrix.container }}:v${{ steps.prep.outputs.version }}
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new
# Temporary fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

View File

@@ -0,0 +1,60 @@
name: "Containers: Update"
on:
workflow_dispatch:
push:
branches:
- main
- staging
paths:
- ".github/workflows/containers.update.yaml"
schedule:
- cron: "0 */4 * * *"
jobs:
get-versions:
name: Get Apps Versions
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
# TODO remove this ref
ref: "staging"
token: ${{ secrets.BOT_TOKEN }}
fetch-depth: 1
- name: Fetch new application versions
run: |
find ./.containers/apps -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do
if test -f "./.containers/apps/${app}/latest-version.sh"; then
version=$(bash "./.containers/apps/${app}/latest-version.sh")
if [[ ! -z "${version}" || "${version}" != "null" ]]; then
echo "${version}" | tee "./.containers/apps/${app}/VERSION" > /dev/null
echo "App: ${app} using version: ${version}"
fi
fi
base=$(cat "./.containers/base/ubuntu/VERSION")
if [[ ! -z "${base}" || "${base}" != "null" ]]; then
echo "${base}" | tee ".containers/apps/${app}/BASE" > /dev/null
echo "App: ${app} using Base: ${base}"
fi
done
- name: Fetch new base versions
run: |
find ./.containers/base -maxdepth 1 -mindepth 1 -type d -exec basename {} \; | while read app; do
if test -f "./.containers/base/${app}/latest-version.sh"; then
version=$(bash "./.containers/base/${app}/latest-version.sh")
if [[ ! -z "${version}" || "${version}" != "null" ]]; then
echo "${version}" | tee "./.containers/base/${app}/VERSION" > /dev/null
echo "${app} ${version}"
fi
fi
done
- name: Add, commit and push to applications version file
run: |
if [[ -n $(git status --porcelain) ]]; then
git config user.name "TrueCharts Bot"
git config user.email "bot@truecharts.org"
git add -A
git commit -sam "Adding new release versions" || exit 0
git push
fi

76
.github/workflows/docs.copy.yaml vendored Normal file
View File

@@ -0,0 +1,76 @@
name: "Docs: Copy"
on:
push:
branches:
- master
- staging
tags-ignore:
- '**'
paths:
- 'library/**/Chart.yaml'
- 'library/**/README.md'
- 'library/**/LICENSE'
- 'library/**/**/CONFIG.md'
- 'library/**/app-readme.md'
- 'charts/**/Chart.yaml'
- 'charts/**/**/README.md'
- 'charts/**/**/CONFIG.md'
- 'charts/**/**/LICENSE'
- 'charts/**/**/app-readme.md'
- '.github/README.md'
- '.github/CODE_OF_CONDUCT.md'
- './LICENSE'
- './NOTICE'
- '.github/workflows/docs.copy.yaml'
jobs:
publish-app-docs:
runs-on: ubuntu-latest
steps:
- name: Checkout-Master
uses: actions/checkout@v2
with:
token: ${{ secrets.BOT_TOKEN }}
- name: install helm-docs
run: |
brew install norwoodj/tap/helm-docs
- name: (re)generate docs
run: |
.tools/gen-helm-docs.sh
- name: Copy general readme to website
run: |
cp .github/README.md docs/about/index.md || echo "readme copy failed, continuing..."
cp .github/CODE_OF_CONDUCT docs/about/code_of_conduct.md || echo "CODE_OF_CONDUCT copy failed, continuing..."
cp .github/CONTRIBUTING docs/development/contributing.md || echo "CONTRIBUTING copy failed, continuing..."
cp LICENSE docs/about/LICENSE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# License<br>\n\n/' docs/about/LICENSE.md
cp NOTICE docs/about/NOTICE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# NOTICE<br>\n\n/' docs/about/NOTICE.md
ls docs/
- name: Copy Apps readme to website
run: |
for chart in charts/*; do
if [ -d "${chart}" ]; then
maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
chartname=$(basename ${chart})
echo "Processing: ${chart} - folder: ${maxfolderversion}"
mkdir -p docs/apps/${chartname} || echo "app path already exists, continuing..."
cp ${chart}/${maxfolderversion}/README.md docs/apps/${chartname}/index.md || echo "readme copy failed, continuing..."
cp ${chart}/${maxfolderversion}/LICENSE docs/apps/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# License<br>\n\n/' docs/apps/${chartname}/LICENSE.md || echo "license edit failed, continuing..."
fi
done
ls docs/apps/
- name: Commit and Push updated docs
run: |
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
git add --all
git commit -sm "Commit docs updates" || exit 0
git push

24
.github/workflows/docs.deploy.yaml vendored Normal file
View File

@@ -0,0 +1,24 @@
name: "Docs: Deploy"
on:
push:
branches:
- master
- staging
paths:
- 'docs/**'
- '.github/workflows/docs.deploy.yaml'
jobs:
build:
name: Deploy Wiki
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v1
with:
token: ${{ secrets.BOT_TOKEN }}
- name: Deploy docs
uses: mhausenblas/mkdocs-deploy-gh-pages@master
env:
GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}

View File

@@ -0,0 +1,67 @@
name: "General: Security Scan"
on:
workflow_dispatch:
push:
branches:
- main
- staging
paths:
- ".github/workflows/general.security-scan.yaml"
schedule:
- cron: "0 0 * * 0"
jobs:
directories: # Job that list subdirectories
runs-on: ubuntu-latest
outputs:
dir: ${{ steps.set-dirs.outputs.dir }} # generate output name dir by using inner step output
steps:
- uses: actions/checkout@v2
with:
# TODO remove this ref
ref: "staging"
token: ${{ secrets.BOT_TOKEN }}
- id: set-dirs # Give it an id to handle to get step outputs in the outputs key above
run: echo "::set-output name=dir::$({ ls -d .containers/apps/*/ & ls -d .containers/base/*/; } | tr '\n' '\0' | xargs -0 -n 1 basename | jq -R -s -c 'split("\n")[:-1]')"
# Define step output named dir base on ls command transformed to JSON thanks to jq
scan_files:
name: Scan Files
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Run Trivy vulnerability scanner in repo mode
uses: aquasecurity/trivy-action@master
with:
scan-type: 'fs'
ignore-unfixed: true
format: 'template'
template: '@/contrib/sarif.tpl'
output: 'trivy-repo-results.sarif'
severity: 'CRITICAL'
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v1
with:
sarif_file: 'trivy-repo-results.sarif'
scan_containers:
name: Scan Containers
runs-on: ubuntu-20.04
needs: [directories] # Depends on previous job
strategy:
matrix:
dir: ${{fromJson(needs.directories.outputs.dir)}} # List matrix strategy from directories dynamically
fail-fast: false
steps:
- name: Container Security scan
uses: Azure/container-scan@v0
with:
image-name: ghcr.io/${{ github.repository_owner }}/${{matrix.dir}}:latest
severity-threshold: HIGH

View File

@@ -1,95 +0,0 @@
name: "Website: Create Docs"
on:
push:
branches:
- master
tags-ignore:
- '**'
jobs:
publish-app-docs:
runs-on: ubuntu-latest
steps:
- name: Checkout-Master
uses: actions/checkout@v2
with:
path: 'master'
token: ${{ secrets.BOT_TOKEN }}
- name: Checkout-Website
uses: actions/checkout@v2
with:
repository: 'truecharts/website'
path: 'website'
token: ${{ secrets.BOT_TOKEN }}
- name: install helm-docs
run: |
brew install norwoodj/tap/helm-docs
- name: (re)generate docs
run: |
cd master
.tools/gen-helm-docs.sh
cd ..
- name: Clean-Website
run: |
rm -Rf website/content
mkdir -p website/content
cp -f website/CNAME website/content/CNAME
- name: Create general website
run: |
cd master
cp -Rf docs/* ../website/content/
cp .github/README.md ../website/content/about/index.md || echo "readme copy failed, continuing..."
cp .github/CODE_OF_CONDUCT ../website/content/about/code_of_conduct.md || echo "CODE_OF_CONDUCT copy failed, continuing..."
cp .github/CONTRIBUTING ../website/content/development/contributing.md || echo "CONTRIBUTING copy failed, continuing..."
cp LICENSE ../website/content/about/LICENSE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# License<br>\n\n/' ../website/content/about/LICENSE.md
cp NOTICE ../website/content/about/NOTICE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# NOTICE<br>\n\n/' ../website/content/about/NOTICE.md
ls ../website/content/
cd ..
- name: Create apps website
run: |
cd master
for chart in charts/*; do
if [ -d "${chart}" ]; then
maxfolderversion=$(ls -l ${chart} | grep ^d | awk '{print $9}' | tail -n 1)
chartname=$(basename ${chart})
echo "Processing: ${chart} - folder: ${maxfolderversion}"
mkdir -p ../website/content/apps/${chartname} || echo "website path already exists, continuing..."
if [ -d "${chart}/${maxfolderversion}/docs" ]; then
cp -f ${chart}/${maxfolderversion}/docs/* ../website/content/apps/${chartname}/
fi
cp ${chart}/${maxfolderversion}/README.md ../website/content/apps/${chartname}/index.md || echo "readme copy failed, continuing..."
cp ${chart}/${maxfolderversion}/LICENSE ../website/content/apps/${chartname}/LICENSE.md || echo "license copy failed, continuing..."
sed -i '1s/^/# License<br>\n\n/' ../website/content/apps/${chartname}/LICENSE.md || echo "license edit failed, continuing..."
fi
done
ls ../website/content/apps/
cd ..
- name: Commit and push updated website
run: |
cd website
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
git add --all
git commit -sm "Publish general website updates" || exit 0
git push
cd ..
- name: Commit and push updated charts
run: |
cd master
git config user.name "TrueCharts-Bot"
git config user.email "bot@truecharts.org"
git add --all
git commit -sm "Publish docs updates" || exit 0
git push