New update-all script + more refactoring (still WIP and undocumented)

This commit is contained in:
Jean Froment 2022-03-06 01:48:28 +01:00
parent cff539464f
commit e8c69b0e3a
6 changed files with 527 additions and 19 deletions

3
.gitignore vendored
View File

@ -1,6 +1,5 @@
.DS_Store
/config
**/tunnel-options.sh
**/.env
**/traefik/http_auth
@ -10,3 +9,5 @@ traefik/custom/dynamic*.yaml
samples/custom*/*.yaml
config.yaml
services/custom/*.yaml
services/custom/*.yml

View File

@ -27,7 +27,28 @@ fi
jq -n '{"services": []}' > config.json
while read -r line ; do
# First, add Traefik as it was not explicitely set by default in old config file (services.conf)
if ! grep -q "traefik" services.conf; then
jq -r '.services[.services| length] |= . +
{
"name": "traefik",
"enabled": true,
"traefik": {
"enabled": true,
"rules": [
{
"host": "traefik.'$(echo '${TRAEFIK_DOMAIN}')'",
"service": "api@internal",
"httpAuth": true,
}
]
}
}' config.json > tmp.json
rm -f config.json
mv tmp.json config.json
fi
cat services.conf | while read line || [[ -n $line ]]; do
key=$(echo $line | sed -r "s/^(.*):.*$/\1/")
enabled="true"
if grep -q "disable" <<< $line; then
@ -41,8 +62,9 @@ while read -r line ; do
# If this service is disabled AND another one in the file is enabled with VPN mode, keep that information
if grep -q "$key-vpn: enable" services.conf; then
if [[ enabled="false" ]]; then
echo "[$0] $key => another service detected enabled with vpn..."
#echo "[$0] $key => another service detected enabled with vpn..."
enableVpn="true"
enabled="true"
fi
fi
@ -50,7 +72,7 @@ while read -r line ; do
# Define if Traefik should be enabled on the service
case $key in
flaresolverr)
flaresolverr|gluetun)
enableTraefik="false"
rules=$(jq -n '[]')
;;
@ -58,7 +80,7 @@ while read -r line ; do
enableTraefik="true"
# If Traefik enabled => define if http auth Traefik middleware must be set by default
case $key in
gluetun|kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli)
kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli)
defaultHttpAuth="false"
;;
*)
@ -69,7 +91,7 @@ while read -r line ; do
internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/")
rules=$(jq -n '[
{
"host": "'"$key"'",
"host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'",
"httpAuth": '"${defaultHttpAuth}"',
"internalPort": '"${internalPort}"',
}
@ -90,18 +112,7 @@ while read -r line ; do
rm -f config.json
mv tmp.json config.json
done < services.conf
# If we should enable Plex with hardware transcoding
# if grep -q -E "plex.*transcoding: enable" services.conf; then
# if grep -q "plex: disable" services.conf; then
# cat config.json | jq -r 'select(.services[].name=="plex") += {"plexHardwareTranscode":"enable"}' > tmp.json
# rm -f config.json
# mv tmp.json config.json
# fi
# fi
#mv config.json config.bak.json
done
# Transform json into yaml, easier to manipulate for the user
cat config.json | yq e -P - > config.yaml

231
config.sample.yaml Normal file
View File

@ -0,0 +1,231 @@
services:
- name: traefik
enabled: true
traefik:
enabled: true
rules:
- host: traefik.${TRAEFIK_DOMAIN}
service: api@internal
httpAuth: true
- name: deluge
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: deluge.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8112
scheme: http
- name: flood
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: flood.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 3000
- name: plex
enabled: true
vpn: false
# uncomment to use custom file with specific parameters for hardware transcoding
# customFile: plex-hardware-transcoding.yaml
traefik:
enabled: true
rules:
- host: plex.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 32400
- name: flaresolverr
enabled: true
vpn: false
traefik:
enabled: false
rules: []
- name: jackett
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: jackett.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 9117
- name: prowlarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: prowlarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 9696
- name: sonarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: sonarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8989
- name: radarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: radarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 7878
- name: bazarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: bazarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 6767
- name: lidarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: lidarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8686
- name: readarr
enabled: false
vpn: false
traefik:
enabled: true
rules:
- host: readarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8787
- name: komga
enabled: false
vpn: false
traefik:
enabled: true
rules:
- host: komga.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 8080
- name: kavita
enabled: false
vpn: false
traefik:
enabled: true
rules:
- host: kavita.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 5000
- name: ombi
enabled: false
vpn: false
traefik:
enabled: true
rules:
- host: ombi.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 3579
- name: overseerr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: overseerr.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 5055
- name: tautulli
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: tautulli.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 8181
- name: jdownloader
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: jdownloader.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 5800
- name: tdarr
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: tdarr.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8265
- name: nextcloud
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: nextcloud.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 443
- name: portainer
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: portainer.${TRAEFIK_DOMAIN}
httpAuth: false
internalPort: 9443
- name: netdata
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: netdata.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 19999
- name: duplicati
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: duplicati.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8200
- name: syncthing
enabled: false
vpn: false
traefik:
enabled: true
rules:
- host: syncthing.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 8384
- name: heimdall
enabled: true
vpn: false
traefik:
enabled: true
rules:
- host: heimdall.${TRAEFIK_DOMAIN}
httpAuth: true
internalPort: 80
- name: gluetun
enabled: false
vpn: false
traefik:
enabled: false
rules: []

24
config/ports Normal file
View File

@ -0,0 +1,24 @@
bazarr: 6767
deluge: 8112
flood: 3000
heimdall: 80
jackett: 9117
jdownloader: 5800
ombi: 3579
overseerr: 5055
plex: 32400
prowlarr: 9696
radarr: 7878
sonarr: 8989
lidarr: 8686
readarr: 8787
komga: 8080
kavita: 5000
nextcloud: 443
portainer: 9443
syncthing: 8384
tdarr: 8265
netdata: 19999
duplicati: 8200
tautulli: 8181
traefik: 80

View File

@ -0,0 +1,4 @@
# Services custom folder
Here, you can place all custom docker-compose files (yaml format) which will be specified in your ``config.yaml``.
All YAML files placed here are ignored in .gitignore.

237
update-all-v2.sh Executable file
View File

@ -0,0 +1,237 @@
#!/bin/bash
set -e
SKIP_PULL=0
DEBUG=0
for i in "$@"; do
case $i in
--no-pull)
SKIP_PULL=1
;;
--debug)
DEBUG=1
;;
*)
echo "[$0] ❌ ERROR: unknown parameter \"$i\""
exit 1
;;
esac
done
cleanup_on_exit() {
rm -f rules.props *-vpn.props config.json
}
trap cleanup_on_exit EXIT
echo-debug() {
[[ ${DEBUG} == "1" ]] && echo "$@"
}
# Cleanup files before start, in case there was a change we start from scratch at every script execution
rm -f config/*-vpn.yaml
# Create/update http_auth file according to values in .env file
source .env
echo "${HTTP_USER}:${HTTP_PASSWORD}" > traefik/http_auth
# Docker-compose settings
export COMPOSE_HTTP_TIMEOUT=240
# Retro-compatibility
[[ -z $HOST_CONFIG_PATH ]] && export HOST_CONFIG_PATH="/data/config"
[[ -z $HOST_MEDIA_PATH ]] && export HOST_MEDIA_PATH="/data/torrents"
[[ -z $DOWNLOAD_SUBFOLDER ]] && export DOWNLOAD_SUBFOLDER="deluge"
if [[ ! -f config.yaml ]]; then
echo "[$0] No config.yaml file found. Copying from sample file..."
cp config.sample.yaml config.yaml
fi
# Alert in case new services have been added (or removed) in sample but active file has not changed
# TODO: adapt to new config structure
# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l)
# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l)
# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then
# echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed."
# diff -yt services.conf services.conf.sample || true
# echo ""
# fi
###############################################################################################
###################################### Pre-flight checks ######################################
###############################################################################################
yq eval -o json config.yaml > config.json
# Check if some services have vpn enabled, that gluetun itself is enabled
nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length')
gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length')
if [[ ${nb_vpn} -gt 0 && ${gluetun_enabled} == 0 ]]; then
echo "[$0] ERROR. ${nb_vpn} VPN-enabled services have been enabled BUT gluetun has not been enabled. Please check your config.yaml file."
echo "[$0] ******* Exiting *******"
exit 1
fi
# Determine what host Flood should connect to
# => If deluge vpn is enabled => gluetun
# => If deluge vpn is disabled => deluge
if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled==true)] | length') -eq 1 ]]; then
# Check that if flood is enabled, deluge should also be enabled
if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==false)] | length') -eq 1 ]]; then
echo "[$0] ERROR. Flood is enabled but Deluge is not. Please either enable Deluge or disable Flood as Flood depends on Deluge."
echo "[$0] ******* Exiting *******"
exit 1
fi
if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then
export DELUGE_HOST="gluetun"
elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then
export DELUGE_HOST="deluge"
fi
# Specific instructions for Flood
# User for Deluge daemon RPC has to be created in deluge auth config file
if [[ ! -z ${FLOOD_PASSWORD} && ${FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON} == true ]]; then
if ! grep -q "flood" $HOST_CONFIG_PATH/deluge/auth; then
echo "flood:${FLOOD_PASSWORD}:10" >> $HOST_CONFIG_PATH/deluge/auth
else
echo "[$0] No need to add user/password for flood as it has already been created."
echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file."
fi
fi
fi
# Apply other arbitrary custom Traefik config files
rm -f $f traefik/custom/dynamic-*
for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do
echo "[$0] Applying custom Traefik config $f..."
cp $f traefik/custom/dynamic-$(basename $f)
done
# Detect Synology devices for Netdata compatibility
if [[ $(cat config.json | jq '[.services[] | select(.name=="netdata" and .enabled==true)] | length') -eq 1 ]]; then
if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then
export OS_RELEASE_FILEPATH="/etc/VERSION"
else
export OS_RELEASE_FILEPATH="/etc/os-release"
fi
fi
###############################################################################################
####################################### SERVICES PARSING ######################################
###############################################################################################
ALL_SERVICES="-f docker-compose.yaml"
# Parse the config.yaml master configuration file
for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do
name=$(echo $json | jq -r .name)
enabled=$(echo $json | jq -r .enabled)
vpn=$(echo $json | jq -r .vpn)
# Skip disabled services
if [[ ${enabled} == "false" ]]; then
echo-debug "[$0] Service $name is disabled. Skipping it."
continue
fi
echo-debug "[$0] ➡️ Parsing service: \"$name\"..."
# Default docker-compose filename is the service name + .yaml.
# Take into account explicit filename if specified in config
customFile=$(echo $json | jq -r .customFile)
file="$name.yaml"
if [[ ${customFile} != "null" ]]; then
file=${customFile}
fi
echo-debug "[$0] File: \"$file\"..."
# Append $file to global list of files which will be passed to docker commands
ALL_SERVICES="${ALL_SERVICES} -f services/${file}"
# For services with VPN enabled, add a docker-compose "override" file specifying that the service network should
# go through gluetun (main vpn client service).
if [[ ${vpn} == "true" ]]; then
echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props
yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml
rm -f ${name}-vpn.props
# Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands
ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml"
fi
###################################### TRAEFIK RULES ######################################
# Skip this part for services which have Traefik rules disabled in config
traefikEnabled=$(echo $json | jq -r .traefik.enabled)
if [[ ${traefikEnabled} == "false" ]]; then
echo-debug "[$0] Traefik is disabled. Skipping rules creation..."
continue
fi
# Loop over all Traefik rules and create the corresponding entries in the generated rules.yaml
echo-debug "[$0] Generating Traefik rules..."
i=0
for rule in $(echo $json | jq -c .traefik.rules[]); do
((i=i+1))
host=$(echo $rule | jq -r .host)
internalPort=$(echo $rule | jq -r .internalPort)
httpAuth=$(echo $rule | jq -r .httpAuth)
echo-debug "[$0] Host ${host}"
echo-debug "[$0] Internal Port ${internalPort}"
echo-debug "[$0] Http Authentication ${httpAuth}"
# If VPN => Traefik rule should redirect to gluetun container
backendHost=${name}
[[ ${vpn} == "true" ]] && backendHost="gluetun"
# Handle custom scheme (default if non-specified is http)
scheme="http"
customScheme=$(echo $rule | jq -r .scheme)
[[ ${customScheme} != "null" ]] && scheme=${customScheme}
# Transform the bash syntax into Traefik/go one => anything.${TRAEFIK_DOMAIN} to anything.{{ env "TRAEFIK_DOMAIN" }}
hostTraefik=$(echo ${host} | sed --regexp-extended 's/^(.*)(\$\{(.*)\})/\1\{\{ env "\3" \}\}/')
ruleId="${name}-${i}"
echo 'http.routers.'"${ruleId}"'.rule: Host(`'${hostTraefik}'`)' >> rules.props
if [[ ${httpAuth} == "true" ]]; then
echo "http.routers.${ruleId}.middlewares.0: common-auth@file" >> rules.props
fi
traefikService=$(echo $rule | jq -r .service)
if [[ ${traefikService} != "null" ]]; then
echo "http.routers.${ruleId}.service: ${traefikService}" >> rules.props
else
echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props
fi
# If the specified service does not contain a "@" => we create it
# If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...)
if echo ${traefikService} | grep -vq "@"; then
echo "http.services.${ruleId}.loadBalancer.servers.0.url: ${scheme}://${backendHost}:${internalPort}" >> rules.props
fi
done
done
# Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik
yq -p=props rules.props > traefik/custom/dynamic-rules.yaml
rm -f rules.props
# echo ${ALL_SERVICES}
if [[ "${SKIP_PULL}" != "1" ]]; then
echo "[$0] ***** Pulling all images... *****"
docker-compose ${ALL_SERVICES} pull
fi
echo "[$0] ***** Recreating containers if required... *****"
docker-compose ${ALL_SERVICES} up -d --remove-orphans
echo "[$0] ***** Done updating containers *****"
echo "[$0] ***** Clean unused images and volumes... *****"
docker image prune -af
docker volume prune -f
echo "[$0] ***** Done! *****"
exit 0