From 30f55a62283b47f1c00f18c1c8535226c2c39785 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 18:32:34 +0100 Subject: [PATCH 01/42] Add komga --- README.md | 1 + services.conf.sample | 1 + services/komga.yaml | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 services/komga.yaml diff --git a/README.md b/README.md index 6af1570..7ad82f7 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ seedbox and personal media server. | Radarr | radarr.yourdomain.com | [linuxserver/radarr](https://hub.docker.com/r/linuxserver/radarr) | *develop* | Movies monitor | | Bazarr | bazarr.yourdomain.com | [linuxserver/bazarr](https://hub.docker.com/r/linuxserver/bazarr) | *latest* | Subtitles monitor | | Lidarr | lidarr.yourdomain.com | [linuxserver/lidarr](https://hub.docker.com/r/linuxserver/lidarr) | *develop* | Music monitor | +| Komga | komga.yourdomain.com | [gotson/komga](https://hub.docker.com/r/gotson/komga) | *latest* | Comic Book Manager | | Ombi | ombi.yourdomain.com | [linuxserver/ombi](https://hub.docker.com/r/linuxserver/ombi) | *latest* | Plex content requests | | Overseerr | overseerr.yourdomain.com | [linuxserver/overseerr](https://hub.docker.com/r/linuxserver/overseerr) | *latest* | Plex content requests | | Jackett | jackett.yourdomain.com | [linuxserver/jackett](https://hub.docker.com/r/linuxserver/jackett) | *latest* | Tracker indexer | diff --git a/services.conf.sample b/services.conf.sample index 3d4078d..e38fd4a 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -8,6 +8,7 @@ sonarr: enable radarr: enable bazarr: enable lidarr: enable +komga: disable ombi: disable overseerr: enable tautulli: enable diff --git a/services/komga.yaml b/services/komga.yaml new file mode 100644 index 0000000..cfbc15a --- /dev/null +++ b/services/komga.yaml @@ -0,0 +1,22 @@ +services: + komga: + image: gotson/komga + container_name: komga + restart: always + volumes: + - configkomga:/config + - torrents:/data + - /etc/localtime:/etc/localtime:ro + environment: + - TZ=${TZ} + user: "${PUID}:${PGID}" + labels: + - "traefik.enable=true" + - "traefik.http.routers.komga.rule=Host(`komga.${TRAEFIK_DOMAIN}`)" + - "traefik.http.routers.komga.middlewares=common-auth@file" + +volumes: + configkomga: + driver: local-persist + driver_opts: + mountpoint: /data/config/komga \ No newline at end of file From ba6a22f931cb4b896c499cb2e6d9c0ac83c82304 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 18:43:39 +0100 Subject: [PATCH 02/42] Komga: remove http auth --- services/komga.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/services/komga.yaml b/services/komga.yaml index cfbc15a..e8cdd88 100644 --- a/services/komga.yaml +++ b/services/komga.yaml @@ -13,7 +13,6 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.komga.rule=Host(`komga.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.komga.middlewares=common-auth@file" volumes: configkomga: From 8efee6c10ce746c1b30dd39493a199981ff320dd Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 18:47:54 +0100 Subject: [PATCH 03/42] Change /data for Komga --- services/komga.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/komga.yaml b/services/komga.yaml index e8cdd88..4ee2571 100644 --- a/services/komga.yaml +++ b/services/komga.yaml @@ -5,7 +5,7 @@ services: restart: always volumes: - configkomga:/config - - torrents:/data + - torrents:/torrents - /etc/localtime:/etc/localtime:ro environment: - TZ=${TZ} From 1dd8c363918f645b550505c31c9893ea545ee2d5 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 18:53:53 +0100 Subject: [PATCH 04/42] Add readarr --- README.md | 1 + services.conf.sample | 1 + services/readarr.yaml | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 services/readarr.yaml diff --git a/README.md b/README.md index 7ad82f7..38ff502 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ seedbox and personal media server. | Radarr | radarr.yourdomain.com | [linuxserver/radarr](https://hub.docker.com/r/linuxserver/radarr) | *develop* | Movies monitor | | Bazarr | bazarr.yourdomain.com | [linuxserver/bazarr](https://hub.docker.com/r/linuxserver/bazarr) | *latest* | Subtitles monitor | | Lidarr | lidarr.yourdomain.com | [linuxserver/lidarr](https://hub.docker.com/r/linuxserver/lidarr) | *develop* | Music monitor | +| Readarr | readarr.yourdomain.com | [linuxserver/readarr](https://hub.docker.com/r/linuxserver/readarr) | *nightly* | Ebook and comic monitor | | Komga | komga.yourdomain.com | [gotson/komga](https://hub.docker.com/r/gotson/komga) | *latest* | Comic Book Manager | | Ombi | ombi.yourdomain.com | [linuxserver/ombi](https://hub.docker.com/r/linuxserver/ombi) | *latest* | Plex content requests | | Overseerr | overseerr.yourdomain.com | [linuxserver/overseerr](https://hub.docker.com/r/linuxserver/overseerr) | *latest* | Plex content requests | diff --git a/services.conf.sample b/services.conf.sample index e38fd4a..f8c9cff 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -8,6 +8,7 @@ sonarr: enable radarr: enable bazarr: enable lidarr: enable +readarr: false komga: disable ombi: disable overseerr: enable diff --git a/services/readarr.yaml b/services/readarr.yaml new file mode 100644 index 0000000..f50cdcc --- /dev/null +++ b/services/readarr.yaml @@ -0,0 +1,23 @@ +services: + readarr: + image: lscr.io/linuxserver/readarr:nightly + container_name: readarr + restart: always + volumes: + - configreadarr:/config + - torrents:/torrents + - /etc/localtime:/etc/localtime:ro + environment: + - PGID=${PGID} + - PUID=${PUID} + - TZ=${TZ} + labels: + - "traefik.enable=true" + - "traefik.http.routers.readarr.rule=Host(`readarr.${TRAEFIK_DOMAIN}`)" + - "traefik.http.routers.readarr.middlewares=common-auth@file" + +volumes: + configreadarr: + driver: local-persist + driver_opts: + mountpoint: /data/config/readarr \ No newline at end of file From a80ee08bf427dbbbef039590a7ebdf33bb231d7e Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 22:16:24 +0100 Subject: [PATCH 05/42] Add Kavita --- README.md | 1 + services.conf.sample | 1 + services/kavita.yaml | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 services/kavita.yaml diff --git a/README.md b/README.md index 38ff502..cf79a29 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ seedbox and personal media server. | Lidarr | lidarr.yourdomain.com | [linuxserver/lidarr](https://hub.docker.com/r/linuxserver/lidarr) | *develop* | Music monitor | | Readarr | readarr.yourdomain.com | [linuxserver/readarr](https://hub.docker.com/r/linuxserver/readarr) | *nightly* | Ebook and comic monitor | | Komga | komga.yourdomain.com | [gotson/komga](https://hub.docker.com/r/gotson/komga) | *latest* | Comic Book Manager | +| Kavita | Kavita.yourdomain.com | [gotson/komga](https://hub.docker.com/r/gotson/komga) | *latest* | Comic Book Manager | | Ombi | ombi.yourdomain.com | [linuxserver/ombi](https://hub.docker.com/r/linuxserver/ombi) | *latest* | Plex content requests | | Overseerr | overseerr.yourdomain.com | [linuxserver/overseerr](https://hub.docker.com/r/linuxserver/overseerr) | *latest* | Plex content requests | | Jackett | jackett.yourdomain.com | [linuxserver/jackett](https://hub.docker.com/r/linuxserver/jackett) | *latest* | Tracker indexer | diff --git a/services.conf.sample b/services.conf.sample index f8c9cff..3d7a5e3 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -10,6 +10,7 @@ bazarr: enable lidarr: enable readarr: false komga: disable +kavita: disable ombi: disable overseerr: enable tautulli: enable diff --git a/services/kavita.yaml b/services/kavita.yaml new file mode 100644 index 0000000..35f7fcb --- /dev/null +++ b/services/kavita.yaml @@ -0,0 +1,22 @@ +services: + kavita: + image: kizaing/kavita:latest + container_name: kavita + restart: always + volumes: + - configkavita:/kavita/config + - torrents:/torrents + - /etc/localtime:/etc/localtime:ro + environment: + - TZ=${TZ} + user: "${PUID}:${PGID}" + labels: + - "traefik.enable=true" + - "traefik.http.routers.kavita.rule=Host(`kavita.${TRAEFIK_DOMAIN}`)" + - "traefik.http.routers.readarr.middlewares=common-auth@file" + +volumes: + configkavita: + driver: local-persist + driver_opts: + mountpoint: /data/config/kavita \ No newline at end of file From befa3942ca3d3f2d6c4224723911f829904a319a Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 22:18:19 +0100 Subject: [PATCH 06/42] Run kavita as default --- services/kavita.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/kavita.yaml b/services/kavita.yaml index 35f7fcb..053ee85 100644 --- a/services/kavita.yaml +++ b/services/kavita.yaml @@ -9,7 +9,7 @@ services: - /etc/localtime:/etc/localtime:ro environment: - TZ=${TZ} - user: "${PUID}:${PGID}" + #user: "${PUID}:${PGID}" labels: - "traefik.enable=true" - "traefik.http.routers.kavita.rule=Host(`kavita.${TRAEFIK_DOMAIN}`)" From b76c94af6fc843cfe8a289ae23096f0bb6de86ae Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 22:21:22 +0100 Subject: [PATCH 07/42] Fix label on kavita for auth --- services/kavita.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/kavita.yaml b/services/kavita.yaml index 053ee85..527f7d4 100644 --- a/services/kavita.yaml +++ b/services/kavita.yaml @@ -13,7 +13,7 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.kavita.rule=Host(`kavita.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.readarr.middlewares=common-auth@file" + - "traefik.http.routers.kavita.middlewares=common-auth@file" volumes: configkavita: From ef286306c41436c0a4ce1f5ed7d5cf84fc2af5c1 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 6 Dec 2021 22:24:05 +0100 Subject: [PATCH 08/42] Remove kavita reverse proxy auth --- services/kavita.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/kavita.yaml b/services/kavita.yaml index 527f7d4..3635c35 100644 --- a/services/kavita.yaml +++ b/services/kavita.yaml @@ -9,11 +9,11 @@ services: - /etc/localtime:/etc/localtime:ro environment: - TZ=${TZ} + # Wait for https://github.com/Kareadita/Kavita/issues/301 to be resolved #user: "${PUID}:${PGID}" labels: - "traefik.enable=true" - "traefik.http.routers.kavita.rule=Host(`kavita.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.kavita.middlewares=common-auth@file" volumes: configkavita: From e4ede925a8ce09b177206f30487a889da9e10334 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 00:24:04 +0100 Subject: [PATCH 09/42] Add VPN support (modularizable) + refactoring + variables changes --- .env.sample | 15 ++++++- .gitignore | 1 + README.md | 6 ++- docker-compose.yaml | 11 +++-- samples/traefik-deluge-vpn.yaml | 12 ++++++ samples/traefik-jdownloader-vpn.yaml | 12 ++++++ services.conf.sample | 5 ++- services/bazarr.yaml | 2 +- services/deluge-vpn.yaml | 21 ++++++++++ services/deluge.yaml | 2 +- services/duplicati.yaml | 10 ++--- services/flaresolverr.yaml | 0 services/flood.yaml | 4 +- services/gluetun.yaml | 34 ++++++++++++++++ services/jackett.yaml | 0 services/jdownloader-vpn.yaml | 20 +++++++++ services/jdownloader.yaml | 2 +- services/kavita.yaml | 2 +- services/komga.yaml | 2 +- services/lidarr.yaml | 2 +- services/netdata.yaml | 0 services/nextcloud.yaml | 6 +-- services/ombi.yaml | 2 +- services/overseerr.yaml | 2 +- services/plex-hardware-transcoding.yaml | 31 ++++++++++++++ services/plex.yaml | 4 +- services/portainer.yaml | 0 services/prowlarr.yaml | 2 +- services/radarr.yaml | 2 +- services/readarr.yaml | 2 +- services/sonarr.yaml | 2 +- services/tautulli.yaml | 2 +- services/tdarr.yaml | 6 +-- services/traefik.yaml | 2 +- update-all.sh | 54 +++++++++++++++++++++++-- 35 files changed, 235 insertions(+), 45 deletions(-) create mode 100755 samples/traefik-deluge-vpn.yaml create mode 100755 samples/traefik-jdownloader-vpn.yaml mode change 100644 => 100755 services/bazarr.yaml create mode 100755 services/deluge-vpn.yaml mode change 100644 => 100755 services/deluge.yaml mode change 100644 => 100755 services/duplicati.yaml mode change 100644 => 100755 services/flaresolverr.yaml mode change 100644 => 100755 services/flood.yaml create mode 100755 services/gluetun.yaml mode change 100644 => 100755 services/jackett.yaml create mode 100755 services/jdownloader-vpn.yaml mode change 100644 => 100755 services/jdownloader.yaml mode change 100644 => 100755 services/kavita.yaml mode change 100644 => 100755 services/komga.yaml mode change 100644 => 100755 services/lidarr.yaml mode change 100644 => 100755 services/netdata.yaml mode change 100644 => 100755 services/nextcloud.yaml mode change 100644 => 100755 services/ombi.yaml mode change 100644 => 100755 services/overseerr.yaml create mode 100755 services/plex-hardware-transcoding.yaml mode change 100644 => 100755 services/plex.yaml mode change 100644 => 100755 services/portainer.yaml mode change 100644 => 100755 services/prowlarr.yaml mode change 100644 => 100755 services/radarr.yaml mode change 100644 => 100755 services/readarr.yaml mode change 100644 => 100755 services/sonarr.yaml mode change 100644 => 100755 services/tautulli.yaml mode change 100644 => 100755 services/tdarr.yaml mode change 100644 => 100755 services/traefik.yaml diff --git a/.env.sample b/.env.sample index 7978b18..b4696b1 100644 --- a/.env.sample +++ b/.env.sample @@ -10,7 +10,11 @@ TZ="Europe/Paris" HTTP_USER=myuser HTTP_PASSWORD='mypassword_encoded' # Keep these simple quotes! -# Containers permissions mapping +# Host paths + containers permissions mapping +HOST_CONFIG_PATH="/data/config" +HOST_MEDIA_PATH="/data/torrents" +# Will be located in $HOST_MEDIA_PATH +DOWNLOAD_SUBFOLDER="deluge" PGID=1000 PUID=1000 @@ -29,4 +33,11 @@ PORTAINER_ADMIN_PASSWORD=h4ckMePleAse # Flood username declared in deluge rpc daemon FLOOD_PASSWORD=myfloodpassword -FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON=false \ No newline at end of file +FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON=false + +# Wireguard custom endpoint +WIREGUARD_ENDPOINT= +WIREGUARD_PORT=51820 +WIREGUARD_PUBLIC_KEY= +WIREGUARD_PRIVATE_KEY= +WIREGUARD_ADDRESS= \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8655bd..579b1b4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ **/traefik/http_auth backup/ services.conf +traefik/custom/dynamic*.yaml \ No newline at end of file diff --git a/README.md b/README.md index cf79a29..05a0032 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ seedbox and personal media server. -----------------------|----------------------------|------------------------------------------------------------------------|-------------------------|---------------------| | Plex | plex.yourdomain.com | [linuxserver/plex](https://hub.docker.com/r/linuxserver/plex) | *latest* | Media Streaming | | Deluge | deluge.yourdomain.com | [linuxserver/deluge](https://hub.docker.com/r/linuxserver/deluge) | *latest* | Torrents downloader | +| Deluge (VPN) | deluge.yourdomain.com | [linuxserver/deluge](https://hub.docker.com/r/linuxserver/deluge) | *latest* | Torrents downloader (behind VPN) | | Flood | flood.yourdomain.com | [jesec/flood](https://hub.docker.com/r/jesec/flood) | *latest* | Web client for Deluge (experimental) | | Sonarr | sonarr.yourdomain.com | [linuxserver/sonarr](https://hub.docker.com/r/linuxserver/sonarr) | *develop* | TV Shows monitor | | Radarr | radarr.yourdomain.com | [linuxserver/radarr](https://hub.docker.com/r/linuxserver/radarr) | *develop* | Movies monitor | @@ -22,6 +23,7 @@ seedbox and personal media server. | Jackett | jackett.yourdomain.com | [linuxserver/jackett](https://hub.docker.com/r/linuxserver/jackett) | *latest* | Tracker indexer | | Prowlarr | prowlarr.yourdomain.com | [linuxserver/prowlarr](https://hub.docker.com/r/linuxserver/prowlarr) | *develop* | Tracker indexer | | JDownloader | jdownloader.yourdomain.com | [jlesage/jdownloader-2](https://hub.docker.com/r/jlesage/jdownloader-2)| *latest* | Direct downloader | +| JDownloader (VPN) | jdownloader.yourdomain.com | [jlesage/jdownloader-2](https://hub.docker.com/r/jlesage/jdownloader-2)| *latest* | Direct downloader (behind VPN) | | Tautulli (plexPy) | tautulli.yourdomain.com | [linuxserver/tautulli](https://hub.docker.com/r/linuxserver/tautulli) | *latest* | Plex stats and admin| | Tdarr | tdarr.yourdomain.com | [haveagitgat/tdarr](https://hub.docker.com/r/haveagitgat/tdarr) | *latest* | Re-encode files | | NextCloud | nextcloud.yourdomain.com | [linuxserver/nextcloud](https://hub.docker.com/r/linuxserver/nextcloud) | *latest* | Files management | @@ -29,6 +31,7 @@ seedbox and personal media server. | Portainer | portainer.yourdomain.com | [portainer/portainer](https://hub.docker.com/r/portainer/portainer) | *latest* | Container management| | Netdata | netdata.yourdomain.com | [netdata/netdata](https://hub.docker.com/r/netdata/netdata) | *latest* | Server monitoring | | Duplicati | duplicati.yourdomain.com | [linuxserver/duplicati](https://hub.docker.com/r/linuxserver/duplicati)| *latest* | Backups | +| Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client (still WIP...) | The front-end reverse proxy (Traefik - **check the next section if you have already the seedbox with Traefik v1**) routes based on the lowest level subdomain (e.g. `deluge.example.com` would route to deluge). Since this is how the router @@ -77,6 +80,7 @@ sudo rm -rf /opt/traefik /tmp/migration ## Configuration Before running, please create the volumes which will be statically mapped to the ones on the host: +For example: ```sh sudo su -c "mkdir /data && mkdir /data/config && mkdir /data/torrents" @@ -106,7 +110,7 @@ this. ## PlexPass Just set the `VERSION` environment variable to `latest` on the Plex service (enabled by default). -See https://hub.docker.com/r/linuxserver/plex. +See [this link](https://hub.docker.com/r/linuxserver/plex). ## Where is my data? diff --git a/docker-compose.yaml b/docker-compose.yaml index 9d62242..97e8dcb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,21 +6,20 @@ version: "3.8" # Common network used by all services networks: - default: - external: - name: "traefik-network" + default: + name: "traefik-network" # Common volumes used by at least 2 services volumes: config: driver: local-persist driver_opts: - mountpoint: /data/config + mountpoint: $HOST_CONFIG_PATH torrents: driver: local-persist driver_opts: - mountpoint: /data/torrents + mountpoint: $HOST_MEDIA_PATH downloads: driver: local-persist driver_opts: - mountpoint: /data/torrents/deluge + mountpoint: $HOST_MEDIA_PATH/$DOWNLOAD_SUBFOLDER diff --git a/samples/traefik-deluge-vpn.yaml b/samples/traefik-deluge-vpn.yaml new file mode 100755 index 0000000..01ec4cb --- /dev/null +++ b/samples/traefik-deluge-vpn.yaml @@ -0,0 +1,12 @@ +http: + routers: + deluge: + rule: 'Host(`deluge.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + service: deluge + services: + deluge: + loadBalancer: + servers: + - url: "http://gluetun:8112" \ No newline at end of file diff --git a/samples/traefik-jdownloader-vpn.yaml b/samples/traefik-jdownloader-vpn.yaml new file mode 100755 index 0000000..2a01bf8 --- /dev/null +++ b/samples/traefik-jdownloader-vpn.yaml @@ -0,0 +1,12 @@ +http: + routers: + jdownloader: + rule: 'Host(`jdownloader.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + service: deluge + services: + jdownloader: + loadBalancer: + servers: + - url: "http://gluetun:5800" \ No newline at end of file diff --git a/services.conf.sample b/services.conf.sample index 3d7a5e3..918c9ba 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -1,6 +1,8 @@ deluge: enable +deluge-vpn: disable flood: enable plex: enable +plex-hardware-transcoding: disable flaresolverr: enable jackett: enable prowlarr: enable @@ -19,4 +21,5 @@ tdarr: enable nextcloud: enable portainer: enable netdata: enable -duplicati: enable \ No newline at end of file +duplicati: enable +gluetun: disable \ No newline at end of file diff --git a/services/bazarr.yaml b/services/bazarr.yaml old mode 100644 new mode 100755 index 568ad3d..214a913 --- a/services/bazarr.yaml +++ b/services/bazarr.yaml @@ -19,4 +19,4 @@ volumes: configbazarr: driver: local-persist driver_opts: - mountpoint: /data/config/bazarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/bazarr \ No newline at end of file diff --git a/services/deluge-vpn.yaml b/services/deluge-vpn.yaml new file mode 100755 index 0000000..7a2f576 --- /dev/null +++ b/services/deluge-vpn.yaml @@ -0,0 +1,21 @@ +services: + deluge: + image: ghcr.io/linuxserver/deluge + container_name: deluge + restart: always + network_mode: "service:gluetun" + volumes: + - torrents:/torrents + - configdeluge:/config + - downloads:/downloads + environment: + - PGID=${PGID} + - PUID=${PUID} + - TZ=${TZ} + # Traefik labels are in Gluetun YAML as deluge must be accessed via Gluetun + +volumes: + configdeluge: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/deluge \ No newline at end of file diff --git a/services/deluge.yaml b/services/deluge.yaml old mode 100644 new mode 100755 index d304ae3..08fa75d --- a/services/deluge.yaml +++ b/services/deluge.yaml @@ -20,4 +20,4 @@ volumes: configdeluge: driver: local-persist driver_opts: - mountpoint: /data/config/deluge \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/deluge \ No newline at end of file diff --git a/services/duplicati.yaml b/services/duplicati.yaml old mode 100644 new mode 100755 index ce6aef0..c344916 --- a/services/duplicati.yaml +++ b/services/duplicati.yaml @@ -10,22 +10,18 @@ services: volumes: - configduplicati:/config - backups:/backups - - alldata:/source + - config:/source labels: - "traefik.enable=true" - "traefik.http.routers.duplicati.rule=Host(`duplicati.${TRAEFIK_DOMAIN}`)" - "traefik.http.routers.duplicati.middlewares=common-auth@file" volumes: - alldata: - driver: local-persist - driver_opts: - mountpoint: /data configduplicati: driver: local-persist driver_opts: - mountpoint: /data/config/duplicati + mountpoint: $HOST_CONFIG_PATH/duplicati backups: driver: local-persist driver_opts: - mountpoint: /data/backups \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/backups \ No newline at end of file diff --git a/services/flaresolverr.yaml b/services/flaresolverr.yaml old mode 100644 new mode 100755 diff --git a/services/flood.yaml b/services/flood.yaml old mode 100644 new mode 100755 index 8a89089..dc647ca --- a/services/flood.yaml +++ b/services/flood.yaml @@ -8,7 +8,7 @@ services: user: ${PUID}:${PGID} command: - --auth=none - - --dehost=deluge + - --dehost=${DELUGE_HOST} - --deport=58846 - --deuser=flood - --depass=${FLOOD_PASSWORD} @@ -29,4 +29,4 @@ volumes: configflood: driver: local-persist driver_opts: - mountpoint: /data/config/flood \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/flood \ No newline at end of file diff --git a/services/gluetun.yaml b/services/gluetun.yaml new file mode 100755 index 0000000..fd4d3cc --- /dev/null +++ b/services/gluetun.yaml @@ -0,0 +1,34 @@ +services: + gluetun: + image: qmcgaw/gluetun + container_name: gluetun + restart: always + cap_add: + - NET_ADMIN + environment: + - PUID=${PUID} + - PGIDq=${PGID} + - TZ=${TZ} + - VPNSP=custom + - VPN_TYPE=wireguard + # For Wireguard + - VPN_ENDPOINT_IP=${WIREGUARD_ENDPOINT} + - VPN_ENDPOINT_PORT=${WIREGUARD_PORT} + - WIREGUARD_PUBLIC_KEY=${WIREGUARD_PUBLIC_KEY} + - WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY} + - WIREGUARD_ADDRESS=${WIREGUARD_ADDRESS} + #labels: + # Control Panel + # - "traefik.enable=true" + # - "traefik.http.routers.gluetun.rule=Host(`gluetun.${TRAEFIK_DOMAIN}`)" + # - "traefik.http.routers.gluetun.middlewares=common-auth@file" + # - "traefik.http.routers.gluetun.service=gluetun" + # - "traefik.http.services.gluetun.loadbalancer.server.port=8000" + # Traefik rules for VPN-enabled services (which have ot be accessed through gluetun) are defined in the samples/ directory + # and are applied automatically in the update-all.sh script. + +volumes: + configgluetun: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/gluetun \ No newline at end of file diff --git a/services/jackett.yaml b/services/jackett.yaml old mode 100644 new mode 100755 diff --git a/services/jdownloader-vpn.yaml b/services/jdownloader-vpn.yaml new file mode 100755 index 0000000..3eb682e --- /dev/null +++ b/services/jdownloader-vpn.yaml @@ -0,0 +1,20 @@ +services: + jdownloader: + image: jlesage/jdownloader-2 + container_name: jdownloader + restart: always + network_mode: "service:gluetun" + volumes: + - configjdownloader:/config + - downloads:/output + environment: + - USER_ID=${PUID} + - GROUP_ID=${PGID} + - TZ=${TZ} + # Traefik routing rules are defined in the samples/ directory and applied automatically when this service is enabled + +volumes: + configjdownloader: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/jdownloader \ No newline at end of file diff --git a/services/jdownloader.yaml b/services/jdownloader.yaml old mode 100644 new mode 100755 index 4850c36..34f2f02 --- a/services/jdownloader.yaml +++ b/services/jdownloader.yaml @@ -20,4 +20,4 @@ volumes: configjdownloader: driver: local-persist driver_opts: - mountpoint: /data/config/jdownloader \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/jdownloader \ No newline at end of file diff --git a/services/kavita.yaml b/services/kavita.yaml old mode 100644 new mode 100755 index 3635c35..034b85f --- a/services/kavita.yaml +++ b/services/kavita.yaml @@ -19,4 +19,4 @@ volumes: configkavita: driver: local-persist driver_opts: - mountpoint: /data/config/kavita \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/kavita \ No newline at end of file diff --git a/services/komga.yaml b/services/komga.yaml old mode 100644 new mode 100755 index 4ee2571..d61f597 --- a/services/komga.yaml +++ b/services/komga.yaml @@ -18,4 +18,4 @@ volumes: configkomga: driver: local-persist driver_opts: - mountpoint: /data/config/komga \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/komga \ No newline at end of file diff --git a/services/lidarr.yaml b/services/lidarr.yaml old mode 100644 new mode 100755 index d13b01f..29cc1cf --- a/services/lidarr.yaml +++ b/services/lidarr.yaml @@ -19,4 +19,4 @@ volumes: configlidarr: driver: local-persist driver_opts: - mountpoint: /data/config/lidarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/lidarr \ No newline at end of file diff --git a/services/netdata.yaml b/services/netdata.yaml old mode 100644 new mode 100755 diff --git a/services/nextcloud.yaml b/services/nextcloud.yaml old mode 100644 new mode 100755 index 094054f..9600d10 --- a/services/nextcloud.yaml +++ b/services/nextcloud.yaml @@ -41,12 +41,12 @@ volumes: nextclouddb: driver: local-persist driver_opts: - mountpoint: /data/nextcloud-db + mountpoint: $HOST_CONFIG_PATH/nextcloud-db confignextcloud: driver: local-persist driver_opts: - mountpoint: /data/config/nextcloud + mountpoint: $HOST_CONFIG_PATH/nextcloud nextclouddata: driver: local-persist driver_opts: - mountpoint: /data/config/nextcloud-data \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/nextcloud-data \ No newline at end of file diff --git a/services/ombi.yaml b/services/ombi.yaml old mode 100644 new mode 100755 index ab41549..e4a47a4 --- a/services/ombi.yaml +++ b/services/ombi.yaml @@ -19,4 +19,4 @@ volumes: configombi: driver: local-persist driver_opts: - mountpoint: /data/config/ombi \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/ombi \ No newline at end of file diff --git a/services/overseerr.yaml b/services/overseerr.yaml old mode 100644 new mode 100755 index 0c9a0bb..1abbb74 --- a/services/overseerr.yaml +++ b/services/overseerr.yaml @@ -17,4 +17,4 @@ volumes: configoverseerr: driver: local-persist driver_opts: - mountpoint: /data/config/overseerr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/overseerr \ No newline at end of file diff --git a/services/plex-hardware-transcoding.yaml b/services/plex-hardware-transcoding.yaml new file mode 100755 index 0000000..44c24c5 --- /dev/null +++ b/services/plex-hardware-transcoding.yaml @@ -0,0 +1,31 @@ +services: + plex: + image: ghcr.io/linuxserver/plex + container_name: plex + restart: always + ports: + - "32400:32400" + - "32400:32400/udp" + - "32469:32469" + - "32469:32469/udp" + devices: + - /dev/dri:/dev/dri # for hardware transcoding + volumes: + - configplex:/config + - torrents:/torrents + - /dev/shm:/transcode + environment: + - PGID=${PGID} + - PUID=${PUID} + - TZ=${TZ} + - VERSION=latest + labels: + - "traefik.enable=true" + - "traefik.http.services.plex-seedbox.loadbalancer.server.port=32400" + - "traefik.http.routers.plex.rule=Host(`plex.${TRAEFIK_DOMAIN}`)" + +volumes: + configplex: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/Plex \ No newline at end of file diff --git a/services/plex.yaml b/services/plex.yaml old mode 100644 new mode 100755 index 33d1b5c..ebe3984 --- a/services/plex.yaml +++ b/services/plex.yaml @@ -8,8 +8,6 @@ services: - "32400:32400/udp" - "32469:32469" - "32469:32469/udp" - - "5353:5353/udp" - - "1900:1900/udp" volumes: - configplex:/config - torrents:/torrents @@ -27,4 +25,4 @@ volumes: configplex: driver: local-persist driver_opts: - mountpoint: /data/config/Plex \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/Plex \ No newline at end of file diff --git a/services/portainer.yaml b/services/portainer.yaml old mode 100644 new mode 100755 diff --git a/services/prowlarr.yaml b/services/prowlarr.yaml old mode 100644 new mode 100755 index 9c2f910..0524a56 --- a/services/prowlarr.yaml +++ b/services/prowlarr.yaml @@ -18,4 +18,4 @@ volumes: configprowlarr: driver: local-persist driver_opts: - mountpoint: /data/config/prowlarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/prowlarr \ No newline at end of file diff --git a/services/radarr.yaml b/services/radarr.yaml old mode 100644 new mode 100755 index 65fc73d..a8d8286 --- a/services/radarr.yaml +++ b/services/radarr.yaml @@ -20,4 +20,4 @@ volumes: configradarr: driver: local-persist driver_opts: - mountpoint: /data/config/radarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/radarr \ No newline at end of file diff --git a/services/readarr.yaml b/services/readarr.yaml old mode 100644 new mode 100755 index f50cdcc..f4b8dd0 --- a/services/readarr.yaml +++ b/services/readarr.yaml @@ -20,4 +20,4 @@ volumes: configreadarr: driver: local-persist driver_opts: - mountpoint: /data/config/readarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/readarr \ No newline at end of file diff --git a/services/sonarr.yaml b/services/sonarr.yaml old mode 100644 new mode 100755 index 6ce4d6f..29100a0 --- a/services/sonarr.yaml +++ b/services/sonarr.yaml @@ -20,4 +20,4 @@ volumes: configsonarr: driver: local-persist driver_opts: - mountpoint: /data/config/sonarr \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/sonarr \ No newline at end of file diff --git a/services/tautulli.yaml b/services/tautulli.yaml old mode 100644 new mode 100755 index 5470396..438e467 --- a/services/tautulli.yaml +++ b/services/tautulli.yaml @@ -18,4 +18,4 @@ volumes: configtautulli: driver: local-persist driver_opts: - mountpoint: /data/config/tautulli \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/tautulli \ No newline at end of file diff --git a/services/tdarr.yaml b/services/tdarr.yaml old mode 100644 new mode 100755 index d5ddffc..01e6639 --- a/services/tdarr.yaml +++ b/services/tdarr.yaml @@ -45,12 +45,12 @@ volumes: configtdarrv2: driver: local-persist driver_opts: - mountpoint: /data/config/tdarrv2/server + mountpoint: $HOST_CONFIG_PATH/tdarrv2/server settingstdarrv2: driver: local-persist driver_opts: - mountpoint: /data/config/tdarrv2/configs + mountpoint: $HOST_CONFIG_PATH/tdarrv2/configs transcodes: driver: local-persist driver_opts: - mountpoint: /data/config/tdarrv2/transcodes \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/tdarrv2/transcodes \ No newline at end of file diff --git a/services/traefik.yaml b/services/traefik.yaml old mode 100644 new mode 100755 index d216921..bf97be0 --- a/services/traefik.yaml +++ b/services/traefik.yaml @@ -27,4 +27,4 @@ volumes: configtraefik: driver: local-persist driver_opts: - mountpoint: /data/config/traefik \ No newline at end of file + mountpoint: $HOST_CONFIG_PATH/traefik \ No newline at end of file diff --git a/update-all.sh b/update-all.sh index c77d1f0..fb7b8be 100755 --- a/update-all.sh +++ b/update-all.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + SKIP_PULL=0 for i in "$@"; do @@ -21,6 +23,11 @@ echo "${HTTP_USER}:${HTTP_PASSWORD}" > traefik/http_auth # Docker-compose settings export COMPOSE_HTTP_TIMEOUT=240 +# Retro-compatibility +[[ -z $HOST_CONFIG_PATH ]] && export HOST_CONFIG_PATH="/data/config" +[[ -z $HOST_MEDIA_PATH ]] && export HOST_MEDIA_PATH="/data/torrents" +[[ -z $DOWNLOAD_SUBFOLDER ]] && export DOWNLOAD_SUBFOLDER="deluge" + if [[ ! -f services.conf ]]; then echo "[$0] No services.conf file found. Copying from sample file..." cp services.conf.sample services.conf @@ -31,9 +38,50 @@ NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." - diff -yt services.conf services.conf.sample + diff -yt services.conf services.conf.sample || true + echo "" fi +# Check if *-vpn services are enabled. If so, check that gluetun is enabled. +if [[ $(cat services.conf | { grep -E ".*vpn: enable" || true; } | wc -l) -ge 1 ]]; then + if [[ $(cat services.conf | { grep "gluetun: enable" || true; } | wc -l) -eq 0 ]]; then + echo "[$0] ERROR. A VPN-enabled service has been enabled BUT gluetun has not been enabled. Please check your services.conf file." + echo "******* Exiting *******" + exit 1 + fi +fi + +# Check if there are no conflict in enabled services (for example, you cannot enable deluge AND deluge-vpn) +for svc in deluge plex jdownloader; do + if [[ $(cat services.conf | { grep -E "${svc}.*: enable" || true; } | wc -l) -gt 1 ]]; then + echo "[$0] ERROR. You cannot enable multiple ${svc^} services simultaneously. Please edit this section in your services.conf file:" + cat services.conf | { grep -E "${svc}.*: enable" || true; } + echo "******* Exiting *******" + exit 1 + fi +done + +# Determine what host Flood should connect to +# => If deluge-vpn is enabled => gluetun +# => If deluge is enabled => deluge +if [[ $(cat services.conf | { grep -E "flood\: enable" || true; } | wc -l) -eq 1 ]]; then + if [[ $(cat services.conf | { grep -E "deluge\-vpn\: enable" || true; } | wc -l) -eq 1 ]]; then + export DELUGE_HOST="gluetun" + elif [[ $(cat services.conf | { grep -E "deluge\: enable" || true; } | wc -l) -eq 1 ]]; then + export DELUGE_HOST="deluge" + fi +fi + +# Apply Traefik dynamic files in traefik conf directory if VPN are enabled for some services +for svc in $(cat services.conf | grep "\-vpn: enable" | sed -E "s/(.*)\: enable/\1/g"); do + if [[ -f samples/traefik-${svc}.yaml ]]; then + echo "[$0] traefik-$svc.yaml file detected in samples/ directory. Applying into Traefik runtime config directory..." + cp samples/traefik-${svc}.yaml traefik/custom/dynamic-${svc}.yaml + else + echo "[$0] No custom traefik file found in samples/directory for app $svc. Skipping..." + fi +done + # Fetch all YAML files disabled_pattern="" while read -r line ; do @@ -46,8 +94,8 @@ ALL_SERVICES="-f docker-compose.yaml $SERVICES" # Specific instructions for Flood # User for Deluge daemon RPC has to be created in deluge auth config file if [[ ! -z ${FLOOD_PASSWORD} && ${FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON} == true ]]; then - if ! grep -q "flood" /data/config/deluge/auth; then - echo "flood:${FLOOD_PASSWORD}:10" >> /data/config/deluge/auth + if ! grep -q "flood" $HOST_CONFIG_PATH/deluge/auth; then + echo "flood:${FLOOD_PASSWORD}:10" >> $HOST_CONFIG_PATH/deluge/auth else echo "[$0] No need to add user/password for flood as it has already been created." echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." From 2099cdbcf5c9f9ec08c7f2c289e7ddfd61f2120e Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 00:27:41 +0100 Subject: [PATCH 10/42] Add missing service in services.conf.sample --- services.conf.sample | 1 + 1 file changed, 1 insertion(+) diff --git a/services.conf.sample b/services.conf.sample index 918c9ba..fe99215 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -17,6 +17,7 @@ ombi: disable overseerr: enable tautulli: enable jdownloader: enable +jdownloader-vpn: disable tdarr: enable nextcloud: enable portainer: enable From 9be95c234e08b9d7e98ca29079f3faf275ba1de4 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 10:31:12 +0100 Subject: [PATCH 11/42] DB runs as user + add notes.md for nextcloud-db volume relocation --- notes.md | 16 ++++++++++++++++ services/nextcloud.yaml | 1 + 2 files changed, 17 insertions(+) create mode 100644 notes.md diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..6533e24 --- /dev/null +++ b/notes.md @@ -0,0 +1,16 @@ +# Nextcloud-db has moved + +Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from +``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*). + +To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema: + +```sh +mv /data/nextcloud-db/ /data/config/ +./update-all.sh +source .env +docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD} +docker restart nextcloud nextcloud-db +``` + +Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI. diff --git a/services/nextcloud.yaml b/services/nextcloud.yaml index 9600d10..332db77 100755 --- a/services/nextcloud.yaml +++ b/services/nextcloud.yaml @@ -3,6 +3,7 @@ services: image: mariadb:10 container_name: nextcloud-db command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW --innodb-file-per-table=1 --skip-innodb-read-only-compressed + user: ${PUID}:${PGID} restart: always environment: - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} From 79a5df698007d9979d9be52abdd27d8082af62aa Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 10:46:30 +0100 Subject: [PATCH 12/42] Add syncthing (work in progress) --- services.conf.sample | 1 + services/syncthing.yaml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 services/syncthing.yaml diff --git a/services.conf.sample b/services.conf.sample index fe99215..5dd2fdb 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -23,4 +23,5 @@ nextcloud: enable portainer: enable netdata: enable duplicati: enable +syncthing: disable gluetun: disable \ No newline at end of file diff --git a/services/syncthing.yaml b/services/syncthing.yaml new file mode 100644 index 0000000..a325613 --- /dev/null +++ b/services/syncthing.yaml @@ -0,0 +1,30 @@ +services: + syncthing: + image: lscr.io/linuxserver/syncthing + container_name: syncthing + restart: unless-stopped + environment: + - PGID=${PGID} + - PUID=${PUID} + - TZ=${TZ} + volumes: + - configsyncthing:/config + - torrents:/torrents + # ports: + # #- 8384:8384 + # - 22000:22000/tcp + # - 22000:22000/udp + # - 21027:21027/udp + labels: + - "traefik.enable=true" + - "traefik.http.routers.syncthing.rule=Host(`syncthing.${TRAEFIK_DOMAIN}`)" + - "traefik.http.routers.syncthing.middlewares=common-auth@file" + - "traefik.http.services.syncthing-seedbox.loadbalancer.server.port=8384" + - "traefik.http.routers.synchro.rule=Host(`synchro.${TRAEFIK_DOMAIN}`)" + - "traefik.http.services.synchro-seedbox.loadbalancer.server.port=22000" + +volumes: + configsyncthing: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/configsyncthing \ No newline at end of file From c3a20d026404d2b7e248c793f04ba0ac812d0f10 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 10:49:08 +0100 Subject: [PATCH 13/42] Fix syncthing labels --- services/syncthing.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/syncthing.yaml b/services/syncthing.yaml index a325613..baf01aa 100644 --- a/services/syncthing.yaml +++ b/services/syncthing.yaml @@ -19,9 +19,11 @@ services: - "traefik.enable=true" - "traefik.http.routers.syncthing.rule=Host(`syncthing.${TRAEFIK_DOMAIN}`)" - "traefik.http.routers.syncthing.middlewares=common-auth@file" - - "traefik.http.services.syncthing-seedbox.loadbalancer.server.port=8384" + - "traefik.http.routers.syncthing.service=syncthing" + - "traefik.http.services.syncthing.loadbalancer.server.port=8384" - "traefik.http.routers.synchro.rule=Host(`synchro.${TRAEFIK_DOMAIN}`)" - - "traefik.http.services.synchro-seedbox.loadbalancer.server.port=22000" + - "traefik.http.routers.synchro.service=synchro" + - "traefik.http.services.synchro.loadbalancer.server.port=22000" volumes: configsyncthing: From 19208bba9f25acc8664f2515eecbdb32103069ee Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 10:53:26 +0100 Subject: [PATCH 14/42] Fix syncthing --- services/syncthing.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/services/syncthing.yaml b/services/syncthing.yaml index baf01aa..6011793 100644 --- a/services/syncthing.yaml +++ b/services/syncthing.yaml @@ -10,20 +10,17 @@ services: volumes: - configsyncthing:/config - torrents:/torrents - # ports: + ports: # #- 8384:8384 - # - 22000:22000/tcp - # - 22000:22000/udp - # - 21027:21027/udp + - 22000:22000/tcp + - 22000:22000/udp + - 21027:21027/udp labels: - "traefik.enable=true" - "traefik.http.routers.syncthing.rule=Host(`syncthing.${TRAEFIK_DOMAIN}`)" - "traefik.http.routers.syncthing.middlewares=common-auth@file" - "traefik.http.routers.syncthing.service=syncthing" - "traefik.http.services.syncthing.loadbalancer.server.port=8384" - - "traefik.http.routers.synchro.rule=Host(`synchro.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.synchro.service=synchro" - - "traefik.http.services.synchro.loadbalancer.server.port=22000" volumes: configsyncthing: From 4166ced7ce33d3ff72599807630b3686d57fa72f Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 14 Feb 2022 16:36:49 +0100 Subject: [PATCH 15/42] Add all config volume to syncthing --- services/syncthing.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/syncthing.yaml b/services/syncthing.yaml index 6011793..24d0d17 100644 --- a/services/syncthing.yaml +++ b/services/syncthing.yaml @@ -10,6 +10,7 @@ services: volumes: - configsyncthing:/config - torrents:/torrents + - config:/allconfig ports: # #- 8384:8384 - 22000:22000/tcp From f5a42cb0bec5086b3966a3ac43bab1ec585c2c76 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Tue, 15 Feb 2022 00:02:49 +0100 Subject: [PATCH 16/42] Add Synology compatibility for Netdata --- services/netdata.yaml | 2 +- update-all.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/netdata.yaml b/services/netdata.yaml index bc118c6..9f090a2 100755 --- a/services/netdata.yaml +++ b/services/netdata.yaml @@ -15,7 +15,7 @@ services: - /sys:/host/sys:ro - /etc/passwd:/host/etc/passwd:ro - /etc/group:/host/etc/group:ro - - /etc/os-release:/host/etc/os-release:ro + - ${OS_RELEASE_FILEPATH}:/host/etc/os-release:ro - /var/run/docker.sock:/var/run/docker.sock:ro labels: - "traefik.enable=true" diff --git a/update-all.sh b/update-all.sh index fb7b8be..af5af6a 100755 --- a/update-all.sh +++ b/update-all.sh @@ -82,6 +82,15 @@ for svc in $(cat services.conf | grep "\-vpn: enable" | sed -E "s/(.*)\: enable/ fi done +# Detect Synology devices for Netdata compatibility +if [[ $(cat services.conf | { grep -E "netdata\: enable" || true; } | wc -l) -eq 1 ]]; then + if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then + export OS_RELEASE_FILEPATH="/etc/VERSION" + else + export OS_RELEASE_FILEPATH="/etc/os-release" + fi +done + # Fetch all YAML files disabled_pattern="" while read -r line ; do From 4bec84badc939839d4f1aa9a797b43487aca6793 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Tue, 15 Feb 2022 00:03:18 +0100 Subject: [PATCH 17/42] Fix typo --- update-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update-all.sh b/update-all.sh index af5af6a..ed2935f 100755 --- a/update-all.sh +++ b/update-all.sh @@ -89,7 +89,7 @@ if [[ $(cat services.conf | { grep -E "netdata\: enable" || true; } | wc -l) -eq else export OS_RELEASE_FILEPATH="/etc/os-release" fi -done +fi # Fetch all YAML files disabled_pattern="" From 34baae3af1a195df434687fd69d68d90c7b8482f Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Fri, 18 Feb 2022 23:48:58 +0100 Subject: [PATCH 18/42] Add support for custom Traefik config files --- .gitignore | 3 ++- samples/custom-traefik/README.md | 3 +++ update-all.sh | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 samples/custom-traefik/README.md diff --git a/.gitignore b/.gitignore index 579b1b4..7f7fda2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ **/traefik/http_auth backup/ services.conf -traefik/custom/dynamic*.yaml \ No newline at end of file +traefik/custom/dynamic*.yaml +samples/custom*/*.yaml \ No newline at end of file diff --git a/samples/custom-traefik/README.md b/samples/custom-traefik/README.md new file mode 100644 index 0000000..b11a2a4 --- /dev/null +++ b/samples/custom-traefik/README.md @@ -0,0 +1,3 @@ +# Custom Traefik config files + +Place in this directory any custom Traefik config file, they will be copied to Traefik folder when running ``update-all.sh`` script. diff --git a/update-all.sh b/update-all.sh index ed2935f..519df41 100755 --- a/update-all.sh +++ b/update-all.sh @@ -82,6 +82,12 @@ for svc in $(cat services.conf | grep "\-vpn: enable" | sed -E "s/(.*)\: enable/ fi done +# Apply other arbitrary custom Traefik config files +for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do + echo "[$0] Applying custom Traefik config $f..." + cp $f traefik/custom/dynamic-$(basename $f) +done + # Detect Synology devices for Netdata compatibility if [[ $(cat services.conf | { grep -E "netdata\: enable" || true; } | wc -l) -eq 1 ]]; then if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then From a39ea3d0d750e27d24ce33849e0f2d535add187f Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Thu, 24 Feb 2022 22:09:30 +0100 Subject: [PATCH 19/42] Add Heimdall --- README.md | 1 + services.conf.sample | 1 + services/heimdall.yaml | 21 +++++++++++++++++++++ tools/wip/clean-fs.sh | 2 +- 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 services/heimdall.yaml diff --git a/README.md b/README.md index 05a0032..534f2b1 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ seedbox and personal media server. | Portainer | portainer.yourdomain.com | [portainer/portainer](https://hub.docker.com/r/portainer/portainer) | *latest* | Container management| | Netdata | netdata.yourdomain.com | [netdata/netdata](https://hub.docker.com/r/netdata/netdata) | *latest* | Server monitoring | | Duplicati | duplicati.yourdomain.com | [linuxserver/duplicati](https://hub.docker.com/r/linuxserver/duplicati)| *latest* | Backups | +| Heimdall | yourdomain.com | [linuxserver/heimdall](https://hub.docker.com/r/linuxserver/heimdall)| *latest* | Main dashboard | | Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client (still WIP...) | The front-end reverse proxy (Traefik - **check the next section if you have already the seedbox with Traefik v1**) routes based on the lowest level subdomain diff --git a/services.conf.sample b/services.conf.sample index 5dd2fdb..a0983a9 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -24,4 +24,5 @@ portainer: enable netdata: enable duplicati: enable syncthing: disable +heimdall: disable gluetun: disable \ No newline at end of file diff --git a/services/heimdall.yaml b/services/heimdall.yaml new file mode 100644 index 0000000..166e1b8 --- /dev/null +++ b/services/heimdall.yaml @@ -0,0 +1,21 @@ +services: + heimdall: + image: lscr.io/linuxserver/heimdall + container_name: heimdall + restart: always + volumes: + - configheimdall:/config + environment: + - PGID=${PGID} + - PUID=${PUID} + - TZ=${TZ} + labels: + - "traefik.enable=true" + - "traefik.http.routers.heimdall.rule=Host(`${TRAEFIK_DOMAIN}`)" + - "traefik.http.routers.heimdall.middlewares=common-auth@file" + +volumes: + configheimdall: + driver: local-persist + driver_opts: + mountpoint: $HOST_CONFIG_PATH/heimdall \ No newline at end of file diff --git a/tools/wip/clean-fs.sh b/tools/wip/clean-fs.sh index cd850a8..d52961c 100755 --- a/tools/wip/clean-fs.sh +++ b/tools/wip/clean-fs.sh @@ -25,7 +25,7 @@ find $1 -maxdepth 10 -type f | grep -v -E "\.webm$|\.flv$|\.vob$|\.ogg$|\.ogv$|\ while true; do read -p "[$0] Do you want to REMOVE these files?" yn case $yn in - [Yy]* ) find $1 -maxdepth 10 -type f | grep -v -E "\.webm$|\.flv$|\.vob$|\.ogg$|\.ogv$|\.drc$|\.gifv$|\.mng$|\.avi$|\.mov$|\.qt$|\.wmv$|\.yuv$|\.rm$|\.rmvb$|/.asf$|\.amv$|\.mp4$|\.m4v$|\.mp*$|\.m?v$|\.svi$|\.3gp$|\.flv$|\.f4v$" | xargs rm -f; break;; + [Yy]* ) find $1 -maxdepth 10 -type f | grep -v -E "\.webm$|\.flv$|\.vob$|\.ogg$|\.ogv$|\.drc$|\.gifv$|\.mng$|\.avi$|\.mov$|\.qt$|\.wmv$|\.yuv$|\.rm$|\.rmvb$|/.asf$|\.amv$|\.mp4$|\.m4v$|\.mp*$|\.m?v$|\.svi$|\.3gp$|\.flv$|\.f4v$" | xargs -0 rm -f; break;; [Nn]* ) echo "[$0] Skipping this part. Continuing...";; * ) echo "[$0] Please answer yes or no.";; esac From cff539464fa95d0d3ed52d8f7406911687138174 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Fri, 4 Mar 2022 00:12:58 +0100 Subject: [PATCH 20/42] Prepare v2 configuration importer (still work in progress) --- .gitignore | 4 +- config-updater.sh | 107 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100755 config-updater.sh diff --git a/.gitignore b/.gitignore index 7f7fda2..fcd2873 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ backup/ services.conf traefik/custom/dynamic*.yaml -samples/custom*/*.yaml \ No newline at end of file +samples/custom*/*.yaml + +config.yaml \ No newline at end of file diff --git a/config-updater.sh b/config-updater.sh new file mode 100755 index 0000000..228acab --- /dev/null +++ b/config-updater.sh @@ -0,0 +1,107 @@ +#!/bin/bash +set -e +################################################################################ +### === config-updater.sh === ### +### Script which takes as input the old format config file (services.conf) ### +### and transforms it in the new format in yaml, using jq and yq ### +################################################################################ + +cleanup_on_exit() { + rm -f tmp.json config.json +} +trap cleanup_on_exit EXIT + +# Check that jq is installed +if ! which jq >/dev/null; then + echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/" + echo "[$0] Also, please make sure it is in the PATH." + exit 1 +fi + +# Check that yq is installed +if ! which yq >/dev/null; then + echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases" + echo "[$0] Also, please make sure it is in the PATH." + exit 1 +fi + +jq -n '{"services": []}' > config.json + +while read -r line ; do + key=$(echo $line | sed -r "s/^(.*):.*$/\1/") + enabled="true" + if grep -q "disable" <<< $line; then + enabled="false" + fi + + # Compatibility for services.conf already on dev (with temporary syntax -vpn) + if grep -q "\-vpn" <<< $line; then continue; fi + + enableVpn="false" + # If this service is disabled AND another one in the file is enabled with VPN mode, keep that information + if grep -q "$key-vpn: enable" services.conf; then + if [[ enabled="false" ]]; then + echo "[$0] $key => another service detected enabled with vpn..." + enableVpn="true" + fi + fi + + if grep -q "\-hardware-transcoding" <<< $line; then continue; fi + + # Define if Traefik should be enabled on the service + case $key in + flaresolverr) + enableTraefik="false" + rules=$(jq -n '[]') + ;; + *) + enableTraefik="true" + # If Traefik enabled => define if http auth Traefik middleware must be set by default + case $key in + gluetun|kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli) + defaultHttpAuth="false" + ;; + *) + defaultHttpAuth="true" + ;; + esac + # Define service default port from bundled config file + internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") + rules=$(jq -n '[ + { + "host": "'"$key"'", + "httpAuth": '"${defaultHttpAuth}"', + "internalPort": '"${internalPort}"', + } + ]') + ;; + esac + + jq -r --argjson RULES "$rules" '.services[.services| length] |= . + + { + "name": "'"$key"'", + "enabled": '"${enabled}"', + "vpn": '"${enableVpn}"', + "traefik": { + "enabled": '"${enableTraefik}"', + "rules": $RULES + } + }' config.json > tmp.json + rm -f config.json + mv tmp.json config.json + +done < services.conf + +# If we should enable Plex with hardware transcoding +# if grep -q -E "plex.*transcoding: enable" services.conf; then +# if grep -q "plex: disable" services.conf; then +# cat config.json | jq -r 'select(.services[].name=="plex") += {"plexHardwareTranscode":"enable"}' > tmp.json +# rm -f config.json +# mv tmp.json config.json +# fi +# fi + +#mv config.json config.bak.json + +# Transform json into yaml, easier to manipulate for the user +cat config.json | yq e -P - > config.yaml \ No newline at end of file From e8c69b0e3a9f2c0e54fac593080f18fd0c14b502 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 01:48:28 +0100 Subject: [PATCH 21/42] New update-all script + more refactoring (still WIP and undocumented) --- .gitignore | 5 +- config-updater.sh | 45 +++++--- config.sample.yaml | 231 +++++++++++++++++++++++++++++++++++++ config/ports | 24 ++++ services/custom/README.md | 4 + update-all-v2.sh | 237 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 527 insertions(+), 19 deletions(-) create mode 100644 config.sample.yaml create mode 100644 config/ports create mode 100644 services/custom/README.md create mode 100755 update-all-v2.sh diff --git a/.gitignore b/.gitignore index fcd2873..a47ffe6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .DS_Store -/config **/tunnel-options.sh **/.env **/traefik/http_auth @@ -9,4 +8,6 @@ services.conf traefik/custom/dynamic*.yaml samples/custom*/*.yaml -config.yaml \ No newline at end of file +config.yaml +services/custom/*.yaml +services/custom/*.yml \ No newline at end of file diff --git a/config-updater.sh b/config-updater.sh index 228acab..9d1ecf9 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -27,7 +27,28 @@ fi jq -n '{"services": []}' > config.json -while read -r line ; do +# First, add Traefik as it was not explicitely set by default in old config file (services.conf) +if ! grep -q "traefik" services.conf; then + jq -r '.services[.services| length] |= . + + { + "name": "traefik", + "enabled": true, + "traefik": { + "enabled": true, + "rules": [ + { + "host": "traefik.'$(echo '${TRAEFIK_DOMAIN}')'", + "service": "api@internal", + "httpAuth": true, + } + ] + } + }' config.json > tmp.json + rm -f config.json + mv tmp.json config.json +fi + +cat services.conf | while read line || [[ -n $line ]]; do key=$(echo $line | sed -r "s/^(.*):.*$/\1/") enabled="true" if grep -q "disable" <<< $line; then @@ -41,8 +62,9 @@ while read -r line ; do # If this service is disabled AND another one in the file is enabled with VPN mode, keep that information if grep -q "$key-vpn: enable" services.conf; then if [[ enabled="false" ]]; then - echo "[$0] $key => another service detected enabled with vpn..." + #echo "[$0] $key => another service detected enabled with vpn..." enableVpn="true" + enabled="true" fi fi @@ -50,7 +72,7 @@ while read -r line ; do # Define if Traefik should be enabled on the service case $key in - flaresolverr) + flaresolverr|gluetun) enableTraefik="false" rules=$(jq -n '[]') ;; @@ -58,7 +80,7 @@ while read -r line ; do enableTraefik="true" # If Traefik enabled => define if http auth Traefik middleware must be set by default case $key in - gluetun|kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli) + kavita|komga|nextcloud|ombi|overseerr|plex|portainer|tautulli) defaultHttpAuth="false" ;; *) @@ -69,7 +91,7 @@ while read -r line ; do internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") rules=$(jq -n '[ { - "host": "'"$key"'", + "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', } @@ -90,18 +112,7 @@ while read -r line ; do rm -f config.json mv tmp.json config.json -done < services.conf - -# If we should enable Plex with hardware transcoding -# if grep -q -E "plex.*transcoding: enable" services.conf; then -# if grep -q "plex: disable" services.conf; then -# cat config.json | jq -r 'select(.services[].name=="plex") += {"plexHardwareTranscode":"enable"}' > tmp.json -# rm -f config.json -# mv tmp.json config.json -# fi -# fi - -#mv config.json config.bak.json +done # Transform json into yaml, easier to manipulate for the user cat config.json | yq e -P - > config.yaml \ No newline at end of file diff --git a/config.sample.yaml b/config.sample.yaml new file mode 100644 index 0000000..1b0eb4d --- /dev/null +++ b/config.sample.yaml @@ -0,0 +1,231 @@ +services: + - name: traefik + enabled: true + traefik: + enabled: true + rules: + - host: traefik.${TRAEFIK_DOMAIN} + service: api@internal + httpAuth: true + - name: deluge + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: deluge.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8112 + scheme: http + - name: flood + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: flood.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 3000 + - name: plex + enabled: true + vpn: false + # uncomment to use custom file with specific parameters for hardware transcoding + # customFile: plex-hardware-transcoding.yaml + traefik: + enabled: true + rules: + - host: plex.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 32400 + - name: flaresolverr + enabled: true + vpn: false + traefik: + enabled: false + rules: [] + - name: jackett + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: jackett.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 9117 + - name: prowlarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: prowlarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 9696 + - name: sonarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: sonarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8989 + - name: radarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: radarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 7878 + - name: bazarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: bazarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 6767 + - name: lidarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: lidarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8686 + - name: readarr + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: readarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8787 + - name: komga + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: komga.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 8080 + - name: kavita + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: kavita.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 5000 + - name: ombi + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: ombi.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 3579 + - name: overseerr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: overseerr.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 5055 + - name: tautulli + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: tautulli.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 8181 + - name: jdownloader + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: jdownloader.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 5800 + - name: tdarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: tdarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8265 + - name: nextcloud + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: nextcloud.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 443 + - name: portainer + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: portainer.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 9443 + - name: netdata + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: netdata.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 19999 + - name: duplicati + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: duplicati.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8200 + - name: syncthing + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: syncthing.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8384 + - name: heimdall + enabled: true + vpn: false + traefik: + enabled: true + rules: + - host: heimdall.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 80 + - name: gluetun + enabled: false + vpn: false + traefik: + enabled: false + rules: [] diff --git a/config/ports b/config/ports new file mode 100644 index 0000000..54f4b74 --- /dev/null +++ b/config/ports @@ -0,0 +1,24 @@ +bazarr: 6767 +deluge: 8112 +flood: 3000 +heimdall: 80 +jackett: 9117 +jdownloader: 5800 +ombi: 3579 +overseerr: 5055 +plex: 32400 +prowlarr: 9696 +radarr: 7878 +sonarr: 8989 +lidarr: 8686 +readarr: 8787 +komga: 8080 +kavita: 5000 +nextcloud: 443 +portainer: 9443 +syncthing: 8384 +tdarr: 8265 +netdata: 19999 +duplicati: 8200 +tautulli: 8181 +traefik: 80 diff --git a/services/custom/README.md b/services/custom/README.md new file mode 100644 index 0000000..c1252f2 --- /dev/null +++ b/services/custom/README.md @@ -0,0 +1,4 @@ +# Services custom folder + +Here, you can place all custom docker-compose files (yaml format) which will be specified in your ``config.yaml``. +All YAML files placed here are ignored in .gitignore. diff --git a/update-all-v2.sh b/update-all-v2.sh new file mode 100755 index 0000000..a784d04 --- /dev/null +++ b/update-all-v2.sh @@ -0,0 +1,237 @@ +#!/bin/bash + +set -e + +SKIP_PULL=0 +DEBUG=0 + +for i in "$@"; do + case $i in + --no-pull) + SKIP_PULL=1 + ;; + --debug) + DEBUG=1 + ;; + *) + echo "[$0] ❌ ERROR: unknown parameter \"$i\"" + exit 1 + ;; + esac +done + +cleanup_on_exit() { + rm -f rules.props *-vpn.props config.json +} +trap cleanup_on_exit EXIT + +echo-debug() { + [[ ${DEBUG} == "1" ]] && echo "$@" +} + +# Cleanup files before start, in case there was a change we start from scratch at every script execution +rm -f config/*-vpn.yaml + +# Create/update http_auth file according to values in .env file +source .env +echo "${HTTP_USER}:${HTTP_PASSWORD}" > traefik/http_auth + +# Docker-compose settings +export COMPOSE_HTTP_TIMEOUT=240 + +# Retro-compatibility +[[ -z $HOST_CONFIG_PATH ]] && export HOST_CONFIG_PATH="/data/config" +[[ -z $HOST_MEDIA_PATH ]] && export HOST_MEDIA_PATH="/data/torrents" +[[ -z $DOWNLOAD_SUBFOLDER ]] && export DOWNLOAD_SUBFOLDER="deluge" + +if [[ ! -f config.yaml ]]; then + echo "[$0] No config.yaml file found. Copying from sample file..." + cp config.sample.yaml config.yaml +fi + +# Alert in case new services have been added (or removed) in sample but active file has not changed +# TODO: adapt to new config structure +# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) +# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) +# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then +# echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." +# diff -yt services.conf services.conf.sample || true +# echo "" +# fi + +############################################################################################### +###################################### Pre-flight checks ###################################### +############################################################################################### +yq eval -o json config.yaml > config.json + +# Check if some services have vpn enabled, that gluetun itself is enabled +nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length') +gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length') +if [[ ${nb_vpn} -gt 0 && ${gluetun_enabled} == 0 ]]; then + echo "[$0] ERROR. ${nb_vpn} VPN-enabled services have been enabled BUT gluetun has not been enabled. Please check your config.yaml file." + echo "[$0] ******* Exiting *******" + exit 1 +fi + +# Determine what host Flood should connect to +# => If deluge vpn is enabled => gluetun +# => If deluge vpn is disabled => deluge +if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled==true)] | length') -eq 1 ]]; then + # Check that if flood is enabled, deluge should also be enabled + if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==false)] | length') -eq 1 ]]; then + echo "[$0] ERROR. Flood is enabled but Deluge is not. Please either enable Deluge or disable Flood as Flood depends on Deluge." + echo "[$0] ******* Exiting *******" + exit 1 + fi + if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then + export DELUGE_HOST="gluetun" + elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then + export DELUGE_HOST="deluge" + fi + + # Specific instructions for Flood + # User for Deluge daemon RPC has to be created in deluge auth config file + if [[ ! -z ${FLOOD_PASSWORD} && ${FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON} == true ]]; then + if ! grep -q "flood" $HOST_CONFIG_PATH/deluge/auth; then + echo "flood:${FLOOD_PASSWORD}:10" >> $HOST_CONFIG_PATH/deluge/auth + else + echo "[$0] No need to add user/password for flood as it has already been created." + echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." + fi + fi + +fi + +# Apply other arbitrary custom Traefik config files +rm -f $f traefik/custom/dynamic-* +for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do + echo "[$0] Applying custom Traefik config $f..." + cp $f traefik/custom/dynamic-$(basename $f) +done + +# Detect Synology devices for Netdata compatibility +if [[ $(cat config.json | jq '[.services[] | select(.name=="netdata" and .enabled==true)] | length') -eq 1 ]]; then + if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then + export OS_RELEASE_FILEPATH="/etc/VERSION" + else + export OS_RELEASE_FILEPATH="/etc/os-release" + fi +fi + +############################################################################################### +####################################### SERVICES PARSING ###################################### +############################################################################################### +ALL_SERVICES="-f docker-compose.yaml" + +# Parse the config.yaml master configuration file +for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do + name=$(echo $json | jq -r .name) + enabled=$(echo $json | jq -r .enabled) + vpn=$(echo $json | jq -r .vpn) + + # Skip disabled services + if [[ ${enabled} == "false" ]]; then + echo-debug "[$0] Service $name is disabled. Skipping it." + continue + fi + + echo-debug "[$0] ➡️ Parsing service: \"$name\"..." + + # Default docker-compose filename is the service name + .yaml. + # Take into account explicit filename if specified in config + customFile=$(echo $json | jq -r .customFile) + file="$name.yaml" + if [[ ${customFile} != "null" ]]; then + file=${customFile} + fi + echo-debug "[$0] File: \"$file\"..." + + # Append $file to global list of files which will be passed to docker commands + ALL_SERVICES="${ALL_SERVICES} -f services/${file}" + + # For services with VPN enabled, add a docker-compose "override" file specifying that the service network should + # go through gluetun (main vpn client service). + if [[ ${vpn} == "true" ]]; then + echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props + yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml + rm -f ${name}-vpn.props + # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands + ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml" + fi + + ###################################### TRAEFIK RULES ###################################### + + # Skip this part for services which have Traefik rules disabled in config + traefikEnabled=$(echo $json | jq -r .traefik.enabled) + if [[ ${traefikEnabled} == "false" ]]; then + echo-debug "[$0] Traefik is disabled. Skipping rules creation..." + continue + fi + + # Loop over all Traefik rules and create the corresponding entries in the generated rules.yaml + echo-debug "[$0] Generating Traefik rules..." + i=0 + for rule in $(echo $json | jq -c .traefik.rules[]); do + ((i=i+1)) + host=$(echo $rule | jq -r .host) + internalPort=$(echo $rule | jq -r .internalPort) + httpAuth=$(echo $rule | jq -r .httpAuth) + echo-debug "[$0] Host ${host}" + echo-debug "[$0] Internal Port ${internalPort}" + echo-debug "[$0] Http Authentication ${httpAuth}" + + # If VPN => Traefik rule should redirect to gluetun container + backendHost=${name} + [[ ${vpn} == "true" ]] && backendHost="gluetun" + + # Handle custom scheme (default if non-specified is http) + scheme="http" + customScheme=$(echo $rule | jq -r .scheme) + [[ ${customScheme} != "null" ]] && scheme=${customScheme} + + # Transform the bash syntax into Traefik/go one => anything.${TRAEFIK_DOMAIN} to anything.{{ env "TRAEFIK_DOMAIN" }} + hostTraefik=$(echo ${host} | sed --regexp-extended 's/^(.*)(\$\{(.*)\})/\1\{\{ env "\3" \}\}/') + + ruleId="${name}-${i}" + echo 'http.routers.'"${ruleId}"'.rule: Host(`'${hostTraefik}'`)' >> rules.props + if [[ ${httpAuth} == "true" ]]; then + echo "http.routers.${ruleId}.middlewares.0: common-auth@file" >> rules.props + fi + + traefikService=$(echo $rule | jq -r .service) + if [[ ${traefikService} != "null" ]]; then + echo "http.routers.${ruleId}.service: ${traefikService}" >> rules.props + else + echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props + fi + + # If the specified service does not contain a "@" => we create it + # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) + if echo ${traefikService} | grep -vq "@"; then + echo "http.services.${ruleId}.loadBalancer.servers.0.url: ${scheme}://${backendHost}:${internalPort}" >> rules.props + fi + + done +done + +# Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik +yq -p=props rules.props > traefik/custom/dynamic-rules.yaml +rm -f rules.props + +# echo ${ALL_SERVICES} + +if [[ "${SKIP_PULL}" != "1" ]]; then + echo "[$0] ***** Pulling all images... *****" + docker-compose ${ALL_SERVICES} pull +fi + +echo "[$0] ***** Recreating containers if required... *****" +docker-compose ${ALL_SERVICES} up -d --remove-orphans +echo "[$0] ***** Done updating containers *****" + +echo "[$0] ***** Clean unused images and volumes... *****" +docker image prune -af +docker volume prune -f + +echo "[$0] ***** Done! *****" +exit 0 \ No newline at end of file From 32da0ba9acd1e491ae19ffdd7859bf183ec4a312 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:07:36 +0100 Subject: [PATCH 22/42] Prepare release v2 - more fixes and refactoring + prepare release notes --- .env.sample | 3 ++ .gitignore | 3 +- UPGRADE_V2.md | 73 ++++++++++++++++++++++++++ config-updater.sh | 22 ++++---- config.sample.yaml | 13 +++-- config/tools.sh | 23 +++++++++ notes.md | 16 ------ update-all.sh => old-update-all.sh | 0 update-all-v2.sh => run-seedbox.sh | 82 ++++++++++++++++++++++-------- services.conf.sample | 10 ++-- services/generated/README.md | 3 ++ traefik/traefik.yaml | 8 +-- 12 files changed, 195 insertions(+), 61 deletions(-) create mode 100644 UPGRADE_V2.md create mode 100755 config/tools.sh delete mode 100644 notes.md rename update-all.sh => old-update-all.sh (100%) rename update-all-v2.sh => run-seedbox.sh (76%) create mode 100644 services/generated/README.md diff --git a/.env.sample b/.env.sample index b4696b1..d108c91 100644 --- a/.env.sample +++ b/.env.sample @@ -1,3 +1,6 @@ +# Internal settings (they will not be passed to running services) +CHECK_FOR_OUTDATED_CONFIG=true + # General Traefik (reverse proxy) settings TRAEFIK_DOMAIN=mydomain.com ACME_MAIL=my-email@my-provider.com diff --git a/.gitignore b/.gitignore index a47ffe6..0dc44b5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ samples/custom*/*.yaml config.yaml services/custom/*.yaml -services/custom/*.yml \ No newline at end of file +services/custom/*.yml +services/generated/*.yaml \ No newline at end of file diff --git a/UPGRADE_V2.md b/UPGRADE_V2.md new file mode 100644 index 0000000..f17fba1 --- /dev/null +++ b/UPGRADE_V2.md @@ -0,0 +1,73 @@ +# Upgrade to V2 + +Seedbox version 2 is here! +Since there are some breaking changes and a configuration structure migration, a major version was mandatory. + +> These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet. + +## What's new? + +* Configuration change to new YAML format + * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format. + * jq (v1.5+) and yq (v4+) are now requirements + * Easier feature switches + * If a service is missing, it won't be enabled by default like before. The config is now more declarative. + * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example. +* VPN support + * With ``gluetun`` service, you can now place any service behind a VPN. + * Default gluetun configuration is Wireguard "custom" mode, but see below... +* Support custom services and docker-compose yaml files + * Place a docker-compose.yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. + * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)). +* Support arbitrary Traefik rules + * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. + * Ideal to forward traffic to other services which do not belong to this seedbox. +* New services: + * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun) + * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall) + * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr) + * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga) + * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita) + +And also: + +* ``update-all.sh`` is now called ``run-seedbox.sh`` but its purpose is the same. +* More checks in ``run-seedbox.sh``. For example, throws an error if Flood is enabled but not Deluge, or if VPN is enabled on a service but the VPN client is not. +* You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)). + * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set. +* ``networks:`` section is now aligned with the new docker compose syntax +* Nextcloud-db has moved. It is now in /data/config (see below how to mitigate the errors). +* Disable Traefik access logs + +## How to migrate + +```sh +./config-updater.sh +# Check the content of your .env file (in comparison with .env.sample which brings new variables) +./run-seedbox.sh +``` + +When everything runs smoothly, you can delete your old configuration file: + +```sh +rm -f services.conf +``` + +> Also, please make sure you have read the next section about Nextcloud Database location. + +## Nextcloud-db has moved + +Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from +``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*). + +To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema: + +```sh +mv /data/nextcloud-db/ /data/config/ +./run-seedbox.sh +source .env +docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD} +docker restart nextcloud nextcloud-db +``` + +Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI. diff --git a/config-updater.sh b/config-updater.sh index 9d1ecf9..7ee7d00 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -11,17 +11,14 @@ cleanup_on_exit() { } trap cleanup_on_exit EXIT -# Check that jq is installed -if ! which jq >/dev/null; then - echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/" - echo "[$0] Also, please make sure it is in the PATH." - exit 1 -fi +# Load common functions +source config/tools.sh -# Check that yq is installed -if ! which yq >/dev/null; then - echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases" - echo "[$0] Also, please make sure it is in the PATH." +# Check that required tools are installed +check_utilities + +if [[ ! -f services.conf ]]; then + echo "[$0] ERROR. Could nof find services.conf. Exiting." exit 1 fi @@ -87,6 +84,10 @@ cat services.conf | while read line || [[ -n $line ]]; do defaultHttpAuth="true" ;; esac + # Define scheme // For nextcloud, scheme must be https + scheme="http" + [[ $key == "nextcloud" ]] && scheme="https" + # Define service default port from bundled config file internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") rules=$(jq -n '[ @@ -94,6 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', + "scheme": '"${scheme}"' } ]') ;; diff --git a/config.sample.yaml b/config.sample.yaml index 1b0eb4d..6c34a39 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -16,7 +16,10 @@ services: - host: deluge.${TRAEFIK_DOMAIN} httpAuth: true internalPort: 8112 - scheme: http + # Uncomment to specify custom schme (http by default) + # scheme: http + # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains) + # disableCertificateGeneration: true - name: flood enabled: true vpn: false @@ -31,6 +34,8 @@ services: vpn: false # uncomment to use custom file with specific parameters for hardware transcoding # customFile: plex-hardware-transcoding.yaml + # You can also place you own file in services/custom/ and call it here like this: + # customFile: custom/my_own_file.yaml traefik: enabled: true rules: @@ -89,7 +94,7 @@ services: httpAuth: true internalPort: 6767 - name: lidarr - enabled: true + enabled: false vpn: false traefik: enabled: true @@ -197,7 +202,7 @@ services: httpAuth: true internalPort: 19999 - name: duplicati - enabled: true + enabled: false vpn: false traefik: enabled: true @@ -220,7 +225,7 @@ services: traefik: enabled: true rules: - - host: heimdall.${TRAEFIK_DOMAIN} + - host: ${TRAEFIK_DOMAIN} httpAuth: true internalPort: 80 - name: gluetun diff --git a/config/tools.sh b/config/tools.sh new file mode 100755 index 0000000..a87b794 --- /dev/null +++ b/config/tools.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +############################################################################## +############################### UTIL FUNCTIONS ############################### +############################################################################## + +check_utilities () { + # Check that jq is installed + if ! which jq >/dev/null; then + echo "[$0] jq does not exist. Install it from here: https://stedolan.github.io/jq/download/" + echo "[$0] Please install jq version 1.5 or above." + echo "[$0] Also, please make sure it is in the PATH." + exit 1 + fi + + # Check that yq is installed + if ! which yq >/dev/null; then + echo "[$0] yq does not exist. Install it from here: https://github.com/mikefarah/yq/releases" + echo "[$0] Please install yq version 4 or above." + echo "[$0] Also, please make sure it is in the PATH." + exit 1 + fi +} \ No newline at end of file diff --git a/notes.md b/notes.md deleted file mode 100644 index 6533e24..0000000 --- a/notes.md +++ /dev/null @@ -1,16 +0,0 @@ -# Nextcloud-db has moved - -Since commit e4ede925a8ce09b177206f30487a889da9e10334, nextcloud-db directory (mapped on /var/lib/mysql) has moved from -``/data/nextcloud-db`` to ``$HOST_CONFIG_PATH/nextcloud-db`` (*/data/config/nextcloud-db by default*). - -To ensure a smooth transition, you will have to move the directory nextcloud-db into the correct new location, then run some commands to fix the schema: - -```sh -mv /data/nextcloud-db/ /data/config/ -./update-all.sh -source .env -docker exec -it nextcloud-db mysql_upgrade -u root -p${MYSQL_ROOT_PASSWORD} -docker restart nextcloud nextcloud-db -``` - -Ensure everything runs nicely by looking at nextcloud-db and nextcloud logs, and by accessing your Nextcloud web UI. diff --git a/update-all.sh b/old-update-all.sh similarity index 100% rename from update-all.sh rename to old-update-all.sh diff --git a/update-all-v2.sh b/run-seedbox.sh similarity index 76% rename from update-all-v2.sh rename to run-seedbox.sh index a784d04..cfe6c5a 100755 --- a/update-all-v2.sh +++ b/run-seedbox.sh @@ -2,6 +2,12 @@ set -e +# Load common functions +source config/tools.sh + +# Check that required tools are installed +check_utilities + SKIP_PULL=0 DEBUG=0 @@ -26,11 +32,19 @@ cleanup_on_exit() { trap cleanup_on_exit EXIT echo-debug() { - [[ ${DEBUG} == "1" ]] && echo "$@" + if [[ ${DEBUG} == "1" ]]; then echo "$@"; fi } -# Cleanup files before start, in case there was a change we start from scratch at every script execution -rm -f config/*-vpn.yaml +############################################################################################### +####################################### Load variables ######################################## +############################################################################################### + +echo "[$0] ***** Checking environment variables and files... *****" + +if [[ ! -f .env ]]; then + echo "[$0] ERROR. \".env\" file not found. Please copy \".env.sample\" and edit its values. Then, re-run this script." + exit 1 +fi # Create/update http_auth file according to values in .env file source .env @@ -49,21 +63,24 @@ if [[ ! -f config.yaml ]]; then cp config.sample.yaml config.yaml fi -# Alert in case new services have been added (or removed) in sample but active file has not changed -# TODO: adapt to new config structure -# NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) -# NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) -# if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then -# echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." -# diff -yt services.conf services.conf.sample || true -# echo "" -# fi - ############################################################################################### ###################################### Pre-flight checks ###################################### ############################################################################################### + +echo "[$0] ***** Checking configuration... *****" + yq eval -o json config.yaml > config.json +if [[ ${CHECK_FOR_OUTDATED_CONFIG} == true ]]; then + nb_services=$(cat config.json | jq '.services | length') + nb_services_sample=$(yq eval -o json config.sample.yaml | jq '.services | length') + if [[ $nb_services_sample -gt $nb_services ]]; then + echo "[$0] There are more services in the config.sample.yaml than in your config.yaml" + echo "[$0] You should check config.sample.yaml because it seems there are new services available for you:" + diff -u config.yaml config.sample.yaml | grep "name:" | grep -E "^\+" || true + fi +fi + # Check if some services have vpn enabled, that gluetun itself is enabled nb_vpn=$(cat config.json | jq '[.services[] | select(.enabled==true and .vpn==true)] | length') gluetun_enabled=$(cat config.json | jq '[.services[] | select(.name=="gluetun" and .enabled==true)] | length') @@ -83,6 +100,7 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled= echo "[$0] ******* Exiting *******" exit 1 fi + # Determine deluge hostname (for flood) based on the VPN status (enabled or not) of deluge if [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==true)] | length') -eq 1 ]]; then export DELUGE_HOST="gluetun" elif [[ $(cat config.json | jq '[.services[] | select(.name=="deluge" and .enabled==true and .vpn==false)] | length') -eq 1 ]]; then @@ -99,11 +117,10 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled= echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." fi fi - fi # Apply other arbitrary custom Traefik config files -rm -f $f traefik/custom/dynamic-* +# rm -f $f traefik/custom/dynamic-* for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do echo "[$0] Applying custom Traefik config $f..." cp $f traefik/custom/dynamic-$(basename $f) @@ -121,6 +138,12 @@ fi ############################################################################################### ####################################### SERVICES PARSING ###################################### ############################################################################################### + +echo "[$0] ***** Generating configuration... *****" + +# Cleanup files before start, in case there was a change we start from scratch at every script execution +rm -f services/generated/*-vpn.yaml + ALL_SERVICES="-f docker-compose.yaml" # Parse the config.yaml master configuration file @@ -153,10 +176,10 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do # go through gluetun (main vpn client service). if [[ ${vpn} == "true" ]]; then echo "services.${name}.network_mode: service:gluetun" > ${name}-vpn.props - yq -p=props ${name}-vpn.props > config/${name}-vpn.yaml + yq -p=props ${name}-vpn.props > services/generated/${name}-vpn.yaml rm -f ${name}-vpn.props # Append config/${name}-vpn.yaml to global list of files which will be passed to docker commands - ALL_SERVICES="${ALL_SERVICES} -f config/${name}-vpn.yaml" + ALL_SERVICES="${ALL_SERVICES} -f services/generated/${name}-vpn.yaml" fi ###################################### TRAEFIK RULES ###################################### @@ -176,9 +199,9 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do host=$(echo $rule | jq -r .host) internalPort=$(echo $rule | jq -r .internalPort) httpAuth=$(echo $rule | jq -r .httpAuth) - echo-debug "[$0] Host ${host}" - echo-debug "[$0] Internal Port ${internalPort}" - echo-debug "[$0] Http Authentication ${httpAuth}" + echo-debug "[$0] Host => ${host}" + echo-debug "[$0] Internal Port => ${internalPort}" + echo-debug "[$0] Http Authentication => ${httpAuth}" # If VPN => Traefik rule should redirect to gluetun container backendHost=${name} @@ -205,6 +228,11 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props fi + disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration) + if [[ ${disableCertificateGeneration} == true ]]; then + echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props + fi + # If the specified service does not contain a "@" => we create it # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) if echo ${traefikService} | grep -vq "@"; then @@ -215,10 +243,22 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do done # Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik +mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml yq -p=props rules.props > traefik/custom/dynamic-rules.yaml rm -f rules.props -# echo ${ALL_SERVICES} +# Post-transformations on the rules file +sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml +sed -i --regexp-extended "s/^(.*: )(Host.*$)/\1'\2'/g" traefik/custom/dynamic-rules.yaml +sed -i --regexp-extended "s/^(.*url: )(.*$)/\1\"\2\"/g" traefik/custom/dynamic-rules.yaml + +rm -f traefik/custom/dynamic-rules-old.yaml + +echo-debug "[$0] Here is the list of all files which are going to be processed: ${ALL_SERVICES}" + +echo "[$0] ***** Config OK. Launching services... *****" + +exit 0 if [[ "${SKIP_PULL}" != "1" ]]; then echo "[$0] ***** Pulling all images... *****" diff --git a/services.conf.sample b/services.conf.sample index a0983a9..3369ae3 100644 --- a/services.conf.sample +++ b/services.conf.sample @@ -9,8 +9,8 @@ prowlarr: enable sonarr: enable radarr: enable bazarr: enable -lidarr: enable -readarr: false +lidarr: disable +readarr: disable komga: disable kavita: disable ombi: disable @@ -18,11 +18,11 @@ overseerr: enable tautulli: enable jdownloader: enable jdownloader-vpn: disable -tdarr: enable +tdarr: disable nextcloud: enable portainer: enable netdata: enable -duplicati: enable +duplicati: disable syncthing: disable -heimdall: disable +heimdall: enable gluetun: disable \ No newline at end of file diff --git a/services/generated/README.md b/services/generated/README.md new file mode 100644 index 0000000..d178d48 --- /dev/null +++ b/services/generated/README.md @@ -0,0 +1,3 @@ +# Generated files + +This directory contains all generated overrides files. They are in .gitignore. diff --git a/traefik/traefik.yaml b/traefik/traefik.yaml index a2dfe1a..1f28c6d 100644 --- a/traefik/traefik.yaml +++ b/traefik/traefik.yaml @@ -2,10 +2,10 @@ api: dashboard: true # Set Access logs timezone -accessLog: - fields: - names: - StartUTC: drop +# accessLog: +# fields: +# names: +# StartUTC: drop providers: docker: From 48fa3a1eb675b858f0f97109bf2c2fb314aa3e28 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:19:36 +0100 Subject: [PATCH 23/42] Remove docker labels, new v2 config.yaml is now mandatory on this branch --- README.md | 52 ++-------- UPGRADE_V2.md => doc/UPGRADE_V2.md | 1 + doc/traefik_v2.md | 26 +++++ old-update-all.sh | 132 ------------------------ samples/traefik-deluge-vpn.yaml | 12 --- samples/traefik-jdownloader-vpn.yaml | 12 --- services/bazarr.yaml | 4 - services/deluge-vpn.yaml | 21 ---- services/deluge.yaml | 4 - services/duplicati.yaml | 4 - services/flood.yaml | 6 -- services/gluetun.yaml | 9 -- services/heimdall.yaml | 4 - services/jackett.yaml | 4 - services/jdownloader-vpn.yaml | 20 ---- services/jdownloader.yaml | 5 - services/kavita.yaml | 3 - services/komga.yaml | 3 - services/lidarr.yaml | 4 - services/netdata.yaml | 4 - services/nextcloud.yaml | 5 - services/ombi.yaml | 4 - services/overseerr.yaml | 3 - services/plex-hardware-transcoding.yaml | 4 - services/plex.yaml | 4 - services/portainer.yaml | 6 +- services/prowlarr.yaml | 4 - services/radarr.yaml | 4 - services/readarr.yaml | 4 - services/sonarr.yaml | 4 - services/syncthing.yaml | 6 -- services/tautulli.yaml | 3 - services/tdarr.yaml | 5 - services/traefik.yaml | 6 -- 34 files changed, 37 insertions(+), 355 deletions(-) rename UPGRADE_V2.md => doc/UPGRADE_V2.md (98%) create mode 100644 doc/traefik_v2.md delete mode 100755 old-update-all.sh delete mode 100755 samples/traefik-deluge-vpn.yaml delete mode 100755 samples/traefik-jdownloader-vpn.yaml delete mode 100755 services/deluge-vpn.yaml delete mode 100755 services/jdownloader-vpn.yaml diff --git a/README.md b/README.md index 534f2b1..056ecab 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,14 @@ A collection of Dockerfiles and a docker-compose configuration to set up a seedbox and personal media server. +⚠️ Version 2 is released, please make sure you read [this V2 Migration Guide](doc/UPGRADE_V2.md) as there are breaking changes! + ## Included Applications | Application | Web Interface | Docker image | Version (image tag) | Notes | -----------------------|----------------------------|------------------------------------------------------------------------|-------------------------|---------------------| | Plex | plex.yourdomain.com | [linuxserver/plex](https://hub.docker.com/r/linuxserver/plex) | *latest* | Media Streaming | | Deluge | deluge.yourdomain.com | [linuxserver/deluge](https://hub.docker.com/r/linuxserver/deluge) | *latest* | Torrents downloader | -| Deluge (VPN) | deluge.yourdomain.com | [linuxserver/deluge](https://hub.docker.com/r/linuxserver/deluge) | *latest* | Torrents downloader (behind VPN) | | Flood | flood.yourdomain.com | [jesec/flood](https://hub.docker.com/r/jesec/flood) | *latest* | Web client for Deluge (experimental) | | Sonarr | sonarr.yourdomain.com | [linuxserver/sonarr](https://hub.docker.com/r/linuxserver/sonarr) | *develop* | TV Shows monitor | | Radarr | radarr.yourdomain.com | [linuxserver/radarr](https://hub.docker.com/r/linuxserver/radarr) | *develop* | Movies monitor | @@ -23,7 +24,6 @@ seedbox and personal media server. | Jackett | jackett.yourdomain.com | [linuxserver/jackett](https://hub.docker.com/r/linuxserver/jackett) | *latest* | Tracker indexer | | Prowlarr | prowlarr.yourdomain.com | [linuxserver/prowlarr](https://hub.docker.com/r/linuxserver/prowlarr) | *develop* | Tracker indexer | | JDownloader | jdownloader.yourdomain.com | [jlesage/jdownloader-2](https://hub.docker.com/r/jlesage/jdownloader-2)| *latest* | Direct downloader | -| JDownloader (VPN) | jdownloader.yourdomain.com | [jlesage/jdownloader-2](https://hub.docker.com/r/jlesage/jdownloader-2)| *latest* | Direct downloader (behind VPN) | | Tautulli (plexPy) | tautulli.yourdomain.com | [linuxserver/tautulli](https://hub.docker.com/r/linuxserver/tautulli) | *latest* | Plex stats and admin| | Tdarr | tdarr.yourdomain.com | [haveagitgat/tdarr](https://hub.docker.com/r/haveagitgat/tdarr) | *latest* | Re-encode files | | NextCloud | nextcloud.yourdomain.com | [linuxserver/nextcloud](https://hub.docker.com/r/linuxserver/nextcloud) | *latest* | Files management | @@ -32,9 +32,9 @@ seedbox and personal media server. | Netdata | netdata.yourdomain.com | [netdata/netdata](https://hub.docker.com/r/netdata/netdata) | *latest* | Server monitoring | | Duplicati | duplicati.yourdomain.com | [linuxserver/duplicati](https://hub.docker.com/r/linuxserver/duplicati)| *latest* | Backups | | Heimdall | yourdomain.com | [linuxserver/heimdall](https://hub.docker.com/r/linuxserver/heimdall)| *latest* | Main dashboard | -| Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client (still WIP...) | +| Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client | -The front-end reverse proxy (Traefik - **check the next section if you have already the seedbox with Traefik v1**) routes based on the lowest level subdomain +The front-end reverse proxy (Traefik - **check [this guide](doc/traefik_v2.md) if you still have the seedbox with Traefik v1**) routes based on the lowest level subdomain (e.g. `deluge.example.com` would route to deluge). Since this is how the router works, it is recommended for you to get a top level domain. If you do not have one, you can edit your domains locally by changing your hosts file or use a @@ -45,38 +45,11 @@ Traefik takes care of valid Let's Encrypt certificates and auto-renewal. Note: Plex is also available directly through the `32400` port without going through the reverse proxy. -## September 2020 - Upgrade to Traefik v2 instructions - -Before upgrading Traefik to version 2, please check the following: - -- In this repo, Traefik v2 upgrade is as seamless as possible (same environment variables than before, out-of-the-box config file...). -- **First, ``git pull`` to grab the latest code.** -- The ``HTTP_PASSWORD`` variable now must be simple-quoted in the .env file. See the updated ``.env.sample`` file (which has also been reorganized) -- Run ``init.sh`` in order to create required Docker objects (network name has changed). -- You can update your acme.json to a Traefik v2-compliant one by doing the following (before launching Traefik v2): - -```sh -mkdir -p /tmp/migration -cd /tmp/migration -sudo cp /opt/traefik/acme.json . -sudo chmod 775 /tmp/migration/acme.json -# Do *NOT* forget the --resolver at the end! (le = Let's Encrypt resolver, see traefik/traefik.yml) -docker run --rm -v ${PWD}:/data -w /data containous/traefik-migration-tool acme -i acme.json -o acme2.json --resolver le -mkdir -p /data/config/traefik -sudo cp acme2.json /data/config/traefik/acme.json -sudo chmod 600 /data/config/traefik/acme.json -# When you already have a backup! -sudo rm -rf /opt/traefik /tmp/migration -``` - -- As from Traefik v2, as Http Authentication is now possible on the Traefik console, the latter is enabled at ``traefik.yourdomain.com``. -- After all this, you can simply do: ``./update-all.sh``! Voilà! - ## Dependencies - [Docker](https://github.com/docker/docker) >= 20.10 -- [Docker Compose](https://github.com/docker/compose) >= 1.28.0 -- [local-persist Docker plugin](https://github.com/MatchbookLab/local-persist): installed directly on host (not in container). This is a volume plugin that extends the default local driver’s functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to always persist, even if the volume is removed via `docker volume rm`. Use *systemd* install for Ubuntu 16.04. +- [Docker Compose](https://github.com/docker/compose) >= 2.2 +- [local-persist Docker plugin](https://github.com/MatchbookLab/local-persist): installed directly on host (not in container). This is a volume plugin that extends the default local driver’s functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to always persist, even if the volume is removed via `docker volume rm`. Use *systemd* install for Ubuntu. ## Configuration @@ -91,22 +64,15 @@ sudo su -c "mkdir /data && mkdir /data/config && mkdir /data/torrents" Edit the `.env` file and change the variables as desired. The variables are all self-explanatory. -**NEW** -You can also disable a service if you do not need it by editing the ``services.conf`` file. -Simply change the "*enable*" key with the "*disable*" one for the service you want to disable. -If you remove a line in this file, it will be considered as "enabled" as all services are enabled by default. - ## Running & updating ```sh -./update-all.sh +./run-seedbox.sh ``` -docker-compose should manage all the volumes and network setup for you. If it -does not, verify that your docker and docker-compose version is updated. +docker-compose should manage all the volumes and network setup for you. If it does not, verify that your docker and docker-compose version is updated. -Make sure you install the dependencies and finish configuration before doing -this. +Make sure you install the dependencies and finish configuration before doing this. ## PlexPass diff --git a/UPGRADE_V2.md b/doc/UPGRADE_V2.md similarity index 98% rename from UPGRADE_V2.md rename to doc/UPGRADE_V2.md index f17fba1..0fdeb28 100644 --- a/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -28,6 +28,7 @@ Since there are some breaking changes and a configuration structure migration, a * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr) * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga) * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita) +* Docker compose v2.2+ is now required And also: diff --git a/doc/traefik_v2.md b/doc/traefik_v2.md new file mode 100644 index 0000000..a0de210 --- /dev/null +++ b/doc/traefik_v2.md @@ -0,0 +1,26 @@ +# September 2020 - Upgrade to Traefik v2 instructions + +Before upgrading Traefik to version 2, please check the following: + +- In this repo, Traefik v2 upgrade is as seamless as possible (same environment variables than before, out-of-the-box config file...). +- **First, ``git pull`` to grab the latest code.** +- The ``HTTP_PASSWORD`` variable now must be simple-quoted in the .env file. See the updated ``.env.sample`` file (which has also been reorganized) +- Run ``init.sh`` in order to create required Docker objects (network name has changed). +- You can update your acme.json to a Traefik v2-compliant one by doing the following (before launching Traefik v2): + +```sh +mkdir -p /tmp/migration +cd /tmp/migration +sudo cp /opt/traefik/acme.json . +sudo chmod 775 /tmp/migration/acme.json +# Do *NOT* forget the --resolver at the end! (le = Let's Encrypt resolver, see traefik/traefik.yml) +docker run --rm -v ${PWD}:/data -w /data containous/traefik-migration-tool acme -i acme.json -o acme2.json --resolver le +mkdir -p /data/config/traefik +sudo cp acme2.json /data/config/traefik/acme.json +sudo chmod 600 /data/config/traefik/acme.json +# When you already have a backup! +sudo rm -rf /opt/traefik /tmp/migration +``` + +- As from Traefik v2, as Http Authentication is now possible on the Traefik console, the latter is enabled at ``traefik.yourdomain.com``. +- After all this, you can simply do: ``./update-all.sh``! Voilà! diff --git a/old-update-all.sh b/old-update-all.sh deleted file mode 100755 index 519df41..0000000 --- a/old-update-all.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash - -set -e - -SKIP_PULL=0 - -for i in "$@"; do - case $i in - --no-pull) - SKIP_PULL=1 - ;; - *) - echo "[$0] ❌ ERROR: unknown parameter \"$i\"" - exit 1 - ;; - esac -done - -# Create/update http_auth file according to values in .env file -source .env -echo "${HTTP_USER}:${HTTP_PASSWORD}" > traefik/http_auth - -# Docker-compose settings -export COMPOSE_HTTP_TIMEOUT=240 - -# Retro-compatibility -[[ -z $HOST_CONFIG_PATH ]] && export HOST_CONFIG_PATH="/data/config" -[[ -z $HOST_MEDIA_PATH ]] && export HOST_MEDIA_PATH="/data/torrents" -[[ -z $DOWNLOAD_SUBFOLDER ]] && export DOWNLOAD_SUBFOLDER="deluge" - -if [[ ! -f services.conf ]]; then - echo "[$0] No services.conf file found. Copying from sample file..." - cp services.conf.sample services.conf -fi - -# Alert in case new services have been added (or removed) in sample but active file has not changed -NB_SERVICES_ACTIVE=$(cat services.conf | wc -l) -NB_SERVICES_ORIG=$(cat services.conf.sample | wc -l) -if [[ ${NB_SERVICES_ACTIVE} != ${NB_SERVICES_ORIG} ]]; then - echo "[$0] Your services.conf file seems outdated. It appears there are new services available, or services that have been removed." - diff -yt services.conf services.conf.sample || true - echo "" -fi - -# Check if *-vpn services are enabled. If so, check that gluetun is enabled. -if [[ $(cat services.conf | { grep -E ".*vpn: enable" || true; } | wc -l) -ge 1 ]]; then - if [[ $(cat services.conf | { grep "gluetun: enable" || true; } | wc -l) -eq 0 ]]; then - echo "[$0] ERROR. A VPN-enabled service has been enabled BUT gluetun has not been enabled. Please check your services.conf file." - echo "******* Exiting *******" - exit 1 - fi -fi - -# Check if there are no conflict in enabled services (for example, you cannot enable deluge AND deluge-vpn) -for svc in deluge plex jdownloader; do - if [[ $(cat services.conf | { grep -E "${svc}.*: enable" || true; } | wc -l) -gt 1 ]]; then - echo "[$0] ERROR. You cannot enable multiple ${svc^} services simultaneously. Please edit this section in your services.conf file:" - cat services.conf | { grep -E "${svc}.*: enable" || true; } - echo "******* Exiting *******" - exit 1 - fi -done - -# Determine what host Flood should connect to -# => If deluge-vpn is enabled => gluetun -# => If deluge is enabled => deluge -if [[ $(cat services.conf | { grep -E "flood\: enable" || true; } | wc -l) -eq 1 ]]; then - if [[ $(cat services.conf | { grep -E "deluge\-vpn\: enable" || true; } | wc -l) -eq 1 ]]; then - export DELUGE_HOST="gluetun" - elif [[ $(cat services.conf | { grep -E "deluge\: enable" || true; } | wc -l) -eq 1 ]]; then - export DELUGE_HOST="deluge" - fi -fi - -# Apply Traefik dynamic files in traefik conf directory if VPN are enabled for some services -for svc in $(cat services.conf | grep "\-vpn: enable" | sed -E "s/(.*)\: enable/\1/g"); do - if [[ -f samples/traefik-${svc}.yaml ]]; then - echo "[$0] traefik-$svc.yaml file detected in samples/ directory. Applying into Traefik runtime config directory..." - cp samples/traefik-${svc}.yaml traefik/custom/dynamic-${svc}.yaml - else - echo "[$0] No custom traefik file found in samples/directory for app $svc. Skipping..." - fi -done - -# Apply other arbitrary custom Traefik config files -for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do - echo "[$0] Applying custom Traefik config $f..." - cp $f traefik/custom/dynamic-$(basename $f) -done - -# Detect Synology devices for Netdata compatibility -if [[ $(cat services.conf | { grep -E "netdata\: enable" || true; } | wc -l) -eq 1 ]]; then - if [[ $(uname -a | { grep synology || true; } | wc -l) -eq 1 ]]; then - export OS_RELEASE_FILEPATH="/etc/VERSION" - else - export OS_RELEASE_FILEPATH="/etc/os-release" - fi -fi - -# Fetch all YAML files -disabled_pattern="" -while read -r line ; do - disabled_pattern="${disabled_pattern} ! -name $line.yaml" -done < <(grep "disable" services.conf | awk -F : '{print $1}' ) - -SERVICES=$(find services -mindepth 1 -maxdepth 1 -name "*.yaml" ${disabled_pattern} | sed -e 's/^/-f /') -ALL_SERVICES="-f docker-compose.yaml $SERVICES" - -# Specific instructions for Flood -# User for Deluge daemon RPC has to be created in deluge auth config file -if [[ ! -z ${FLOOD_PASSWORD} && ${FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON} == true ]]; then - if ! grep -q "flood" $HOST_CONFIG_PATH/deluge/auth; then - echo "flood:${FLOOD_PASSWORD}:10" >> $HOST_CONFIG_PATH/deluge/auth - else - echo "[$0] No need to add user/password for flood as it has already been created." - echo "[$0] Consider setting FLOOD_AUTOCREATE_USER_IN_DELUGE_DAEMON variable to false in .env file." - fi -fi - -if [[ "${SKIP_PULL}" != "1" ]]; then - echo "[$0] ***** Pulling all images... *****" - docker-compose ${ALL_SERVICES} pull -fi - -echo "[$0] ***** Recreating containers if required... *****" -docker-compose ${ALL_SERVICES} up -d --remove-orphans -echo "[$0] ***** Done updating containers *****" -echo "[$0] ***** Clean unused images and volumes... *****" -docker image prune -af -docker volume prune -f -echo "[$0] ***** Done! *****" -exit 0 \ No newline at end of file diff --git a/samples/traefik-deluge-vpn.yaml b/samples/traefik-deluge-vpn.yaml deleted file mode 100755 index 01ec4cb..0000000 --- a/samples/traefik-deluge-vpn.yaml +++ /dev/null @@ -1,12 +0,0 @@ -http: - routers: - deluge: - rule: 'Host(`deluge.{{ env "TRAEFIK_DOMAIN" }}`)' - middlewares: - - common-auth@file - service: deluge - services: - deluge: - loadBalancer: - servers: - - url: "http://gluetun:8112" \ No newline at end of file diff --git a/samples/traefik-jdownloader-vpn.yaml b/samples/traefik-jdownloader-vpn.yaml deleted file mode 100755 index 2a01bf8..0000000 --- a/samples/traefik-jdownloader-vpn.yaml +++ /dev/null @@ -1,12 +0,0 @@ -http: - routers: - jdownloader: - rule: 'Host(`jdownloader.{{ env "TRAEFIK_DOMAIN" }}`)' - middlewares: - - common-auth@file - service: deluge - services: - jdownloader: - loadBalancer: - servers: - - url: "http://gluetun:5800" \ No newline at end of file diff --git a/services/bazarr.yaml b/services/bazarr.yaml index 214a913..b543402 100755 --- a/services/bazarr.yaml +++ b/services/bazarr.yaml @@ -10,10 +10,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.bazarr.rule=Host(`bazarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.bazarr.middlewares=common-auth@file" volumes: configbazarr: diff --git a/services/deluge-vpn.yaml b/services/deluge-vpn.yaml deleted file mode 100755 index 7a2f576..0000000 --- a/services/deluge-vpn.yaml +++ /dev/null @@ -1,21 +0,0 @@ -services: - deluge: - image: ghcr.io/linuxserver/deluge - container_name: deluge - restart: always - network_mode: "service:gluetun" - volumes: - - torrents:/torrents - - configdeluge:/config - - downloads:/downloads - environment: - - PGID=${PGID} - - PUID=${PUID} - - TZ=${TZ} - # Traefik labels are in Gluetun YAML as deluge must be accessed via Gluetun - -volumes: - configdeluge: - driver: local-persist - driver_opts: - mountpoint: $HOST_CONFIG_PATH/deluge \ No newline at end of file diff --git a/services/deluge.yaml b/services/deluge.yaml index 08fa75d..21244aa 100755 --- a/services/deluge.yaml +++ b/services/deluge.yaml @@ -11,10 +11,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.deluge.rule=Host(`deluge.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.deluge.middlewares=common-auth@file" volumes: configdeluge: diff --git a/services/duplicati.yaml b/services/duplicati.yaml index c344916..f2c24d3 100755 --- a/services/duplicati.yaml +++ b/services/duplicati.yaml @@ -11,10 +11,6 @@ services: - configduplicati:/config - backups:/backups - config:/source - labels: - - "traefik.enable=true" - - "traefik.http.routers.duplicati.rule=Host(`duplicati.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.duplicati.middlewares=common-auth@file" volumes: configduplicati: diff --git a/services/flood.yaml b/services/flood.yaml index dc647ca..aa672c8 100755 --- a/services/flood.yaml +++ b/services/flood.yaml @@ -18,12 +18,6 @@ services: - torrents:/torrents - downloads:/downloads - /etc/localtime:/etc/localtime:ro - labels: - - "traefik.enable=true" - - "traefik.http.routers.flood.rule=Host(`flood.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.flood.middlewares=common-auth@file" - - "traefik.http.routers.flood.service=flood-service" - - "traefik.http.services.flood-service.loadbalancer.server.port=3000" volumes: configflood: diff --git a/services/gluetun.yaml b/services/gluetun.yaml index fd4d3cc..ab02491 100755 --- a/services/gluetun.yaml +++ b/services/gluetun.yaml @@ -17,15 +17,6 @@ services: - WIREGUARD_PUBLIC_KEY=${WIREGUARD_PUBLIC_KEY} - WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY} - WIREGUARD_ADDRESS=${WIREGUARD_ADDRESS} - #labels: - # Control Panel - # - "traefik.enable=true" - # - "traefik.http.routers.gluetun.rule=Host(`gluetun.${TRAEFIK_DOMAIN}`)" - # - "traefik.http.routers.gluetun.middlewares=common-auth@file" - # - "traefik.http.routers.gluetun.service=gluetun" - # - "traefik.http.services.gluetun.loadbalancer.server.port=8000" - # Traefik rules for VPN-enabled services (which have ot be accessed through gluetun) are defined in the samples/ directory - # and are applied automatically in the update-all.sh script. volumes: configgluetun: diff --git a/services/heimdall.yaml b/services/heimdall.yaml index 166e1b8..b059aba 100644 --- a/services/heimdall.yaml +++ b/services/heimdall.yaml @@ -9,10 +9,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.heimdall.rule=Host(`${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.heimdall.middlewares=common-auth@file" volumes: configheimdall: diff --git a/services/jackett.yaml b/services/jackett.yaml index ca3b1f8..c6c3f44 100755 --- a/services/jackett.yaml +++ b/services/jackett.yaml @@ -12,7 +12,3 @@ services: - PUID=${PUID} - TZ=${TZ} - RUN_OPTS='-FlareSolverrURL="http://flaresolverr:8191"' - labels: - - "traefik.enable=true" - - "traefik.http.routers.jackett.rule=Host(`jackett.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.jackett.middlewares=common-auth@file" diff --git a/services/jdownloader-vpn.yaml b/services/jdownloader-vpn.yaml deleted file mode 100755 index 3eb682e..0000000 --- a/services/jdownloader-vpn.yaml +++ /dev/null @@ -1,20 +0,0 @@ -services: - jdownloader: - image: jlesage/jdownloader-2 - container_name: jdownloader - restart: always - network_mode: "service:gluetun" - volumes: - - configjdownloader:/config - - downloads:/output - environment: - - USER_ID=${PUID} - - GROUP_ID=${PGID} - - TZ=${TZ} - # Traefik routing rules are defined in the samples/ directory and applied automatically when this service is enabled - -volumes: - configjdownloader: - driver: local-persist - driver_opts: - mountpoint: $HOST_CONFIG_PATH/jdownloader \ No newline at end of file diff --git a/services/jdownloader.yaml b/services/jdownloader.yaml index 34f2f02..b9b7dc2 100755 --- a/services/jdownloader.yaml +++ b/services/jdownloader.yaml @@ -10,11 +10,6 @@ services: - USER_ID=${PUID} - GROUP_ID=${PGID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.services.jdownloader-seedbox.loadbalancer.server.port=5800" - - "traefik.http.routers.jdownloader.rule=Host(`jdownloader.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.jdownloader.middlewares=common-auth@file" volumes: configjdownloader: diff --git a/services/kavita.yaml b/services/kavita.yaml index 034b85f..8b12b9e 100755 --- a/services/kavita.yaml +++ b/services/kavita.yaml @@ -11,9 +11,6 @@ services: - TZ=${TZ} # Wait for https://github.com/Kareadita/Kavita/issues/301 to be resolved #user: "${PUID}:${PGID}" - labels: - - "traefik.enable=true" - - "traefik.http.routers.kavita.rule=Host(`kavita.${TRAEFIK_DOMAIN}`)" volumes: configkavita: diff --git a/services/komga.yaml b/services/komga.yaml index d61f597..c4945fa 100755 --- a/services/komga.yaml +++ b/services/komga.yaml @@ -10,9 +10,6 @@ services: environment: - TZ=${TZ} user: "${PUID}:${PGID}" - labels: - - "traefik.enable=true" - - "traefik.http.routers.komga.rule=Host(`komga.${TRAEFIK_DOMAIN}`)" volumes: configkomga: diff --git a/services/lidarr.yaml b/services/lidarr.yaml index 29cc1cf..87242c1 100755 --- a/services/lidarr.yaml +++ b/services/lidarr.yaml @@ -10,10 +10,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.lidarr.rule=Host(`lidarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.lidarr.middlewares=common-auth@file" volumes: configlidarr: diff --git a/services/netdata.yaml b/services/netdata.yaml index 9f090a2..3d64f3a 100755 --- a/services/netdata.yaml +++ b/services/netdata.yaml @@ -17,7 +17,3 @@ services: - /etc/group:/host/etc/group:ro - ${OS_RELEASE_FILEPATH}:/host/etc/os-release:ro - /var/run/docker.sock:/var/run/docker.sock:ro - labels: - - "traefik.enable=true" - - "traefik.http.routers.netdata.rule=Host(`netdata.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.netdata.middlewares=common-auth@file" \ No newline at end of file diff --git a/services/nextcloud.yaml b/services/nextcloud.yaml index 332db77..f45253f 100755 --- a/services/nextcloud.yaml +++ b/services/nextcloud.yaml @@ -32,11 +32,6 @@ services: - nextclouddata:/data - torrents:/torrents - config:/seedbox-config - labels: - - "traefik.enable=true" - - "traefik.http.routers.nextcloud.rule=Host(`nextcloud.${TRAEFIK_DOMAIN}`)" - - "traefik.http.services.nextcloud-seedbox.loadbalancer.server.scheme=https" - - "traefik.http.services.nextcloud-seedbox.loadbalancer.server.port=443" volumes: nextclouddb: diff --git a/services/ombi.yaml b/services/ombi.yaml index e4a47a4..15f9f21 100755 --- a/services/ombi.yaml +++ b/services/ombi.yaml @@ -10,10 +10,6 @@ services: - PUID=${PUID} - TZ=${TZ} - BASE_URL=/ - labels: - - "traefik.enable=true" - - "traefik.http.routers.ombi.rule=Host(`ombi.${TRAEFIK_DOMAIN}`)" - #- "traefik.http.routers.ombi.middlewares=common-auth@file" volumes: configombi: diff --git a/services/overseerr.yaml b/services/overseerr.yaml index 1abbb74..2ed8948 100755 --- a/services/overseerr.yaml +++ b/services/overseerr.yaml @@ -9,9 +9,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.overseerr.rule=Host(`overseerr.${TRAEFIK_DOMAIN}`)" volumes: configoverseerr: diff --git a/services/plex-hardware-transcoding.yaml b/services/plex-hardware-transcoding.yaml index 44c24c5..7d8b671 100755 --- a/services/plex-hardware-transcoding.yaml +++ b/services/plex-hardware-transcoding.yaml @@ -19,10 +19,6 @@ services: - PUID=${PUID} - TZ=${TZ} - VERSION=latest - labels: - - "traefik.enable=true" - - "traefik.http.services.plex-seedbox.loadbalancer.server.port=32400" - - "traefik.http.routers.plex.rule=Host(`plex.${TRAEFIK_DOMAIN}`)" volumes: configplex: diff --git a/services/plex.yaml b/services/plex.yaml index ebe3984..bd66677 100755 --- a/services/plex.yaml +++ b/services/plex.yaml @@ -16,10 +16,6 @@ services: - PUID=${PUID} - TZ=${TZ} - VERSION=latest - labels: - - "traefik.enable=true" - - "traefik.http.services.plex-seedbox.loadbalancer.server.port=32400" - - "traefik.http.routers.plex.rule=Host(`plex.${TRAEFIK_DOMAIN}`)" volumes: configplex: diff --git a/services/portainer.yaml b/services/portainer.yaml index a9da06f..493692c 100755 --- a/services/portainer.yaml +++ b/services/portainer.yaml @@ -5,8 +5,4 @@ services: restart: always volumes: - /var/run/docker.sock:/var/run/docker.sock - command: --admin-password ${PORTAINER_ADMIN_PASSWORD} --host=unix:///var/run/docker.sock - labels: - - "traefik.enable=true" - - "traefik.http.routers.portainer.rule=Host(`portainer.${TRAEFIK_DOMAIN}`)" - - "traefik.http.services.portainer-seedbox.loadbalancer.server.port=9000" \ No newline at end of file + command: --admin-password ${PORTAINER_ADMIN_PASSWORD} --host=unix:///var/run/docker.sock \ No newline at end of file diff --git a/services/prowlarr.yaml b/services/prowlarr.yaml index 0524a56..5ec3dc4 100755 --- a/services/prowlarr.yaml +++ b/services/prowlarr.yaml @@ -9,10 +9,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.prowlarr.rule=Host(`prowlarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.prowlarr.middlewares=common-auth@file" volumes: configprowlarr: diff --git a/services/radarr.yaml b/services/radarr.yaml index a8d8286..a151a40 100755 --- a/services/radarr.yaml +++ b/services/radarr.yaml @@ -11,10 +11,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.radarr.rule=Host(`radarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.radarr.middlewares=common-auth@file" volumes: configradarr: diff --git a/services/readarr.yaml b/services/readarr.yaml index f4b8dd0..8461025 100755 --- a/services/readarr.yaml +++ b/services/readarr.yaml @@ -11,10 +11,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.readarr.rule=Host(`readarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.readarr.middlewares=common-auth@file" volumes: configreadarr: diff --git a/services/sonarr.yaml b/services/sonarr.yaml index 29100a0..1526db6 100755 --- a/services/sonarr.yaml +++ b/services/sonarr.yaml @@ -11,10 +11,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.sonarr.rule=Host(`sonarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.sonarr.middlewares=common-auth@file" volumes: configsonarr: diff --git a/services/syncthing.yaml b/services/syncthing.yaml index 24d0d17..d69766c 100644 --- a/services/syncthing.yaml +++ b/services/syncthing.yaml @@ -16,12 +16,6 @@ services: - 22000:22000/tcp - 22000:22000/udp - 21027:21027/udp - labels: - - "traefik.enable=true" - - "traefik.http.routers.syncthing.rule=Host(`syncthing.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.syncthing.middlewares=common-auth@file" - - "traefik.http.routers.syncthing.service=syncthing" - - "traefik.http.services.syncthing.loadbalancer.server.port=8384" volumes: configsyncthing: diff --git a/services/tautulli.yaml b/services/tautulli.yaml index 438e467..0f02d81 100755 --- a/services/tautulli.yaml +++ b/services/tautulli.yaml @@ -10,9 +10,6 @@ services: - PGID=${PGID} - PUID=${PUID} - TZ=${TZ} - labels: - - "traefik.enable=true" - - "traefik.http.routers.tautulli.rule=Host(`tautulli.${TRAEFIK_DOMAIN}`)" volumes: configtautulli: diff --git a/services/tdarr.yaml b/services/tdarr.yaml index 01e6639..7dd179b 100755 --- a/services/tdarr.yaml +++ b/services/tdarr.yaml @@ -15,11 +15,6 @@ services: - transcodes:/temp - configtdarrv2:/app/server - settingstdarrv2:/app/configs - labels: - - "traefik.enable=true" - - "traefik.http.services.tdarrv2-seedbox.loadbalancer.server.port=8265" - - "traefik.http.routers.tdarrv2.rule=Host(`tdarr.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.tdarrv2.middlewares=common-auth@file" tdarr_node: image: haveagitgat/tdarr_node:latest diff --git a/services/traefik.yaml b/services/traefik.yaml index bf97be0..b454ec1 100755 --- a/services/traefik.yaml +++ b/services/traefik.yaml @@ -16,12 +16,6 @@ services: environment: - TRAEFIK_DOMAIN=${TRAEFIK_DOMAIN} - TZ=${TZ} - labels: - - "traefik.enable=true" - # Docker labels for enabling Traefik dashboard - - "traefik.http.routers.traefik.rule=Host(`traefik.${TRAEFIK_DOMAIN}`)" - - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.middlewares=common-auth@file" volumes: configtraefik: From 96770579cbc664c7ce2bacd1f28789b65982be8b Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:21:54 +0100 Subject: [PATCH 24/42] Fix config-updater script --- config-updater.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config-updater.sh b/config-updater.sh index 7ee7d00..eb5abef 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -95,7 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', - "scheme": '"${scheme}"' + "scheme": "'"${scheme}"'" } ]') ;; From 3accf3d8bddd3e99b46ba81cc10dac17007e33c4 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:24:57 +0100 Subject: [PATCH 25/42] Fix run-seedbox script --- run-seedbox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-seedbox.sh b/run-seedbox.sh index cfe6c5a..741e661 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -243,7 +243,7 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do done # Convert properties files into Traefik-ready YAML and place it in the correct folder loaded by Traefik -mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml +mv traefik/custom/dynamic-rules.yaml traefik/custom/dynamic-rules-old.yaml || true yq -p=props rules.props > traefik/custom/dynamic-rules.yaml rm -f rules.props From d1e2bfa212af81c609d32934a915010bdca39d37 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:25:18 +0100 Subject: [PATCH 26/42] Another fix --- run-seedbox.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/run-seedbox.sh b/run-seedbox.sh index 741e661..df1991f 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -258,8 +258,6 @@ echo-debug "[$0] Here is the list of all files which are going to be processed: echo "[$0] ***** Config OK. Launching services... *****" -exit 0 - if [[ "${SKIP_PULL}" != "1" ]]; then echo "[$0] ***** Pulling all images... *****" docker-compose ${ALL_SERVICES} pull From 18caa638bd2ca209cf1aed40da6fa5670de35dd3 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 18:38:44 +0100 Subject: [PATCH 27/42] Rename custom files for Traefik rules (allow deletion on script run without conflict with generated rules) --- .gitignore | 2 ++ run-seedbox.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0dc44b5..5657fae 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,9 @@ **/traefik/http_auth backup/ services.conf + traefik/custom/dynamic*.yaml +traefik/custom/custom-*.yaml samples/custom*/*.yaml config.yaml diff --git a/run-seedbox.sh b/run-seedbox.sh index df1991f..7169c1a 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -120,10 +120,10 @@ if [[ $(cat config.json | jq '[.services[] | select(.name=="flood" and .enabled= fi # Apply other arbitrary custom Traefik config files -# rm -f $f traefik/custom/dynamic-* +rm -f $f traefik/custom/custom-* for f in `find samples/custom-traefik -maxdepth 1 -mindepth 1 -type f | grep -E "\.yml$|\.yaml$" | sort`; do echo "[$0] Applying custom Traefik config $f..." - cp $f traefik/custom/dynamic-$(basename $f) + cp $f traefik/custom/custom-$(basename $f) done # Detect Synology devices for Netdata compatibility From 75edd0940d78a97693f0c9f88cb1c1ae973aa26f Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 20:52:50 +0100 Subject: [PATCH 28/42] Rename scheme to internalScheme --- config-updater.sh | 6 +++--- config.sample.yaml | 2 +- run-seedbox.sh | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config-updater.sh b/config-updater.sh index eb5abef..f150423 100755 --- a/config-updater.sh +++ b/config-updater.sh @@ -85,8 +85,8 @@ cat services.conf | while read line || [[ -n $line ]]; do ;; esac # Define scheme // For nextcloud, scheme must be https - scheme="http" - [[ $key == "nextcloud" ]] && scheme="https" + internalScheme="http" + [[ $key == "nextcloud" ]] && internalScheme="https" # Define service default port from bundled config file internalPort=$(cat config/ports | { grep $key || true; } | sed -r "s/^${key}: (.*)$/\1/") @@ -95,7 +95,7 @@ cat services.conf | while read line || [[ -n $line ]]; do "host": "'"$key"'.'$(echo '${TRAEFIK_DOMAIN}')'", "httpAuth": '"${defaultHttpAuth}"', "internalPort": '"${internalPort}"', - "scheme": "'"${scheme}"'" + "internalScheme": "'"${internalScheme}"'" } ]') ;; diff --git a/config.sample.yaml b/config.sample.yaml index 6c34a39..b1078b6 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -17,7 +17,7 @@ services: httpAuth: true internalPort: 8112 # Uncomment to specify custom schme (http by default) - # scheme: http + # internalScheme: http # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains) # disableCertificateGeneration: true - name: flood diff --git a/run-seedbox.sh b/run-seedbox.sh index 7169c1a..4d53565 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -208,9 +208,9 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do [[ ${vpn} == "true" ]] && backendHost="gluetun" # Handle custom scheme (default if non-specified is http) - scheme="http" - customScheme=$(echo $rule | jq -r .scheme) - [[ ${customScheme} != "null" ]] && scheme=${customScheme} + internalScheme="http" + customInternalScheme=$(echo $rule | jq -r .internalScheme) + [[ ${customInternalScheme} != "null" ]] && internalScheme=${customInternalScheme} # Transform the bash syntax into Traefik/go one => anything.${TRAEFIK_DOMAIN} to anything.{{ env "TRAEFIK_DOMAIN" }} hostTraefik=$(echo ${host} | sed --regexp-extended 's/^(.*)(\$\{(.*)\})/\1\{\{ env "\3" \}\}/') @@ -236,7 +236,7 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do # If the specified service does not contain a "@" => we create it # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) if echo ${traefikService} | grep -vq "@"; then - echo "http.services.${ruleId}.loadBalancer.servers.0.url: ${scheme}://${backendHost}:${internalPort}" >> rules.props + echo "http.services.${ruleId}.loadBalancer.servers.0.url: ${internalScheme}://${backendHost}:${internalPort}" >> rules.props fi done From 130bee09a8e5a90667f0d5e4be3dcac71dad79e1 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 20:59:53 +0100 Subject: [PATCH 29/42] Test new flag httpOnly in config --- run-seedbox.sh | 6 ++++++ traefik/traefik.yaml | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/run-seedbox.sh b/run-seedbox.sh index 4d53565..f2c6947 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -228,11 +228,17 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props fi + # TODO: leave it or remove it? disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration) if [[ ${disableCertificateGeneration} == true ]]; then echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props fi + httpOnly=$(echo $rule | jq -r .httpOnly) + if [[ ${httpOnly} == true ]]; then + echo "http.routers.${ruleId}.entryPoints.0: insecure" >> rules.props + fi + # If the specified service does not contain a "@" => we create it # If the service has a @, it means it is defined elsewhere so we do not create it (custom file, @internal...) if echo ${traefikService} | grep -vq "@"; then diff --git a/traefik/traefik.yaml b/traefik/traefik.yaml index 1f28c6d..91d7823 100644 --- a/traefik/traefik.yaml +++ b/traefik/traefik.yaml @@ -19,11 +19,11 @@ providers: entryPoints: insecure: address: ":80" - http: - redirections: - entryPoint: - to: secure - scheme: https + # http: + # redirections: + # entryPoint: + # to: secure + # scheme: https secure: address: ":443" http: From 4d9b4214e5b90a28fc589ba2e623b0c40610b95d Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 22:16:00 +0100 Subject: [PATCH 30/42] Handle https redirection for regular services ar router level --- run-seedbox.sh | 10 +++++++++- traefik/custom/middlewares.yaml | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/run-seedbox.sh b/run-seedbox.sh index f2c6947..a44219e 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -194,6 +194,7 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do # Loop over all Traefik rules and create the corresponding entries in the generated rules.yaml echo-debug "[$0] Generating Traefik rules..." i=0 + middlewareCount=0 for rule in $(echo $json | jq -c .traefik.rules[]); do ((i=i+1)) host=$(echo $rule | jq -r .host) @@ -218,7 +219,8 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do ruleId="${name}-${i}" echo 'http.routers.'"${ruleId}"'.rule: Host(`'${hostTraefik}'`)' >> rules.props if [[ ${httpAuth} == "true" ]]; then - echo "http.routers.${ruleId}.middlewares.0: common-auth@file" >> rules.props + echo "http.routers.${ruleId}.middlewares.${middlewareCount}: common-auth@file" >> rules.props + ((middlewareCount=middlewareCount+1)) fi traefikService=$(echo $rule | jq -r .service) @@ -234,9 +236,15 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props fi + # Check if httpOnly flag is enabled + # If enabled => Specify to use only "insecure" (port 80) entrypoint + # If not => use all entryPoints (by not specifying any) but force redirection to https httpOnly=$(echo $rule | jq -r .httpOnly) if [[ ${httpOnly} == true ]]; then echo "http.routers.${ruleId}.entryPoints.0: insecure" >> rules.props + else + echo "http.routers.${ruleId}.middlewares.${middlewareCount}: redirect-to-https" >> rules.props + ((middlewareCount=middlewareCount+1)) fi # If the specified service does not contain a "@" => we create it diff --git a/traefik/custom/middlewares.yaml b/traefik/custom/middlewares.yaml index da09072..e0d2a75 100644 --- a/traefik/custom/middlewares.yaml +++ b/traefik/custom/middlewares.yaml @@ -15,3 +15,7 @@ http: frameDeny: true contentTypeNosniff: true browserXssFilter: true + redirect-to-https: + redirectScheme: + scheme: https + permanent: false From 94772a812959329057df15a85380e8674bfcbb94 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 22:27:22 +0100 Subject: [PATCH 31/42] Fix middleware counter init in loop --- run-seedbox.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run-seedbox.sh b/run-seedbox.sh index a44219e..3c3e26a 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -194,7 +194,6 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do # Loop over all Traefik rules and create the corresponding entries in the generated rules.yaml echo-debug "[$0] Generating Traefik rules..." i=0 - middlewareCount=0 for rule in $(echo $json | jq -c .traefik.rules[]); do ((i=i+1)) host=$(echo $rule | jq -r .host) @@ -218,6 +217,8 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do ruleId="${name}-${i}" echo 'http.routers.'"${ruleId}"'.rule: Host(`'${hostTraefik}'`)' >> rules.props + + middlewareCount=0 if [[ ${httpAuth} == "true" ]]; then echo "http.routers.${ruleId}.middlewares.${middlewareCount}: common-auth@file" >> rules.props ((middlewareCount=middlewareCount+1)) From bfacdaa0d7967ca5f16f3a79876dce8d41a2764b Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 23:28:23 +0100 Subject: [PATCH 32/42] Add documentation for configuration, fix sample config --- config.sample.yaml | 2 +- doc/UPGRADE_V2.md | 31 ++++--- doc/configuration.md | 211 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 doc/configuration.md diff --git a/config.sample.yaml b/config.sample.yaml index b1078b6..82eb600 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -183,6 +183,7 @@ services: - host: nextcloud.${TRAEFIK_DOMAIN} httpAuth: false internalPort: 443 + internalScheme: https - name: portainer enabled: true vpn: false @@ -230,7 +231,6 @@ services: internalPort: 80 - name: gluetun enabled: false - vpn: false traefik: enabled: false rules: [] diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index 0fdeb28..a7637ed 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -1,34 +1,37 @@ # Upgrade to V2 -Seedbox version 2 is here! +**Seedbox version 2 is here!** 🔥 + Since there are some breaking changes and a configuration structure migration, a major version was mandatory. > These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet. ## What's new? -* Configuration change to new YAML format +* **Configuration change to new YAML format** * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format. - * jq (v1.5+) and yq (v4+) are now requirements + * ⚠️ jq (v1.5+) and yq (v4+) are now requirements * Easier feature switches * If a service is missing, it won't be enabled by default like before. The config is now more declarative. * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example. -* VPN support + * New config syntax documented in the [Configuration Guide](./configuration.md). +* **VPN support** * With ``gluetun`` service, you can now place any service behind a VPN. * Default gluetun configuration is Wireguard "custom" mode, but see below... -* Support custom services and docker-compose yaml files - * Place a docker-compose.yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. +* **Support custom services and docker-compose yaml files** + * Place a docker-compose yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)). -* Support arbitrary Traefik rules + * More details in the [Configuration Guide](./configuration.md). +* **Support arbitrary Traefik rules** * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. * Ideal to forward traffic to other services which do not belong to this seedbox. -* New services: +* **New services** * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun) * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall) * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr) * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga) * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita) -* Docker compose v2.2+ is now required +* ⚠️ Docker compose v2.2+ is now required And also: @@ -37,14 +40,20 @@ And also: * You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)). * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set. * ``networks:`` section is now aligned with the new docker compose syntax -* Nextcloud-db has moved. It is now in /data/config (see below how to mitigate the errors). +* ⚠️ Nextcloud-db has moved. It is now in ``/data/config`` (or somewhere else if you set the new variables for host paths) (see below how to mitigate the errors). [See the dedicated section below](#nextcloud-db-has-moved). * Disable Traefik access logs +* New flag ``--debug`` for ``run-seedbox.sh`` to see what is happening during configuration parsing. + +## Some reading about configuration + +📖 Do not forget to read the [Configuration Guide](./configuration.md). ## How to migrate ```sh ./config-updater.sh # Check the content of your .env file (in comparison with .env.sample which brings new variables) +# Also, check your generated config.yaml and read the config documentation (in doc/configuration.md) ./run-seedbox.sh ``` @@ -54,7 +63,7 @@ When everything runs smoothly, you can delete your old configuration file: rm -f services.conf ``` -> Also, please make sure you have read the next section about Nextcloud Database location. +> ⚠️ Also, please make sure you have read the next section about Nextcloud Database location. ## Nextcloud-db has moved diff --git a/doc/configuration.md b/doc/configuration.md new file mode 100644 index 0000000..334a755 --- /dev/null +++ b/doc/configuration.md @@ -0,0 +1,211 @@ +# Seedbox configuration + +Almost the whole stack can be configured by using the main configuration item: ``config.yaml``. +Here is what it looks like: + +```yaml +# List of all services +services: + # Name of the service + - name: traefik + # Flag indicating if the service will be created + enabled: true + # Define traefik behavior for this service + traefik: + # Enable or disable Traefik routing. For example, if your service is a DB, disable Traefik. + enabled: true + # Routing rules, which will be processed and rendered as Traefik "dynamic configuration" via file provider + rules: + # Host to match request. Any environment variable is supported here, as long as there are braces around it. + - host: traefik.${TRAEFIK_DOMAIN} + # Traefik service to match (if it is a particular one). Here the "api@internal" service is internal to Traefik (dashboard access). + service: api@internal + # Enable http authentication + httpAuth: true + # Another service + - name: deluge + enabled: true + # Enable VPN (default to false). Service "gluetun" must be configured and enabled (with proper variables set in .env) to be able to use vpn mode on any service. + vpn: true + traefik: + enabled: true + rules: + - host: deluge.${TRAEFIK_DOMAIN} + httpAuth: true + # Internal container port on which we want to bind the Traefik routing + internalPort: 8112 + # Another service + - name: flaresolverr + enabled: true + # Do not use VPN (same as nothing as false is default) + vpn: false + traefik: + # This service is not reachable directly (no UI). So Traefik is disabled and rules are an empty array. + enabled: false + # Optional, won't be evaluated as Traefik is disabled + rules: [] + # Another service with 2 Traefik rules + - name: sonarr + enabled: true + vpn: false + traefik: + enabled: true + rules: + # First "regular" routing rule + - host: sonarr.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8989 + # Another rule which bypasses certificate generation using Let's Encrypt (ACME challenge). + - host: sonarr-unsecure.${TRAEFIK_DOMAIN} + httpAuth: true + internalPort: 8989 + # Using this flag, sonarr-unsecure.domain.com (for example) will be accesisble ONLY via http protocol + httpOnly: true + # Another service with backend using https + - name: nextcloud + enabled: false + vpn: false + traefik: + enabled: true + rules: + - host: nextcloud.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 443 + # Specify that the routing will be on https://nextcloud:443 (internally), while by default services expose only http. + # Nextcloud is known to be an exception and exposes only port 443 with SSL + internalScheme: https +``` + +The provided ``config.sample.yaml`` is good enough to get started and will be used if no ``config.yaml`` is found. + +Some general rules: + +* In order to enable VPN (``vpn: true``) on a service, you must configure and enable gluetun service. +* By default, all services match "http://service_name:port" for routing. +* By default, when ``httpOnly`` is false or not set, service will be accessible from both http and https, but a redirection will be created from http to https. + * And when ``httpOnly`` is set to true, the service will be accessible ONLY via http, bypassing certificate generation. It is useful when you do not want Traefik to handle certificates for this service. + +## Environment variables + +Also, do not forget to edit your ``.env`` file, which is where all the data which will be sent to containers (passwords, tokens, uid for disk permission...) lives. + +## Add your own service + +Let's say you want to add a container nginx without interfering or creating conflicts in this git repository. That's possible. + +Start by creating a files named nginx.yaml in the [services/custom/](services/custom/) directory: + +```yaml +services: + nginx: + image: nginx:latest + container_name: nginx + restart: always + # ... +``` + +Then, enable it in your ``config.yaml``: + +```yaml +services: + # ... + - name: nginx + enabled: true + vpn: false + # Specify the path to your custom docker-compose file, relative to the "services" directory + customFile: custom/nginx.yaml + traefik: + enabled: true + rules: + - host: nginx.${TRAEFIK_DOMAIN} + httpAuth: false + # 80 because official nginx image uses this port + internalPort: 80 + # ... +``` + +...and you're set! + +Please note that the ``customFile`` flag **MUST** be relative to the "services" directory, because in some cases, some alternatives yaml files for bundled services are provided in the services directory. +For example, in this repository is provided a custom "plex-hardware-transcoding.yaml" file, with all the necessary adaptations to make Plex run with hardware transcoding enabled. Just add the ``customFile`` field in the ``plex`` service and this file will be used, instead of the default "plex.yaml". + +## Integration with other services (custom Traefik config) + +You can also add you own Traefik configuration to integrate with local services on your LAN. +Just put your Traefik configuration file in the [samples/custom-traefik/](../samples/custom-traefik/) directory. +All files will be copied in the Traefik configuration directory on each ``run-seedbox`` execution. +Example: + +```yaml +http: + routers: + synology-admin: + rule: 'Host(`synology-admin.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + service: admin + services: + synology-admin: + loadBalancer: + servers: + - url: "https://your-nas-hostname-on-your-local-network:5001" +``` + +## Disable HTTPS completely + +If you want to handle your certificates on a firewall or another reverse proxy somewhere else on your network, it is now possible. +You just have to set ``httpOnly: true`` on all your services in ``config.yaml``. +Then, make the machine which acts as reverse proxy (and handles certificates) points on every Traefik URL with the correpsonding certificate, or make a wildcard redirection, based on your reverse proxy. + +## How does it work? + +Behind the scenes, the ``run-seedbox.sh`` script will parse your ``config.yaml`` file and will generate a Traefik dynamic configuration file, which looks like this: + +```yaml +http: + routers: + deluge-1: + rule: 'Host(`deluge.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + - redirect-to-https + service: deluge-1 + sonarr-1: + rule: 'Host(`sonarr.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + - redirect-to-https + service: sonarr-1 + sonarr-2: + rule: 'Host(`sonarr-unsecure.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - common-auth@file + service: sonarr-2 + entryPoints: + - insecure + nextcloud: + rule: 'Host(`nextcloud.{{ env "TRAEFIK_DOMAIN" }}`)' + middlewares: + - redirect-to-https + service: nextcloud-1 + services: + deluge-1: + loadBalancer: + servers: + # Gluetun is automatically set by run-seedbox.sh (instead of "deluge") because vpn was enabled on this service + - url: "http://gluetun:8112" + sonarr-1: + loadBalancer: + servers: + - url: "http://sonarr:8989" + sonarr-2: + loadBalancer: + servers: + - url: "http://sonarr:8989" + nextcloud-1: + loadBalancer: + servers: + - url: "https://nextcloud:443" +``` + +This file will be automatically placed in [traefik/custom/](../traefik/custom/) directory (mounted by Traefik container) so the config will dynamically apply. This file is updated on each ``run-seedbox.sh`` execution. From 36304749dab7966fcdfd806cf572aaf6e3c59fc8 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 23:34:05 +0100 Subject: [PATCH 33/42] Add doc --- README.md | 21 ++++++++++++--------- doc/UPGRADE_V2.md | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 056ecab..0ad51c9 100644 --- a/README.md +++ b/README.md @@ -34,24 +34,21 @@ seedbox and personal media server. | Heimdall | yourdomain.com | [linuxserver/heimdall](https://hub.docker.com/r/linuxserver/heimdall)| *latest* | Main dashboard | | Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client | -The front-end reverse proxy (Traefik - **check [this guide](doc/traefik_v2.md) if you still have the seedbox with Traefik v1**) routes based on the lowest level subdomain - (e.g. `deluge.example.com` would route to deluge). Since this is how the router -works, it is recommended for you to get a top level domain. If you do not have -one, you can edit your domains locally by changing your hosts file or use a -browser plugin that changes the host header. +The front-end reverse proxy (Traefik - **check [this guide](doc/traefik_v2.md) if you still have the seedbox with Traefik v1**) routes based on the lowest level subdomain (e.g. `deluge.example.com` would route to deluge). Since this is how the router works, it is recommended for you to get a top level domain. If you do not have one, you can edit your domains locally by changing your hosts file or use a browser plugin that changes the host header. Traefik takes care of valid Let's Encrypt certificates and auto-renewal. -Note: Plex is also available directly through the `32400` port without going -through the reverse proxy. +Note: Plex is also available directly through the `32400` port without going through the reverse proxy. ## Dependencies - [Docker](https://github.com/docker/docker) >= 20.10 - [Docker Compose](https://github.com/docker/compose) >= 2.2 - [local-persist Docker plugin](https://github.com/MatchbookLab/local-persist): installed directly on host (not in container). This is a volume plugin that extends the default local driver’s functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to always persist, even if the volume is removed via `docker volume rm`. Use *systemd* install for Ubuntu. +- [jq](https://stedolan.github.io/jq/download/) >= 1.5 +- [yq](https://github.com/mikefarah/yq/releases) > 4 -## Configuration +## Set up for the first time Before running, please create the volumes which will be statically mapped to the ones on the host: For example: @@ -64,13 +61,19 @@ sudo su -c "mkdir /data && mkdir /data/config && mkdir /data/torrents" Edit the `.env` file and change the variables as desired. The variables are all self-explanatory. +## Configuration + +The configuration lives in the ``config.yaml`` file. + +All you need to know is located in the [Configuration Guide](doc/configuration.md). + ## Running & updating ```sh ./run-seedbox.sh ``` -docker-compose should manage all the volumes and network setup for you. If it does not, verify that your docker and docker-compose version is updated. +All services and synamic configuration will be automatically created without further action from your part. Make sure you install the dependencies and finish configuration before doing this. diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index a7637ed..5db93b3 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -20,7 +20,7 @@ Since there are some breaking changes and a configuration structure migration, a * Default gluetun configuration is Wireguard "custom" mode, but see below... * **Support custom services and docker-compose yaml files** * Place a docker-compose yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. - * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a customFile on plex service - see [config.sample.yaml](config.sample.yaml)). + * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a ``customFile`` on plex service - see [config.sample.yaml](../config.sample.yaml)). * More details in the [Configuration Guide](./configuration.md). * **Support arbitrary Traefik rules** * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. From af77d8c0f9b6a247d82146e4bc35a9ae44d53210 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 23:34:49 +0100 Subject: [PATCH 34/42] Typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ad51c9..61ed213 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Note: Plex is also available directly through the `32400` port without going thr - [Docker Compose](https://github.com/docker/compose) >= 2.2 - [local-persist Docker plugin](https://github.com/MatchbookLab/local-persist): installed directly on host (not in container). This is a volume plugin that extends the default local driver’s functionality by allowing you specify a mountpoint anywhere on the host, which enables the files to always persist, even if the volume is removed via `docker volume rm`. Use *systemd* install for Ubuntu. - [jq](https://stedolan.github.io/jq/download/) >= 1.5 -- [yq](https://github.com/mikefarah/yq/releases) > 4 +- [yq](https://github.com/mikefarah/yq/releases) >= 4 ## Set up for the first time From 44ee173213855a502d8003a36654d33e9cc6b228 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Sun, 6 Mar 2022 23:48:58 +0100 Subject: [PATCH 35/42] Clean + add doc --- doc/UPGRADE_V2.md | 6 ++++++ run-seedbox.sh | 10 +++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index 5db93b3..f7e5655 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -25,6 +25,12 @@ Since there are some breaking changes and a configuration structure migration, a * **Support arbitrary Traefik rules** * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. * Ideal to forward traffic to other services which do not belong to this seedbox. +* **Disable certificates for some domains** + * Using the flag ``httpOnly: true`` on a service, access any service in unsecure mode, delegating certificates management on a higher level (reverse proxy, firewall...). More details in the [Configuration Guide](./configuration.md). +* **Multiple hosts for any services** + * The new config structure allows for more customization, for example you can now have many routes on the same service. Let's say, a local unsecured route + a secured one for remote access. Or anything you want. +* **More customization** + * Such as http authentication which is no more hardcoded but configurable for each service. * **New services** * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun) * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall) diff --git a/run-seedbox.sh b/run-seedbox.sh index 3c3e26a..bddd2e4 100755 --- a/run-seedbox.sh +++ b/run-seedbox.sh @@ -231,12 +231,6 @@ for json in $(yq eval -o json config.yaml | jq -c ".services[]"); do echo "http.routers.${ruleId}.service: ${ruleId}" >> rules.props fi - # TODO: leave it or remove it? - disableCertificateGeneration=$(echo $rule | jq -r .disableCertificateGeneration) - if [[ ${disableCertificateGeneration} == true ]]; then - echo "http.routers.${ruleId}.tls: EMPTYMAP" >> rules.props - fi - # Check if httpOnly flag is enabled # If enabled => Specify to use only "insecure" (port 80) entrypoint # If not => use all entryPoints (by not specifying any) but force redirection to https @@ -263,8 +257,10 @@ yq -p=props rules.props > traefik/custom/dynamic-rules.yaml rm -f rules.props # Post-transformations on the rules file -sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml +# sed -i "s/EMPTYMAP/{}/g" traefik/custom/dynamic-rules.yaml +# Add simple quotes around Host rule sed -i --regexp-extended "s/^(.*: )(Host.*$)/\1'\2'/g" traefik/custom/dynamic-rules.yaml +# Add double quotes around the backend traefik service sed -i --regexp-extended "s/^(.*url: )(.*$)/\1\"\2\"/g" traefik/custom/dynamic-rules.yaml rm -f traefik/custom/dynamic-rules-old.yaml From 9de9a3276621dbbd8c8d037535c3625154bb6437 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 00:04:01 +0100 Subject: [PATCH 36/42] Add VPN section in configuration.md --- doc/configuration.md | 31 +++++++++++++++++++++++++++++++ samples/custom-traefik/README.md | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/doc/configuration.md b/doc/configuration.md index 334a755..de0be0e 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -209,3 +209,34 @@ http: ``` This file will be automatically placed in [traefik/custom/](../traefik/custom/) directory (mounted by Traefik container) so the config will dynamically apply. This file is updated on each ``run-seedbox.sh`` execution. + +## VPN + +In order to hide a service behind a VPN, just enable ``gluetun`` service. + +By default, the file used is [gluetun.yaml](../services/gluetun.yaml), which is in "Wireguard custom" mode, meaning you must have somewhere a Wireguard server running and access to its client configuration. But you can add your own config to match your requirements. See sections below. + +### Default mode - Wireguard custom + +* Edit the .env and replace the Wireguard variables with your own (take them in ``.env.sample``). +* Enable ``gluetun`` service. +* Enable vn (``vpn: true``) on any service. +* Run ``./run-seedbox.sh``. +* The service now uses Wireguard. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. + +### Your own mode (VPN provider supported by gluetun) + +* Create a ``gluetun-custom.yaml`` in the [services/custom/](../services/custom/) directory. You can duplicate [this one](../services/gluetun.yaml) to avoid starting from scratch. +* Adapt it to your needs (variables, mode...) according to your provider. + * Add all variables you may need (used in your custom yaml file in your ``.env`` file (replacing the wireguard ones). +* Edit your ``config.yaml`` and add ``customFile: custom/gluetun-custom.yaml`` in the ``gluetun`` section. +* Enable vn (``vpn: true``) on any service. +* Run ``./run-seedbox.sh``. +* The service now uses your VPN by tunneling via gluetun container. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. + +### How does VPN is handled? + +Behind the scenes, the ``run-seedbox.sh`` script will mainly add 2 overrides when enabling VPN on a service: + +* Adds a file in [services/generated/](../services/generated/) which adds a ``network_mode: gluetun`` for your service. +* Specify in Traefik rule that the backend host is gluetun instead of the service directly. diff --git a/samples/custom-traefik/README.md b/samples/custom-traefik/README.md index b11a2a4..60eebd1 100644 --- a/samples/custom-traefik/README.md +++ b/samples/custom-traefik/README.md @@ -1,3 +1,3 @@ # Custom Traefik config files -Place in this directory any custom Traefik config file, they will be copied to Traefik folder when running ``update-all.sh`` script. +Place in this directory any custom Traefik config file, they will be copied to Traefik folder when running ``run-seedbox.sh`` script. From 7ce8e0cf6183518086ea12bb631a616ffae9b88b Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 00:09:15 +0100 Subject: [PATCH 37/42] More documentation --- doc/UPGRADE_V2.md | 8 +++++--- doc/configuration.md | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index f7e5655..15b1b74 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -14,19 +14,21 @@ Since there are some breaking changes and a configuration structure migration, a * Easier feature switches * If a service is missing, it won't be enabled by default like before. The config is now more declarative. * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example. - * New config syntax documented in the [Configuration Guide](./configuration.md). + * New config syntax documented in the [Configuration Guide](./configuration.md#configuration-structure-and-parameters). * **VPN support** * With ``gluetun`` service, you can now place any service behind a VPN. * Default gluetun configuration is Wireguard "custom" mode, but see below... + * More details in the [VPN section of the Configuration Guide](./configuration.md#vpn). * **Support custom services and docker-compose yaml files** * Place a docker-compose yaml file in ``services/custom/`` directory, add a service in your config.yaml specifying a ``customFile``, and you are set. * Support Plex hardware transcoding using a custom-file, already available in the ``services`` directory (just specify a ``customFile`` on plex service - see [config.sample.yaml](../config.sample.yaml)). - * More details in the [Configuration Guide](./configuration.md). + * More details in the [Configuration Guide](./configuration.md#add-your-own-service). * **Support arbitrary Traefik rules** * Place a Traefik YAML in ``samples/custom-traefik/`` directory, it will be copied in the Traefik configuration folder. * Ideal to forward traffic to other services which do not belong to this seedbox. + * More details in [this section of the Configuration Guide](./configuration.md#integration-with-other-services-custom-traefik-config) * **Disable certificates for some domains** - * Using the flag ``httpOnly: true`` on a service, access any service in unsecure mode, delegating certificates management on a higher level (reverse proxy, firewall...). More details in the [Configuration Guide](./configuration.md). + * Using the flag ``httpOnly: true`` on a service, access any service in unsecure mode, delegating certificates management on a higher level (reverse proxy, firewall...). More details in the [Configuration Guide](./configuration.md#disable-https-completely). * **Multiple hosts for any services** * The new config structure allows for more customization, for example you can now have many routes on the same service. Let's say, a local unsecured route + a secured one for remote access. Or anything you want. * **More customization** diff --git a/doc/configuration.md b/doc/configuration.md index de0be0e..104d98c 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,5 +1,7 @@ # Seedbox configuration +## Configuration structure and parameters + Almost the whole stack can be configured by using the main configuration item: ``config.yaml``. Here is what it looks like: From 5eefdb0e99c817fe46adf67afe8510a9075223dc Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 00:38:30 +0100 Subject: [PATCH 38/42] Typos + additions in doc --- config.sample.yaml | 5 +++-- doc/UPGRADE_V2.md | 1 + doc/configuration.md | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index 82eb600..c300c54 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -19,7 +19,7 @@ services: # Uncomment to specify custom schme (http by default) # internalScheme: http # Uncomment to *NOT* generate LetsEncrypt certificate (useful for local domains) - # disableCertificateGeneration: true + # httpOnly: true - name: flood enabled: true vpn: false @@ -221,7 +221,8 @@ services: httpAuth: true internalPort: 8384 - name: heimdall - enabled: true + # Beware to the host when enabling it! + enabled: false vpn: false traefik: enabled: true diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index 15b1b74..7e2562a 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -39,6 +39,7 @@ Since there are some breaking changes and a configuration structure migration, a * ``Readarr``: [Ebook and comic monitor](https://github.com/Readarr/Readarr) * ``Komga``: [Comic Book Manager](https://github.com/gotson/komga) * ``Kavita``: [Comic / Book Manager](https://github.com/Kareadita/Kavita) + * ``Syncthing``: [P2P files synchronization](https://github.com/linuxserver/docker-syncthing) * ⚠️ Docker compose v2.2+ is now required And also: diff --git a/doc/configuration.md b/doc/configuration.md index 104d98c..d5bc28d 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -95,7 +95,7 @@ Also, do not forget to edit your ``.env`` file, which is where all the data whic Let's say you want to add a container nginx without interfering or creating conflicts in this git repository. That's possible. -Start by creating a files named nginx.yaml in the [services/custom/](services/custom/) directory: +Start by creating a file named nginx.yaml in the [services/custom/](services/custom/) directory: ```yaml services: @@ -230,7 +230,7 @@ By default, the file used is [gluetun.yaml](../services/gluetun.yaml), which is * Create a ``gluetun-custom.yaml`` in the [services/custom/](../services/custom/) directory. You can duplicate [this one](../services/gluetun.yaml) to avoid starting from scratch. * Adapt it to your needs (variables, mode...) according to your provider. - * Add all variables you may need (used in your custom yaml file in your ``.env`` file (replacing the wireguard ones). + * Add all variables you may need (used in your custom yaml file) in your ``.env`` file (replacing the wireguard ones). * Edit your ``config.yaml`` and add ``customFile: custom/gluetun-custom.yaml`` in the ``gluetun`` section. * Enable vn (``vpn: true``) on any service. * Run ``./run-seedbox.sh``. From 53f4d6ab879f1c0e573c7420750fbd2d210dbf5d Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 19:13:57 +0100 Subject: [PATCH 39/42] Improve documentations --- README.md | 59 ++++++++++++++++++++++++++++++++------------ doc/UPGRADE_V2.md | 4 +-- doc/configuration.md | 4 +-- services.conf.sample | 28 --------------------- 4 files changed, 46 insertions(+), 49 deletions(-) delete mode 100644 services.conf.sample diff --git a/README.md b/README.md index 61ed213..4f82ec3 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,33 @@ -# Seedbox +

Seedbox

+

+ An extensive and hackable collection of containerized services to set up a seedbox and personal media server. +

+
-A collection of Dockerfiles and a docker-compose configuration to set up a -seedbox and personal media server. +## ✨ Features -⚠️ Version 2 is released, please make sure you read [this V2 Migration Guide](doc/UPGRADE_V2.md) as there are breaking changes! +* Easy to configure personal media server without needing too much technical skills +* Compatible with multiple systems (Linux servers, desktops, Synology NAS...) +* Automatic HTTPS ceritificates management and renewal + * Support for HTTP only too if required for your use-case +* Everything is hackable + * Add your own services + * Disable the ones you do not want + * Customize or add your own routing rules to integrate with existing services + * Tweak any service to your need by using custom file parameter on any service +* VPN support with multiple providers + * Hide the service(s) of your choice behind a VPN tunnel + * Non mandatory +* Declarative configuration +* Persistent data for your media +* Install & update using the same script +* Start with the [Configuration Guide](doc/configuration.md) -## Included Applications +## ⚠️ News + +Version 2 is released, please make sure you read [this V2 Migration Guide](doc/UPGRADE_V2.md) as there are breaking changes! + +## 📦 Included Applications | Application | Web Interface | Docker image | Version (image tag) | Notes | -----------------------|----------------------------|------------------------------------------------------------------------|-------------------------|---------------------| @@ -27,12 +49,17 @@ seedbox and personal media server. | Tautulli (plexPy) | tautulli.yourdomain.com | [linuxserver/tautulli](https://hub.docker.com/r/linuxserver/tautulli) | *latest* | Plex stats and admin| | Tdarr | tdarr.yourdomain.com | [haveagitgat/tdarr](https://hub.docker.com/r/haveagitgat/tdarr) | *latest* | Re-encode files | | NextCloud | nextcloud.yourdomain.com | [linuxserver/nextcloud](https://hub.docker.com/r/linuxserver/nextcloud) | *latest* | Files management | -| NextCloud-db (MariaDB) | not reachable | [mariadb](https://hub.docker.com/r/_/mariadb) | *10* | DB for Nextcloud | +| NextCloud-db (MariaDB) | *not reachable* | [mariadb](https://hub.docker.com/r/_/mariadb) | *10* | DB for Nextcloud | | Portainer | portainer.yourdomain.com | [portainer/portainer](https://hub.docker.com/r/portainer/portainer) | *latest* | Container management| | Netdata | netdata.yourdomain.com | [netdata/netdata](https://hub.docker.com/r/netdata/netdata) | *latest* | Server monitoring | | Duplicati | duplicati.yourdomain.com | [linuxserver/duplicati](https://hub.docker.com/r/linuxserver/duplicati)| *latest* | Backups | | Heimdall | yourdomain.com | [linuxserver/heimdall](https://hub.docker.com/r/linuxserver/heimdall)| *latest* | Main dashboard | +| Syncthing | syncthing.yourdomain.com | [linuxserver/syncthing](https://hub.docker.com/r/linuxserver/syncthing) | *latest* | P2P files sharing | +| Traefik | traefik.yourdomain.com | [traefik](https://hub.docker.com/_/traefik) | *latest* | Traefik reverse proxy (access to admin dashboard) | | Gluetun | - | [qmcgaw/gluetun](https://hub.docker.com/r/qmcgaw/gluetun)| *latest* | VPN client | +| *Any application you want!* | *whatever.yourdomain.com* | *Any image* | *Any tag* | *Any service - See the [Configuration Guide](doc/configuration.md)* | + +## 🌐 Traefik The front-end reverse proxy (Traefik - **check [this guide](doc/traefik_v2.md) if you still have the seedbox with Traefik v1**) routes based on the lowest level subdomain (e.g. `deluge.example.com` would route to deluge). Since this is how the router works, it is recommended for you to get a top level domain. If you do not have one, you can edit your domains locally by changing your hosts file or use a browser plugin that changes the host header. @@ -40,7 +67,12 @@ Traefik takes care of valid Let's Encrypt certificates and auto-renewal. Note: Plex is also available directly through the `32400` port without going through the reverse proxy. -## Dependencies +You can also add your own Traefik rules to integrate with other services (deployed wihthin docker or somewhere else on your LAN, or even on the Internet). +Check the [Configuration Guide](doc/configuration.md). + +## ⚙️ Installation + +### Dependencies - [Docker](https://github.com/docker/docker) >= 20.10 - [Docker Compose](https://github.com/docker/compose) >= 2.2 @@ -48,7 +80,7 @@ Note: Plex is also available directly through the `32400` port without going thr - [jq](https://stedolan.github.io/jq/download/) >= 1.5 - [yq](https://github.com/mikefarah/yq/releases) >= 4 -## Set up for the first time +### Prepare your host Before running, please create the volumes which will be statically mapped to the ones on the host: For example: @@ -61,13 +93,13 @@ sudo su -c "mkdir /data && mkdir /data/config && mkdir /data/torrents" Edit the `.env` file and change the variables as desired. The variables are all self-explanatory. -## Configuration +### Review the configuration The configuration lives in the ``config.yaml`` file. All you need to know is located in the [Configuration Guide](doc/configuration.md). -## Running & updating +### Running & updating ```sh ./run-seedbox.sh @@ -77,12 +109,7 @@ All services and synamic configuration will be automatically created without fur Make sure you install the dependencies and finish configuration before doing this. -## PlexPass - -Just set the `VERSION` environment variable to `latest` on the Plex service (enabled by default). -See [this link](https://hub.docker.com/r/linuxserver/plex). - -## Where is my data? +### Where is my data? All data is saved in the docker volumes `seedbox_config` or `seedbox_torrents`. diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index 7e2562a..d70766c 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -4,8 +4,6 @@ Since there are some breaking changes and a configuration structure migration, a major version was mandatory. -> These releases notes are still a work-in-progress as V2 is not fully tested and finalized yet. - ## What's new? * **Configuration change to new YAML format** @@ -66,7 +64,7 @@ And also: ./run-seedbox.sh ``` -When everything runs smoothly, you can delete your old configuration file: +When everything runs smoothly, you can delete your old configuration file which is now useless: ```sh rm -f services.conf diff --git a/doc/configuration.md b/doc/configuration.md index d5bc28d..6141952 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -129,7 +129,7 @@ services: ...and you're set! Please note that the ``customFile`` flag **MUST** be relative to the "services" directory, because in some cases, some alternatives yaml files for bundled services are provided in the services directory. -For example, in this repository is provided a custom "plex-hardware-transcoding.yaml" file, with all the necessary adaptations to make Plex run with hardware transcoding enabled. Just add the ``customFile`` field in the ``plex`` service and this file will be used, instead of the default "plex.yaml". +For example, in this repository is provided a custom "plex-hardware-transcoding.yaml" file, with all the necessary adaptations to make Plex run with hardware transcoding enabled. Just add the ``customFile: plex-hardware-transcoding.yaml`` field in the ``plex`` service and this file will be used, instead of the default "plex.yaml". ## Integration with other services (custom Traefik config) @@ -232,7 +232,7 @@ By default, the file used is [gluetun.yaml](../services/gluetun.yaml), which is * Adapt it to your needs (variables, mode...) according to your provider. * Add all variables you may need (used in your custom yaml file) in your ``.env`` file (replacing the wireguard ones). * Edit your ``config.yaml`` and add ``customFile: custom/gluetun-custom.yaml`` in the ``gluetun`` section. -* Enable vn (``vpn: true``) on any service. +* Enable vpn (``vpn: true``) on any service. * Run ``./run-seedbox.sh``. * The service now uses your VPN by tunneling via gluetun container. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. diff --git a/services.conf.sample b/services.conf.sample deleted file mode 100644 index 3369ae3..0000000 --- a/services.conf.sample +++ /dev/null @@ -1,28 +0,0 @@ -deluge: enable -deluge-vpn: disable -flood: enable -plex: enable -plex-hardware-transcoding: disable -flaresolverr: enable -jackett: enable -prowlarr: enable -sonarr: enable -radarr: enable -bazarr: enable -lidarr: disable -readarr: disable -komga: disable -kavita: disable -ombi: disable -overseerr: enable -tautulli: enable -jdownloader: enable -jdownloader-vpn: disable -tdarr: disable -nextcloud: enable -portainer: enable -netdata: enable -duplicati: disable -syncthing: disable -heimdall: enable -gluetun: disable \ No newline at end of file From 2b4bc46d5c86c27bca6545141b19411ed35a892a Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 22:46:00 +0100 Subject: [PATCH 40/42] Prepare v2 release --- config.sample.yaml | 1 + doc/UPGRADE_V2.md | 2 +- doc/configuration.md | 156 ++++++++++++++++++++++++++--------- doc/traefik_v2.md | 2 + services/generated/README.md | 2 +- 5 files changed, 120 insertions(+), 43 deletions(-) diff --git a/config.sample.yaml b/config.sample.yaml index c300c54..9eb973b 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -230,6 +230,7 @@ services: - host: ${TRAEFIK_DOMAIN} httpAuth: true internalPort: 80 + # Gluetun is the VPN client service. - name: gluetun enabled: false traefik: diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index d70766c..62877c0 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -8,7 +8,7 @@ Since there are some breaking changes and a configuration structure migration, a * **Configuration change to new YAML format** * Run ``config-updater.sh`` to migrate your old services.conf to the new config.yaml format. - * ⚠️ jq (v1.5+) and yq (v4+) are now requirements + * ⚠️ ``jq`` (v1.5+) and ``yq`` (v4+) are now requirements * Easier feature switches * If a service is missing, it won't be enabled by default like before. The config is now more declarative. * Traefik routing rules are now dynamically generated in a file in Traefik config directory, so no more Docker labels. They became hard to maintain due to all possibilities caused by VPN support or custom files for example. diff --git a/doc/configuration.md b/doc/configuration.md index 6141952..b90d90e 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,5 +1,44 @@ # Seedbox configuration +## Table of content + +* [General principles](#general-principles) +* [Configuration structure and parameters](#configuration-structure-and-parameters) +* [Environment variables](#environment-variables) +* [Add your own service](#add-your-own-service) + * [Particular case: Plex with Hardware Transcoding](#particular-case-plex-with-hardware-transcoding) +* [Integration with other services (custom Traefik config)](#integration-with-other-services-custom-traefik-config) +* [Disable HTTPS completely](#disable-https-completely) +* [VPN](#vpn) + * [Default mode - Wireguard custom](#default-mode---wireguard-custom) + * [Your own mode (VPN provider supported by gluetun)](#your-own-mode-vpn-provider-supported-by-gluetun) + * [How does VPN is handled?](#how-does-vpn-is-handled) +* [Make the services communicate with each other](#make-the-services-communicate-with-each-other) +* [How does the configuration work?](#how-does-the-configuration-work) + +## General principles + +Every service (Plex, Sonarr, Jackett, Nextcloud...) is defined in a dedicated file (in docker-compose format) in the [services](../services/) directory. + +All your configuration such as: + +* Which services must be enabled +* What docker-compose file they are using if a a particular one must be applied instead of the default one (which is the one with the same name as the service) +* Routing rules (example: ``sonarr.yourdomain.com`` for Sonarr) +* If http authentication must be enabled (example: *enable* for Sonarr, *disable* for Nextcloud has it has built-in authentication) +* Some other parameters (see below) + +... is located in ``config.yaml``. If you are starting fresh, copy ``config.sample.yaml`` into ``config.yaml``. If you don't, on the first run, ``./run-seedbox.sh`` will copy the sample file for you. + +Then, all your "sensitive" (or "personal") data configuration (passwords, usernames, domain name, paths on the machine for persistent data...) is located in ``.env`` file. + +When running ``./run-seedbox.sh``: + +* ``.env`` is loaded +* ``config.yaml`` is parsed, some checks are performed +* For each service, if enabled, Traefik rules are generated ([see this section](#how-does-the-configuration-work)) dynamically +* docker-compose commands (pull, up...) are executed against all relevant files + ## Configuration structure and parameters Almost the whole stack can be configured by using the main configuration item: ``config.yaml``. @@ -10,9 +49,9 @@ Here is what it looks like: services: # Name of the service - name: traefik - # Flag indicating if the service will be created + # Flag indicating if the service will be created or not enabled: true - # Define traefik behavior for this service + # Define traefik (reverse proxy) behavior for this service traefik: # Enable or disable Traefik routing. For example, if your service is a DB, disable Traefik. enabled: true @@ -20,7 +59,7 @@ services: rules: # Host to match request. Any environment variable is supported here, as long as there are braces around it. - host: traefik.${TRAEFIK_DOMAIN} - # Traefik service to match (if it is a particular one). Here the "api@internal" service is internal to Traefik (dashboard access). + # Traefik service to match (if it is a particular one). Here the "api@internal" service is internal to Traefik (dashboard access). If not specified, a "traefik service" with the same name as the docker service will be created. service: api@internal # Enable http authentication httpAuth: true @@ -33,6 +72,7 @@ services: enabled: true rules: - host: deluge.${TRAEFIK_DOMAIN} + # No service is defined here => a traefik service named "deluge" will be created automatically httpAuth: true # Internal container port on which we want to bind the Traefik routing internalPort: 8112 @@ -61,7 +101,7 @@ services: - host: sonarr-unsecure.${TRAEFIK_DOMAIN} httpAuth: true internalPort: 8989 - # Using this flag, sonarr-unsecure.domain.com (for example) will be accesisble ONLY via http protocol + # Using this flag, sonarr-unsecure.domain.com (for example) will be accessible ONLY via http protocol httpOnly: true # Another service with backend using https - name: nextcloud @@ -126,10 +166,36 @@ services: # ... ``` -...and you're set! +...and you're set! Just run ``./run-seedbox.sh`` to apply configuration changes. -Please note that the ``customFile`` flag **MUST** be relative to the "services" directory, because in some cases, some alternatives yaml files for bundled services are provided in the services directory. -For example, in this repository is provided a custom "plex-hardware-transcoding.yaml" file, with all the necessary adaptations to make Plex run with hardware transcoding enabled. Just add the ``customFile: plex-hardware-transcoding.yaml`` field in the ``plex`` service and this file will be used, instead of the default "plex.yaml". +> Please note that the ``customFile`` flag **MUST** be relative to the "services" directory, because in some cases, some alternatives yaml files for bundled services are provided in the services directory. See the next section for more details. + +### Particular case: Plex with Hardware Transcoding + +This project provides a custom [plex-hardware-transcoding.yaml](../services/plex-hardware-transcoding.yaml) file, with all the necessary adaptations to make Plex run with hardware transcoding enabled. + +Just add the ``customFile: plex-hardware-transcoding.yaml`` field in the ``plex`` service and this file will be used, instead of the default "plex.yaml": + +```yaml +services: + # ... + - name: plex + enabled: true + vpn: false + # Here is the change: by default, without this flag, the applied file was "plex.yaml" + customFile: plex-hardware-transcoding.yaml + traefik: + enabled: true + rules: + - host: plex.${TRAEFIK_DOMAIN} + httpAuth: false + internalPort: 32400 + # ... +``` + +Then, run ``./run-seedbox.sh`` to make these changes taken into account and the new container Plex will have hardware transcoding. + +> Note that you also have to enable Hardware Transcoding in your Plex Server settings after the container has started. ## Integration with other services (custom Traefik config) @@ -157,9 +223,48 @@ http: If you want to handle your certificates on a firewall or another reverse proxy somewhere else on your network, it is now possible. You just have to set ``httpOnly: true`` on all your services in ``config.yaml``. -Then, make the machine which acts as reverse proxy (and handles certificates) points on every Traefik URL with the correpsonding certificate, or make a wildcard redirection, based on your reverse proxy. +Then, make the machine which acts as reverse proxy (and handles certificates) points on every Traefik URL with the corresponding certificate, or make a wildcard redirection, based on your reverse proxy. -## How does it work? +## VPN + +In order to hide a service behind a VPN, just enable ``gluetun`` service. + +By default, the file used is [gluetun.yaml](../services/gluetun.yaml), which is in "Wireguard custom" mode, meaning you must have somewhere a Wireguard server running and access to its client configuration. But you can add your own config to match your requirements. See sections below. + +### Default mode - Wireguard custom + +* Edit the ``.env`` file and replace the Wireguard variables with your own (take them in ``.env.sample``). +* Enable ``gluetun`` service. +* Enable vpn (``vpn: true``) on any service. +* Run ``./run-seedbox.sh``. +* The service now uses Wireguard. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. + +### Your own mode (VPN provider supported by gluetun) + +* Create a ``gluetun-custom.yaml`` in the [services/custom/](../services/custom/) directory. You can duplicate [this one](../services/gluetun.yaml) to avoid starting from scratch. +* Adapt it to your needs (variables, mode...) according to your provider. + * Add all variables you may need (used in your custom yaml file) in your ``.env`` file (replacing the wireguard ones). +* Edit your ``config.yaml`` and add ``customFile: custom/gluetun-custom.yaml`` in the ``gluetun`` section. +* Enable vpn (``vpn: true``) on any service. +* Run ``./run-seedbox.sh``. +* The service now uses your VPN by tunneling via gluetun container. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. + +### How does VPN is handled? + +Behind the scenes, the ``run-seedbox.sh`` script will mainly add 2 overrides when enabling VPN on a service: + +* Adds a file in [services/generated/](../services/generated/) which adds a ``network_mode: gluetun`` for your service. +* Specify in Traefik rule that the backend host is gluetun instead of the service directly. + +## Make the services communicate with each other + +With docker-compose, all services are in the same Docker network (it is called ``traefik-network`` and is defined [here](../docker-compose.yaml)). Docker provides DNS resolution in the same network based on the name of the services, which act as hostnames. + +So, for example, in order to setup Deluge in Sonarr, just add ``http://deluge:8112`` in the Download Clients settings section in Sonarr. + +⚠️ If you are trying to contact a container which has ``vpn`` flag enabled, you will have to point your config to ``gluetun`` instead, which acts as relay to contact the service. So if Deluge is behind the VPN, add ``http://gluetun:8112`` in Sonarr instead. + +## How does the configuration work? Behind the scenes, the ``run-seedbox.sh`` script will parse your ``config.yaml`` file and will generate a Traefik dynamic configuration file, which looks like this: @@ -210,35 +315,4 @@ http: - url: "https://nextcloud:443" ``` -This file will be automatically placed in [traefik/custom/](../traefik/custom/) directory (mounted by Traefik container) so the config will dynamically apply. This file is updated on each ``run-seedbox.sh`` execution. - -## VPN - -In order to hide a service behind a VPN, just enable ``gluetun`` service. - -By default, the file used is [gluetun.yaml](../services/gluetun.yaml), which is in "Wireguard custom" mode, meaning you must have somewhere a Wireguard server running and access to its client configuration. But you can add your own config to match your requirements. See sections below. - -### Default mode - Wireguard custom - -* Edit the .env and replace the Wireguard variables with your own (take them in ``.env.sample``). -* Enable ``gluetun`` service. -* Enable vn (``vpn: true``) on any service. -* Run ``./run-seedbox.sh``. -* The service now uses Wireguard. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. - -### Your own mode (VPN provider supported by gluetun) - -* Create a ``gluetun-custom.yaml`` in the [services/custom/](../services/custom/) directory. You can duplicate [this one](../services/gluetun.yaml) to avoid starting from scratch. -* Adapt it to your needs (variables, mode...) according to your provider. - * Add all variables you may need (used in your custom yaml file) in your ``.env`` file (replacing the wireguard ones). -* Edit your ``config.yaml`` and add ``customFile: custom/gluetun-custom.yaml`` in the ``gluetun`` section. -* Enable vpn (``vpn: true``) on any service. -* Run ``./run-seedbox.sh``. -* The service now uses your VPN by tunneling via gluetun container. If gluetun is down or if the VPN link is broken, your service won't have any access to Internet. - -### How does VPN is handled? - -Behind the scenes, the ``run-seedbox.sh`` script will mainly add 2 overrides when enabling VPN on a service: - -* Adds a file in [services/generated/](../services/generated/) which adds a ``network_mode: gluetun`` for your service. -* Specify in Traefik rule that the backend host is gluetun instead of the service directly. +This file will be automatically placed in [traefik/custom/](../traefik/custom/) directory (mounted by Traefik container) so the config will dynamically apply. This file is updated on each ``run-seedbox.sh`` execution. \ No newline at end of file diff --git a/doc/traefik_v2.md b/doc/traefik_v2.md index a0de210..3b00b4f 100644 --- a/doc/traefik_v2.md +++ b/doc/traefik_v2.md @@ -1,5 +1,7 @@ # September 2020 - Upgrade to Traefik v2 instructions +> This guide is useful if you already are using the seedbox but did not update before September 2020. + Before upgrading Traefik to version 2, please check the following: - In this repo, Traefik v2 upgrade is as seamless as possible (same environment variables than before, out-of-the-box config file...). diff --git a/services/generated/README.md b/services/generated/README.md index d178d48..b2d4f84 100644 --- a/services/generated/README.md +++ b/services/generated/README.md @@ -1,3 +1,3 @@ # Generated files -This directory contains all generated overrides files. They are in .gitignore. +This directory contains all generated overrides files. They are in .gitignore so there should be no conflict when running ``git pull``. From c5cf3a16dbae713d2f7e0de98220bb0a0a4d0a25 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 22:48:13 +0100 Subject: [PATCH 41/42] Tiny doc fixes --- doc/UPGRADE_V2.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index 62877c0..d7c42f3 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -31,6 +31,7 @@ Since there are some breaking changes and a configuration structure migration, a * The new config structure allows for more customization, for example you can now have many routes on the same service. Let's say, a local unsecured route + a secured one for remote access. Or anything you want. * **More customization** * Such as http authentication which is no more hardcoded but configurable for each service. + * Configurable paths on host for persistent data * **New services** * ``Gluetun``: [VPN client (see above)](https://github.com/qdm12/gluetun) * ``Heimdall``: [Dashboard](https://github.com/linuxserver/Heimdall) @@ -45,7 +46,7 @@ And also: * ``update-all.sh`` is now called ``run-seedbox.sh`` but its purpose is the same. * More checks in ``run-seedbox.sh``. For example, throws an error if Flood is enabled but not Deluge, or if VPN is enabled on a service but the VPN client is not. * You can now specify where your data lives on your host through new environments variables (see [.env.sample](.env.sample)). - * This change is backward-compatible as the run-seedbox.sh script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set. + * This change is backward-compatible as the ``run-seedbox.sh`` script will default to the old "/data/torrents" and "/data/config" paths if these variables are not set. * ``networks:`` section is now aligned with the new docker compose syntax * ⚠️ Nextcloud-db has moved. It is now in ``/data/config`` (or somewhere else if you set the new variables for host paths) (see below how to mitigate the errors). [See the dedicated section below](#nextcloud-db-has-moved). * Disable Traefik access logs From b929a52bc2a3da7ef322ff6e0bb5740f6f7c82c6 Mon Sep 17 00:00:00 2001 From: Jean Froment Date: Mon, 7 Mar 2022 22:51:29 +0100 Subject: [PATCH 42/42] Release note --- doc/UPGRADE_V2.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/UPGRADE_V2.md b/doc/UPGRADE_V2.md index d7c42f3..916b86d 100644 --- a/doc/UPGRADE_V2.md +++ b/doc/UPGRADE_V2.md @@ -51,6 +51,7 @@ And also: * ⚠️ Nextcloud-db has moved. It is now in ``/data/config`` (or somewhere else if you set the new variables for host paths) (see below how to mitigate the errors). [See the dedicated section below](#nextcloud-db-has-moved). * Disable Traefik access logs * New flag ``--debug`` for ``run-seedbox.sh`` to see what is happening during configuration parsing. +* Releases are named after LOST mythology. I exhausted all the characters of Person of Interest, so that's time for a change. Only geeks will get it, I know. ## Some reading about configuration