.gitignore

@@ -1,2 +1,2 @@
 /duplicity_passphrase.txt
-/google_client_secret.json
+/google_client_config.yml

Dockerfile

@@ -4,10 +4,11 @@ FROM alpine:latest AS final
 # Install duplicity
 # RUN pacman --noconfirm -Syu duplicity python-pip python-pydrive2
 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
-RUN apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl
+RUN \
-RUN pip install --upgrade pip --break-system-packages
+  apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl && \
-RUN pip install google-auth-oauthlib google-api-python-client --break-system-packages
+  pip install --upgrade pip --break-system-packages && \
-RUN apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
+  pip install pydrive2 --break-system-packages && \
+  apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
 
 WORKDIR /usr/lib/duplicity
 ENV HOME="/usr/lib/duplicity"

README.md

@@ -2,7 +2,7 @@
 
 ![](.media/icon-128x128_round.png)
 
-# Gestalt Amadeus
+# Docker Duplicity Backup
 
 Backup solution for Docker volumes based on Duplicity   
 
@@ -10,150 +10,191 @@ Backup solution for Docker volumes based on Duplicity
 
 ## Usage
 
+> [!CAUTION] 
+>
+> Killed by Google :tm:
+>
+> New instructions soon
+
 > [!NOTE]
 >
-> Other backends are available, but haven't been tested. Please let me know if you want to try using them so I can help you out with setting them up!
+> The following instructions assume Google Drive is used as a storage backend; refer to [duplicity's man page](https://duplicity.us/stable/duplicity.1.html) to find out how to configure different backends!
 
-### Backup with Google Drive
+### Backup
 
-1. Create a new directory somewhere on your system to use to store certain configuration files; it can be anywhere, but for the purposes of this guide, it'll be referred to as `$ga_config_dir`, and will be located in `/srv/docker/.ga`:
+1. Create two new volumes in Docker with the names `duplicity_credentials` and `duplicity_cache`:
 
-    ```bash
+    ```console
-    export ga_config_dir="/srv/docker/.ga"
+    # docker volume create duplicity_credentials
-    mkdir --verbose --parents "$ga_config_dir"
+    # docker volume create duplicity_cache
     ```
 
-1. Create a new file inside `$ga_config_dir` secret with the name `ga_passphrase.txt`, which will contain the password used to encrypt backups before uploading them to Google Drive:
+2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files:
 
-    ```bash
+    ```console
-    cat "/dev/urandom" | LC_ALL="C" tr --delete --complement '[:graph:]' | head --bytes 32 > "$ga_config_dir/ga_passphrase.txt"
+    # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt
     ```
 
-1. [Use the Google Cloud Console to create new OAuth credentials](https://console.cloud.google.com/apis/credentials) for a ***Desktop Application***.
+3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials)
 
-1. Download the resulting JSON credential file, and move it inside `$ga_config_dir` with the name `ga_gdrive_client_secret.json`:
+4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it:
 
-    ```bash
+    ```console
-    mv --verbose --interactive ./client_secret* "$ga_config_dir/ga_gdrive_client_secret.json"
+    # edit /root/secrets/backup/client_config.yml
-
-1. Create a new Docker volume with the name `ga_cache`, which will be used to temporarily store previous backups:
-
-    ```bash
-    docker volume create "ga_cache"
     ```
 
-1. Create a new Docker volume with the name `ga_credentials`, which will be use to store Google Drive API credentials:
+    ```yml
-
+    client_config_backend: settings
-    ```bash
+    client_config:
-    docker volume create "ga_credentials"
+        client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE"
+        client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
+    save_credentials: True
+    save_credentials_backend: file
+    save_credentials_file: "/var/lib/duplicity/credentials"
+    get_refresh_token: True
     ```
 
-1. Create a new directory in Google Drive, open it, and copy the final part of the URL:
+5. Add the following keys to the `compose.yml` file of the project you want to backup:
 
-    ```text
+    ```console
-    https://drive.google.com/drive/u/0/folders/1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB
+    # edit ./compose.yml
-                                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-                                                         copy this part         
     ```
 
-1. Add your Gestalt Amadeus configuration at the top of your `compose.yml` project:
+    1. Connect the previously created `duplicity_credentials` volume to the project:
 
-    ```yaml
+        ```yml
-    x-gestalt-amadeus:
+        volumes:
-        # Set this to "restore" to recover files from the last available backup.
+            duplicity_credentials:
-        x-ga-mode: &ga_mode
+                external: true
-            "backup"
-        # The URL where your backups should be uploaded to.
-        # For Google Drive, replace:
-        # - `1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB` with the final part of the URL you've previously copied
-        # - `111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com` with the value of the `.installed.client_id` key of the Google client_secret file you've previously downloaded
-        x-ga-backup-to: &ga_backup_to
-            "gdrive://111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com/${COMPOSE_PROJECT_NAME}?myDriveFolderID=1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB"
-        # If you're planning to use ntfy, set this to the full URL of the topic you'd like to receive notifications at.
-        # If you don't want to use ntfy, set this to an empty string, "".
-        x-ga-ntfy: &ga_ntfy
-            "https://ntfy.sh/phil_alerts"
-        # The path to the `ga_passphrase.txt` file.
-        x-ga-passphrase: &ga_passphrase
-            "/srv/docker/.ga/ga_passphrase.txt"
-        # The path to the `ga_gdrive_client_secret.json` file.
-        x-ga-gdrive-client-secret: &ga_gdrive_client_secret
-            "/srv/docker/.ga/ga_gdrive_client_secret.json"
         ```
 
-1. Merge the following keys with the rest of your existent `compose.yml` project:
+    2. Setup the two previously created files as Docker secrets:
 
-    ```yaml
+        ```yml
+        secrets:
+            duplicity_passphrase:
+                file: "/root/secrets/duplicity/passphrase.txt"
+            google_client_config:
+                file: "/root/secrets/duplicity/client_config.yml"
+        ```
+
+    3. Add the following service:
+
+        ```yml
         services:
-        ga:
+            duplicity:
-            image: "ghcr.io/steffo99/gestalt-amadeus:2"
+                image: "ghcr.io/steffo99/backup-duplicity:latest"
                 restart: unless-stopped
-            network_mode: host
+                secrets:
-            stdin_open: true
+                    - google_client_config
-            tty: true
+                    - duplicity_passphrase
                 volumes:
-                - type: bind
+                    - "duplicity_credentials:/var/lib/duplicity" 
-                  source: "."
+                    # Mount whatever you want to backup in subdirectories of /mnt
-                  target: "/mnt"
+                    - ".:/mnt/compose"  # Backup the current directory?
-                - type: volume
+                    - "data:/mnt/data"  # Backup a named volume?
-                  source: ga_credentials
-                  target: "/var/lib/duplicity"
-                - type: volume
-                  source: ga_cache
-                  target: "/usr/lib/duplicity/.cache/duplicity"
                 environment:
-                MODE: *ga_mode
+                    MODE: "backup"  # Change this to "restore" to restore the latest backup
-                DUPLICITY_TARGET_URL: *ga_backup_to
+                    DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this"  # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT
-                NTFY: *ga_ntfy
+                    # Don't touch these, they allow the program to read the secrets
-                NTFY_TAGS: "host-${HOSTNAME:-${hostname:-undefined}},${COMPOSE_PROJECT_NAME}"
+                    DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
-                DUPLICITY_PASSPHRASE_FILE: "/run/secrets/ga_passphrase"
+                    GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
-                GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/ga_gdrive_client_secret"
+        ```
-                GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
-                GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
-                GOOGLE_OAUTH_LOCAL_SERVER_PORT: "8080"
-            secrets:
-                - ga_passphrase
-                - ga_gdrive_client_secret
 
+6. Log in to Google Drive and perform an initial backup with:
+
+    ```console
+    # docker compose run -i --entrypoint=/bin/sh duplicity /etc/periodic/daily/backup.sh
+    ```
+
+7. Properly start the container with:
+
+    ```console
+    # docker compose up -d && docker compose logs -f
+    ```
+
+### Restore
+
+1. Create a new volume in Docker with the name `duplicity_credentials`:
+
+    ```console
+    # docker volume create duplicity_credentials
+    ```
+
+2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files:
+
+    ```console
+    # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt
+    ```
+
+3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials)
+
+4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it:
+
+    ```console
+    # edit /root/secrets/backup/client_config.yml
+    ```
+
+    ```yml
+    client_config_backend: settings
+    client_config:
+        client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE"
+        client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
+    save_credentials: True
+    save_credentials_backend: file
+    save_credentials_file: "/var/lib/duplicity/credentials"
+    get_refresh_token: True
+    ```
+
+5. Add the following keys to the `compose.yml` file of the project you want to backup:
+
+    ```console
+    # edit ./compose.yml
+    ```
+
+    1. Connect the previously created `duplicity_credentials` volume to the project:
+
+        ```yml
         volumes:
-        ga_cache:
+            duplicity_credentials:
-            external: true
-        ga_credentials:
                 external: true
+        ```
 
+    2. Setup the two previously created files as Docker secrets:
+
+        ```yml
         secrets:
-        ga_passphrase:
+            duplicity_passphrase:
-            file: *ga_passphrase
+                file: "/root/secrets/duplicity/passphrase.txt"
-        ga_gdrive_client_secret:
+            google_client_config:
-            file: *ga_gdrive_client_secret
+                file: "/root/secrets/duplicity/client_config.yml"
         ```
 
-1. Bring up the Compose project:
+    3. Add the following service:
 
-    ```bash
+        ```yml
-    docker compose up --detach
+        services:
+            duplicity:
+                image: "ghcr.io/steffo99/backup-duplicity:latest"
+                restart: no
+                secrets:
+                    - google_client_config
+                    - duplicity_passphrase
+                volumes:
+                    - "duplicity_credentials:/var/lib/duplicity" 
+                    # Mount whatever you want to backup in subdirectories of /mnt
+                    - ".:/mnt/compose"  # Backup the current directory?
+                    - "data:/mnt/data"  # Backup a named volume?
+                environment:
+                    MODE: "restore"  # Change this to "restore" to restore the latest backup
+                    DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this"  # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT
+                    # Don't touch these, they allow the program to read the secrets
+                    DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
+                    GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
         ```
 
-1. Pay attention to the logs; if this is the first container you're setting up Gestalt Automata on the host, you'll be asked to login with Google before the backup can proceed:
+6. Log in to Google Drive and perform the restore with:
 
-    ```bash
+    ```console
-    docker compose logs --follow ga
+    # docker compose run -i --entrypoint=/bin/sh duplicity /usr/lib/backup-duplicity/restore.sh
     ```
-
-    ```log
-    duplicity-1  | Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth
-    ```
-
-    Complete the authentication to proceed.
-
-    > For authentication to work correctly after [Google's removal of the OOB Flow](https://developers.google.com/identity/protocols/oauth2/resources/oob-migration), your `http://localhost:8080` address needs to match the `http://localhost:8080` of the Gestalt Amadeus container.
-    > 
-    > This is not an issue if you can launch a browser on the same machine you're configuring Gestalt Amadeus, but it might be troublesome for non-graphical servers, where this is not possible.
-    >
-    > To apply a quick band-aid to the issue, you can temporarily set up an SSH tunnel towards the server for the duration of the setup process:
-    >
-    > ```bash
-    > ssh -L 8080:8080 yourserver
-    > ```
-
-1. You should be done! Make sure backups are appearing in the Google Drive directory you've configured.

backup.sh

@@ -2,8 +2,6 @@
 
 set -e
 
-hostname=$(cat /etc/hostname)
-
 # Get secrets from files
 # Insecure, but there's not much I can do about it
 # It's duplicity's fault!
@@ -18,14 +16,11 @@ if [ -n "${NTFY}" ]; then
 		--header "X-Title: Backup started" \
 		--data "Duplicity is attempting to perform a backup to **${DUPLICITY_TARGET_URL}**..." \
 		--header "X-Priority: min" \
-		--header "X-Tags: arrow_heading_up,duplicity,container-${hostname},${NTFY_TAGS}" \
+		--header "X-Tags: arrow_heading_up,${NTFY_TAGS}" \
-		--header "Content-Type: text/markdown" \
+		--header "Content-Type: text/markdown"
-		>/dev/null
 fi
 
-echo "Running duplicity..."
 duplicity \
-	backup \
 	--allow-source-mismatch \
 	--full-if-older-than "${DUPLICITY_FULL_IF_OLDER_THAN}" \
 	/mnt \
@@ -42,9 +37,8 @@ if [ -n "${NTFY}" ]; then
 				--header "X-Title: Backup complete" \
 				--data "Duplicity has successfully performed a backup to **${DUPLICITY_TARGET_URL}**!" \
 				--header "X-Priority: low" \
-				--header "X-Tags: white_check_mark,duplicity,container-${hostname},${NTFY_TAGS}" \
+				--header "X-Tags: white_check_mark,${NTFY_TAGS}" \
-				--header "Content-Type: text/markdown" \
+				--header "Content-Type: text/markdown"
-				>/dev/null
 		;;
 		*)
 			echo "Sending ntfy backup failed notification..." >> /dev/stderr
@@ -53,9 +47,8 @@ if [ -n "${NTFY}" ]; then
 				--header "X-Title: Backup failed" \
 				--data "Duplicity failed to perform a backup to **${DUPLICITY_TARGET_URL}**, and exited with status code **${backup_result}**." \
 				--header "X-Priority: max" \
-				--header "X-Tags: sos,duplicity,container-${hostname},${NTFY_TAGS}" \
+				--header "X-Tags: sos,${NTFY_TAGS}" \
-				--header "Content-Type: text/markdown" \
+				--header "Content-Type: text/markdown"
-				>/dev/null
 		;;
 	esac
 fi

compose.test.yml

@@ -1,41 +0,0 @@
-secrets:
-    ga_passphrase:
-        external: true
-    ga_gdrive_client_secret:
-        external: true
-
-volumes:
-    ga_credentials:
-        external: true
-    ga_cache:
-        external: true
-
-services:
-    ga:
-        build:
-            context: "."
-        network_mode: host
-        stdin_open: true
-        tty: true
-        restart: unless-stopped
-        volumes:
-            - type: bind
-              source: "./exampledata"
-              target: "/mnt"
-            - type: volume
-              source: ga_credentials
-              target: "/var/lib/duplicity"
-            - type: volume
-              source: ga_cache
-              target: "/usr/lib/duplicity/.cache/duplicity"        
-        environment:
-            MODE: "backup"
-            DUPLICITY_TARGET_URL: "gdrive://641079776729-da3fi7a2kgk5jkutsjdcnhugqolu40mo.apps.googleusercontent.com/this?myDriveFolderID=1_8rQ4E8ssoN-guFrGs7CC2IFofXBaimi"
-            GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/google_client_secret"
-            DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
-            GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
-            GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
-            GOOGLE_OAUTH_LOCAL_SERVER_PORT: "80"
-        secrets:
-            - ga_passphrase
-            - ga_gdrive_client_secret

gdrive.docker-compose.yml

@@ -0,0 +1,32 @@
+secrets:
+  google_client_config:
+    file: "./google_client_config.yml"
+  duplicity_passphrase:
+    file: "./duplicity_passphrase.txt"
+
+volumes:
+  duplicity_credentials:
+    external: true
+  duplicity_cache:
+    external: true
+
+services:
+  duplicity:
+    image: "ghcr.io/steffo99/backup-duplicity:latest"
+    entrypoint: "/bin/sh"
+    command: "/etc/periodic/daily/backup.sh"
+    restart: unless-stopped
+    volumes:
+      - "./exampledata:/mnt/example"
+      - "duplicity_credentials:/var/lib/duplicity"
+      - "duplicity_cache:/usr/lib/duplicity/.cache/duplicity"
+    environment:
+      MODE: "backup"
+      DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
+      DUPLICITY_TARGET_URL: "pydrive://641079776729-90s4tnli0ao913ajrpv8cp3c4kkk77j5.apps.googleusercontent.com/Duplicity/this"
+      GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
+      NTFY: "https://ntfy.sh/garasauto"
+      NTFY_TAGS: "garasauto"
+    secrets:
+      - google_client_config
+      - duplicity_passphrase

restore.sh

@@ -9,7 +9,6 @@ export PASSPHRASE=$(cat "${DUPLICITY_PASSPHRASE_FILE}")
 
 echo "Launched in restore mode, restoring backup..." >> /dev/stderr
 duplicity \
-	restore \
 	--force \
 	--allow-source-mismatch \
 	"${DUPLICITY_TARGET_URL}" \