too much nier automata music

.gitignore

@@ -1,2 +1,2 @@
 /duplicity_passphrase.txt
-/google_client_config.yml
+/google_client_secret.json

Dockerfile

@@ -4,11 +4,10 @@ FROM alpine:latest AS final
 # Install duplicity
 # RUN pacman --noconfirm -Syu duplicity python-pip python-pydrive2
 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
-RUN \
+RUN apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl
-  apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl && \
+RUN pip install --upgrade pip --break-system-packages
-  pip install --upgrade pip --break-system-packages && \
+RUN pip install google-auth-oauthlib google-api-python-client --break-system-packages
-  pip install pydrive2 --break-system-packages && \
+RUN apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
-  apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
 
 WORKDIR /usr/lib/duplicity
 ENV HOME="/usr/lib/duplicity"

README.md

@@ -2,7 +2,7 @@
 
 ![](.media/icon-128x128_round.png)
 
-# Docker Duplicity Backup
+# Gestalt Amadeus
 
 Backup solution for Docker volumes based on Duplicity
 
@@ -10,191 +10,150 @@ Backup solution for Docker volumes based on Duplicity
 
 ## Usage
 
-> [!CAUTION] 
->
-> Killed by Google :tm:
->
-> New instructions soon
-
 > [!NOTE]
 >
-> The following instructions assume Google Drive is used as a storage backend; refer to [duplicity's man page](https://duplicity.us/stable/duplicity.1.html) to find out how to configure different backends!
+> Other backends are available, but haven't been tested. Please let me know if you want to try using them so I can help you out with setting them up!
 
-### Backup
+### Backup with Google Drive
 
-1. Create two new volumes in Docker with the names `duplicity_credentials` and `duplicity_cache`:
+1. Create a new directory somewhere on your system to use to store certain configuration files; it can be anywhere, but for the purposes of this guide, it'll be referred to as `$ga_config_dir`, and will be located in `/srv/docker/.ga`:
 
-    ```console
+    ```bash
-    # docker volume create duplicity_credentials
+    export ga_config_dir="/srv/docker/.ga"
-    # docker volume create duplicity_cache
+    mkdir --verbose --parents "$ga_config_dir"
     ```
 
-2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files:
+1. Create a new file inside `$ga_config_dir` secret with the name `ga_passphrase.txt`, which will contain the password used to encrypt backups before uploading them to Google Drive:
 
-    ```console
+    ```bash
-    # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt
+    cat "/dev/urandom" | LC_ALL="C" tr --delete --complement '[:graph:]' | head --bytes 32 > "$ga_config_dir/ga_passphrase.txt"
     ```
 
-3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials)
+1. [Use the Google Cloud Console to create new OAuth credentials](https://console.cloud.google.com/apis/credentials) for a ***Desktop Application***.
 
-4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it:
+1. Download the resulting JSON credential file, and move it inside `$ga_config_dir` with the name `ga_gdrive_client_secret.json`:
 
-    ```console
+    ```bash
-    # edit /root/secrets/backup/client_config.yml
+    mv --verbose --interactive ./client_secret* "$ga_config_dir/ga_gdrive_client_secret.json"
+
+1. Create a new Docker volume with the name `ga_cache`, which will be used to temporarily store previous backups:
+
+    ```bash
+    docker volume create "ga_cache"
     ```
 
-    ```yml
+1. Create a new Docker volume with the name `ga_credentials`, which will be use to store Google Drive API credentials:
-    client_config_backend: settings
+
-    client_config:
+    ```bash
-        client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE"
+    docker volume create "ga_credentials"
-        client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
-    save_credentials: True
-    save_credentials_backend: file
-    save_credentials_file: "/var/lib/duplicity/credentials"
-    get_refresh_token: True
     ```
 
-5. Add the following keys to the `compose.yml` file of the project you want to backup:
+1. Create a new directory in Google Drive, open it, and copy the final part of the URL:
 
-    ```console
+    ```text
-    # edit ./compose.yml
+    https://drive.google.com/drive/u/0/folders/1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB
+                                               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+                                                         copy this part         
     ```
 
-    1. Connect the previously created `duplicity_credentials` volume to the project:
+1. Add your Gestalt Amadeus configuration at the top of your `compose.yml` project:
 
-        ```yml
+    ```yaml
-        volumes:
+    x-gestalt-amadeus:
-            duplicity_credentials:
+        # Set this to "restore" to recover files from the last available backup.
-                external: true
+        x-ga-mode: &ga_mode
+            "backup"
+        # The URL where your backups should be uploaded to.
+        # For Google Drive, replace:
+        # - `1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB` with the final part of the URL you've previously copied
+        # - `111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com` with the value of the `.installed.client_id` key of the Google client_secret file you've previously downloaded
+        x-ga-backup-to: &ga_backup_to
+            "gdrive://111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com/${COMPOSE_PROJECT_NAME}?myDriveFolderID=1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB"
+        # If you're planning to use ntfy, set this to the full URL of the topic you'd like to receive notifications at.
+        # If you don't want to use ntfy, set this to an empty string, "".
+        x-ga-ntfy: &ga_ntfy
+            "https://ntfy.sh/phil_alerts"
+        # The path to the `ga_passphrase.txt` file.
+        x-ga-passphrase: &ga_passphrase
+            "/srv/docker/.ga/ga_passphrase.txt"
+        # The path to the `ga_gdrive_client_secret.json` file.
+        x-ga-gdrive-client-secret: &ga_gdrive_client_secret
+            "/srv/docker/.ga/ga_gdrive_client_secret.json"
     ```
 
-    2. Setup the two previously created files as Docker secrets:
+1. Merge the following keys with the rest of your existent `compose.yml` project:
 
-        ```yml
+    ```yaml
-        secrets:
-            duplicity_passphrase:
-                file: "/root/secrets/duplicity/passphrase.txt"
-            google_client_config:
-                file: "/root/secrets/duplicity/client_config.yml"
-        ```
-
-    3. Add the following service:
-
-        ```yml
     services:
-            duplicity:
+        ga:
-                image: "ghcr.io/steffo99/backup-duplicity:latest"
+            image: "ghcr.io/steffo99/gestalt-amadeus:2"
             restart: unless-stopped
-                secrets:
+            network_mode: host
-                    - google_client_config
+            stdin_open: true
-                    - duplicity_passphrase
+            tty: true
             volumes:
-                    - "duplicity_credentials:/var/lib/duplicity" 
+                - type: bind
-                    # Mount whatever you want to backup in subdirectories of /mnt
+                  source: "."
-                    - ".:/mnt/compose"  # Backup the current directory?
+                  target: "/mnt"
-                    - "data:/mnt/data"  # Backup a named volume?
+                - type: volume
+                  source: ga_credentials
+                  target: "/var/lib/duplicity"
+                - type: volume
+                  source: ga_cache
+                  target: "/usr/lib/duplicity/.cache/duplicity"
             environment:
-                    MODE: "backup"  # Change this to "restore" to restore the latest backup
+                MODE: *ga_mode
-                    DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this"  # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT
+                DUPLICITY_TARGET_URL: *ga_backup_to
-                    # Don't touch these, they allow the program to read the secrets
+                NTFY: *ga_ntfy
-                    DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
+                NTFY_TAGS: "host-${HOSTNAME:-${hostname:-undefined}},${COMPOSE_PROJECT_NAME}"
-                    GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
+                DUPLICITY_PASSPHRASE_FILE: "/run/secrets/ga_passphrase"
-        ```
+                GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/ga_gdrive_client_secret"
+                GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
+                GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
+                GOOGLE_OAUTH_LOCAL_SERVER_PORT: "8080"
+            secrets:
+                - ga_passphrase
+                - ga_gdrive_client_secret
     
-6. Log in to Google Drive and perform an initial backup with:
-
-    ```console
-    # docker compose run -i --entrypoint=/bin/sh duplicity /etc/periodic/daily/backup.sh
-    ```
-
-7. Properly start the container with:
-
-    ```console
-    # docker compose up -d && docker compose logs -f
-    ```
-
-### Restore
-
-1. Create a new volume in Docker with the name `duplicity_credentials`:
-
-    ```console
-    # docker volume create duplicity_credentials
-    ```
-
-2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files:
-
-    ```console
-    # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt
-    ```
-
-3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials)
-
-4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it:
-
-    ```console
-    # edit /root/secrets/backup/client_config.yml
-    ```
-
-    ```yml
-    client_config_backend: settings
-    client_config:
-        client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE"
-        client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
-    save_credentials: True
-    save_credentials_backend: file
-    save_credentials_file: "/var/lib/duplicity/credentials"
-    get_refresh_token: True
-    ```
-
-5. Add the following keys to the `compose.yml` file of the project you want to backup:
-
-    ```console
-    # edit ./compose.yml
-    ```
-
-    1. Connect the previously created `duplicity_credentials` volume to the project:
-
-        ```yml
     volumes:
-            duplicity_credentials:
+        ga_cache:
+            external: true
+        ga_credentials:
             external: true
-        ```
     
-    2. Setup the two previously created files as Docker secrets:
-
-        ```yml
     secrets:
-            duplicity_passphrase:
+        ga_passphrase:
-                file: "/root/secrets/duplicity/passphrase.txt"
+            file: *ga_passphrase
-            google_client_config:
+        ga_gdrive_client_secret:
-                file: "/root/secrets/duplicity/client_config.yml"
+            file: *ga_gdrive_client_secret
     ```
 
-    3. Add the following service:
+1. Bring up the Compose project:
 
-        ```yml
+    ```bash
-        services:
+    docker compose up --detach
-            duplicity:
-                image: "ghcr.io/steffo99/backup-duplicity:latest"
-                restart: no
-                secrets:
-                    - google_client_config
-                    - duplicity_passphrase
-                volumes:
-                    - "duplicity_credentials:/var/lib/duplicity" 
-                    # Mount whatever you want to backup in subdirectories of /mnt
-                    - ".:/mnt/compose"  # Backup the current directory?
-                    - "data:/mnt/data"  # Backup a named volume?
-                environment:
-                    MODE: "restore"  # Change this to "restore" to restore the latest backup
-                    DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this"  # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT
-                    # Don't touch these, they allow the program to read the secrets
-                    DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
-                    GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
     ```
 
-6. Log in to Google Drive and perform the restore with:
+1. Pay attention to the logs; if this is the first container you're setting up Gestalt Automata on the host, you'll be asked to login with Google before the backup can proceed:
 
-    ```console
+    ```bash
-    # docker compose run -i --entrypoint=/bin/sh duplicity /usr/lib/backup-duplicity/restore.sh
+    docker compose logs --follow ga
     ```
+
+    ```log
+    duplicity-1  | Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth
+    ```
+
+    Complete the authentication to proceed.
+
+    > For authentication to work correctly after [Google's removal of the OOB Flow](https://developers.google.com/identity/protocols/oauth2/resources/oob-migration), your `http://localhost:8080` address needs to match the `http://localhost:8080` of the Gestalt Amadeus container.
+    > 
+    > This is not an issue if you can launch a browser on the same machine you're configuring Gestalt Amadeus, but it might be troublesome for non-graphical servers, where this is not possible.
+    >
+    > To apply a quick band-aid to the issue, you can temporarily set up an SSH tunnel towards the server for the duration of the setup process:
+    >
+    > ```bash
+    > ssh -L 8080:8080 yourserver
+    > ```
+
+1. You should be done! Make sure backups are appearing in the Google Drive directory you've configured.

backup.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+hostname=$(cat /etc/hostname)
+
 # Get secrets from files
 # Insecure, but there's not much I can do about it
 # It's duplicity's fault!
@@ -16,11 +18,14 @@ if [ -n "${NTFY}" ]; then
 		--header "X-Title: Backup started" \
 		--data "Duplicity is attempting to perform a backup to **${DUPLICITY_TARGET_URL}**..." \
 		--header "X-Priority: min" \
-		--header "X-Tags: arrow_heading_up,${NTFY_TAGS}" \
+		--header "X-Tags: arrow_heading_up,duplicity,container-${hostname},${NTFY_TAGS}" \
-		--header "Content-Type: text/markdown"
+		--header "Content-Type: text/markdown" \
+		>/dev/null
 fi
 
+echo "Running duplicity..."
 duplicity \
+	backup \
 	--allow-source-mismatch \
 	--full-if-older-than "${DUPLICITY_FULL_IF_OLDER_THAN}" \
 	/mnt \
@@ -37,8 +42,9 @@ if [ -n "${NTFY}" ]; then
 				--header "X-Title: Backup complete" \
 				--data "Duplicity has successfully performed a backup to **${DUPLICITY_TARGET_URL}**!" \
 				--header "X-Priority: low" \
-				--header "X-Tags: white_check_mark,${NTFY_TAGS}" \
+				--header "X-Tags: white_check_mark,duplicity,container-${hostname},${NTFY_TAGS}" \
-				--header "Content-Type: text/markdown"
+				--header "Content-Type: text/markdown" \
+				>/dev/null
 		;;
 		*)
 			echo "Sending ntfy backup failed notification..." >> /dev/stderr
@@ -47,8 +53,9 @@ if [ -n "${NTFY}" ]; then
 				--header "X-Title: Backup failed" \
 				--data "Duplicity failed to perform a backup to **${DUPLICITY_TARGET_URL}**, and exited with status code **${backup_result}**." \
 				--header "X-Priority: max" \
-				--header "X-Tags: sos,${NTFY_TAGS}" \
+				--header "X-Tags: sos,duplicity,container-${hostname},${NTFY_TAGS}" \
-				--header "Content-Type: text/markdown"
+				--header "Content-Type: text/markdown" \
+				>/dev/null
 		;;
 	esac
 fi

compose.test.yml

@@ -0,0 +1,41 @@
+secrets:
+    ga_passphrase:
+        external: true
+    ga_gdrive_client_secret:
+        external: true
+
+volumes:
+    ga_credentials:
+        external: true
+    ga_cache:
+        external: true
+
+services:
+    ga:
+        build:
+            context: "."
+        network_mode: host
+        stdin_open: true
+        tty: true
+        restart: unless-stopped
+        volumes:
+            - type: bind
+              source: "./exampledata"
+              target: "/mnt"
+            - type: volume
+              source: ga_credentials
+              target: "/var/lib/duplicity"
+            - type: volume
+              source: ga_cache
+              target: "/usr/lib/duplicity/.cache/duplicity"        
+        environment:
+            MODE: "backup"
+            DUPLICITY_TARGET_URL: "gdrive://641079776729-da3fi7a2kgk5jkutsjdcnhugqolu40mo.apps.googleusercontent.com/this?myDriveFolderID=1_8rQ4E8ssoN-guFrGs7CC2IFofXBaimi"
+            GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/google_client_secret"
+            DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
+            GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
+            GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
+            GOOGLE_OAUTH_LOCAL_SERVER_PORT: "80"
+        secrets:
+            - ga_passphrase
+            - ga_gdrive_client_secret

gdrive.docker-compose.yml

@@ -1,32 +0,0 @@
-secrets:
-  google_client_config:
-    file: "./google_client_config.yml"
-  duplicity_passphrase:
-    file: "./duplicity_passphrase.txt"
-
-volumes:
-  duplicity_credentials:
-    external: true
-  duplicity_cache:
-    external: true
-
-services:
-  duplicity:
-    image: "ghcr.io/steffo99/backup-duplicity:latest"
-    entrypoint: "/bin/sh"
-    command: "/etc/periodic/daily/backup.sh"
-    restart: unless-stopped
-    volumes:
-      - "./exampledata:/mnt/example"
-      - "duplicity_credentials:/var/lib/duplicity"
-      - "duplicity_cache:/usr/lib/duplicity/.cache/duplicity"
-    environment:
-      MODE: "backup"
-      DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
-      DUPLICITY_TARGET_URL: "pydrive://641079776729-90s4tnli0ao913ajrpv8cp3c4kkk77j5.apps.googleusercontent.com/Duplicity/this"
-      GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
-      NTFY: "https://ntfy.sh/garasauto"
-      NTFY_TAGS: "garasauto"
-    secrets:
-      - google_client_config
-      - duplicity_passphrase

restore.sh

@@ -9,6 +9,7 @@ export PASSPHRASE=$(cat "${DUPLICITY_PASSPHRASE_FILE}")
 
 echo "Launched in restore mode, restoring backup..." >> /dev/stderr
 duplicity \
+	restore \
 	--force \
 	--allow-source-mismatch \
 	"${DUPLICITY_TARGET_URL}" \