From f1b7a0166f8e9157156f8d0f0e82f0a170985d78 Mon Sep 17 00:00:00 2001 From: Stefano Pigozzi Date: Fri, 25 Oct 2024 06:35:59 +0200 Subject: [PATCH] Revived from Google --- .gitignore | 2 +- Dockerfile | 9 +- README.md | 267 +++++++++++++++----------------------- backup.sh | 18 ++- compose.yml | 42 ++++++ gdrive.docker-compose.yml | 32 ----- 6 files changed, 167 insertions(+), 203 deletions(-) create mode 100644 compose.yml delete mode 100644 gdrive.docker-compose.yml diff --git a/.gitignore b/.gitignore index 10908f0..8a4a348 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /duplicity_passphrase.txt -/google_client_config.yml +/google_client_secret.json diff --git a/Dockerfile b/Dockerfile index a328451..a6dc297 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,10 @@ FROM alpine:latest AS final # Install duplicity # RUN pacman --noconfirm -Syu duplicity python-pip python-pydrive2 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -RUN \ - apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl && \ - pip install --upgrade pip --break-system-packages && \ - pip install pydrive2 --break-system-packages && \ - apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev +RUN apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl +RUN pip install --upgrade pip --break-system-packages +RUN pip install google-auth-oauthlib google-api-python-client --break-system-packages +RUN apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev WORKDIR /usr/lib/duplicity ENV HOME="/usr/lib/duplicity" diff --git a/README.md b/README.md index 2bbfb80..f9037c6 100644 --- a/README.md +++ b/README.md @@ -2,199 +2,148 @@ ![](.media/icon-128x128_round.png) -# Docker Duplicity Backup +# Gestalt Amadeus -Backup solution for Docker volumes based on Duplicity +Backup solution for Docker volumes based on Duplicity ## Usage -> [!CAUTION] +### Backup with Google Drive + +> [!Note] > -> Killed by Google :tm: -> -> New instructions soon +> Other backends are available, but haven't been tested. Please let me know if you want to try using them so I can help you out with setting them up! -> [!NOTE] -> -> The following instructions assume Google Drive is used as a storage backend; refer to [duplicity's man page](https://duplicity.us/stable/duplicity.1.html) to find out how to configure different backends! +1. Create a new Docker volume with the name `ga_cache`, which Duplicity will use to temporarily store previous backups: -### Backup - -1. Create two new volumes in Docker with the names `duplicity_credentials` and `duplicity_cache`: - - ```console - # docker volume create duplicity_credentials - # docker volume create duplicity_cache + ```bash + docker volume create "ga_cache" ``` -2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files: +1. Create a new Docker volume with the name `ga_credentials`, which Duplicity will use to store Google Drive API credentials: - ```console - # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt + ```bash + docker volume create "ga_credentials" ``` -3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials) +1. Create a new Docker secret with the name `ga_passphrase` containing the password that will be used to encrypt backups before uploading them: -4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it: - - ```console - # edit /root/secrets/backup/client_config.yml + ```bash + # This command will generate a secure random password, print it to the console, and use it to create a Docker secret + cat /dev/urandom | LC_ALL="C" tr --delete --complement '[:graph:]' | head --bytes 32 | tee "/dev/stderr" | docker secret create "ga_passphrase" - ``` - ```yml - client_config_backend: settings - client_config: - client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE" - client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE" - save_credentials: True - save_credentials_backend: file - save_credentials_file: "/var/lib/duplicity/credentials" - get_refresh_token: True +1. [Use the Google Cloud Console to create new OAuth credentials](https://console.cloud.google.com/apis/credentials) for a ***Desktop Application***. + +1. Download the JSON credential file, and use it to create a new Docker secret with the name `ga_gdrive_client_secret`: + + ```bash + docker secret create "ga_gdrive_client_secret" ./client_secret* ``` -5. Add the following keys to the `compose.yml` file of the project you want to backup: +1. Create a new directory in Google Drive, open it, and copy the final part of the URL: - ```console - # edit ./compose.yml + ```text + https://drive.google.com/drive/u/0/folders/1_8rQ4E8ssoN-guFrGs7CC2IFofXBaimi + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + copy this part ``` - 1. Connect the previously created `duplicity_credentials` volume to the project: - - ```yml - volumes: - duplicity_credentials: - external: true - ``` - - 2. Setup the two previously created files as Docker secrets: - - ```yml - secrets: - duplicity_passphrase: - file: "/root/secrets/duplicity/passphrase.txt" - google_client_config: - file: "/root/secrets/duplicity/client_config.yml" - ``` - - 3. Add the following service: - - ```yml - services: - duplicity: - image: "ghcr.io/steffo99/backup-duplicity:latest" - restart: unless-stopped - secrets: - - google_client_config - - duplicity_passphrase - volumes: - - "duplicity_credentials:/var/lib/duplicity" - # Mount whatever you want to backup in subdirectories of /mnt - - ".:/mnt/compose" # Backup the current directory? - - "data:/mnt/data" # Backup a named volume? - environment: - MODE: "backup" # Change this to "restore" to restore the latest backup - DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this" # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT - # Don't touch these, they allow the program to read the secrets - DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase" - GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config" - ``` - -6. Log in to Google Drive and perform an initial backup with: - - ```console - # docker compose run -i --entrypoint=/bin/sh duplicity /etc/periodic/daily/backup.sh +1. Add your Gestalt Amadeus configuration in your Compose project at `compose.yml`: + ```yaml + x-gestalt-automata: + # Set this to "restore" to recover files from the last available backup. + ga_mode: &ga_mode + "backup" + # The URL where your backups should be uploaded to. + # For Google Drive, replace: + # - `1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB` with the final part of the URL you've previously copied + # - `111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com` with the value of the `.installed.client_id` key of the Google client_secret file you've previously downloaded + ga_backup_to: &ga_backup_to + "gdrive://111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com/${COMPOSE_PROJECT_NAME}?myDriveFolderID=1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB" + # If you're planning to use ntfy, set this to the full URL of the topic you'd like to receive notifications at. + # An example: `ntfy.sh/ko7OC50phzmh1ZMQ` + ga_ntfy: &ntfy + "" ``` -7. Properly start the container with: +1. Merge the following keys to your Compose project at `compose.yml`: - ```console - # docker compose up -d && docker compose logs -f + ```yaml + services: + ga: + image: "" + restart: unless-stopped + network_mode: host + stdin_open: true + tty: true + volumes: + - type: bind + source: "." + target: "/mnt" + - type: volume + source: ga_credentials + target: "/var/lib/duplicity" + - type: volume + source: ga_cache + target: "/usr/lib/duplicity/.cache/duplicity" + environment: + MODE: *ga_mode + DUPLICITY_TARGET_URL: *ga_backup_to + NTFY: *ga_ntfy + NTFY_TAGS: "host-${HOSTNAME},${COMPOSE_PROJECT_NAME}" + DUPLICITY_PASSPHRASE_FILE: "/run/secrets/ga_passphrase" + GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/ga_gdrive_client_secret" + GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost" + GOOGLE_OAUTH_LOCAL_SERVER_PORT: "80" + secrets: + - ga_passphrase + - ga_gdrive_client_secret + + volumes: + ga_cache: + external: true + ga_credentials: + external: true + + secrets: + ga_passphrase: + external: true + ga_gdrive_client_secret: + external: true ``` -### Restore +1. Bring up the Compose project: -1. Create a new volume in Docker with the name `duplicity_credentials`: - - ```console - # docker volume create duplicity_credentials + ```bash + docker compose up --detach ``` -2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files: +1. Pay attention to the logs; if this is the first container you're setting up Gestalt Automata on the host, you'll be asked to login with Google before the backup can proceed: - ```console - # echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt + ```bash + docker compose logs --follow ga ``` -3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials) - -4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it: - - ```console - # edit /root/secrets/backup/client_config.yml + ```log + duplicity-1 | Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth ``` - ```yml - client_config_backend: settings - client_config: - client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE" - client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE" - save_credentials: True - save_credentials_backend: file - save_credentials_file: "/var/lib/duplicity/credentials" - get_refresh_token: True - ``` + Complete the authentication to proceed. -5. Add the following keys to the `compose.yml` file of the project you want to backup: + > [!Caution] + > + > For authentication to work correctly after [Google's removal of the OOB Flow](https://developers.google.com/identity/protocols/oauth2/resources/oob-migration), your `http://localhost:80` address needs to match the `http://localhost:80` of the Gestalt Amadeus container. + > + > This is not an issue if you can launch a browser on the same machine you're configuring Gestalt Amadeus, but it might be troublesome for non-graphical servers, where this is not possible. + > + > As a quick band-aid to the issue, you can temporarily set up an SSH tunnel towards the server for the duration of the authentication process: + > + > ``` + > # This unfortunately requires root access, since the port we have to tunnel, 80, has a number lower than 1024. + > sudo ssh -L 80:80 yourserver + > ``` - ```console - # edit ./compose.yml - ``` - - 1. Connect the previously created `duplicity_credentials` volume to the project: - - ```yml - volumes: - duplicity_credentials: - external: true - ``` - - 2. Setup the two previously created files as Docker secrets: - - ```yml - secrets: - duplicity_passphrase: - file: "/root/secrets/duplicity/passphrase.txt" - google_client_config: - file: "/root/secrets/duplicity/client_config.yml" - ``` - - 3. Add the following service: - - ```yml - services: - duplicity: - image: "ghcr.io/steffo99/backup-duplicity:latest" - restart: no - secrets: - - google_client_config - - duplicity_passphrase - volumes: - - "duplicity_credentials:/var/lib/duplicity" - # Mount whatever you want to backup in subdirectories of /mnt - - ".:/mnt/compose" # Backup the current directory? - - "data:/mnt/data" # Backup a named volume? - environment: - MODE: "restore" # Change this to "restore" to restore the latest backup - DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this" # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT - # Don't touch these, they allow the program to read the secrets - DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase" - GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config" - ``` - -6. Log in to Google Drive and perform the restore with: - - ```console - # docker compose run -i --entrypoint=/bin/sh duplicity /usr/lib/backup-duplicity/restore.sh - ``` +1. You should be done! Make sure backups are appearing in the Google Drive directory you've configured. diff --git a/backup.sh b/backup.sh index 66e098d..4753ab1 100755 --- a/backup.sh +++ b/backup.sh @@ -2,6 +2,8 @@ set -e +hostname=$(cat /etc/hostname) + # Get secrets from files # Insecure, but there's not much I can do about it # It's duplicity's fault! @@ -16,10 +18,12 @@ if [ -n "${NTFY}" ]; then --header "X-Title: Backup started" \ --data "Duplicity is attempting to perform a backup to **${DUPLICITY_TARGET_URL}**..." \ --header "X-Priority: min" \ - --header "X-Tags: arrow_heading_up,${NTFY_TAGS}" \ - --header "Content-Type: text/markdown" + --header "X-Tags: arrow_heading_up,duplicity,container-${hostname},${NTFY_TAGS}" \ + --header "Content-Type: text/markdown" \ + >/dev/null fi +echo "Running duplicity..." duplicity \ backup \ --allow-source-mismatch \ @@ -38,8 +42,9 @@ if [ -n "${NTFY}" ]; then --header "X-Title: Backup complete" \ --data "Duplicity has successfully performed a backup to **${DUPLICITY_TARGET_URL}**!" \ --header "X-Priority: low" \ - --header "X-Tags: white_check_mark,${NTFY_TAGS}" \ - --header "Content-Type: text/markdown" + --header "X-Tags: white_check_mark,duplicity,container-${hostname},${NTFY_TAGS}" \ + --header "Content-Type: text/markdown" \ + >/dev/null ;; *) echo "Sending ntfy backup failed notification..." >> /dev/stderr @@ -48,8 +53,9 @@ if [ -n "${NTFY}" ]; then --header "X-Title: Backup failed" \ --data "Duplicity failed to perform a backup to **${DUPLICITY_TARGET_URL}**, and exited with status code **${backup_result}**." \ --header "X-Priority: max" \ - --header "X-Tags: sos,${NTFY_TAGS}" \ - --header "Content-Type: text/markdown" + --header "X-Tags: sos,duplicity,container-${hostname},${NTFY_TAGS}" \ + --header "Content-Type: text/markdown" \ + >/dev/null ;; esac fi diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..c9a6d0f --- /dev/null +++ b/compose.yml @@ -0,0 +1,42 @@ +secrets: + google_client_secret: + external: true + duplicity_passphrase: + external: true + +volumes: + duplicity_credentials: + external: true + duplicity_cache: + external: true + +services: + ga: + build: + context: "." + network_mode: host + stdin_open: true + tty: true + restart: unless-stopped + volumes: + - "./exampledata:/mnt/exampledata" + - "ga_credentials:/var/lib/duplicity" + - "ga_cache:/usr/lib/duplicity/.cache/duplicity" + environment: + # Change this to "restore" to restore from an existing backup + MODE: "backup" + # Change the URL here to the Client ID specified in google_client_secret.json + DUPLICITY_TARGET_URL: "gdrive://641079776729-da3fi7a2kgk5jkutsjdcnhugqolu40mo.apps.googleusercontent.com/this?myDriveFolderID=1_8rQ4E8ssoN-guFrGs7CC2IFofXBaimi" + # The URL to send ntfy notifications at + NTFY: "" + # Tags to append to ntfy notifications for this service + NTFY_TAGS: "${COMPOSE_PROJECT_NAME}" + #=== These shouldn't be edited. ===# + GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/google_client_secret" + DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase" + GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials" + GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost" + GOOGLE_OAUTH_LOCAL_SERVER_PORT: "80" + secrets: + - google_client_secret + - duplicity_passphrase diff --git a/gdrive.docker-compose.yml b/gdrive.docker-compose.yml deleted file mode 100644 index 1dbd7de..0000000 --- a/gdrive.docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -secrets: - google_client_config: - file: "./google_client_config.yml" - duplicity_passphrase: - file: "./duplicity_passphrase.txt" - -volumes: - duplicity_credentials: - external: true - duplicity_cache: - external: true - -services: - duplicity: - image: "ghcr.io/steffo99/backup-duplicity:latest" - entrypoint: "/bin/sh" - command: "/etc/periodic/daily/backup.sh" - restart: unless-stopped - volumes: - - "./exampledata:/mnt/example" - - "duplicity_credentials:/var/lib/duplicity" - - "duplicity_cache:/usr/lib/duplicity/.cache/duplicity" - environment: - MODE: "backup" - DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase" - DUPLICITY_TARGET_URL: "pydrive://641079776729-90s4tnli0ao913ajrpv8cp3c4kkk77j5.apps.googleusercontent.com/Duplicity/this" - GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config" - NTFY: "https://ntfy.sh/garasauto" - NTFY_TAGS: "garasauto" - secrets: - - google_client_config - - duplicity_passphrase