1
Fork 0
mirror of https://github.com/Steffo99/backup-duplicity.git synced 2024-12-22 19:44:22 +00:00

Compare commits

...

13 commits

7 changed files with 174 additions and 199 deletions

2
.gitignore vendored
View file

@ -1,2 +1,2 @@
/duplicity_passphrase.txt /duplicity_passphrase.txt
/google_client_config.yml /google_client_secret.json

View file

@ -4,11 +4,10 @@ FROM alpine:latest AS final
# Install duplicity # Install duplicity
# RUN pacman --noconfirm -Syu duplicity python-pip python-pydrive2 # RUN pacman --noconfirm -Syu duplicity python-pip python-pydrive2
ENV CARGO_NET_GIT_FETCH_WITH_CLI=true ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
RUN \ RUN apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl
apk add py3-pip python3-dev gcc libffi-dev musl-dev openssl-dev pkgconfig duplicity rust cargo git curl && \ RUN pip install --upgrade pip --break-system-packages
pip install --upgrade pip --break-system-packages && \ RUN pip install google-auth-oauthlib google-api-python-client --break-system-packages
pip install pydrive2 --break-system-packages && \ RUN apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
apk del rust musl-dev libffi-dev gcc python3-dev cargo git pkgconfig openssl-dev
WORKDIR /usr/lib/duplicity WORKDIR /usr/lib/duplicity
ENV HOME="/usr/lib/duplicity" ENV HOME="/usr/lib/duplicity"

259
README.md
View file

@ -2,7 +2,7 @@
![](.media/icon-128x128_round.png) ![](.media/icon-128x128_round.png)
# Docker Duplicity Backup # Gestalt Amadeus
Backup solution for Docker volumes based on Duplicity Backup solution for Docker volumes based on Duplicity
@ -10,191 +10,150 @@ Backup solution for Docker volumes based on Duplicity
## Usage ## Usage
> [!CAUTION]
>
> Killed by Google :tm:
>
> New instructions soon
> [!NOTE] > [!NOTE]
> >
> The following instructions assume Google Drive is used as a storage backend; refer to [duplicity's man page](https://duplicity.us/stable/duplicity.1.html) to find out how to configure different backends! > Other backends are available, but haven't been tested. Please let me know if you want to try using them so I can help you out with setting them up!
### Backup ### Backup with Google Drive
1. Create two new volumes in Docker with the names `duplicity_credentials` and `duplicity_cache`: 1. Create a new directory somewhere on your system to use to store certain configuration files; it can be anywhere, but for the purposes of this guide, it'll be referred to as `$ga_config_dir`, and will be located in `/srv/docker/.ga`:
```console ```bash
# docker volume create duplicity_credentials export ga_config_dir="/srv/docker/.ga"
# docker volume create duplicity_cache mkdir --verbose --parents "$ga_config_dir"
``` ```
2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files: 1. Create a new file inside `$ga_config_dir` secret with the name `ga_passphrase.txt`, which will contain the password used to encrypt backups before uploading them to Google Drive:
```console ```bash
# echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt cat "/dev/urandom" | LC_ALL="C" tr --delete --complement '[:graph:]' | head --bytes 32 > "$ga_config_dir/ga_passphrase.txt"
``` ```
3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials) 1. [Use the Google Cloud Console to create new OAuth credentials](https://console.cloud.google.com/apis/credentials) for a ***Desktop Application***.
4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it: 1. Download the resulting JSON credential file, and move it inside `$ga_config_dir` with the name `ga_gdrive_client_secret.json`:
```console ```bash
# edit /root/secrets/backup/client_config.yml mv --verbose --interactive ./client_secret* "$ga_config_dir/ga_gdrive_client_secret.json"
1. Create a new Docker volume with the name `ga_cache`, which will be used to temporarily store previous backups:
```bash
docker volume create "ga_cache"
``` ```
```yml 1. Create a new Docker volume with the name `ga_credentials`, which will be use to store Google Drive API credentials:
client_config_backend: settings
client_config: ```bash
client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE" docker volume create "ga_credentials"
client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
save_credentials: True
save_credentials_backend: file
save_credentials_file: "/var/lib/duplicity/credentials"
get_refresh_token: True
``` ```
5. Add the following keys to the `compose.yml` file of the project you want to backup: 1. Create a new directory in Google Drive, open it, and copy the final part of the URL:
```console ```text
# edit ./compose.yml https://drive.google.com/drive/u/0/folders/1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
copy this part
``` ```
1. Connect the previously created `duplicity_credentials` volume to the project: 1. Add your Gestalt Amadeus configuration at the top of your `compose.yml` project:
```yml ```yaml
volumes: x-gestalt-amadeus:
duplicity_credentials: # Set this to "restore" to recover files from the last available backup.
external: true x-ga-mode: &ga_mode
"backup"
# The URL where your backups should be uploaded to.
# For Google Drive, replace:
# - `1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB` with the final part of the URL you've previously copied
# - `111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com` with the value of the `.installed.client_id` key of the Google client_secret file you've previously downloaded
x-ga-backup-to: &ga_backup_to
"gdrive://111111111111-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.apps.googleusercontent.com/${COMPOSE_PROJECT_NAME}?myDriveFolderID=1_AAAAAAAAAA-BBBBBBBBBBBBBBBBBBBB"
# If you're planning to use ntfy, set this to the full URL of the topic you'd like to receive notifications at.
# If you don't want to use ntfy, set this to an empty string, "".
x-ga-ntfy: &ga_ntfy
"https://ntfy.sh/phil_alerts"
# The path to the `ga_passphrase.txt` file.
x-ga-passphrase: &ga_passphrase
"/srv/docker/.ga/ga_passphrase.txt"
# The path to the `ga_gdrive_client_secret.json` file.
x-ga-gdrive-client-secret: &ga_gdrive_client_secret
"/srv/docker/.ga/ga_gdrive_client_secret.json"
``` ```
2. Setup the two previously created files as Docker secrets: 1. Merge the following keys with the rest of your existent `compose.yml` project:
```yml ```yaml
secrets:
duplicity_passphrase:
file: "/root/secrets/duplicity/passphrase.txt"
google_client_config:
file: "/root/secrets/duplicity/client_config.yml"
```
3. Add the following service:
```yml
services: services:
duplicity: ga:
image: "ghcr.io/steffo99/backup-duplicity:latest" image: "ghcr.io/steffo99/gestalt-amadeus:2"
restart: unless-stopped restart: unless-stopped
secrets: network_mode: host
- google_client_config stdin_open: true
- duplicity_passphrase tty: true
volumes: volumes:
- "duplicity_credentials:/var/lib/duplicity" - type: bind
# Mount whatever you want to backup in subdirectories of /mnt source: "."
- ".:/mnt/compose" # Backup the current directory? target: "/mnt"
- "data:/mnt/data" # Backup a named volume? - type: volume
source: ga_credentials
target: "/var/lib/duplicity"
- type: volume
source: ga_cache
target: "/usr/lib/duplicity/.cache/duplicity"
environment: environment:
MODE: "backup" # Change this to "restore" to restore the latest backup MODE: *ga_mode
DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this" # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT DUPLICITY_TARGET_URL: *ga_backup_to
# Don't touch these, they allow the program to read the secrets NTFY: *ga_ntfy
DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase" NTFY_TAGS: "host-${HOSTNAME:-${hostname:-undefined}},${COMPOSE_PROJECT_NAME}"
GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config" DUPLICITY_PASSPHRASE_FILE: "/run/secrets/ga_passphrase"
``` GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/ga_gdrive_client_secret"
GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
GOOGLE_OAUTH_LOCAL_SERVER_PORT: "8080"
secrets:
- ga_passphrase
- ga_gdrive_client_secret
6. Log in to Google Drive and perform an initial backup with:
```console
# docker compose run -i --entrypoint=/bin/sh duplicity /etc/periodic/daily/backup.sh
```
7. Properly start the container with:
```console
# docker compose up -d && docker compose logs -f
```
### Restore
1. Create a new volume in Docker with the name `duplicity_credentials`:
```console
# docker volume create duplicity_credentials
```
2. Create a new file in the host system with the name `/root/secrets/backup/passphrase.txt`, and enter in it a secure passphrase to use to encrypt files:
```console
# echo 'CorrectHorseBatteryStaple' >> /root/secrets/backup/passphrase.txt
```
3. [Obtain *Desktop Application* OAuth credentials from the Google Cloud Console.](https://console.cloud.google.com/apis/credentials)
4. Create a new file in the host system with the name `/root/secrets/backup/client_config.yml`, and enter the following content in it:
```console
# edit /root/secrets/backup/client_config.yml
```
```yml
client_config_backend: settings
client_config:
client_id: "YOUR_GOOGLE_CLIENT_ID_GOES_HERE"
client_secret: "YOUR_GOOGLE_CLIENT_SECRET_GOES_HERE"
save_credentials: True
save_credentials_backend: file
save_credentials_file: "/var/lib/duplicity/credentials"
get_refresh_token: True
```
5. Add the following keys to the `compose.yml` file of the project you want to backup:
```console
# edit ./compose.yml
```
1. Connect the previously created `duplicity_credentials` volume to the project:
```yml
volumes: volumes:
duplicity_credentials: ga_cache:
external: true
ga_credentials:
external: true external: true
```
2. Setup the two previously created files as Docker secrets:
```yml
secrets: secrets:
duplicity_passphrase: ga_passphrase:
file: "/root/secrets/duplicity/passphrase.txt" file: *ga_passphrase
google_client_config: ga_gdrive_client_secret:
file: "/root/secrets/duplicity/client_config.yml" file: *ga_gdrive_client_secret
``` ```
3. Add the following service: 1. Bring up the Compose project:
```yml ```bash
services: docker compose up --detach
duplicity:
image: "ghcr.io/steffo99/backup-duplicity:latest"
restart: no
secrets:
- google_client_config
- duplicity_passphrase
volumes:
- "duplicity_credentials:/var/lib/duplicity"
# Mount whatever you want to backup in subdirectories of /mnt
- ".:/mnt/compose" # Backup the current directory?
- "data:/mnt/data" # Backup a named volume?
environment:
MODE: "restore" # Change this to "restore" to restore the latest backup
DUPLICITY_TARGET_URL: "pydrive://YOUR_GOOGLE_CLIENT_ID_GOES_HERE/Duplicity/this" # Change this to the Drive directory you want to backup files to https://man.archlinux.org/man/duplicity.1.en#URL_FORMAT
# Don't touch these, they allow the program to read the secrets
DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
``` ```
6. Log in to Google Drive and perform the restore with: 1. Pay attention to the logs; if this is the first container you're setting up Gestalt Automata on the host, you'll be asked to login with Google before the backup can proceed:
```console ```bash
# docker compose run -i --entrypoint=/bin/sh duplicity /usr/lib/backup-duplicity/restore.sh docker compose logs --follow ga
``` ```
```log
duplicity-1 | Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth
```
Complete the authentication to proceed.
> For authentication to work correctly after [Google's removal of the OOB Flow](https://developers.google.com/identity/protocols/oauth2/resources/oob-migration), your `http://localhost:8080` address needs to match the `http://localhost:8080` of the Gestalt Amadeus container.
>
> This is not an issue if you can launch a browser on the same machine you're configuring Gestalt Amadeus, but it might be troublesome for non-graphical servers, where this is not possible.
>
> To apply a quick band-aid to the issue, you can temporarily set up an SSH tunnel towards the server for the duration of the setup process:
>
> ```bash
> ssh -L 8080:8080 yourserver
> ```
1. You should be done! Make sure backups are appearing in the Google Drive directory you've configured.

View file

@ -2,6 +2,8 @@
set -e set -e
hostname=$(cat /etc/hostname)
# Get secrets from files # Get secrets from files
# Insecure, but there's not much I can do about it # Insecure, but there's not much I can do about it
# It's duplicity's fault! # It's duplicity's fault!
@ -16,11 +18,14 @@ if [ -n "${NTFY}" ]; then
--header "X-Title: Backup started" \ --header "X-Title: Backup started" \
--data "Duplicity is attempting to perform a backup to **${DUPLICITY_TARGET_URL}**..." \ --data "Duplicity is attempting to perform a backup to **${DUPLICITY_TARGET_URL}**..." \
--header "X-Priority: min" \ --header "X-Priority: min" \
--header "X-Tags: arrow_heading_up,${NTFY_TAGS}" \ --header "X-Tags: arrow_heading_up,duplicity,container-${hostname},${NTFY_TAGS}" \
--header "Content-Type: text/markdown" --header "Content-Type: text/markdown" \
>/dev/null
fi fi
echo "Running duplicity..."
duplicity \ duplicity \
backup \
--allow-source-mismatch \ --allow-source-mismatch \
--full-if-older-than "${DUPLICITY_FULL_IF_OLDER_THAN}" \ --full-if-older-than "${DUPLICITY_FULL_IF_OLDER_THAN}" \
/mnt \ /mnt \
@ -37,8 +42,9 @@ if [ -n "${NTFY}" ]; then
--header "X-Title: Backup complete" \ --header "X-Title: Backup complete" \
--data "Duplicity has successfully performed a backup to **${DUPLICITY_TARGET_URL}**!" \ --data "Duplicity has successfully performed a backup to **${DUPLICITY_TARGET_URL}**!" \
--header "X-Priority: low" \ --header "X-Priority: low" \
--header "X-Tags: white_check_mark,${NTFY_TAGS}" \ --header "X-Tags: white_check_mark,duplicity,container-${hostname},${NTFY_TAGS}" \
--header "Content-Type: text/markdown" --header "Content-Type: text/markdown" \
>/dev/null
;; ;;
*) *)
echo "Sending ntfy backup failed notification..." >> /dev/stderr echo "Sending ntfy backup failed notification..." >> /dev/stderr
@ -47,8 +53,9 @@ if [ -n "${NTFY}" ]; then
--header "X-Title: Backup failed" \ --header "X-Title: Backup failed" \
--data "Duplicity failed to perform a backup to **${DUPLICITY_TARGET_URL}**, and exited with status code **${backup_result}**." \ --data "Duplicity failed to perform a backup to **${DUPLICITY_TARGET_URL}**, and exited with status code **${backup_result}**." \
--header "X-Priority: max" \ --header "X-Priority: max" \
--header "X-Tags: sos,${NTFY_TAGS}" \ --header "X-Tags: sos,duplicity,container-${hostname},${NTFY_TAGS}" \
--header "Content-Type: text/markdown" --header "Content-Type: text/markdown" \
>/dev/null
;; ;;
esac esac
fi fi

41
compose.test.yml Normal file
View file

@ -0,0 +1,41 @@
secrets:
ga_passphrase:
external: true
ga_gdrive_client_secret:
external: true
volumes:
ga_credentials:
external: true
ga_cache:
external: true
services:
ga:
build:
context: "."
network_mode: host
stdin_open: true
tty: true
restart: unless-stopped
volumes:
- type: bind
source: "./exampledata"
target: "/mnt"
- type: volume
source: ga_credentials
target: "/var/lib/duplicity"
- type: volume
source: ga_cache
target: "/usr/lib/duplicity/.cache/duplicity"
environment:
MODE: "backup"
DUPLICITY_TARGET_URL: "gdrive://641079776729-da3fi7a2kgk5jkutsjdcnhugqolu40mo.apps.googleusercontent.com/this?myDriveFolderID=1_8rQ4E8ssoN-guFrGs7CC2IFofXBaimi"
GOOGLE_CLIENT_SECRET_JSON_FILE: "/run/secrets/google_client_secret"
DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
GOOGLE_CREDENTIALS_FILE: "/var/lib/duplicity/google_credentials"
GOOGLE_OAUTH_LOCAL_SERVER_HOST: "localhost"
GOOGLE_OAUTH_LOCAL_SERVER_PORT: "80"
secrets:
- ga_passphrase
- ga_gdrive_client_secret

View file

@ -1,32 +0,0 @@
secrets:
google_client_config:
file: "./google_client_config.yml"
duplicity_passphrase:
file: "./duplicity_passphrase.txt"
volumes:
duplicity_credentials:
external: true
duplicity_cache:
external: true
services:
duplicity:
image: "ghcr.io/steffo99/backup-duplicity:latest"
entrypoint: "/bin/sh"
command: "/etc/periodic/daily/backup.sh"
restart: unless-stopped
volumes:
- "./exampledata:/mnt/example"
- "duplicity_credentials:/var/lib/duplicity"
- "duplicity_cache:/usr/lib/duplicity/.cache/duplicity"
environment:
MODE: "backup"
DUPLICITY_PASSPHRASE_FILE: "/run/secrets/duplicity_passphrase"
DUPLICITY_TARGET_URL: "pydrive://641079776729-90s4tnli0ao913ajrpv8cp3c4kkk77j5.apps.googleusercontent.com/Duplicity/this"
GOOGLE_DRIVE_SETTINGS: "/run/secrets/google_client_config"
NTFY: "https://ntfy.sh/garasauto"
NTFY_TAGS: "garasauto"
secrets:
- google_client_config
- duplicity_passphrase

View file

@ -9,6 +9,7 @@ export PASSPHRASE=$(cat "${DUPLICITY_PASSPHRASE_FILE}")
echo "Launched in restore mode, restoring backup..." >> /dev/stderr echo "Launched in restore mode, restoring backup..." >> /dev/stderr
duplicity \ duplicity \
restore \
--force \ --force \
--allow-source-mismatch \ --allow-source-mismatch \
"${DUPLICITY_TARGET_URL}" \ "${DUPLICITY_TARGET_URL}" \