From eaeeb4ed6c07e2f681583db6b55080a31c167a10 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Tue, 28 Jan 2025 16:50:33 +0100 Subject: [PATCH 1/9] feat(nextcloud): Add simple restic backup --- roles/nextcloud/defaults/main.yml | 6 ++++++ roles/nextcloud/templates/docker-stack.yml.j2 | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/roles/nextcloud/defaults/main.yml b/roles/nextcloud/defaults/main.yml index 732cb18..3481219 100644 --- a/roles/nextcloud/defaults/main.yml +++ b/roles/nextcloud/defaults/main.yml @@ -42,3 +42,9 @@ nextcloud_smtp_from_domain: "{{ server_domain }}" # nextcloud_s3_ssl: true # nextcloud_s3_region: eu-central-1 # nextcloud_s3_usepath_style: true + +nextcloud_backup_db_enable: false +# nextcloud_backup_db_repo: s3.eu-central-1.wasabisys.com/myrepo +# nextcloud_backup_db_key: +# nextcloud_backup_db_secret: +# nextcloud_backup_db_timezone: US/Chicago diff --git a/roles/nextcloud/templates/docker-stack.yml.j2 b/roles/nextcloud/templates/docker-stack.yml.j2 index 80f28fa..07a57f4 100644 --- a/roles/nextcloud/templates/docker-stack.yml.j2 +++ b/roles/nextcloud/templates/docker-stack.yml.j2 @@ -160,6 +160,26 @@ services: networks: - backend +{% if nextcloud_backup_db_enable is not undefined and not false %} + backup: + image: mazzolino/restic + environment: + - "TZ={{ nextcloud_backup_db_timezone }}" + # go-cron starts w seconds + - "BACKUP_CRON=0 30 3 * * *" + - "RESTIC_REPOSITORY={{ nextcloud_backup_db_repo }}" + - "AWS_ACCESS_KEY_ID={{ nextcloud_backup_db_key }}" + - "AWS_SECRET_ACCESS_KEY={{ nextcloud_backup_db_secret }}" + - "RESTIC_PASSWORD={{ nextcloud_backup_db_pass }}" + - "RESTIC_BACKUP_TAGS=nextcloud-db" + - "RESTIC_BACKUP_SOURCES=/mnt/volumes" + volumes: + - db:/mnt/volumes/nextcloud_db:ro + - data:/mnt/volumes/nextcloud_data:ro + networks: + - backend +{% endif %} + # metrics: # image: telegraf # hostname: "${HOSTNAME:-vmi352583.contaboserver.net}" From 135aadf3a08a43c724dcdf54b53598e319e2a98b Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 18:45:22 +0100 Subject: [PATCH 2/9] feat(restic): Add restic backup maintenance stack Sets up regular backup maintenance for a restic (S3) backend, and enables global variables for other roles to use for their individual backup. Example found in nextcloud role. --- roles/nextcloud/defaults/main.yml | 10 ++-- roles/nextcloud/templates/docker-stack.yml.j2 | 24 +++++---- roles/restic/README.md | 49 +++++++++++++++++++ roles/restic/defaults/main.yml | 14 ++++++ roles/restic/meta/main.yml | 11 +++++ roles/restic/tasks/main.yml | 11 +++++ roles/restic/templates/docker-stack.yml.j2 | 31 ++++++++++++ roles/restic/vars/main.yml | 8 +++ 8 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 roles/restic/README.md create mode 100644 roles/restic/defaults/main.yml create mode 100644 roles/restic/meta/main.yml create mode 100644 roles/restic/tasks/main.yml create mode 100644 roles/restic/templates/docker-stack.yml.j2 create mode 100644 roles/restic/vars/main.yml diff --git a/roles/nextcloud/defaults/main.yml b/roles/nextcloud/defaults/main.yml index 3481219..916c7ee 100644 --- a/roles/nextcloud/defaults/main.yml +++ b/roles/nextcloud/defaults/main.yml @@ -18,6 +18,10 @@ nextcloud_redis_password: myredispass nextcloud_db_username: nextcloud nextcloud_db_password: secretnextcloud +# run restic backups +nextcloud_backup_enable: false +nextcloud_backup_cron: 0 30 3 * * * + nextcloud_php_memory_limit: 5G # maximum ram php may use nextcloud_php_upload_limit: 15G # maximum size of (web) uploaded files @@ -42,9 +46,3 @@ nextcloud_smtp_from_domain: "{{ server_domain }}" # nextcloud_s3_ssl: true # nextcloud_s3_region: eu-central-1 # nextcloud_s3_usepath_style: true - -nextcloud_backup_db_enable: false -# nextcloud_backup_db_repo: s3.eu-central-1.wasabisys.com/myrepo -# nextcloud_backup_db_key: -# nextcloud_backup_db_secret: -# nextcloud_backup_db_timezone: US/Chicago diff --git a/roles/nextcloud/templates/docker-stack.yml.j2 b/roles/nextcloud/templates/docker-stack.yml.j2 index 07a57f4..ff5f6ce 100644 --- a/roles/nextcloud/templates/docker-stack.yml.j2 +++ b/roles/nextcloud/templates/docker-stack.yml.j2 @@ -160,24 +160,22 @@ services: networks: - backend -{% if nextcloud_backup_db_enable is not undefined and not false %} +{% if backup_enable is not undefined and not false and nextcloud_backup_enable is not undefined and not false %} backup: image: mazzolino/restic environment: - - "TZ={{ nextcloud_backup_db_timezone }}" + - "TZ={{ restic_timezone }}" # go-cron starts w seconds - - "BACKUP_CRON=0 30 3 * * *" - - "RESTIC_REPOSITORY={{ nextcloud_backup_db_repo }}" - - "AWS_ACCESS_KEY_ID={{ nextcloud_backup_db_key }}" - - "AWS_SECRET_ACCESS_KEY={{ nextcloud_backup_db_secret }}" - - "RESTIC_PASSWORD={{ nextcloud_backup_db_pass }}" - - "RESTIC_BACKUP_TAGS=nextcloud-db" - - "RESTIC_BACKUP_SOURCES=/mnt/volumes" + - "BACKUP_CRON={{ nextcloud_backup_cron }}" + - "RESTIC_REPOSITORY={{ restic_repo }}" + - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" + - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" + - "RESTIC_PASSWORD={{ restic_pass }}" + - "RESTIC_BACKUP_TAGS=nextcloud" + - "RESTIC_BACKUP_SOURCES=/volumes" volumes: - - db:/mnt/volumes/nextcloud_db:ro - - data:/mnt/volumes/nextcloud_data:ro - networks: - - backend + - db:/volumes/nextcloud_db:ro + - data:/volumes/nextcloud_data:ro {% endif %} # metrics: diff --git a/roles/restic/README.md b/roles/restic/README.md new file mode 100644 index 0000000..8849990 --- /dev/null +++ b/roles/restic/README.md @@ -0,0 +1,49 @@ +# restic + +Backup maintenance stack. + +Takes care of regularly pruning the backup repository and checking its integrity. +Currently only supports S3 as a backend. + +## Defaults + +```yaml +restic_timezone: US/Chicago +``` + +The timezone to be used for the cronjob. + +```yaml +restic_version: latest +``` + +The docker image version to be used in stack creation. + +```yaml +restic_repo: s3.eu-central-1.wasabisys.com/myrepo +restic_pass: +``` + +The repository url and the restic repository password. +See the restic documentation for more information. + +```yaml +restic_s3_key: +restic_s3_secret: +``` + +The restic S3 credentials, i.e. the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. + +```yaml +restic_prune_cron: 0 0 4 * * * +restic_forget_args: --prune --keep-last 14 --keep-daily 2 --keep-weekly 2 +``` + +The default prune and forget cronjob schedule and arguments: Prune the repository every day at 4:00 AM and keep the last 14 snapshots, 2 daily snapshots and 2 weekly snapshots. + +```yaml +restic_check_cron: 0 15 5 * * * +restic_check_args: --read-data-subset=5% +``` + +The default check cronjob schedule and arguments: Check the repository integrity every day at 5:15 AM and in addition to structural checks, read 5 randomly chosen % for a data integrity check. diff --git a/roles/restic/defaults/main.yml b/roles/restic/defaults/main.yml new file mode 100644 index 0000000..8022df5 --- /dev/null +++ b/roles/restic/defaults/main.yml @@ -0,0 +1,14 @@ +--- +restic_version: latest + +# restic_repo: s3.eu-central-1.wasabisys.com/myrepo +# restic_pass: +# restic_s3_key: +# restic_s3_secret: +restic_timezone: "{{ server_timezone | default('US/Chicago') }}" + +restic_prune_cron: 0 0 4 * * * +restic_forget_args: --prune --keep-last 14 --keep-daily 2 --keep-weekly 2 + +restic_check_cron: 0 15 5 * * * +restic_check_args: --read-data-subset=5% diff --git a/roles/restic/meta/main.yml b/roles/restic/meta/main.yml new file mode 100644 index 0000000..dd49542 --- /dev/null +++ b/roles/restic/meta/main.yml @@ -0,0 +1,11 @@ +--- +galaxy_info: + author: Marty Oehme + description: Installs a restic-based backup maintenance stack. Only supports S3 atm. + license: GPL-3.0-only + min_ansible_version: "2.9" + galaxy_tags: [] + +dependencies: + - docker-swarm + - caddy_id diff --git a/roles/restic/tasks/main.yml b/roles/restic/tasks/main.yml new file mode 100644 index 0000000..5bb5027 --- /dev/null +++ b/roles/restic/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy restic to swarm + community.general.docker_stack: + name: "{{ stack_name }}" + state: present + prune: yes + compose: + - "{{ stack_compose }}" + become: true + tags: + - docker-swarm diff --git a/roles/restic/templates/docker-stack.yml.j2 b/roles/restic/templates/docker-stack.yml.j2 new file mode 100644 index 0000000..822572d --- /dev/null +++ b/roles/restic/templates/docker-stack.yml.j2 @@ -0,0 +1,31 @@ +version: '3.4' + +services: + prune: + image: "{{ stack_image }}:{{ restic_version }}" + hostname: docker + environment: + - "TZ={{ restic_timezone }}" + - SKIP_INIT: "true" + # go-cron starts w seconds + - "PRUNE_CRON={{ restic_prune_cron }}" + - RESTIC_FORGET_ARGS: "{{ restic_forget_args }}" + - "RESTIC_REPOSITORY={{ restic_repo }}" + - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" + - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" + - "RESTIC_PASSWORD={{ restic_pass }}" + + check: + image: "{{ stack_image }}:{{ restic_version }}" + hostname: docker + environment: + - "TZ={{ restic_timezone }}" + - SKIP_INIT: "true" + - RUN_ON_STARTUP: "false" + # go-cron starts w seconds + - "CHECK_CRON={{ restic_check_cron }}" + - RESTIC_CHECK_ARGS: "{{ restic_check_args }}" + - "RESTIC_REPOSITORY={{ restic_repo }}" + - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" + - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" + - "RESTIC_PASSWORD={{ restic_pass }}" diff --git a/roles/restic/vars/main.yml b/roles/restic/vars/main.yml new file mode 100644 index 0000000..8b3dcf5 --- /dev/null +++ b/roles/restic/vars/main.yml @@ -0,0 +1,8 @@ +--- +stack_name: restic + +stack_image: "mazzolino/restic" + +stack_compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" + +backup_enable: true From af4cfc5a4b8c5965c30cae3c4d7fb94e51201627 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 18:57:52 +0100 Subject: [PATCH 3/9] fix(nextcloud): Default to backups enabled Backups should be enabled by default if available. --- roles/nextcloud/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nextcloud/defaults/main.yml b/roles/nextcloud/defaults/main.yml index 916c7ee..4881068 100644 --- a/roles/nextcloud/defaults/main.yml +++ b/roles/nextcloud/defaults/main.yml @@ -19,7 +19,7 @@ nextcloud_db_username: nextcloud nextcloud_db_password: secretnextcloud # run restic backups -nextcloud_backup_enable: false +nextcloud_backup_enable: true nextcloud_backup_cron: 0 30 3 * * * nextcloud_php_memory_limit: 5G # maximum ram php may use From 557f20d7b40193260800f294e488ea636015892e Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 18:58:12 +0100 Subject: [PATCH 4/9] feat(shaarli): Add backups Add restic backup functionality for shaarli data. --- roles/shaarli/defaults/main.yml | 4 ++++ roles/shaarli/templates/docker-stack.yml.j2 | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/roles/shaarli/defaults/main.yml b/roles/shaarli/defaults/main.yml index a654c5f..45f1bef 100644 --- a/roles/shaarli/defaults/main.yml +++ b/roles/shaarli/defaults/main.yml @@ -7,3 +7,7 @@ shaarli_use_https: true # the subdomain link shaarli will be reachable under subdomain_alias: links + +# should we back up the data? +shaarli_backup_enable: true +shaarli_backup_cron: 0 45 3 * * * diff --git a/roles/shaarli/templates/docker-stack.yml.j2 b/roles/shaarli/templates/docker-stack.yml.j2 index 545d20e..a90157a 100644 --- a/roles/shaarli/templates/docker-stack.yml.j2 +++ b/roles/shaarli/templates/docker-stack.yml.j2 @@ -15,6 +15,23 @@ services: - data:/var/www/shaarli/data - cache:/var/www/shaarli/cache +{% if backup_enable is not undefined and not false and shaarli_backup_enable is not undefined and not false %} + backup: + image: mazzolino/restic + environment: + - "TZ={{ restic_timezone }}" + # go-cron starts w seconds + - "BACKUP_CRON={{ shaarli_backup_cron }}" + - "RESTIC_REPOSITORY={{ restic_repo }}" + - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" + - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" + - "RESTIC_PASSWORD={{ restic_pass }}" + - "RESTIC_BACKUP_TAGS=shaarli" + - "RESTIC_BACKUP_SOURCES=/volumes" + volumes: + - data:/volumes/shaarli_data:ro +{% endif %} + volumes: data: cache: From 1a3fd9160e31df1489107bd05613de1b4bafbc12 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 18:59:58 +0100 Subject: [PATCH 5/9] fix(restic): Add role to site deployment --- site.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/site.yml b/site.yml index 892a4e2..19f483d 100644 --- a/site.yml +++ b/site.yml @@ -24,6 +24,12 @@ tags: - caddy + - name: Install restic backup management + import_role: + role: restic + tags: + - restic + - name: Grab caddy container id for all following services import_role: role: caddy_id From 0d7e99763fcb428b849909719daeea46b1ccd225 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 21:34:58 +0100 Subject: [PATCH 6/9] feat(nextcloud): Add caddy server HSTS preload, webfinger --- roles/nextcloud/files/Caddyfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/nextcloud/files/Caddyfile b/roles/nextcloud/files/Caddyfile index a54f3f5..a56ed59 100644 --- a/roles/nextcloud/files/Caddyfile +++ b/roles/nextcloud/files/Caddyfile @@ -10,7 +10,7 @@ header { # enable HSTS - Strict-Transport-Security max-age=31536000; + Strict-Transport-Security max-age=31536000;includeSubDomains;preload; Permissions-Policy interest-cohort=() X-Content-Type-Options nosniff X-Frame-Options SAMEORIGIN @@ -18,11 +18,13 @@ X-XSS-Protection "1; mode=block" X-Permitted-Cross-Domain-Policies none X-Robots-Tag "noindex, nofollow" - -X-Powered-By } + # client support (e.g. os x calendar / contacts) redir /.well-known/carddav /remote.php/dav 301 redir /.well-known/caldav /remote.php/dav 301 + redir /.well-known/webfinger /index.php/.well-known/webfinger 301 + redir /.well-known/nodeinfo /index.php/.well-known/nodeinfo 301 # Uncomment this block if you use the high speed files backend: https://github.com/nextcloud/notify_push #handle_path /push/* { From a4ccdb98840907975a1672142cd78d9877db3027 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 21:35:25 +0100 Subject: [PATCH 7/9] fix(restic): Fix docker stack environment variables --- roles/restic/templates/docker-stack.yml.j2 | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/roles/restic/templates/docker-stack.yml.j2 b/roles/restic/templates/docker-stack.yml.j2 index 822572d..f64be0a 100644 --- a/roles/restic/templates/docker-stack.yml.j2 +++ b/roles/restic/templates/docker-stack.yml.j2 @@ -1,15 +1,14 @@ -version: '3.4' - services: prune: image: "{{ stack_image }}:{{ restic_version }}" hostname: docker environment: - "TZ={{ restic_timezone }}" - - SKIP_INIT: "true" + - "SKIP_INIT=true" + - "RUN_ON_STARTUP=true" # go-cron starts w seconds - "PRUNE_CRON={{ restic_prune_cron }}" - - RESTIC_FORGET_ARGS: "{{ restic_forget_args }}" + - "RESTIC_FORGET_ARGS={{ restic_forget_args }}" - "RESTIC_REPOSITORY={{ restic_repo }}" - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" @@ -20,11 +19,11 @@ services: hostname: docker environment: - "TZ={{ restic_timezone }}" - - SKIP_INIT: "true" - - RUN_ON_STARTUP: "false" + - "SKIP_INIT=true" + - "RUN_ON_STARTUP=false" # go-cron starts w seconds - "CHECK_CRON={{ restic_check_cron }}" - - RESTIC_CHECK_ARGS: "{{ restic_check_args }}" + - "RESTIC_CHECK_ARGS={{ restic_check_args }}" - "RESTIC_REPOSITORY={{ restic_repo }}" - "AWS_ACCESS_KEY_ID={{ restic_s3_key }}" - "AWS_SECRET_ACCESS_KEY={{ restic_s3_secret }}" From 90e45cacda6a52c8b4e2c0bbc96b4fdcc42485bd Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 21:35:44 +0100 Subject: [PATCH 8/9] chore(restic): Do not require caddy id for the role --- roles/restic/meta/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/restic/meta/main.yml b/roles/restic/meta/main.yml index dd49542..0c765ab 100644 --- a/roles/restic/meta/main.yml +++ b/roles/restic/meta/main.yml @@ -8,4 +8,3 @@ galaxy_info: dependencies: - docker-swarm - - caddy_id From 7543170f75308bda7c728c55d2f58c78f33f1008 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Mon, 3 Feb 2025 21:36:18 +0100 Subject: [PATCH 9/9] chore(restic): By default run check every Sunday night And check a larger subset of the data with 15%. --- roles/restic/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/restic/defaults/main.yml b/roles/restic/defaults/main.yml index 8022df5..48fdbc7 100644 --- a/roles/restic/defaults/main.yml +++ b/roles/restic/defaults/main.yml @@ -10,5 +10,5 @@ restic_timezone: "{{ server_timezone | default('US/Chicago') }}" restic_prune_cron: 0 0 4 * * * restic_forget_args: --prune --keep-last 14 --keep-daily 2 --keep-weekly 2 -restic_check_cron: 0 15 5 * * * -restic_check_args: --read-data-subset=5% +restic_check_cron: 0 30 4 * * SUN +restic_check_args: --read-data-subset=15%