From 2d01350fa5d9d655e251228b8bd87e16004bb20a Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 20:28:44 +0100 Subject: [PATCH 01/14] Switch to new landingpage and remove old blog New landingpage includes the blog itself to better integrate with the main page. Also runs on astro not on hugo which I am a little more familiar with. --- roles/blog/README.md | 37 ----------------- roles/blog/defaults/main.yml | 11 ----- roles/blog/handlers/main.yml | 53 ------------------------ roles/blog/meta/main.yml | 14 ------- roles/blog/tasks/main.yml | 24 ----------- roles/blog/templates/docker-stack.yml.j2 | 20 --------- roles/blog/templates/upstream.json.j2 | 40 ------------------ roles/blog/vars/main.yml | 7 ---- roles/landingpage/defaults/main.yml | 3 +- roles/landingpage/vars/main.yml | 2 +- site.yml | 5 --- 11 files changed, 2 insertions(+), 214 deletions(-) delete mode 100644 roles/blog/README.md delete mode 100644 roles/blog/defaults/main.yml delete mode 100644 roles/blog/handlers/main.yml delete mode 100644 roles/blog/meta/main.yml delete mode 100644 roles/blog/tasks/main.yml delete mode 100644 roles/blog/templates/docker-stack.yml.j2 delete mode 100644 roles/blog/templates/upstream.json.j2 delete mode 100644 roles/blog/vars/main.yml diff --git a/roles/blog/README.md b/roles/blog/README.md deleted file mode 100644 index d649b50..0000000 --- a/roles/blog/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# landingpage - -The public face of my server. -Not much to see here honestly, -just a few simple lines of html explaining what this server is about and how to contact me. - -I don't see anybody else benefiting massively from this role but me, -but if you want the same web presence go for it I suppose 😉 - -## Defaults - -``` -landingpage_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" -``` - -The on-target directory where the proxy configuration file should be stashed. - -``` -landingpage_use_https: true -``` - -Whether the service should be reachable through http (port 80) or through https (port 443) and provision an https certificate. Usually you will want this to stay `true`. - -``` -landingpage_version: latest -``` - -The docker image version to be used in stack creation. - -``` -subdomain_alias: www -``` - -If the deployed container should be served over a uri that is not the stack name. -By default, it will be set to `www.yourdomain.com` - -if this option is not set it will be served on `landingpage.yourdomain.com` instead. - diff --git a/roles/blog/defaults/main.yml b/roles/blog/defaults/main.yml deleted file mode 100644 index fcf9a0f..0000000 --- a/roles/blog/defaults/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -# never got around to removing the master tag from the images -blog_version: master - -blog_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" - -blog_use_https: true - -# the subdomain link blog will be reachable under -subdomain_alias: blog diff --git a/roles/blog/handlers/main.yml b/roles/blog/handlers/main.yml deleted file mode 100644 index 29ca5c2..0000000 --- a/roles/blog/handlers/main.yml +++ /dev/null @@ -1,53 +0,0 @@ -## Register reverse proxy -- name: Ensure upstream directory exists - ansible.builtin.file: - path: "{{ blog_upstream_file_dir }}" - state: directory - mode: '0755' - become: true - listen: "update blog upstream" - -- name: Update upstream template - ansible.builtin.template: - src: upstream.json.j2 - dest: "{{ blog_upstream_file_dir }}/upstream.json" - become: true - listen: "update blog upstream" - -# figure out if upstream id exists -- name: check {{ stack_name }} upstream - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - curl localhost:2019/id/{{ stack_name }}_upstream/ - changed_when: False - register: result - become: true - listen: "update blog upstream" - -# upstream already exists, patch it -- name: remove old {{ stack_name }} upstream - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - curl -X DELETE localhost:2019/id/{{ stack_name }}_upstream/ - become: true - when: (result.stdout | from_json)['error'] is not defined - listen: "update blog upstream" - -# upstream has to be created -- name: add {{ stack_name }} upstream - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - curl -X POST -H "Content-Type: application/json" -d @{{ blog_upstream_file_dir }}/upstream.json localhost:2019/config/apps/http/servers/{{ (blog_use_https == True) | ternary(caddy_https_server_name, caddy_http_server_name) }}/routes/0/ - become: true - listen: "update blog upstream" - -- name: Ensure upstream directory is gone again - ansible.builtin.file: - path: "{{ blog_upstream_file_dir }}" - state: absent - become: true - listen: "update blog upstream" - diff --git a/roles/blog/meta/main.yml b/roles/blog/meta/main.yml deleted file mode 100644 index fbb1340..0000000 --- a/roles/blog/meta/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- - -galaxy_info: - author: Marty Oehme - description: Installs my personal public facing landing page as a docker stack service - license: GPL-3.0-only - min_ansible_version: 2.9 - galaxy_tags: [] - - -dependencies: - - docker - - docker-swarm - - caddy diff --git a/roles/blog/tasks/main.yml b/roles/blog/tasks/main.yml deleted file mode 100644 index 0637ae3..0000000 --- a/roles/blog/tasks/main.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -## install blog container -- name: Check upstream status - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - curl localhost:2019/id/{{ stack_name }}_upstream/ - register: result - changed_when: (result.stdout | from_json) != (lookup('template', 'upstream.json.j2') | from_yaml) - become: true - notify: "update blog upstream" - -- name: Deploy blog to swarm - community.general.docker_stack: - name: "{{ stack_name }}" - state: present - prune: yes - compose: - - "{{ stack_compose }}" - become: true - tags: - - docker-swarm - notify: "update blog upstream" - diff --git a/roles/blog/templates/docker-stack.yml.j2 b/roles/blog/templates/docker-stack.yml.j2 deleted file mode 100644 index 1d83e61..0000000 --- a/roles/blog/templates/docker-stack.yml.j2 +++ /dev/null @@ -1,20 +0,0 @@ -version: '3.4' - -services: - app: - image: "{{ stack_image }}:{{ blog_version }}" - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost"] - interval: 1m - timeout: 10s - retries: 3 - start_period: 1m - entrypoint: sh -c "/docker-entrypoint.sh nginx -g 'daemon off;'" - networks: - - "{{ docker_swarm_public_network_name }}" - -networks: - "{{ docker_swarm_public_network_name }}": - external: true - - diff --git a/roles/blog/templates/upstream.json.j2 b/roles/blog/templates/upstream.json.j2 deleted file mode 100644 index a8af36a..0000000 --- a/roles/blog/templates/upstream.json.j2 +++ /dev/null @@ -1,40 +0,0 @@ -{ - "@id": "{{ stack_name }}_upstream", -{% if server_domain is not undefined and not none %} - "match": [ - { - "host": [ -{% if subdomain_alias is not undefined and not none %} - "{{ subdomain_alias }}.{{ server_domain }}" -{% else %} - "{{ stack_name }}.{{ server_domain }}" -{% endif %} - ] - } - ], -{% else %} - "match": [ - { - "path": [ -{% if subdomain_alias is not undefined and not none %} - "/{{ subdomain_alias }}*" -{% else %} - "/{{ stack_name }}*" -{% endif %} - , - "/" - ] - } - ], -{% endif %} - "handle": [ - { - "handler": "reverse_proxy", - "upstreams": [ - { - "dial": "{{ stack_name }}_app:80" - } - ] - } - ] -} diff --git a/roles/blog/vars/main.yml b/roles/blog/vars/main.yml deleted file mode 100644 index 565e61f..0000000 --- a/roles/blog/vars/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -stack_name: blog - -stack_image: "registry.gitlab.com/cloud-serve/blog" - -stack_compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" diff --git a/roles/landingpage/defaults/main.yml b/roles/landingpage/defaults/main.yml index 2c47345..b47f57f 100644 --- a/roles/landingpage/defaults/main.yml +++ b/roles/landingpage/defaults/main.yml @@ -1,7 +1,6 @@ --- -# never got around to removing the master tag from the images -landingpage_version: master +landingpage_version: latest landingpage_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" diff --git a/roles/landingpage/vars/main.yml b/roles/landingpage/vars/main.yml index e3616a9..df7f3d7 100644 --- a/roles/landingpage/vars/main.yml +++ b/roles/landingpage/vars/main.yml @@ -2,6 +2,6 @@ stack_name: landingpage -stack_image: "registry.gitlab.com/cloud-serve/landing" +stack_image: "martyo/cloudserve-landing" stack_compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" diff --git a/site.yml b/site.yml index 62f01e9..a99b30b 100644 --- a/site.yml +++ b/site.yml @@ -63,11 +63,6 @@ role: landingpage tags: landingpage - - name: Install my personal blog - import_role: - role: blog - tags: blog - - name: Install gitea import_role: role: gitea From d4dbeb4eb4e35a12b277a4328495508e4a1bc0c6 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 20:31:15 +0100 Subject: [PATCH 02/14] Improve gitea stability on first launch When launching many containers gitea admin waiting still sometimes gets stuck. This should provide a bandaid for now. Also improve the container detection. --- roles/gitea/tasks/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/gitea/tasks/main.yml b/roles/gitea/tasks/main.yml index cf790d3..d930ed7 100644 --- a/roles/gitea/tasks/main.yml +++ b/roles/gitea/tasks/main.yml @@ -94,9 +94,9 @@ register: gitea_deployment notify: "update gitea upstream" -- name: Wait 30 seconds for gitea to become healthy +- name: Wait a minute for gitea to become healthy wait_for: - timeout: 30 + timeout: 60 delegate_to: localhost when: gitea_deployment is changed @@ -104,7 +104,7 @@ ansible.builtin.command: cmd: docker ps -q -f name={{ stack_name }}_app become: true - until: gitea_app_container_name['rc'] == 0 and gitea_app_container_name['stdout'] | length >= 1 + until: gitea_app_container_name['rc'] | default('') == 0 and gitea_app_container_name['stdout'] | length >= 1 retries: 10 delay: 10 changed_when: False @@ -116,9 +116,9 @@ user: git command: > gitea admin user list --admin - until: gitea_admin_list is defined and gitea_admin_list['rc'] == 0 + until: gitea_admin_list is defined and gitea_admin_list['rc'] | default('') == 0 retries: 15 - delay: 10 + delay: 20 become: true register: gitea_admin_list changed_when: gitea_admin_list['stdout_lines'] | length <= 1 and 'Username' in gitea_admin_list['stdout'] From 9cf43d0d5d8ebc455db2364a880fe85e997d6b09 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 20:34:07 +0100 Subject: [PATCH 03/14] Fix new stat module checksum option In the module get_md5 has been replaced by get_checksum. --- roles/system-upgrade/tasks/Ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/system-upgrade/tasks/Ubuntu.yml b/roles/system-upgrade/tasks/Ubuntu.yml index 983e6b5..dc33abf 100644 --- a/roles/system-upgrade/tasks/Ubuntu.yml +++ b/roles/system-upgrade/tasks/Ubuntu.yml @@ -21,7 +21,7 @@ register: reboot_required_file stat: path: /var/run/reboot-required - get_md5: no + get_checksum: false tags: - os - reboot From a52cab2f61907c398dbb4b022ca1c2a2241701d0 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 20:34:41 +0100 Subject: [PATCH 04/14] Refactor wallabag stack name and repo variables Brought in line with other stack naming schemes. --- roles/wallabag/README.md | 2 +- roles/wallabag/defaults/main.yml | 2 +- roles/wallabag/handlers/main.yml | 11 +++++------ roles/wallabag/tasks/main.yml | 8 +++----- roles/wallabag/templates/upstream.json.j2 | 8 ++++---- roles/wallabag/vars/main.yml | 6 +++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/roles/wallabag/README.md b/roles/wallabag/README.md index 44eefa9..c1eceff 100644 --- a/roles/wallabag/README.md +++ b/roles/wallabag/README.md @@ -6,7 +6,7 @@ Contains only a single deployed image and a couple of simple variables to set. ## Variables ``` -wallabag_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack.name }}" +wallabag_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" ``` The on-target directory where the proxy configuration file should be stashed. diff --git a/roles/wallabag/defaults/main.yml b/roles/wallabag/defaults/main.yml index a1b8116..dcc02b2 100644 --- a/roles/wallabag/defaults/main.yml +++ b/roles/wallabag/defaults/main.yml @@ -2,7 +2,7 @@ wallabag_version: latest -wallabag_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack.name }}" +wallabag_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" wallabag_use_https: true diff --git a/roles/wallabag/handlers/main.yml b/roles/wallabag/handlers/main.yml index 1f6268b..a104112 100644 --- a/roles/wallabag/handlers/main.yml +++ b/roles/wallabag/handlers/main.yml @@ -15,28 +15,28 @@ listen: "update wallabag upstream" # figure out if upstream id exists -- name: check {{ stack.name }} upstream +- name: check {{ stack_name }} upstream community.docker.docker_container_exec: container: "{{ caddy_container_id }}" command: > - curl localhost:2019/id/{{ stack.name }}_upstream/ + curl localhost:2019/id/{{ stack_name }}_upstream/ changed_when: False register: result become: true listen: "update wallabag upstream" # upstream already exists, patch it -- name: remove old {{ stack.name }} upstream +- name: remove old {{ stack_name }} upstream community.docker.docker_container_exec: container: "{{ caddy_container_id }}" command: > - curl -X DELETE localhost:2019/id/{{ stack.name }}_upstream/ + curl -X DELETE localhost:2019/id/{{ stack_name }}_upstream/ become: true when: (result.stdout | from_json)['error'] is not defined listen: "update wallabag upstream" # upstream has to be created -- name: add {{ stack.name }} upstream +- name: add {{ stack_name }} upstream community.docker.docker_container_exec: container: "{{ caddy_container_id }}" command: > @@ -50,4 +50,3 @@ state: absent become: true listen: "update wallabag upstream" - diff --git a/roles/wallabag/tasks/main.yml b/roles/wallabag/tasks/main.yml index 6c789d6..34e5859 100644 --- a/roles/wallabag/tasks/main.yml +++ b/roles/wallabag/tasks/main.yml @@ -4,7 +4,7 @@ community.docker.docker_container_exec: container: "{{ caddy_container_id }}" command: > - curl localhost:2019/id/{{ stack.name }}_upstream/ + curl localhost:2019/id/{{ stack_name }}_upstream/ register: result changed_when: (result.stdout | from_json) != (lookup('template', 'upstream.json.j2') | from_yaml) become: true @@ -12,14 +12,12 @@ - name: Deploy wallabag to swarm community.general.docker_stack: - name: "{{ stack.name }}" + name: "{{ stack_name }}" state: present prune: yes compose: - - "{{ stack.compose }}" - when: stack is defined + - "{{ stack_compose }}" become: true tags: - docker-swarm notify: "update wallabag upstream" - diff --git a/roles/wallabag/templates/upstream.json.j2 b/roles/wallabag/templates/upstream.json.j2 index 6db9d1a..a20061f 100644 --- a/roles/wallabag/templates/upstream.json.j2 +++ b/roles/wallabag/templates/upstream.json.j2 @@ -1,5 +1,5 @@ { - "@id": "{{ stack.name }}_upstream", + "@id": "{{ stack_name }}_upstream", {% if server_domain is not undefined and not none %} "match": [ { @@ -7,7 +7,7 @@ {% if subdomain_alias is not undefined and not none %} "{{ subdomain_alias }}.{{ server_domain }}" {% else %} - "{{ stack.name }}.{{ server_domain }}" + "{{ stack_name }}.{{ server_domain }}" {% endif %} ] } @@ -19,7 +19,7 @@ {% if subdomain_alias is not undefined and not none %} "/{{ subdomain_alias }}*" {% else %} - "/{{ stack.name }}*" + "/{{ stack_name }}*" {% endif %} ] } @@ -30,7 +30,7 @@ "handler": "reverse_proxy", "upstreams": [ { - "dial": "{{ stack.name }}_app:80" + "dial": "{{ stack_name }}_app:80" } ] } diff --git a/roles/wallabag/vars/main.yml b/roles/wallabag/vars/main.yml index 7e281a4..d270fd5 100644 --- a/roles/wallabag/vars/main.yml +++ b/roles/wallabag/vars/main.yml @@ -1,7 +1,7 @@ --- -stack: - name: wallabag - compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" +stack_name: wallabag stack_image: "wallabag/wallabag" + +stack_compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" From 388a1d8cfc862d7a016e5569f140cd562b144cc3 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 20:35:51 +0100 Subject: [PATCH 05/14] Separate caddy container id grabbing into own role Since other roles often rely on this not an actual new caddy server installation we should probably have it as its own little role. --- roles/caddy/tasks/main.yml | 42 +--------- roles/caddy_id/README.md | 84 ++++++++++++++++++++ roles/caddy_id/meta/main.yml | 5 ++ roles/caddy_id/tasks/main.yml | 39 +++++++++ roles/caddy_id/templates/config.json.j2 | 72 +++++++++++++++++ roles/caddy_id/templates/docker-stack.yml.j2 | 30 +++++++ roles/caddy_id/vars/main.yml | 5 ++ site.yml | 10 ++- 8 files changed, 246 insertions(+), 41 deletions(-) create mode 100644 roles/caddy_id/README.md create mode 100644 roles/caddy_id/meta/main.yml create mode 100644 roles/caddy_id/tasks/main.yml create mode 100644 roles/caddy_id/templates/config.json.j2 create mode 100644 roles/caddy_id/templates/docker-stack.yml.j2 create mode 100644 roles/caddy_id/vars/main.yml diff --git a/roles/caddy/tasks/main.yml b/roles/caddy/tasks/main.yml index c2577ee..61f1abe 100644 --- a/roles/caddy/tasks/main.yml +++ b/roles/caddy/tasks/main.yml @@ -5,9 +5,9 @@ ansible.builtin.file: path: "{{ caddy_caddyfile_dir }}" state: directory - mode: '0755' + mode: "0755" become: true - tags: + tags: - fs - name: Ensure Caddyfile exists @@ -30,44 +30,6 @@ become: true tags: - docker-swarm - -- name: Get caddy container info - ansible.builtin.command: - cmd: docker ps -q -f name={{ caddy_stack.name }} - become: true - # bringing up the container takes some time, we have to wait - until: caddy_container_info['rc'] == 0 and caddy_container_info['stdout'] | length >= 1 - retries: 5 - delay: 10 - changed_when: False - register: caddy_container_info - -- name: Register caddy container id - ansible.builtin.set_fact: caddy_container_id={{ caddy_container_info['stdout'] }} - notify: - - debug caddy container - -# FIXME this should be taken care of in Dockerfile not here -- name: Ensure caddy curl available - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - apk add curl - become: true - register: result - changed_when: "'Installing' in result.stdout" - -- name: Ensure caddy api is responsive - community.docker.docker_container_exec: - container: "{{ caddy_container_id }}" - command: > - curl localhost:2019/config/ - become: true - until: result.rc == 0 - when: caddy_use_api == True - changed_when: False - register: result - # TODO FIXME UP # - name: Allow access to services # firewalld: diff --git a/roles/caddy_id/README.md b/roles/caddy_id/README.md new file mode 100644 index 0000000..88871db --- /dev/null +++ b/roles/caddy_id/README.md @@ -0,0 +1,84 @@ +# Caddy + +Caddy is the reverse proxy for all other services running on the infrastructure. +It was chosen for its relative ease of use, +interactible API and https-by-default setup. + +## Variables + +``` +caddy_caddyfile_dir: "{{ docker_stack_files_dir }}/caddy" +``` + +Sets up the on-target directory where important caddy files should be stored. + +``` +caddy_email: +``` + +Which e-mail should be used to provision https certificates with. I believe theoretically caddy will work and provision you with certificates even without providing an e-mail, but I would strongly urge providing one. + +``` +caddy_tls_use_staging: no +``` + +If turned on will use the staging servers of the acme certificate service, which is useful for testing and playing around with https (due to higher API limits and less severe restrictions). + +``` +caddy_use_api: yes +``` + +If turned off, will turn off the admin api for caddy. Should only be used if no other services are intended to be provisioned on the target, since most other service stacks rely on the API to set up their proxy targets. + +``` +caddy_use_debug: no +``` + +If true, will turn on caddy's debug logging. + +``` +caddy_use_https: yes +``` + +If turned off will turn of all auto-provisioning of https certificates by caddy. + +``` +caddy_version: alpine +``` + +Sets the docker image version to be used. + + +## Internal variables + +```yaml +caddy_stack: + name: caddy + compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" +``` + +Defines the actual docker stack which will later run on the target. +The name can be changed and will be used as a proxy target (`caddy.mydomain.com` or `192.168.1.1/caddy`) --- +though to be clear there is no intention currently to expose the caddy to the web at the moment.\ +The compose option defines which template to use for the `docker-stack.yml` file. You can either change options for the stack in the template file, +or directly here like the following: + +```yaml + compose: + - "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}" + - version: '3' + services: + another-container: + image: nginx:latest +# ... +``` + +```yaml +caddy_http_server_name: http +``` + +```yaml +caddy_https_server_name: https +``` + +The internal representation of the http and https servers respectively. diff --git a/roles/caddy_id/meta/main.yml b/roles/caddy_id/meta/main.yml new file mode 100644 index 0000000..5863772 --- /dev/null +++ b/roles/caddy_id/meta/main.yml @@ -0,0 +1,5 @@ +--- + +dependencies: + - docker + - docker-swarm diff --git a/roles/caddy_id/tasks/main.yml b/roles/caddy_id/tasks/main.yml new file mode 100644 index 0000000..adbfc5c --- /dev/null +++ b/roles/caddy_id/tasks/main.yml @@ -0,0 +1,39 @@ +--- +# get the caddy container id for all other containers + +- name: Get caddy container info + ansible.builtin.command: + cmd: docker ps -q -f name={{ caddy_stack.name }} + become: true + # bringing up the container takes some time, we have to wait + until: caddy_container_info['rc'] | default('') == 0 and caddy_container_info['stdout'] | length >= 1 + retries: 5 + delay: 10 + changed_when: False + register: caddy_container_info + +- name: Register caddy container id + ansible.builtin.set_fact: caddy_container_id={{ caddy_container_info['stdout'] }} + notify: + - debug caddy container + +# FIXME this should be taken care of in Dockerfile not here +- name: Ensure caddy curl available + community.docker.docker_container_exec: + container: "{{ caddy_container_id }}" + command: > + apk add curl + become: true + register: result + changed_when: "'Installing' in result.stdout" + +- name: Ensure caddy api is responsive + community.docker.docker_container_exec: + container: "{{ caddy_container_id }}" + command: > + curl localhost:2019/config/ + become: true + until: result.rc | default('') == 0 + when: caddy_use_api == True + changed_when: False + register: result diff --git a/roles/caddy_id/templates/config.json.j2 b/roles/caddy_id/templates/config.json.j2 new file mode 100644 index 0000000..b104a25 --- /dev/null +++ b/roles/caddy_id/templates/config.json.j2 @@ -0,0 +1,72 @@ +{ +{% if caddy_use_api is sameas false %} + "admin": { + "disabled": true + }, +{% endif %} +{% if caddy_use_debug is sameas true %} + "logging": { + "logs": { + "default": { + "level": "DEBUG" + } + } + }, +{% endif %} + "apps": { + "http": { + "servers": { + "{{ caddy_http_server_name }}": { + "listen": [ + ":80" + ], + "routes": [] +{% if caddy_use_https is sameas false %}, + "automatic_https": { + "disable": true + } +{% endif %} + }, + "{{ caddy_https_server_name }}": { + "listen": [ + ":443" + ], + "routes": [] +{% if caddy_use_https is sameas false %}, + "automatic_https": { + "disable": true + } +{% endif %} + } + } + } +{% if caddy_use_https is sameas true %}, + "tls": { + "automation": { + "policies": [ + { + "subjects": [], + "issuers": [ + { + {% if caddy_tls_use_staging is sameas true %} + "ca": "https://acme-staging-v02.api.letsencrypt.org/directory", + {% endif %} + {%- if caddy_email is not undefined and not none %} + "email": "{{ caddy_email }}", + {% endif %} + "module": "acme" + }, + { + {%- if caddy_email is not undefined and not none %} + "email": "{{ caddy_email }}", + {% endif %} + "module": "zerossl" + } + ] + } + ] + } + } +{% endif %} + } +} diff --git a/roles/caddy_id/templates/docker-stack.yml.j2 b/roles/caddy_id/templates/docker-stack.yml.j2 new file mode 100644 index 0000000..9c1ecf1 --- /dev/null +++ b/roles/caddy_id/templates/docker-stack.yml.j2 @@ -0,0 +1,30 @@ +version: "3.7" + +services: + app: + image: caddy:{{ caddy_version }} + command: caddy run --config /etc/caddy/config.json + healthcheck: + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://localhost:2019/metrics"] + interval: 1m + timeout: 10s + retries: 3 + start_period: 1m + ports: + - "80:80" + - "443:443" + volumes: + - "{{ caddy_caddyfile_dir }}:/etc/caddy" + - "{{ docker_stack_files_dir }}:/stacks:ro" + - data:/data + - config:/config + networks: + - "{{ docker_swarm_public_network_name }}" + +volumes: + data: + config: + +networks: + "{{ docker_swarm_public_network_name }}": + external: true diff --git a/roles/caddy_id/vars/main.yml b/roles/caddy_id/vars/main.yml new file mode 100644 index 0000000..7e60722 --- /dev/null +++ b/roles/caddy_id/vars/main.yml @@ -0,0 +1,5 @@ +--- +caddy_stack: + name: caddy + +caddy_use_api: yes # if no turns off api interface; it is *required* for other swarm roles to be routed diff --git a/site.yml b/site.yml index a99b30b..4b8e657 100644 --- a/site.yml +++ b/site.yml @@ -21,7 +21,15 @@ - name: Install caddy reverse proxy import_role: role: caddy - tags: caddy + tags: + - caddy + + - name: Grab caddy container id for all following services + import_role: + role: caddy_id + tags: + - caddy_id + - always - name: Install wallabag import_role: From 26cceccfd992b780d858a01bf659ad15ecaee2f2 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 22:49:43 +0100 Subject: [PATCH 06/14] Update Nextcloud internal Caddyfile Add suggested security improvements and static file caching. --- roles/nextcloud/files/Caddyfile | 58 +++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/roles/nextcloud/files/Caddyfile b/roles/nextcloud/files/Caddyfile index 8a86c1c..a54f3f5 100644 --- a/roles/nextcloud/files/Caddyfile +++ b/roles/nextcloud/files/Caddyfile @@ -1,16 +1,34 @@ -:80 { - root * /var/www/html - file_server +{ + servers { + trusted_proxies static 10.0.0.0/8 + } +} + +:80 { + encode zstd gzip + root * /var/www/html - php_fastcgi app:9000 header { # enable HSTS Strict-Transport-Security max-age=31536000; + Permissions-Policy interest-cohort=() + X-Content-Type-Options nosniff + X-Frame-Options SAMEORIGIN + Referrer-Policy no-referrer + X-XSS-Protection "1; mode=block" + X-Permitted-Cross-Domain-Policies none + X-Robots-Tag "noindex, nofollow" + -X-Powered-By } redir /.well-known/carddav /remote.php/dav 301 redir /.well-known/caldav /remote.php/dav 301 + # Uncomment this block if you use the high speed files backend: https://github.com/nextcloud/notify_push + #handle_path /push/* { + # reverse_proxy unix//run/notify_push/notify_push.sock # I love Unix sockets, but you can do :7867 also + #} + # .htaccess / data / config / ... shouldn't be accessible from outside @forbidden { path /.htaccess @@ -25,8 +43,36 @@ path /occ path /console.php } + handle @forbidden { + respond 404 + } - respond @forbidden 404 + handle { + root * /var/www/html + php_fastcgi app:9000 { + # Tells nextcloud to remove /index.php from URLs in links + env front_controller_active true + env modHeadersAvailable true # Avoid sending the security headers twice + } + } + # From .htaccess, set cache for versioned static files (cache-busting) + @immutable { + path *.css *.js *.mjs *.svg *.gif *.png *.jpg *.ico *.wasm *.tflite + query v=* + } + header @immutable Cache-Control "max-age=15778463, immutable" + + # From .htaccess, set cache for normal static files + @static { + path *.css *.js *.mjs *.svg *.gif *.png *.jpg *.ico *.wasm *.tflite + not query v=* + } + header @static Cache-Control "max-age=15778463" + + # From .htaccess, cache fonts for 1 week + @woff2 path *.woff2 + header @woff2 Cache-Control "max-age=604800" + + file_server } - From bc7796710adf58da6dd534876524d9461a52b829 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Fri, 8 Dec 2023 22:50:01 +0100 Subject: [PATCH 07/14] Pin Nextcloud version to current stable release --- roles/nextcloud/defaults/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/nextcloud/defaults/main.yml b/roles/nextcloud/defaults/main.yml index 10858f8..e97d9e0 100644 --- a/roles/nextcloud/defaults/main.yml +++ b/roles/nextcloud/defaults/main.yml @@ -1,7 +1,7 @@ --- # set preferred application version -nextcloud_version: fpm-alpine +nextcloud_version: 27-fpm-alpine # set preferred postgres version nextcloud_db_version: 12-alpine @@ -41,4 +41,3 @@ nextcloud_smtp_from_domain: "{{ server_domain }}" # nextcloud_s3_ssl: true # nextcloud_s3_region: eu-central-1 # nextcloud_s3_usepath_style: true - From d3f65a07fbcf2ab59a8a272086e3ba71d07c4bdb Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:04:28 +0200 Subject: [PATCH 08/14] Fix wget healthchecks to not use localhost For a reason, current wget versions error out when using localhost instead of 127.0.0.1 as the healthcheck for docker services. Probably has something to do with dns resolution - either on docker or wget end, but have not looked to deep into it. --- roles/caddy/templates/docker-stack.yml.j2 | 2 +- roles/caddy_id/templates/docker-stack.yml.j2 | 2 +- roles/gitea/templates/docker-stack.yml.j2 | 2 +- roles/landingpage/templates/docker-stack.yml.j2 | 2 +- roles/nextcloud/templates/docker-stack.yml.j2 | 2 +- roles/ntfy/templates/docker-stack.yml.j2 | 2 +- roles/searx/templates/docker-stack.yml.j2 | 2 +- roles/shaarli/templates/docker-stack.yml.j2 | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/caddy/templates/docker-stack.yml.j2 b/roles/caddy/templates/docker-stack.yml.j2 index 9c1ecf1..21a4c5a 100644 --- a/roles/caddy/templates/docker-stack.yml.j2 +++ b/roles/caddy/templates/docker-stack.yml.j2 @@ -5,7 +5,7 @@ services: image: caddy:{{ caddy_version }} command: caddy run --config /etc/caddy/config.json healthcheck: - test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://localhost:2019/metrics"] + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://127.0.0.1:2019/metrics"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/caddy_id/templates/docker-stack.yml.j2 b/roles/caddy_id/templates/docker-stack.yml.j2 index 9c1ecf1..21a4c5a 100644 --- a/roles/caddy_id/templates/docker-stack.yml.j2 +++ b/roles/caddy_id/templates/docker-stack.yml.j2 @@ -5,7 +5,7 @@ services: image: caddy:{{ caddy_version }} command: caddy run --config /etc/caddy/config.json healthcheck: - test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://localhost:2019/metrics"] + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://127.0.0.1:2019/metrics"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/gitea/templates/docker-stack.yml.j2 b/roles/gitea/templates/docker-stack.yml.j2 index fa4cff9..fff54a8 100644 --- a/roles/gitea/templates/docker-stack.yml.j2 +++ b/roles/gitea/templates/docker-stack.yml.j2 @@ -4,7 +4,7 @@ services: app: image: "{{ stack_image }}:{{ gitea_version }}" healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:3000"] + test: ["CMD", "wget", "--spider", "-q", "127.0.0.1:3000"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/landingpage/templates/docker-stack.yml.j2 b/roles/landingpage/templates/docker-stack.yml.j2 index b2525ab..fdaa2b4 100644 --- a/roles/landingpage/templates/docker-stack.yml.j2 +++ b/roles/landingpage/templates/docker-stack.yml.j2 @@ -4,7 +4,7 @@ services: app: image: "{{ stack_image }}:{{ landingpage_version }}" healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost"] + test: ["CMD", "wget", "--spider", "-q", "127.0.0.1"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/nextcloud/templates/docker-stack.yml.j2 b/roles/nextcloud/templates/docker-stack.yml.j2 index d97632a..6886e6d 100644 --- a/roles/nextcloud/templates/docker-stack.yml.j2 +++ b/roles/nextcloud/templates/docker-stack.yml.j2 @@ -7,7 +7,7 @@ services: - backend - "{{ docker_swarm_public_network_name }}" healthcheck: - test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://localhost:2019/metrics"] + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://127.0.0.1:2019/metrics"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/ntfy/templates/docker-stack.yml.j2 b/roles/ntfy/templates/docker-stack.yml.j2 index ffa1dc9..dce3b29 100644 --- a/roles/ntfy/templates/docker-stack.yml.j2 +++ b/roles/ntfy/templates/docker-stack.yml.j2 @@ -4,7 +4,7 @@ services: app: image: "{{ stack_image }}:{{ ntfy_version }}" healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost"] + test: ["CMD", "wget", "--spider", "-q", "127.0.0.1"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/searx/templates/docker-stack.yml.j2 b/roles/searx/templates/docker-stack.yml.j2 index 6f9c774..9c1cd2d 100644 --- a/roles/searx/templates/docker-stack.yml.j2 +++ b/roles/searx/templates/docker-stack.yml.j2 @@ -6,7 +6,7 @@ services: networks: - "{{ docker_swarm_public_network_name }}" healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080"] + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080"] interval: 1m timeout: 10s retries: 3 diff --git a/roles/shaarli/templates/docker-stack.yml.j2 b/roles/shaarli/templates/docker-stack.yml.j2 index dc15fa2..545d20e 100644 --- a/roles/shaarli/templates/docker-stack.yml.j2 +++ b/roles/shaarli/templates/docker-stack.yml.j2 @@ -4,7 +4,7 @@ services: app: image: "{{ stack_image }}:{{ shaarli_version }}" healthcheck: - test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://localhost:80"] + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://127.0.0.1:80"] interval: 1m timeout: 10s retries: 3 From 948ca7517acfd5c71918321910cc98f412feab00 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:05:01 +0200 Subject: [PATCH 09/14] Always update docker requirements to latest versions --- roles/docker/tasks/Ubuntu.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/docker/tasks/Ubuntu.yml b/roles/docker/tasks/Ubuntu.yml index 53ea490..92751ce 100644 --- a/roles/docker/tasks/Ubuntu.yml +++ b/roles/docker/tasks/Ubuntu.yml @@ -1,7 +1,7 @@ - name: Ensure requirements installed ansible.builtin.package: name: "{{ requisites }}" - state: present + state: latest update_cache: yes tags: - apt @@ -11,11 +11,14 @@ - name: Ensure docker GPG apt key exists apt_key: - url: https://download.docker.com/linux/ubuntu/gpg + url: "https://download.docker.com/linux/ubuntu/gpg" state: present tags: - apt - repository + # FIXME: Needs a 'until:' defined for the retries to actually work + retries: 3 + delay: 5 become: true - name: Ensure docker repository exists @@ -40,7 +43,7 @@ - name: Ensure docker requisites for python installed pip: - name: + name: - docker - jsondiff - pyyaml From ff49856107deb4ff1ffa4d66220afe582be6b69c Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:05:15 +0200 Subject: [PATCH 10/14] Pint Nextcloud to current stable version --- roles/nextcloud/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nextcloud/defaults/main.yml b/roles/nextcloud/defaults/main.yml index e97d9e0..37e73ba 100644 --- a/roles/nextcloud/defaults/main.yml +++ b/roles/nextcloud/defaults/main.yml @@ -1,7 +1,7 @@ --- # set preferred application version -nextcloud_version: 27-fpm-alpine +nextcloud_version: 28-fpm-alpine # set preferred postgres version nextcloud_db_version: 12-alpine From 7fb14b07a875ef8fb7c1708b1e209438786db519 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:06:10 +0200 Subject: [PATCH 11/14] Remove nextcloud db readiness check We instead just wait for the db to be up with the usual docker wait commands. A little more brittle but the old method ceased to work. --- roles/nextcloud/templates/docker-stack.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nextcloud/templates/docker-stack.yml.j2 b/roles/nextcloud/templates/docker-stack.yml.j2 index 6886e6d..ebf5b80 100644 --- a/roles/nextcloud/templates/docker-stack.yml.j2 +++ b/roles/nextcloud/templates/docker-stack.yml.j2 @@ -31,7 +31,7 @@ services: start_period: 5m # needed for db to be up, # see https://help.nextcloud.com/t/failed-to-install-nextcloud-with-docker-compose/83681/15 - entrypoint: sh -c "while !(nc -z db 5432); do sleep 30; done; /entrypoint.sh php-fpm" + # entrypoint: sh -c "while !(nc -z db 5432); do sleep 30; done; /entrypoint.sh php-fpm" environment: - NEXTCLOUD_ADMIN_USER={{ nextcloud_app_admin_username }} - NEXTCLOUD_ADMIN_PASSWORD={{ nextcloud_app_admin_password }} From 38b32a66e5fad34b6eb482fa43d0f6c04a443e76 Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:07:10 +0200 Subject: [PATCH 12/14] Reduce gitea healthy-await delay We waited for 60 seconds previously which is exactly when the supplied ssh key would disappear in my setup. So instead we wait for slightly shorter (55 seconds) to ease this for me. --- roles/gitea/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/gitea/tasks/main.yml b/roles/gitea/tasks/main.yml index d930ed7..e0fce1d 100644 --- a/roles/gitea/tasks/main.yml +++ b/roles/gitea/tasks/main.yml @@ -96,7 +96,7 @@ - name: Wait a minute for gitea to become healthy wait_for: - timeout: 60 + timeout: 55 delegate_to: localhost when: gitea_deployment is changed From b3d84b607533b0ebc0cccdb1eb1da97a02d7b03e Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:07:22 +0200 Subject: [PATCH 13/14] Set Nextcloud php upload limit to 2GB --- roles/nextcloud/templates/docker-stack.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/nextcloud/templates/docker-stack.yml.j2 b/roles/nextcloud/templates/docker-stack.yml.j2 index ebf5b80..df37989 100644 --- a/roles/nextcloud/templates/docker-stack.yml.j2 +++ b/roles/nextcloud/templates/docker-stack.yml.j2 @@ -41,6 +41,7 @@ services: - POSTGRES_DB={{ nextcloud_db_username }} - POSTGRES_USER={{ nextcloud_db_username }} - POSTGRES_PASSWORD={{ nextcloud_db_password }} + - PHP_UPLOAD_LIMIT=2048M {% if nextcloud_trusted_domains is not undefined and not none %} - NEXTCLOUD_TRUSTED_DOMAINS={{ nextcloud_trusted_domains }} {% endif %} From b6e30811dc1fccb55d989672fb514a602a60fded Mon Sep 17 00:00:00 2001 From: Marty Oehme Date: Thu, 11 Apr 2024 13:08:06 +0200 Subject: [PATCH 14/14] Fix shaarli version and image source Shaarli images moved a while ago and received a different tag naming scheme. So we changed to the new repository and renamed the version from latest to release. --- roles/shaarli/defaults/main.yml | 2 +- roles/shaarli/vars/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/shaarli/defaults/main.yml b/roles/shaarli/defaults/main.yml index 6d676c3..9dd0b02 100644 --- a/roles/shaarli/defaults/main.yml +++ b/roles/shaarli/defaults/main.yml @@ -1,6 +1,6 @@ --- -shaarli_version: latest +shaarli_version: release # they offer: latest and release (stable) versions shaarli_upstream_file_dir: "{{ docker_stack_files_dir }}/{{ stack_name }}" diff --git a/roles/shaarli/vars/main.yml b/roles/shaarli/vars/main.yml index c23e67a..e557c0a 100644 --- a/roles/shaarli/vars/main.yml +++ b/roles/shaarli/vars/main.yml @@ -2,6 +2,6 @@ stack_name: shaarli -stack_image: "shaarli/shaarli" +stack_image: "ghcr.io/shaarli/shaarli" stack_compose: "{{ lookup('template', 'docker-stack.yml.j2') | from_yaml }}"