backup

1---
2# Back up one /docker/<svc> directory as one snapshot (tags: <svc>, docker).
3#
4# The containers of the compose project that are RUNNING are recorded,
5# stopped for a consistent copy, and exactly those are started again in
6# `always` - also when the backup fails. A stack that was deliberately down
7# (or a directory without containers) is backed up without starting anything.
8
9- name: "{{ svc }} - Find its running containers"
10  community.docker.docker_host_info:
11    containers: true
12    containers_filters:
13      label: "com.docker.compose.project={{ svc }}"
14      status: running
15  register: _svc_info
16
17- name: "{{ svc }} - Prepare"
18  ansible.builtin.set_fact:
19    _svc_running: "{{ _svc_info.containers | map(attribute='Names') | map('first') | map('regex_replace', '^/', '') | sort }}"
20    _svc_stop: "{{ backup_stop_containers | bool and svc not in backup_hot_services }}"
21    _svc_paths: >-
22      {{ [backup_docker_root ~ '/' ~ svc] + (backup_service_extra_paths[svc] | default([])) }}
23    _svc_excludes: >-
24      {{ backup_global_excludes
25         + ((backup_service_excludes[svc] | default([])) | map('regex_replace', '^', backup_docker_root ~ '/' ~ svc ~ '/') | list) }}
26    _svc_error: ""
27    _svc_summary: {}
28    _svc_stopped: false
29
30- name: "{{ svc }} - Back up"
31  block:
32    - name: "{{ svc }} - Stop running containers ({{ _svc_running | join(', ') or 'none' }})"
33      ansible.builtin.command:
34        argv: "{{ ['docker', 'stop', '--time', '60'] + _svc_running }}"
35      when:
36        - _svc_stop | bool
37        - _svc_running | length > 0
38      register: _svc_stop_result
39
40    - name: "{{ svc }} - Remember that containers were stopped"
41      ansible.builtin.set_fact:
42        _svc_stopped: "{{ _svc_stop_result is changed }}"
43
44    - name: "{{ svc }} - restic backup"
45      ansible.builtin.command:
46        argv: >-
47          {{ [backup_restic_bin, 'backup', '--json', '--quiet',
48              '--host', inventory_hostname, '--tag', svc, '--tag', 'docker',
49              '--one-file-system', '--exclude-caches', '--exclude-if-present', '.nobackup']
50             + (_svc_excludes | map('regex_replace', '^', '--exclude=') | list)
51             + restic_backup_args
52             + _svc_paths }}
53      environment:
54        RESTIC_REPOSITORY: "{{ restic_repo }}"
55        RESTIC_PASSWORD: "{{ restic_password }}"
56      register: _svc_backup
57      changed_when: true
58
59    - name: "{{ svc }} - Read the snapshot summary"
60      ansible.builtin.set_fact:
61        _svc_summary: >-
62          {{ (_svc_backup.stdout_lines | select('search', '"message_type":"summary"') | list | last | default('{}')) | from_json }}
63
64  rescue:
65    - name: "{{ svc }} - Record the failure"
66      ansible.builtin.set_fact:
67        _svc_error: >-
68          {{ (ansible_failed_result.stderr | default(ansible_failed_result.msg | default('unknown error')))
69             | regex_replace('rest:[^ ]*', 'rest:<repo>') | truncate(300) }}
70
71  always:
72    - name: "{{ svc }} - Start the containers that were running"
73      ansible.builtin.command:
74        argv: "{{ ['docker', 'start'] + _svc_running }}"
75      when: _svc_stopped | bool
76      register: _svc_start
77      failed_when: false
78
79    - name: "{{ svc }} - Wait for restarted containers to be healthy"
80      community.docker.docker_host_info:
81        containers: true
82        containers_filters:
83          label: "com.docker.compose.project={{ svc }}"
84      register: _svc_health
85      until: >-
86        (_svc_health.containers | selectattr('Names', 'defined')
87         | selectattr('Status', 'search', '\(health: starting\)') | list | length) == 0
88      retries: "{{ [(backup_health_timeout_sec | int) // (backup_health_interval_sec | int), 1] | max }}"
89      delay: "{{ backup_health_interval_sec | int }}"
90      when:
91        - _svc_stopped | bool
92        - backup_health_check | bool
93      failed_when: false
94
95    - name: "{{ svc }} - Record the result"
96      ansible.builtin.set_fact:
97        backup_results: >-
98          {{ backup_results + [{
99               'name': svc, 'kind': 'docker',
100               'status': ('failed' if _svc_error else
101                          ('restart-failed' if (_svc_start.rc | default(0)) != 0 else 'ok')),
102               'error': _svc_error or ((_svc_start.stderr | default('')) | truncate(300)),
103               'snapshot': _svc_summary.snapshot_id | default(''),
104               'added_bytes': _svc_summary.data_added | default(0),
105               'processed_bytes': _svc_summary.total_bytes_processed | default(0),
106               'files': _svc_summary.total_files_processed | default(0),
107               'seconds': (_svc_summary.total_duration | default(0)) | round(1),
108               'stopped': _svc_running if _svc_stopped | bool else [] }] }}
109