backup

1---
2# Repository maintenance and reporting - once per backup run, on the host
3# that serves the repository (fast local access).
4#
5# Expects `backup_client_hosts`: the hosts that were supposed to back up.
6# A host without `backup_results` (unreachable, crashed) counts as missed.
7#
8# The run fails at the very end - after the report is published - if any
9# item failed, a host was missed, or forget/check reported a problem.
10
11- name: Install the pinned restic client
12  ansible.builtin.include_tasks: restic.yml
13
14- name: Collect the results of all hosts
15  ansible.builtin.set_fact:
16    _run_results: >-
17      {%- set out = [] -%}
18      {%- for h in backup_client_hosts -%}
19      {%-   for r in hostvars[h].backup_results | default([]) -%}
20      {%-     set _ = out.append(r | combine({'host': h})) -%}
21      {%-   endfor -%}
22      {%- endfor -%}
23      {{ out }}
24    _run_missed: >-
25      {%- set out = [] -%}
26      {%- for h in backup_client_hosts if hostvars[h].backup_results is not defined -%}
27      {%-   set _ = out.append(h) -%}
28      {%- endfor -%}
29      {{ out }}
30
31- name: Forget expired snapshots{{ ' and prune' if backup_prune else '' }}
32  ansible.builtin.command:
33    argv: >-
34      {{ [backup_restic_bin, 'forget', '--group-by', 'host,tags']
35         + backup_retention_args + (['--prune'] if backup_prune | bool else []) }}
36  environment:
37    RESTIC_REPOSITORY: "{{ restic_repo }}"
38    RESTIC_PASSWORD: "{{ restic_password }}"
39  register: _forget
40  changed_when: "'remove ' in _forget.stdout"
41  failed_when: false
42
43- name: Check repository integrity (reading {{ backup_check_read_subset }} of the data)
44  ansible.builtin.command:
45    argv: "{{ [backup_restic_bin, 'check', '--read-data-subset=' ~ backup_check_read_subset] }}"
46  environment:
47    RESTIC_REPOSITORY: "{{ restic_repo }}"
48    RESTIC_PASSWORD: "{{ restic_password }}"
49  register: _check
50  changed_when: false
51  failed_when: false
52
53- name: Copy new snapshots off-site
54  ansible.builtin.command:
55    argv: "{{ [backup_restic_bin, 'copy', '--from-repo', restic_repo] }}"
56  environment:
57    RESTIC_REPOSITORY: "{{ backup_offsite_repo }}"
58    RESTIC_PASSWORD: "{{ backup_offsite_password }}"
59    RESTIC_FROM_PASSWORD: "{{ restic_password }}"
60  register: _offsite
61  changed_when: true
62  failed_when: false
63  when: backup_offsite_repo | length > 0
64
65- name: Read all snapshots
66  ansible.builtin.command:
67    argv: ["{{ backup_restic_bin }}", snapshots, --json]
68  environment:
69    RESTIC_REPOSITORY: "{{ restic_repo }}"
70    RESTIC_PASSWORD: "{{ restic_password }}"
71  register: _snapshots
72  changed_when: false
73
74- name: Read the repository size
75  ansible.builtin.command:
76    argv: ["{{ backup_restic_bin }}", stats, --mode, raw-data, --json]
77  environment:
78    RESTIC_REPOSITORY: "{{ restic_repo }}"
79    RESTIC_PASSWORD: "{{ restic_password }}"
80  register: _stats
81  changed_when: false
82  failed_when: false
83
84# One entry per host + service. A snapshot's service is its tag that is not
85# a kind marker (docker/data/appliance); tag order is not relied on. Groups
86# whose host/service no longer exist age out through retention on their own.
87- name: Build the per-service backup health
88  ansible.builtin.set_fact:
89    _health: >-
90      {%- set kinds = ['docker', 'data', 'appliance'] -%}
91      {%- set grp = {} -%}
92      {%- for s in _snapshots.stdout | from_json -%}
93      {%-   set svc = (s.tags | default([]) | reject('in', kinds) | list + ['untagged']) | first -%}
94      {%-   set key = s.hostname ~ '/' ~ svc -%}
95      {%-   set _ = grp.update({key: grp.get(key, []) + [s]}) -%}
96      {%- endfor -%}
97      {%- set out = [] -%}
98      {%- for key, snaps in grp | dictsort -%}
99      {%-   set ordered = snaps | sort(attribute='time') -%}
100      {%-   set latest = ordered | last -%}
101      {#- restic writes local time with its offset (or Z); keep the offset -#}
102      {%-   set off = latest.time[-6:] if latest.time[-6:-5] in ['+', '-'] else '+00:00' -%}
103      {%-   set dt = (latest.time[:19] ~ off) | to_datetime('%Y-%m-%dT%H:%M:%S%z') -%}
104      {%-   set age = ((now().timestamp() - dt.timestamp()) / 86400) | round(1) -%}
105      {%-   set _ = out.append({
106              'host': latest.hostname,
107              'service': key.split('/', 1)[1],
108              'kind': (latest.tags | default([]) | select('in', kinds) | list + ['legacy']) | first,
109              'last': dt.isoformat(),
110              'age_days': age,
111              'snapshots': ordered | length,
112              'oldest': (ordered | first).time[:10],
113              'size_mb': ((latest.summary | default({})).total_bytes_processed | default(0) / 1048576) | round(1),
114              'added_mb': ((latest.summary | default({})).data_added | default(0) / 1048576) | round(1)}) -%}
115      {%- endfor -%}
116      {{ out }}
117
118- name: List the failed items
119  ansible.builtin.set_fact:
120    _run_errors: >-
121      {%- set out = [] -%}
122      {%- for r in _run_results if r.status != 'ok' -%}
123      {%-   set _ = out.append({'host': r.host, 'name': r.name, 'status': r.status, 'error': r.error}) -%}
124      {%- endfor -%}
125      {{ out }}
126
127- name: Assemble the run report
128  ansible.builtin.set_fact:
129    _report: >-
130      {{ {
131        'status': ('ok' if (_run_results | rejectattr('status', 'equalto', 'ok') | list | length == 0
132                            and _run_missed | length == 0 and _forget.rc == 0 and _check.rc == 0
133                            and (_offsite.rc | default(0)) == 0) else 'failed'),
134        'finished': now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ'),
135        'items_ok': _run_results | selectattr('status', 'equalto', 'ok') | list | length,
136        'items_failed': _run_results | rejectattr('status', 'equalto', 'ok') | map(attribute='name') | list,
137        'missed_hosts': _run_missed,
138        'added_mb': ((_run_results | map(attribute='added_bytes') | sum) / 1048576) | round(1),
139        'forget': 'ok' if _forget.rc == 0 else 'failed (rc ' ~ _forget.rc ~ ')',
140        'check': 'ok' if _check.rc == 0 else 'failed (rc ' ~ _check.rc ~ ')',
141        'offsite': ('not configured' if backup_offsite_repo | length == 0 else
142                    ('ok' if (_offsite.rc | default(1)) == 0 else 'failed')),
143        'repo_size_gb': (((_stats.stdout | default('{}') | from_json).total_size | default(0)) / 1073741824) | round(2),
144        'snapshot_count': _snapshots.stdout | from_json | length,
145        'services': _health,
146        'errors': _run_errors
147      } }}
148
149- name: Publish the report to Home Assistant
150  ansible.builtin.include_tasks: report.yml
151  when:
152    - backup_report_enabled | bool
153    - backup_report_ha_url | length > 0
154    - backup_report_ha_token | length > 0
155
156- name: Show the run report
157  ansible.builtin.debug:
158    msg: |
159      Backup run: {{ _report.status | upper }}
160      Items ok: {{ _report.items_ok }}, failed: {{ _report.items_failed | join(', ') or 'none' }}
161      Missed hosts: {{ _report.missed_hosts | join(', ') or 'none' }}
162      Added this run: {{ _report.added_mb }} MB | repository: {{ _report.repo_size_gb }} GB, {{ _report.snapshot_count }} snapshots
163      forget: {{ _report.forget }} | check: {{ _report.check }} | off-site: {{ _report.offsite }}
164      {% for e in _report.errors %}
165      - {{ e.host }}/{{ e.name }}: {{ e.error }}
166      {% endfor %}
167      {% if _check.rc != 0 %}
168      check output: {{ (_check.stderr | default('') ~ _check.stdout | default('')) | truncate(800) }}
169      {% endif %}
170
171- name: Fail the run if anything went wrong
172  ansible.builtin.fail:
173    msg: "Backup run failed - see the report above"
174  when: _report.status != 'ok'
175