system

1---
2# Persistent systemd journal.
3#
4# **Why this exists.** `Storage=volatile` keeps the journal in RAM, so it is
5# wiped on every boot -- which means the logs describing *why* a machine went
6# down are destroyed by the reboot that follows. Found 2026-08-30 on
7# hifi-node-dining after an unexplained outage: the machine had a
8# `/var/log/journal` directory (so it *looked* persistent) while journald was
9# configured volatile, and the entire previous boot was gone.
10#
11# The directory alone does not make it persistent -- `Storage=` decides, and
12# `auto` only uses the directory if it already exists. Setting it explicitly
13# is what makes this deterministic rather than dependent on whether something
14# happened to create the directory first.
15#
16# **Bounded, because most of these are SD cards.** An unbounded journal on a
17# Pi will happily grow to 10% of the filesystem and write continuously, which
18# is exactly the wear pattern SD cards fail on. The caps below are per-machine
19# overridable, and the retention limit means an idle node is not still holding
20# last spring's logs.
21
22- name: Create the persistent journal directory
23  ansible.builtin.file:
24    path: /var/log/journal
25    state: directory
26    owner: root
27    group: systemd-journal
28    # 2755: setgid so journald's group owns new files, which is what
29    # `systemd-tmpfiles --create systemd.conf` would set up anyway.
30    mode: '2755'
31  become: true
32
33- name: Create journald.conf.d
34  ansible.builtin.file:
35    path: /etc/systemd/journald.conf.d
36    state: directory
37    mode: '0755'
38  become: true
39
40# A drop-in rather than editing journald.conf, so a distribution upgrade
41# rewriting the shipped file cannot silently revert this -- the same reason
42# the umask task uses user.conf.d.
43#
44# **The `99-` prefix is load-bearing.** Drop-ins are merged in lexical order
45# across *all* the drop-in directories, and the last assignment of a key wins.
46# Raspberry Pi OS ships `/usr/lib/systemd/journald.conf.d/40-rpi-volatile-storage.conf`
47# containing `Storage=volatile`, so a `10-` prefixed drop-in is applied first
48# and then overridden by the vendor's. Measured on dnd-table 2026-09-11: the
49# file was present and correct, Ansible reported changed=1, and
50# `systemd-analyze cat-config` showed `Storage=persistent` followed by
51# `Storage=volatile` -- with /var/log/journal still empty. Sorting after the
52# vendor drop-in is the whole fix.
53- name: Remove the superseded 10- journal drop-in
54  ansible.builtin.file:
55    path: /etc/systemd/journald.conf.d/10-persistent.conf
56    state: absent
57  become: true
58  notify: restart journald
59
60- name: Configure a persistent, bounded journal
61  ansible.builtin.copy:
62    content: |
63      # Managed by Ansible (roles/system/tasks/journal.yml).
64      #
65      # Storage=persistent: survive reboots. A volatile journal loses exactly
66      # the logs you need after an unexplained restart.
67      [Journal]
68      Storage=persistent
69      SystemMaxUse={{ system_journal_max_use }}
70      SystemMaxFileSize={{ system_journal_max_file }}
71      MaxRetentionSec={{ system_journal_retention }}
72      # Cap the write rate rather than let a looping service chew the card.
73      RateLimitIntervalSec={{ system_journal_ratelimit_interval }}
74      RateLimitBurst={{ system_journal_ratelimit_burst }}
75    dest: /etc/systemd/journald.conf.d/99-persistent.conf
76    mode: '0644'
77  become: true
78  notify: restart journald
79
80# **Restart before flushing, or the flush is a no-op.** Handlers normally run
81# at the end of the play, which puts `journalctl --flush` *before* journald has
82# read the new config -- so it flushes a daemon that is still volatile, keeps
83# writing to /run/log/journal, and leaves the machine no better off until the
84# next reboot. Measured on dnd-table 2026-09-11: the drop-in was correct and
85# ordered correctly, the assertion below passed, and journald was still writing
86# to tmpfs because the restart had not happened yet.
87- name: Flush handlers (restart journald before flushing)
88  ansible.builtin.meta: flush_handlers
89
90# `systemctl restart systemd-journald` applies the config but does NOT migrate
91# the in-RAM journal to disk, so the boot that prompted this change is still
92# lost. Flushing explicitly is what makes persistence start now rather than at
93# the next reboot.
94- name: Flush the volatile journal to disk
95  ansible.builtin.command: journalctl --flush
96  become: true
97  changed_when: false
98
99# A crossing assertion: writing the drop-in is not the same as journald having
100# read it. The 10-/40-rpi ordering bug above wrote a correct file and changed
101# nothing, and only the *effective* merged config showed it. `cat-config`
102# concatenates every source in load order, so the LAST Storage= line is the
103# one that wins -- which is what this checks.
104- name: Read the effective journald configuration
105  ansible.builtin.shell:
106    cmd: >
107      set -o pipefail;
108      systemd-analyze cat-config systemd/journald.conf
109      | grep -E '^Storage=' | tail -1
110    executable: /bin/bash
111  become: true
112  register: system_journal_effective
113  changed_when: false
114  failed_when: false
115
116- name: Require the effective journal storage to be persistent
117  ansible.builtin.assert:
118    that:
119      - "'Storage=persistent' in system_journal_effective.stdout"
120    fail_msg: >-
121      journald's effective Storage is
122      '{{ system_journal_effective.stdout | default('(unreadable)') | trim }}',
123      not persistent. A later drop-in is overriding
124      /etc/systemd/journald.conf.d/99-persistent.conf -- check
125      /usr/lib/systemd/journald.conf.d/ for a vendor file sorting after 99-,
126      because the journal will be wiped by the next reboot.
127    success_msg: "journald storage is persistent (effective, not just configured)"
128
129# The config being right is still not the journal being on disk. `--header`
130# names the files the running daemon has open, which is the only reading that
131# distinguishes "configured persistent" from "writing to /var/log/journal".
132- name: Read where journald is actually writing
133  ansible.builtin.shell:
134    cmd: >
135      set -o pipefail;
136      journalctl --header | grep -m1 -i '^File path:'
137    executable: /bin/bash
138  become: true
139  register: system_journal_path
140  changed_when: false
141  failed_when: false
142
143- name: Require journald to be writing to persistent storage
144  ansible.builtin.assert:
145    that:
146      - "'/var/log/journal' in system_journal_path.stdout"
147    fail_msg: >-
148      journald is writing to
149      '{{ system_journal_path.stdout | default('(unreadable)') | trim }}'
150      rather than /var/log/journal. /run/log/journal is tmpfs and is lost on
151      reboot -- the config is right but the daemon has not adopted it, which
152      means the restart or the flush did not happen.
153    success_msg: "journald is writing to /var/log/journal (verified, not inferred)"
154