connectivity

1#!/bin/bash
2# ==============================================================================
3# Docker Restart All Script
4# ==============================================================================
5#
6# Description: Restarts all connectivity Docker services in proper order
7# Usage: ./docker-restart-all.sh [graceful|force|quiet|validate]
8#
9# This script is automatically generated by Ansible - DO NOT EDIT MANUALLY
10# Template: docker-restart-all.sh.j2
11#
12# ==============================================================================
13
14set -euo pipefail
15
16# Configuration
17DOCKER_COMPOSE_DIR="{{ docker_base_path }}"
18LOG_FILE="/var/log/docker-restart-all.log"
19STARTUP_DELAY={{ service_startup_delay | default(10) }}
20SHUTDOWN_TIMEOUT=30
21
22# Service order (dependency order for startup, reverse for shutdown)
23SERVICES=(
24    "{{ unbound_service_name }}"
25    "{{ pihole_service_name }}"
26    "{{ nginx_proxy_db_service_name }}"
27    "{{ nginx_proxy_service_name }}"
28    "{{ wireguard_service_name }}"
29)
30
31# Logging function
32log() {
33    if [[ "${1:-}" != "quiet" ]]; then
34        echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
35    else
36        echo "$(date '+%Y-%m-%d %H:%M:%S') - $2" >> "${LOG_FILE}"
37    fi
38}
39
40# Error handling function
41error_exit() {
42    log "ERROR: $1"
43    exit 1
44}
45
46# Check if container exists
47container_exists() {
48    docker inspect "$1" >/dev/null 2>&1
49}
50
51# Check if container is running
52container_running() {
53    docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null | grep -q "true"
54}
55
56# Wait for container to become healthy
57wait_for_healthy() {
58    local container="$1"
59    local timeout="${2:-60}"
60    local start_time=$(date +%s)
61    
62    log "Waiting for ${container} to become healthy (timeout: ${timeout}s)"
63    
64    while [[ $(($(date +%s) - start_time)) -lt timeout ]]; do
65        if docker inspect -f '{{.State.Health.Status}}' "${container}" 2>/dev/null | grep -q "healthy"; then
66            log "${container} is healthy"
67            return 0
68        fi
69        sleep 5
70    done
71    
72    log "WARNING: ${container} did not become healthy within ${timeout} seconds"
73    return 1
74}
75
76# Stop a specific service
77stop_service() {
78    local service="$1"
79    local timeout="${2:-${SHUTDOWN_TIMEOUT}}"
80    local force="${3:-false}"
81    
82    local compose_file="${DOCKER_COMPOSE_DIR}/${service}/docker-compose.yml"
83    
84    if ! container_exists "${service}"; then
85        log "${service} does not exist, skipping stop"
86        return 0
87    fi
88    
89    if ! container_running "${service}"; then
90        log "${service} is not running, skipping stop"
91        return 0
92    fi
93    
94    if [[ "${force}" == "true" ]]; then
95        log "Force stopping ${service}"
96        docker rm -f "${service}" 2>/dev/null || true
97    else
98        log "Stopping ${service} (timeout: ${timeout}s)"
99        
100        if [[ -f "${compose_file}" ]]; then
101            docker-compose -f "${compose_file}" down --timeout ${timeout} 2>/dev/null || true
102        else
103            docker stop --time ${timeout} "${service}" 2>/dev/null || true
104        fi
105    fi
106    
107    # Verify service is stopped
108    if container_running "${service}"; then
109        log "WARNING: ${service} is still running after stop attempt"
110        return 1
111    else
112        log "${service} stopped successfully"
113        return 0
114    fi
115}
116
117# Start a specific service
118start_service() {
119    local service="$1"
120    local force="${2:-false}"
121    
122    local compose_file="${DOCKER_COMPOSE_DIR}/${service}/docker-compose.yml"
123    
124    if [[ ! -f "${compose_file}" ]]; then
125        log "WARNING: Compose file not found for ${service}, skipping"
126        return 1
127    fi
128    
129    if container_running "${service}"; then
130        if [[ "${force}" == "true" ]]; then
131            log "Restarting ${service} (forced)"
132            docker-compose -f "${compose_file}" down --timeout 30 2>/dev/null || true
133            sleep 2
134            docker-compose -f "${compose_file}" up -d 2>/dev/null || true
135        else
136            log "${service} is already running, skipping start"
137            return 0
138        fi
139    else
140        log "Starting ${service}"
141        docker-compose -f "${compose_file}" up -d 2>/dev/null || true
142    fi
143    
144    # Wait for service to become healthy
145    wait_for_healthy "${service}" 60 || true
146    
147    return 0
148}
149
150# Graceful restart (stop all, then start all)
151graceful_restart() {
152    local force="${1:-false}"
153    local quiet="${2:-false}"
154    
155    log "Starting graceful restart of all services"
156    
157    # Stop all services in reverse order
158    local stopped_count=0
159    local failed_stop_count=0
160    
161    for ((i=${#SERVICES[@]}-1; i>=0; i--)); do
162        local service="${SERVICES[i]}"
163        if stop_service "${service}" "${SHUTDOWN_TIMEOUT}" "${force}" "${quiet}"; then
164            stopped_count=$((stopped_count + 1))
165        else
166            failed_stop_count=$((failed_stop_count + 1))
167        fi
168    done
169    
170    log "Shutdown completed: ${stopped_count} stopped, ${failed_stop_count} failed"
171    
172    # Start all services in proper order
173    local started_count=0
174    local failed_start_count=0
175    
176    for service in "${SERVICES[@]}"; do
177        if start_service "${service}" "${force}" "${quiet}"; then
178            started_count=$((started_count + 1))
179        else
180            failed_start_count=$((failed_start_count + 1))
181        fi
182        
183        # Add delay between service starts
184        if [[ ${STARTUP_DELAY} -gt 0 ]]; then
185            sleep ${STARTUP_DELAY}
186        fi
187    done
188    
189    # Summary
190    log "Restart completed: ${started_count} services started, ${failed_start_count} failed to start"
191    
192    if [[ ${failed_start_count} -gt 0 ]]; then
193        return 1
194    fi
195    
196    return 0
197}
198
199# Rolling restart (restart one service at a time)
200rolling_restart() {
201    local force="${1:-false}"
202    local quiet="${2:-false}"
203    
204    log "Starting rolling restart of services"
205    
206    local restarted_count=0
207    local failed_count=0
208    
209    for service in "${SERVICES[@]}"; do
210        # Stop the service
211        if stop_service "${service}" "${SHUTDOWN_TIMEOUT}" "${force}" "${quiet}"; then
212            log "Successfully stopped ${service}"
213        else
214            log "WARNING: Failed to stop ${service}"
215            failed_count=$((failed_count + 1))
216            continue
217        fi
218        
219        # Start the service
220        if start_service "${service}" "${force}" "${quiet}"; then
221            log "Successfully started ${service}"
222            restarted_count=$((restarted_count + 1))
223        else
224            log "ERROR: Failed to start ${service}"
225            failed_count=$((failed_count + 1))
226        fi
227        
228        # Add delay between service restarts
229        if [[ ${STARTUP_DELAY} -gt 0 ]]; then
230            sleep ${STARTUP_DELAY}
231        fi
232    done
233    
234    # Summary
235    log "Rolling restart completed: ${restarted_count} services restarted, ${failed_count} failed"
236    
237    if [[ ${failed_count} -gt 0 ]]; then
238        return 1
239    fi
240    
241    return 0
242}
243
244# Show service status
245show_status() {
246    log "Current service status:"
247    
248    for service in "${SERVICES[@]}"; do
249        if container_exists "${service}"; then
250            local status=$(docker inspect -f '{{.State.Status}}' "${service}" 2>/dev/null || echo "unknown")
251            local health=$(docker inspect -f '{{.State.Health.Status}}' "${service}" 2>/dev/null || echo "no healthcheck")
252            local restarts=$(docker inspect -f '{{.RestartCount}}' "${service}" 2>/dev/null || echo "0")
253            
254            echo "${service}: status=${status}, health=${health}, restarts=${restarts}"
255        else
256            echo "${service}: does not exist"
257        fi
258    done
259}
260
261# Validate service health
262validate_health() {
263    log "Validating service health"
264    
265    local healthy_count=0
266    local unhealthy_count=0
267    
268    for service in "${SERVICES[@]}"; do
269        if container_running "${service}"; then
270            local health=$(docker inspect -f '{{.State.Health.Status}}' "${service}" 2>/dev/null || echo "no healthcheck")
271            
272            if [[ "${health}" == "healthy" ]]; then
273                healthy_count=$((healthy_count + 1))
274                log "${service}: HEALTHY"
275            else
276                unhealthy_count=$((unhealthy_count + 1))
277                log "WARNING: ${service}: ${health}"
278            fi
279        else
280            log "WARNING: ${service}: NOT RUNNING"
281            unhealthy_count=$((unhealthy_count + 1))
282        fi
283    done
284    
285    log "Health validation: ${healthy_count} healthy, ${unhealthy_count} unhealthy/stopped"
286    
287    if [[ ${unhealthy_count} -eq 0 ]]; then
288        return 0
289    else
290        return 1
291    fi
292}
293
294# Show usage
295usage() {
296    cat << EOF
297Docker Restart All Script
298
299Usage: $0 [option]
300
301Options:
302  graceful    Graceful restart (stop all, then start all)
303  rolling     Rolling restart (one service at a time)
304  force       Force restart (remove and recreate containers)
305  quiet       Quiet mode (minimal output)
306  status      Show current service status
307  validate    Validate service health
308  help        Show this help message
309
310Service Order:
311  Startup:  1. {{ unbound_service_name }} → 2. {{ pihole_service_name }} → 3. {{ nginx_proxy_db_service_name }} → 4. {{ nginx_proxy_service_name }} → 5. {{ wireguard_service_name }}
312  Shutdown: 1. {{ wireguard_service_name }} → 2. {{ nginx_proxy_service_name }} → 3. {{ nginx_proxy_db_service_name }} → 4. {{ pihole_service_name }} → 5. {{ unbound_service_name }}
313
314Startup delay: ${STARTUP_DELAY} seconds between services
315EOF
316}
317
318# Main execution
319main() {
320    local option="${1:-graceful}"
321    
322    case "${option}" in
323        graceful)
324            graceful_restart "false" "false"
325            validate_health
326            ;;
327        rolling)
328            rolling_restart "false" "false"
329            validate_health
330            ;;
331        force)
332            graceful_restart "true" "false"
333            validate_health
334            ;;
335        quiet)
336            graceful_restart "false" "quiet"
337            ;;
338        status)
339            show_status
340            ;;
341        validate)
342            validate_health
343            ;;
344        help|*)
345            usage
346            ;;
347    esac
348}
349
350# Run main function with all arguments
351main "$@"