Linux for Defenders 08: Capstone — A Daily Health-Check Script in Bash, Kept in Git

$ mkdir ~/healthcheck && cd ~/healthcheck
$ git init -b main
$ git config user.name "Your Name"
$ git config user.email "you@example.test"

#!/bin/bash
# healthcheck.sh - daily Linux health check (StashGrid Linux for Defenders 08)
# Takes a snapshot of the things labs 01-07 taught you to watch, and compares it
# with a saved baseline. Exit 0 = nothing changed, 1 = something changed, 2 = setup problem.
set -u
STATE=/var/lib/healthcheck
BASE="$STATE/baseline.txt"
NOW="$STATE/latest.txt"

snapshot() {
  echo "## 01 who can become root"
  grep -rhvE '^\s*(#|$|Defaults)' /etc/sudoers /etc/sudoers.d/ 2>/dev/null | sort
  echo "sudo group: $(getent group sudo | cut -d: -f4)"
  echo "## 02 enabled services"
  systemctl list-unit-files --type=service --state=enabled --no-legend | awk '{print $1}' | sort
  echo "## 03 scheduled jobs"
  for u in $(cut -d: -f1 /etc/passwd); do
    crontab -l -u "$u" 2>/dev/null | grep -vE '^\s*(#|$)' | sed "s|^|crontab $u: |"
  done
  grep -H -vE '^\s*(#|$)' /etc/cron.d/* | sed 's|^|cron.d: |'
  systemctl list-unit-files --type=timer --state=enabled --no-legend | awk '{print "timer: " $1}' | sort
  echo "## 04 firewall"
  ufw status | head -1
  echo "## 05 ssh settings"
  sshd -T 2>/dev/null | grep -E '^(permitrootlogin|passwordauthentication|maxauthtries) '
  echo "## 06 programs with extra rights (SUID/SGID)"
  find / -xdev -perm /6000 -type f 2>/dev/null | sort
  echo "## 07 world-writable files"
  find / -xdev -type f -perm -0002 2>/dev/null | sort
}

info() {
  echo "security updates waiting: $(apt list --upgradable 2>/dev/null | grep -c -- '-security')"
  echo "failed SSH logins, last 24h: $(journalctl -u ssh --since -24h --no-pager 2>/dev/null | grep -cE 'Invalid user|Failed password')"
}

if [ "$(id -u)" -ne 0 ]; then echo "run me as root (sudo)"; exit 2; fi
mkdir -p "$STATE" && chmod 700 "$STATE"
snapshot > "$NOW"

if [ "${1:-}" = "--baseline" ]; then
  cp "$NOW" "$BASE"
  echo "baseline saved: $(wc -l < "$BASE") lines in $BASE"
  exit 0
fi
if [ ! -f "$BASE" ]; then echo "no baseline yet: run with --baseline first"; exit 2; fi

info
if diff -u "$BASE" "$NOW" > "$STATE/changes.diff"; then
  echo "OK: nothing changed since the baseline"
  exit 0
else
  echo "CHANGED since the baseline (+ new, - gone):"
  grep -E '^[+-][^+-]' "$STATE/changes.diff"
  exit 1
fi

$ chmod 755 healthcheck.sh
$ git add healthcheck.sh && git commit -m "First version of the daily health check"
$ git log --oneline
827202c First version of the daily health check
$ sudo install -o root -g root -m 0755 healthcheck.sh /usr/local/sbin/healthcheck
$ sudo healthcheck --baseline
baseline saved: 64 lines in /var/lib/healthcheck/baseline.txt
$ sudo head -6 /var/lib/healthcheck/baseline.txt
## 01 who can become root
%admin ALL=(ALL) ALL
%sudo	ALL=(ALL:ALL) ALL
@includedir /etc/sudoers.d
root	ALL=(ALL:ALL) ALL
sudo group: ubuntu,alex

$ sudo healthcheck; echo "exit=$?"
security updates waiting: 0
failed SSH logins, last 24h: 0
OK: nothing changed since the baseline
exit=0

$ sudo tee /etc/systemd/system/healthcheck.service >/dev/null <<'EOF'
[Unit]
Description=Daily Linux health check (StashGrid)

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/healthcheck
EOF
$ sudo tee /etc/systemd/system/healthcheck.timer >/dev/null <<'EOF'
[Unit]
Description=Run the health check every morning

[Timer]
OnCalendar=*-*-* 07:00:00
Persistent=true

[Install]
WantedBy=timers.target
EOF
$ sudo systemctl daemon-reload
$ sudo systemctl enable --now healthcheck.timer
Created symlink /etc/systemd/system/timers.target.wants/healthcheck.timer → /etc/systemd/system/healthcheck.timer.
$ systemctl list-timers healthcheck.timer --no-pager
NEXT                        LEFT LAST PASSED UNIT              ACTIVATES
Wed 2026-10-21 07:00:00 UTC  11h -         - healthcheck.timer healthcheck.service

$ echo 'sam ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/90-temp >/dev/null; sudo chmod 440 /etc/sudoers.d/90-temp
$ echo '*/10 * * * * root /bin/bash -c "id > /tmp/marker"' | sudo tee /etc/cron.d/sysmon >/dev/null
$ sudo chmod 777 /usr/local/bin/nightly-backup.sh
$ sudo systemctl start healthcheck.service
Job for healthcheck.service failed because the control process exited with error code.
$ journalctl -u healthcheck.service -t healthcheck -o cat --no-pager | tail -7
security updates waiting: 0
failed SSH logins, last 24h: 0
CHANGED since the baseline (+ new, - gone):
+sam ALL=(ALL) NOPASSWD: ALL
+cron.d: sysmon
+timer: healthcheck.timer
+/usr/local/bin/nightly-backup.sh
$ systemctl --failed --no-legend --no-pager
● healthcheck.service loaded failed failed Daily Linux health check (StashGrid)

$ sudo mv /etc/cron.d/sysmon /root/removed-sysmon.cron
$ sudo mv /etc/sudoers.d/90-temp /root/removed-90-temp.sudoers
$ sudo chmod 755 /usr/local/bin/nightly-backup.sh
$ sudo healthcheck
security updates waiting: 0
failed SSH logins, last 24h: 0
CHANGED since the baseline (+ new, - gone):
+timer: healthcheck.timer
$ sudo healthcheck --baseline
baseline saved: 65 lines in /var/lib/healthcheck/baseline.txt
$ sudo healthcheck
security updates waiting: 0
failed SSH logins, last 24h: 0
OK: nothing changed since the baseline
$ sudo systemctl reset-failed healthcheck.service

$ git diff
-  ls /etc/cron.d/ | sed 's|^|cron.d: |'
+  grep -H -vE '^\s*(#|$)' /etc/cron.d/* | sed 's|^|cron.d: |'
$ git commit -am "Watch the contents of /etc/cron.d, not just file names"
$ git log --oneline
63229c8 Watch the contents of /etc/cron.d, not just file names
827202c First version of the daily health check
$ sudo install -o root -g root -m 0755 healthcheck.sh /usr/local/sbin/healthcheck
$ sudo healthcheck --baseline
baseline saved: 66 lines in /var/lib/healthcheck/baseline.txt