Fix v19 first-boot races, live lockouts, and startup delay

changes/turnkey.changelog

@@ -15,8 +15,10 @@ turnkey-core-19.0 (1) turnkey; urgency=low
      work but fully functional.
 
   * Improved fail2ban config:
-    - Increased default findtime (10 minutes) & bumped maxretry (3) to minimize
-      risk of user accidentally locking themself out.
+    - Disable fail2ban while an appliance is running from a non-persistent
+      live ISO.
+    - Allow 10 retries within 10 minutes and limit bans to 10 minutes to
+      reduce accidental lockouts on installed systems.
     - Removed redundant v18.x custom patches.
 
   * Include 'zstd' by default to support smaller initramfs that unpacks faster.

overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py

@@ -39,9 +39,10 @@ def __init__(self) -> None:
         shutil.chown("/run/mysqld", user="mysql", group="mysql")
 
         self.selfstarted = False
-        if not self._is_alive():
+        state = self._state()
+        if state != "active":
             self._start()
-            self.selfstarted = True
+            self.selfstarted = state not in ("activating", "reloading")
 
         self.connect()
 
@@ -53,15 +54,15 @@ def connect(self) -> None:
         )
         self.connected = True
 
-    def _is_alive(self) -> bool:
-        return (
-            subprocess.run(
-                # don't use systemctl path - build time uses wrapper
-                ["systemctl", "is-active", "--quiet", "mariadb"],  # noqa: S607
-                check=False,
-            ).returncode
-            == 0
+    def _state(self) -> str:
+        state = subprocess.run(
+            # don't use systemctl path - build time uses wrapper
+            ["systemctl", "is-active", "mariadb"],  # noqa: S607
+            check=False,
+            stdout=subprocess.PIPE,
+            text=True,
         )
+        return state.stdout.strip()
 
     def _start(self) -> None:
         start_mysql = subprocess.run(

overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local

@@ -8,9 +8,9 @@
 
 [DEFAULT]
 ignoreip    = 127.0.0.1/8 ::1  
-bantime     = 3600
+bantime     = 600
 findtime    = 600  # 10 minutes
-maxretry    = 3
+maxretry    = 10
 backend     = systemd
 
 [sshd]

overlays/turnkey.d/fail2ban/etc/systemd/system/fail2ban.service.d/turnkey-live.conf

@@ -0,0 +1,3 @@
+[Unit]
+ConditionKernelCommandLine=!boot=live
+ConditionKernelCommandLine=!boot=casper

overlays/turnkey.d/inithooks/usr/lib/inithooks/run

@@ -0,0 +1,150 @@
+#!/bin/bash
+# Executed by init script
+
+# load/set general global vars
+INITHOOKS_DEFAULT="${INITHOOKS_DEFAULT:-/etc/default/inithooks}"
+# - give shellcheck explict repo path for linting package
+# shellcheck source=default/inithooks
+source "$INITHOOKS_DEFAULT"
+TERM=${TERM:-linux}
+RUN_FIRSTBOOT="${RUN_FIRSTBOOT,,}"
+TKLINFO="${TKLINFO:-/var/lib/turnkey-info}"
+REDIRECT_OUTPUT="${REDIRECT_OUTPUT,,}"
+PID=
+REBOOT_REQUIRED=
+
+# load preseeds if they exist - although preseeds file should always be empty
+# unless RUN_FIRSTBOOT=true (firstboot will wipe preseeds file)
+if [[ -f $INITHOOKS_CONF ]]; then
+    # hide this shellcheck warning for now, although we probably should
+    # include an example conf file?!
+    # shellcheck source=/dev/null
+    source "$INITHOOKS_CONF"
+    export INITHOOKS_CONF="$INITHOOKS_CONF"
+fi
+
+# ensure that log file exists and has appropriate permissions
+export INITHOOKS_LOGFILE="${INITHOOKS_LOGFILE:-/var/log/inithooks.log}"
+mkdir -p "$(dirname "$INITHOOKS_LOGFILE")"
+touch "$INITHOOKS_LOGFILE"
+chmod 640 "$INITHOOKS_LOGFILE"
+
+wait_for_boot() {
+    # wait up to 10 secs for system to be running before starting; minimizes chance
+    # of journal overwriting inithook dialog/confconsole
+    logger -t inithooks "systemctl is-system-running: $(systemctl is-system-running)"
+    for count in {1..10}; do
+        if [[ "$(systemctl is-system-running)" == "starting" ]]; then
+            logger -t inithooks "Waiting for boot to finish ($count/10 seconds)"
+            sleep 1
+        fi
+    done
+}
+
+log() {
+    # log to journal as well as $INITHOOKS_LOGFILE
+    local level=$1 # err|warn|info|debug
+    shift
+    logger -t inithooks -p "${level,,}" "$@"
+    if [[ -f "$INITHOOKS_LOGFILE" ]]; then
+        echo "${level^^}: $*" >> "$INITHOOKS_LOGFILE"
+    fi
+}
+
+if [[ "$REDIRECT_OUTPUT" == "true" ]]; then
+    # on xen redirection is performed by the inithooks-xen service
+    # on lxc and other headless deployments, redirection is handled below
+    # otherwise redirection is handled by inithooks service and redirected to
+    # tty8
+
+    if [[ ! -f "$TKLINFO/xen" ]]; then
+        TTY=$(cat /sys/devices/virtual/tty/tty0/active)
+        if [[ -z $TTY ]]; then
+            TTY=console
+        fi
+        tail -f "$INITHOOKS_LOGFILE" > "/dev/$TTY" &
+        PID="$!"
+    fi
+fi
+
+exec_scripts() {
+    local script_dir=$1
+    local firstboot=$2
+    local boot_wait_complete=
+    local script_executable=
+    local script=
+    [[ -d "$script_dir" ]] || return 0
+    readarray -d '' all_scripts < <(find "$script_dir" \( -type f -or -type l \) -print0 | sort -z)
+    for script_executable in "${all_scripts[@]}"; do
+        # this is already sourced above is it needed again here?
+        if [[ -e $INITHOOKS_CONF ]]; then
+            # as per above shellcheck $INITHOOKS_CONF note
+            # shellcheck source=/dev/null
+            source "$INITHOOKS_CONF"
+        fi
+        script=$(basename "$script_executable")
+        if [[ -n "$firstboot" && -z "$boot_wait_complete" ]]; then
+            # if firstboot, then only run <30 scripts - then wait
+            prefix="${script:0:2}"
+            if [[ $prefix =~ ^[0-9]+$ ]] && (( 10#$prefix >= 30 )); then
+                wait_for_boot
+                boot_wait_complete=true
+            fi
+        fi
+        if [[ ! -x "$script_executable" ]]; then
+            log warn "[$script] skipping"
+            continue
+        fi
+        log info "[$script] running"
+        "$script_executable"
+        exit_code=$?
+        if [[ "$exit_code" -eq 0 ]]; then
+            log info "[$script] successfully completed"
+        elif [[ "$script" = "95secupdates" ]] && [[ "$exit_code" -eq 2 ]]; then
+            log info "[$script] detected live system - skipping"
+        elif [[ "$script" = "95secupdates" ]] && [[ "$exit_code" -eq 42 ]]; then
+            REBOOT_REQUIRED=true
+            log warn "[$script] reboot is required"
+        else
+            log err "[$script] failed - exit code $exit_code"
+        fi
+    done
+    return 0
+}
+
+if [[ "$RUN_FIRSTBOOT" == "true" ]]; then
+    log info "Running firstboot scripts"
+    exec_scripts "$INITHOOKS_PATH/firstboot.d" firstboot
+fi
+
+# ensure everyboot scripts only run once per boot
+if [[ ! -f /run/inithooks-complete ]]; then
+    log info "Running everyboot scripts"
+    exec_scripts "$INITHOOKS_PATH/everyboot.d"
+    touch /run/inithooks-complete
+fi
+
+if [[ -n "$PID" ]];  then
+    log info "Killing inithooks pid $PID"
+    kill -9 $PID || true
+fi
+
+log info "Inithooks run completed"
+if [[ -n "$REBOOT_REQUIRED" ]]; then
+    log err "Rebooting now to ensure all security updates are applied"
+    systemctl reboot
+    exit 0
+fi
+
+if [[ "$REDIRECT_OUTPUT" == "true" ]]; then
+    log info "Inithooks exiting."
+else
+    # ensure confconsole --usage isn't overwritten on reboots
+    wait_for_boot
+    log info "Inithooks starting Confconsole"
+    sleep 2 # anyway to replace this?
+    log info "Confconsole started, Inithooks exiting"
+    confconsole --usage
+fi
+
+exit 0

tests/test_fail2ban_policy.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+import configparser
+import pathlib
+import unittest
+
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+JAIL = ROOT / "overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local"
+LIVE_DROPIN = (
+    ROOT
+    / "overlays/turnkey.d/fail2ban/etc/systemd/system"
+    / "fail2ban.service.d/turnkey-live.conf"
+)
+
+
+class Fail2banPolicyTests(unittest.TestCase):
+    def test_installed_system_policy_allows_human_retries(self):
+        config = configparser.ConfigParser(inline_comment_prefixes=("#", ";"))
+        config.read(JAIL)
+
+        defaults = config["DEFAULT"]
+        self.assertEqual(defaults.getint("maxretry"), 10)
+        self.assertEqual(defaults.getint("findtime"), 600)
+        self.assertEqual(defaults.getint("bantime"), 600)
+
+    def test_live_boot_modes_skip_fail2ban(self):
+        conditions = {
+            line.strip()
+            for line in LIVE_DROPIN.read_text().splitlines()
+            if line.startswith("ConditionKernelCommandLine=")
+        }
+        self.assertEqual(
+            conditions,
+            {
+                "ConditionKernelCommandLine=!boot=live",
+                "ConditionKernelCommandLine=!boot=casper",
+            },
+        )
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/test_inithooks_wait_policy.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+import unittest
+
+
+RUNNER = (
+    Path(__file__).resolve().parents[1]
+    / "overlays/turnkey.d/inithooks/usr/lib/inithooks/run"
+)
+
+
+class InithooksWaitPolicyTests(unittest.TestCase):
+    def test_late_firstboot_hooks_share_one_startup_wait(self):
+        runner = RUNNER.read_text()
+
+        self.assertIn("local boot_wait_complete=", runner)
+        self.assertIn(
+            '[[ -n "$firstboot" && -z "$boot_wait_complete" ]]',
+            runner,
+        )
+        self.assertIn("boot_wait_complete=true", runner)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_mysqlconf_service_state.py

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+
+import importlib.util
+from pathlib import Path
+import subprocess
+import sys
+import types
+import unittest
+from unittest import mock
+
+
+MYSQLCONF = (
+    Path(__file__).resolve().parents[1]
+    / "overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py"
+)
+
+pymysql = types.ModuleType("pymysql")
+pymysql.connect = mock.Mock()
+pymysql.cursors = types.SimpleNamespace(DictCursor=object)
+sys.modules.setdefault("pymysql", pymysql)
+sys.modules.setdefault("pymysql.cursors", pymysql.cursors)
+
+libinithooks = types.ModuleType("libinithooks")
+dialog_wrapper = types.ModuleType("libinithooks.dialog_wrapper")
+dialog_wrapper.Dialog = mock.Mock()
+sys.modules.setdefault("libinithooks", libinithooks)
+sys.modules.setdefault("libinithooks.dialog_wrapper", dialog_wrapper)
+
+spec = importlib.util.spec_from_file_location("mysqlconf", MYSQLCONF)
+mysqlconf = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mysqlconf)
+
+
+class MySQLServiceStateTests(unittest.TestCase):
+    def make_mysql(self, state):
+        commands = []
+
+        def run(command, **kwargs):
+            commands.append(command)
+            if command[1:3] == ["is-active", "mariadb"]:
+                return subprocess.CompletedProcess(command, 0, stdout=state + "\n")
+            return subprocess.CompletedProcess(command, 0)
+
+        with mock.patch.object(mysqlconf.os, "makedirs"), \
+                mock.patch.object(mysqlconf.shutil, "chown"), \
+                mock.patch.object(mysqlconf.subprocess, "run", side_effect=run), \
+                mock.patch.object(mysqlconf.MySQL, "connect"):
+            database = mysqlconf.MySQL()
+            database._stop()
+            database.selfstarted = False
+
+        return commands
+
+    def test_already_active_service_is_left_running(self):
+        commands = self.make_mysql("active")
+        self.assertEqual(commands, [["systemctl", "is-active", "mariadb"]])
+
+    def test_already_activating_service_is_not_stopped(self):
+        commands = self.make_mysql("activating")
+        self.assertEqual(commands, [
+            ["systemctl", "is-active", "mariadb"],
+            ["systemctl", "start", "mariadb"],
+        ])
+
+    def test_inactive_service_is_stopped_after_temporary_use(self):
+        commands = self.make_mysql("inactive")
+        self.assertEqual(commands, [
+            ["systemctl", "is-active", "mariadb"],
+            ["systemctl", "start", "mariadb"],
+            ["systemctl", "stop", "mariadb"],
+        ])
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)