From 0a517f895b39da102de94e28516840927c00da62 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 2 Sep 2026 14:38:45 +0000 Subject: [PATCH 1/3] Preserve MariaDB during concurrent boot activation The v19 MySQL helper treated systemd activating state as an inactive service. A first-boot hook racing MariaDB startup would therefore mark the service as self-started and stop it when the helper exited, causing later application hooks such as WordPress initialization to fail. Record the unit state before waiting for startup and only stop MariaDB when the helper actually initiated an inactive service. Build-time temporary startup behavior remains unchanged. Verified with focused active, activating, and inactive state tests, plus a live AWS boot-race reproduction that left MariaDB active and allowed the WordPress hook to complete. --- .../mysql/usr/lib/inithooks/bin/mysqlconf.py | 21 +++--- tests/test_mysqlconf_service_state.py | 75 +++++++++++++++++++ 2 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 tests/test_mysqlconf_service_state.py diff --git a/overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py b/overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py index 65292c7a..a2f4af02 100755 --- a/overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py +++ b/overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py @@ -39,9 +39,10 @@ def __init__(self) -> None: shutil.chown("/run/mysqld", user="mysql", group="mysql") self.selfstarted = False - if not self._is_alive(): + state = self._state() + if state != "active": self._start() - self.selfstarted = True + self.selfstarted = state not in ("activating", "reloading") self.connect() @@ -53,15 +54,15 @@ def connect(self) -> None: ) self.connected = True - def _is_alive(self) -> bool: - return ( - subprocess.run( - # don't use systemctl path - build time uses wrapper - ["systemctl", "is-active", "--quiet", "mariadb"], # noqa: S607 - check=False, - ).returncode - == 0 + def _state(self) -> str: + state = subprocess.run( + # don't use systemctl path - build time uses wrapper + ["systemctl", "is-active", "mariadb"], # noqa: S607 + check=False, + stdout=subprocess.PIPE, + text=True, ) + return state.stdout.strip() def _start(self) -> None: start_mysql = subprocess.run( diff --git a/tests/test_mysqlconf_service_state.py b/tests/test_mysqlconf_service_state.py new file mode 100644 index 00000000..366f32b1 --- /dev/null +++ b/tests/test_mysqlconf_service_state.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +import importlib.util +from pathlib import Path +import subprocess +import sys +import types +import unittest +from unittest import mock + + +MYSQLCONF = ( + Path(__file__).resolve().parents[1] + / "overlays/mysql/usr/lib/inithooks/bin/mysqlconf.py" +) + +pymysql = types.ModuleType("pymysql") +pymysql.connect = mock.Mock() +pymysql.cursors = types.SimpleNamespace(DictCursor=object) +sys.modules.setdefault("pymysql", pymysql) +sys.modules.setdefault("pymysql.cursors", pymysql.cursors) + +libinithooks = types.ModuleType("libinithooks") +dialog_wrapper = types.ModuleType("libinithooks.dialog_wrapper") +dialog_wrapper.Dialog = mock.Mock() +sys.modules.setdefault("libinithooks", libinithooks) +sys.modules.setdefault("libinithooks.dialog_wrapper", dialog_wrapper) + +spec = importlib.util.spec_from_file_location("mysqlconf", MYSQLCONF) +mysqlconf = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mysqlconf) + + +class MySQLServiceStateTests(unittest.TestCase): + def make_mysql(self, state): + commands = [] + + def run(command, **kwargs): + commands.append(command) + if command[1:3] == ["is-active", "mariadb"]: + return subprocess.CompletedProcess(command, 0, stdout=state + "\n") + return subprocess.CompletedProcess(command, 0) + + with mock.patch.object(mysqlconf.os, "makedirs"), \ + mock.patch.object(mysqlconf.shutil, "chown"), \ + mock.patch.object(mysqlconf.subprocess, "run", side_effect=run), \ + mock.patch.object(mysqlconf.MySQL, "connect"): + database = mysqlconf.MySQL() + database._stop() + database.selfstarted = False + + return commands + + def test_already_active_service_is_left_running(self): + commands = self.make_mysql("active") + self.assertEqual(commands, [["systemctl", "is-active", "mariadb"]]) + + def test_already_activating_service_is_not_stopped(self): + commands = self.make_mysql("activating") + self.assertEqual(commands, [ + ["systemctl", "is-active", "mariadb"], + ["systemctl", "start", "mariadb"], + ]) + + def test_inactive_service_is_stopped_after_temporary_use(self): + commands = self.make_mysql("inactive") + self.assertEqual(commands, [ + ["systemctl", "is-active", "mariadb"], + ["systemctl", "start", "mariadb"], + ["systemctl", "stop", "mariadb"], + ]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From a7d17ea0ac8f928814f3f3a77772d82cefb1fbf7 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 2 Sep 2026 15:27:30 +0000 Subject: [PATCH 2/3] Relax fail2ban outside persistent deployments Skip fail2ban when the kernel command line identifies a live ISO session, where a ban cannot provide durable protection and can lock an evaluator out of the appliance. Installed systems continue to run fail2ban, but now allow ten retries within ten minutes and limit a ban to ten minutes. Add focused policy tests for both the live-boot conditions and the installed system thresholds. The fail2ban and MariaDB service-state tests pass together. --- changes/turnkey.changelog | 6 ++- .../fail2ban/etc/fail2ban/jail.local | 4 +- .../fail2ban.service.d/turnkey-live.conf | 3 ++ tests/test_fail2ban_policy.py | 43 +++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 overlays/turnkey.d/fail2ban/etc/systemd/system/fail2ban.service.d/turnkey-live.conf create mode 100644 tests/test_fail2ban_policy.py diff --git a/changes/turnkey.changelog b/changes/turnkey.changelog index 20fcef6f..cd3b8877 100644 --- a/changes/turnkey.changelog +++ b/changes/turnkey.changelog @@ -15,8 +15,10 @@ turnkey-core-19.0 (1) turnkey; urgency=low work but fully functional. * Improved fail2ban config: - - Increased default findtime (10 minutes) & bumped maxretry (3) to minimize - risk of user accidentally locking themself out. + - Disable fail2ban while an appliance is running from a non-persistent + live ISO. + - Allow 10 retries within 10 minutes and limit bans to 10 minutes to + reduce accidental lockouts on installed systems. - Removed redundant v18.x custom patches. * Include 'zstd' by default to support smaller initramfs that unpacks faster. diff --git a/overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local b/overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local index ccd1b765..803b8f18 100644 --- a/overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local +++ b/overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local @@ -8,9 +8,9 @@ [DEFAULT] ignoreip = 127.0.0.1/8 ::1 -bantime = 3600 +bantime = 600 findtime = 600 # 10 minutes -maxretry = 3 +maxretry = 10 backend = systemd [sshd] diff --git a/overlays/turnkey.d/fail2ban/etc/systemd/system/fail2ban.service.d/turnkey-live.conf b/overlays/turnkey.d/fail2ban/etc/systemd/system/fail2ban.service.d/turnkey-live.conf new file mode 100644 index 00000000..fa36eb9e --- /dev/null +++ b/overlays/turnkey.d/fail2ban/etc/systemd/system/fail2ban.service.d/turnkey-live.conf @@ -0,0 +1,3 @@ +[Unit] +ConditionKernelCommandLine=!boot=live +ConditionKernelCommandLine=!boot=casper diff --git a/tests/test_fail2ban_policy.py b/tests/test_fail2ban_policy.py new file mode 100644 index 00000000..cedb272d --- /dev/null +++ b/tests/test_fail2ban_policy.py @@ -0,0 +1,43 @@ +#!/usr/bin/python3 + +import configparser +import pathlib +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +JAIL = ROOT / "overlays/turnkey.d/fail2ban/etc/fail2ban/jail.local" +LIVE_DROPIN = ( + ROOT + / "overlays/turnkey.d/fail2ban/etc/systemd/system" + / "fail2ban.service.d/turnkey-live.conf" +) + + +class Fail2banPolicyTests(unittest.TestCase): + def test_installed_system_policy_allows_human_retries(self): + config = configparser.ConfigParser(inline_comment_prefixes=("#", ";")) + config.read(JAIL) + + defaults = config["DEFAULT"] + self.assertEqual(defaults.getint("maxretry"), 10) + self.assertEqual(defaults.getint("findtime"), 600) + self.assertEqual(defaults.getint("bantime"), 600) + + def test_live_boot_modes_skip_fail2ban(self): + conditions = { + line.strip() + for line in LIVE_DROPIN.read_text().splitlines() + if line.startswith("ConditionKernelCommandLine=") + } + self.assertEqual( + conditions, + { + "ConditionKernelCommandLine=!boot=live", + "ConditionKernelCommandLine=!boot=casper", + }, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 6d1cc931a12e1dabb9440571553b2ca9e7a9d686 Mon Sep 17 00:00:00 2001 From: Liraz Siri Date: Wed, 2 Sep 2026 16:36:41 +0000 Subject: [PATCH 3/3] Avoid repeated first-boot startup waits Inithooks 2.3.6 waits up to ten seconds before every firstboot hook numbered 30 or later. Because the inithooks service keeps systemd in the starting state, a Hub-launched WordPress appliance spent 150 seconds in repeated waits while its security update hook needed only three seconds. Overlay the upstream wait-once correction into v19 appliance roots until the fixed inithooks package reaches the archive. This applies the acceleration to ISO boots and to cloud images built from those ISOs. --- .../turnkey.d/inithooks/usr/lib/inithooks/run | 150 ++++++++++++++++++ tests/test_inithooks_wait_policy.py | 24 +++ 2 files changed, 174 insertions(+) create mode 100755 overlays/turnkey.d/inithooks/usr/lib/inithooks/run create mode 100644 tests/test_inithooks_wait_policy.py diff --git a/overlays/turnkey.d/inithooks/usr/lib/inithooks/run b/overlays/turnkey.d/inithooks/usr/lib/inithooks/run new file mode 100755 index 00000000..a43d2eef --- /dev/null +++ b/overlays/turnkey.d/inithooks/usr/lib/inithooks/run @@ -0,0 +1,150 @@ +#!/bin/bash +# Executed by init script + +# load/set general global vars +INITHOOKS_DEFAULT="${INITHOOKS_DEFAULT:-/etc/default/inithooks}" +# - give shellcheck explict repo path for linting package +# shellcheck source=default/inithooks +source "$INITHOOKS_DEFAULT" +TERM=${TERM:-linux} +RUN_FIRSTBOOT="${RUN_FIRSTBOOT,,}" +TKLINFO="${TKLINFO:-/var/lib/turnkey-info}" +REDIRECT_OUTPUT="${REDIRECT_OUTPUT,,}" +PID= +REBOOT_REQUIRED= + +# load preseeds if they exist - although preseeds file should always be empty +# unless RUN_FIRSTBOOT=true (firstboot will wipe preseeds file) +if [[ -f $INITHOOKS_CONF ]]; then + # hide this shellcheck warning for now, although we probably should + # include an example conf file?! + # shellcheck source=/dev/null + source "$INITHOOKS_CONF" + export INITHOOKS_CONF="$INITHOOKS_CONF" +fi + +# ensure that log file exists and has appropriate permissions +export INITHOOKS_LOGFILE="${INITHOOKS_LOGFILE:-/var/log/inithooks.log}" +mkdir -p "$(dirname "$INITHOOKS_LOGFILE")" +touch "$INITHOOKS_LOGFILE" +chmod 640 "$INITHOOKS_LOGFILE" + +wait_for_boot() { + # wait up to 10 secs for system to be running before starting; minimizes chance + # of journal overwriting inithook dialog/confconsole + logger -t inithooks "systemctl is-system-running: $(systemctl is-system-running)" + for count in {1..10}; do + if [[ "$(systemctl is-system-running)" == "starting" ]]; then + logger -t inithooks "Waiting for boot to finish ($count/10 seconds)" + sleep 1 + fi + done +} + +log() { + # log to journal as well as $INITHOOKS_LOGFILE + local level=$1 # err|warn|info|debug + shift + logger -t inithooks -p "${level,,}" "$@" + if [[ -f "$INITHOOKS_LOGFILE" ]]; then + echo "${level^^}: $*" >> "$INITHOOKS_LOGFILE" + fi +} + +if [[ "$REDIRECT_OUTPUT" == "true" ]]; then + # on xen redirection is performed by the inithooks-xen service + # on lxc and other headless deployments, redirection is handled below + # otherwise redirection is handled by inithooks service and redirected to + # tty8 + + if [[ ! -f "$TKLINFO/xen" ]]; then + TTY=$(cat /sys/devices/virtual/tty/tty0/active) + if [[ -z $TTY ]]; then + TTY=console + fi + tail -f "$INITHOOKS_LOGFILE" > "/dev/$TTY" & + PID="$!" + fi +fi + +exec_scripts() { + local script_dir=$1 + local firstboot=$2 + local boot_wait_complete= + local script_executable= + local script= + [[ -d "$script_dir" ]] || return 0 + readarray -d '' all_scripts < <(find "$script_dir" \( -type f -or -type l \) -print0 | sort -z) + for script_executable in "${all_scripts[@]}"; do + # this is already sourced above is it needed again here? + if [[ -e $INITHOOKS_CONF ]]; then + # as per above shellcheck $INITHOOKS_CONF note + # shellcheck source=/dev/null + source "$INITHOOKS_CONF" + fi + script=$(basename "$script_executable") + if [[ -n "$firstboot" && -z "$boot_wait_complete" ]]; then + # if firstboot, then only run <30 scripts - then wait + prefix="${script:0:2}" + if [[ $prefix =~ ^[0-9]+$ ]] && (( 10#$prefix >= 30 )); then + wait_for_boot + boot_wait_complete=true + fi + fi + if [[ ! -x "$script_executable" ]]; then + log warn "[$script] skipping" + continue + fi + log info "[$script] running" + "$script_executable" + exit_code=$? + if [[ "$exit_code" -eq 0 ]]; then + log info "[$script] successfully completed" + elif [[ "$script" = "95secupdates" ]] && [[ "$exit_code" -eq 2 ]]; then + log info "[$script] detected live system - skipping" + elif [[ "$script" = "95secupdates" ]] && [[ "$exit_code" -eq 42 ]]; then + REBOOT_REQUIRED=true + log warn "[$script] reboot is required" + else + log err "[$script] failed - exit code $exit_code" + fi + done + return 0 +} + +if [[ "$RUN_FIRSTBOOT" == "true" ]]; then + log info "Running firstboot scripts" + exec_scripts "$INITHOOKS_PATH/firstboot.d" firstboot +fi + +# ensure everyboot scripts only run once per boot +if [[ ! -f /run/inithooks-complete ]]; then + log info "Running everyboot scripts" + exec_scripts "$INITHOOKS_PATH/everyboot.d" + touch /run/inithooks-complete +fi + +if [[ -n "$PID" ]]; then + log info "Killing inithooks pid $PID" + kill -9 $PID || true +fi + +log info "Inithooks run completed" +if [[ -n "$REBOOT_REQUIRED" ]]; then + log err "Rebooting now to ensure all security updates are applied" + systemctl reboot + exit 0 +fi + +if [[ "$REDIRECT_OUTPUT" == "true" ]]; then + log info "Inithooks exiting." +else + # ensure confconsole --usage isn't overwritten on reboots + wait_for_boot + log info "Inithooks starting Confconsole" + sleep 2 # anyway to replace this? + log info "Confconsole started, Inithooks exiting" + confconsole --usage +fi + +exit 0 diff --git a/tests/test_inithooks_wait_policy.py b/tests/test_inithooks_wait_policy.py new file mode 100644 index 00000000..29c7d4db --- /dev/null +++ b/tests/test_inithooks_wait_policy.py @@ -0,0 +1,24 @@ +from pathlib import Path +import unittest + + +RUNNER = ( + Path(__file__).resolve().parents[1] + / "overlays/turnkey.d/inithooks/usr/lib/inithooks/run" +) + + +class InithooksWaitPolicyTests(unittest.TestCase): + def test_late_firstboot_hooks_share_one_startup_wait(self): + runner = RUNNER.read_text() + + self.assertIn("local boot_wait_complete=", runner) + self.assertIn( + '[[ -n "$firstboot" && -z "$boot_wait_complete" ]]', + runner, + ) + self.assertIn("boot_wait_complete=true", runner) + + +if __name__ == "__main__": + unittest.main()