diff --git a/agents-api.php b/agents-api.php index 4112eb1..e7fb2fd 100644 --- a/agents-api.php +++ b/agents-api.php @@ -347,6 +347,7 @@ require_once AGENTS_API_PATH . 'src/Workflows/register-action-scheduler-listener.php'; require_once AGENTS_API_PATH . 'src/Routines/class-wp-agent-routine.php'; require_once AGENTS_API_PATH . 'src/Routines/interface-wp-agent-routine-backend.php'; +require_once AGENTS_API_PATH . 'src/Routines/class-wp-agent-routine-option-lock.php'; require_once AGENTS_API_PATH . 'src/Routines/class-wp-agent-routine-registry.php'; require_once AGENTS_API_PATH . 'src/Routines/class-wp-agent-routine-action-scheduler-bridge.php'; require_once AGENTS_API_PATH . 'src/Routines/register-routines.php'; diff --git a/docs/channels-workflows-operations.md b/docs/channels-workflows-operations.md index b1ceb82..5cc95f9 100644 --- a/docs/channels-workflows-operations.md +++ b/docs/channels-workflows-operations.md @@ -424,11 +424,11 @@ Failure reporting is uniform: a missing ability (`ability_missing`), a denied pe Durable scheduling sits behind a contract: `WP_Agent_Routine_Backend` (in `src/Routines/`) is the only scheduling surface the registry consumes — availability, register/repair/unregister, pause/resume, run-now, pause-state and generation reads, and `pending_by_routine()` / `cancel()` for reconcile (handles are opaque ints). `register()` is the cheap declaration entry point; `repair()` is the authoritative check-and-repair that never trusts recorded state. The registry resolves one backend per request through the `wp_agent_routine_backend` filter; the default is the Action Scheduler bridge (`WP_Agent_Routine_Action_Scheduler_Bridge`) when Action Scheduler is present, and `null` otherwise. With no backend, routines are still registered and their lifecycle hooks still fire, but nothing is scheduled and `reconcile()` reports a `_scheduler` error. Consumers replace the backend by filtering in their own `WP_Agent_Routine_Backend` implementation. -`register()` is idempotent **and cheap to call on every boot, even for hundreds of persisted routines — and the cheap path touches Action Scheduler not at all**. The bridge keeps one persisted record of what it last scheduled: a single non-autoloaded option (`agents_routine_schedules`) mapping routine id → `['signature' => string, 'action_id' => int]`, where the signature is exactly what the matcher compares — trigger type plus interval seconds or cron expression. When a declaration's signature matches its record, `register()` returns before issuing a single `as_get_scheduled_actions()` query or `fetch_action()` call; reading the record costs one `get_option()` per request, independent of routine count. Only a missing or changed signature reaches the store, serialized through a short single-flight per-routine lock (`agents_routine_schedule_lock_`, `add_option()`-based, 30-second stale takeover) so two concurrent processes can never both check-then-schedule — the race that used to mint duplicate chains; a losing caller no-ops. A consumer that must re-declare all of its routines on every request (the in-memory registry does not persist across requests) no longer pays any per-routine scheduler I/O once every record matches. +`register()` is idempotent **and cheap to call on every boot, even for hundreds of persisted routines — and the cheap path touches Action Scheduler not at all**. The bridge keeps one persisted record of what it last scheduled: a single non-autoloaded option (`agents_routine_schedules`) mapping routine id → `['signature' => string, 'action_id' => int]`, where the signature is exactly what the matcher compares — trigger type plus interval seconds or cron expression. When a declaration's signature matches its record, `register()` returns before issuing a single `as_get_scheduled_actions()` query or `fetch_action()` call; reading the record costs one `get_option()` per request, independent of routine count. Only a missing or changed signature reaches the store, serialized through a short single-flight per-routine lock (`agents_routine_schedule_lock_`, the same shared atomic `$wpdb` compare-and-set as the reconcile lock — see the shared lock helper below — with a 30-second stale takeover) so two concurrent processes can never both check-then-schedule — the race that used to mint duplicate chains; a losing caller no-ops. A consumer that must re-declare all of its routines on every request (the in-memory registry does not persist across requests) no longer pays any per-routine scheduler I/O once every record matches. Whenever a `register()` or reconcile call does reach Action Scheduler, it enforces **one chain per routine**: among the routine's pending recurring actions whose recurrence matches, exactly the lowest-id chain that is live (at or above the routine's current watermark) is kept and every other pending recurring action for that routine is cancelled by id — healing duplicate chains left by earlier versions or crashed requests. An action already claimed by an in-flight worker is not part of the pending set (Action Scheduler marks claimed actions in-progress), so a collapse never breaks a running worker. When no live matching chain exists (missing schedule, changed interval/expression, or a chain that is entirely below the current watermark), a fresh chain is scheduled, the generation/watermark fence is advanced, and the record is rewritten. -The trade-off is deliberate: plain declarations trust the record, so an out-of-band deletion (pruned store, restored backup, manually deleted row) is invisible to them; `WP_Agent_Routine_Registry::reconcile()` — via the backend contract's `repair( $routine )` method, which never trusts recorded state — is what repairs that drift. +The trade-off is deliberate: plain declarations trust the record, so an out-of-band deletion (pruned store, restored backup, manually deleted row) is invisible to them. A lost chain is restored within one hour by the WP-Cron routine repair (below), and on demand by `WP_Agent_Routine_Registry::reconcile()` — via the backend contract's `repair( $routine )` method, which never trusts recorded state. ### Routine generation + watermark fencing @@ -444,12 +444,20 @@ Earlier versions stamped a generation onto every stored action individually, in Routines registered with the same interval would all fire in the same second. `WP_Agent_Routine` accepts `stagger => bool|int` (default `true` for interval routines, `false` for cron expressions, where the expression already *is* the slot; an int is an explicit max window in seconds). `WP_Agent_Routine::stagger_offset()` computes `crc32( 'agents_routine_stagger_' . $id ) % min( interval, max_window )`, capped by `WP_Agent_Routine::MAX_STAGGER_SECONDS` (one hour). The bridge adds the offset to the first-run timestamp. The offset depends only on the routine id, so re-registration always lands the routine back in the same slot — and, since the offset only affects the first-run timestamp and never the recurrence itself, it never causes an unchanged re-register to be mistaken for a schedule mismatch. +### Hourly WP-Cron routine repair + +Agents API declares ONE WordPress cron event on `init`: hook `wp_agent_routines_reconcile`, recurrence `hourly` (in `src/Routines/register-routine-bridge-sync.php`, alongside the declaration listeners). The declaration is guarded by `wp_next_scheduled()`, so any number of `init` fires schedule exactly one event, and the guard itself reads the autoloaded `cron` option — it costs no queries per request. Firing the event runs `WP_Agent_Routine_Registry::reconcile()` in apply mode (the lock-protected path) and fires `wp_agent_routines_reconcile_completed` with the reconcile report (`enqueued`/`removed`/`unchanged`/`errors`) so consumers can log or alert on it. + +This is the scheduled counterpart to the record fast path above: since declarations trust the persisted schedule record, a chain lost out of band (cancelled in the Action Scheduler admin UI, a pruned or restored table) stays lost until something authoritative runs. The repair lives in WP-Cron rather than Action Scheduler on purpose — losing Action Scheduler chains cannot also take out the mechanism that restores them — and it runs through `$wpdb`, not the options API, so a stale or cached-away lock row can never wedge it. There is deliberately no unschedule-on-deactivation hook and no self-repair machinery for the event itself: agents-api ships far more often as a Composer-embedded library inside a consumer plugin than as its own activatable plugin, so there is no deactivation path that would fire for most real installs, and a leftover event after removal is inert. + ### Routine reconcile `WP_Agent_Routine_Registry::reconcile( array $opts = [] )` repairs drift between the registry and the Action Scheduler store. For every registered, non-paused routine it calls the backend's authoritative `repair()`: pending-action coverage is verified against the real store by logical identity (never against the schedule record), duplicate matching chains are collapsed to one live chain, a missing or entirely-stale schedule is re-enqueued, and the schedule record is (re)written so later declarations fast-path again; pending routine actions whose logical `routine_id` is not registered (or is durably paused) are unscheduled as orphans. It returns `array( 'enqueued' => [ids], 'removed' => [ids], 'unchanged' => [ids], 'errors' => [id => message] )`; `$opts['dry_run']` reports the same shape without writing. `pending_by_routine()` (the one bulk read reconcile uses) always forces a fresh fetch of the Action Scheduler store — reconcile's entire purpose is authoritative drift detection, so trusting recorded state would defeat the point; that fresh read also refreshes the request-scoped map `repair()` walks, so a reconcile run pays exactly one bulk read. The run is serialized through a `$wpdb`-level compare-and-set lock (`agents_routine_reconcile_lock`; a single `INSERT ... ON DUPLICATE KEY UPDATE` against `wp_options`) that never trusts `get_option()`/the options cache for the decision — only the real DB row. A lock older than five minutes is treated as stale and taken over atomically in the same query, and the lock is always released (a direct `DELETE`, again bypassing the cache) in a `finally`. This closes a production failure mode where a killed request left the lock option cached (e.g. in Redis) with no backing DB row: `add_option()` always failed because the cache said the option existed, the stale check read that same cached value, and `delete_option()` found nothing in the DB to delete — the lock stayed stuck "held" until the cache entry was manually cleared. Reading and writing exclusively through `$wpdb`, with an explicit `wp_cache_delete()` after every write, means a stale cache entry with no backing row can never block acquisition. +Both routine locks — this one and the bridge's per-routine schedule lock — share one internal helper, `AgentsAPI\AI\Routines\WP_Agent_Routine_Option_Lock` (`acquire( $option_name, $ttl )` / `release( $option_name )`): one atomic query decides insert, stale takeover, or "someone fresh holds it", so two processes that both observe the same stale lock cannot both take it over — only the first write changes the row and the second reports zero affected rows. + Pause state is durable: the bridge maintains the `agents_routine_paused` option so reconcile can distinguish "unscheduled on purpose" from "missing by drift" across requests, even though the registry itself stays in-memory. `unregister()` and `pause()` also drop the routine's entry from the `agents_routine_schedules` record, so a later `register()`/`resume()` cannot fast-path on a signature whose action no longer exists. The `agents/reconcile-routines` ability (`show_in_rest: true`, annotations `destructive: true, idempotent: true`) exposes the same operation to ability consumers: input `{ dry_run?: bool }`, output the reconcile report, permission `current_user_can( 'manage_options' )` filterable via `agents_reconcile_routines_permission`. diff --git a/src/Routines/class-wp-agent-routine-action-scheduler-bridge.php b/src/Routines/class-wp-agent-routine-action-scheduler-bridge.php index c38bec3..1333076 100644 --- a/src/Routines/class-wp-agent-routine-action-scheduler-bridge.php +++ b/src/Routines/class-wp-agent-routine-action-scheduler-bridge.php @@ -811,39 +811,22 @@ private function cancel_superseded( int $keep_id, array $existing, bool $recurri * schedule step must not run concurrently in two processes, or both see * "no matching chain" and both schedule one (the duplicate-chain race). * - * An `add_option()`-based lock: the insert succeeds only when no lock row - * exists, so acquisition is atomic in MySQL against `option_name`'s - * unique key. A lock older than {@see SCHEDULE_LOCK_TTL} is stale (its - * holder crashed mid-schedule) and is taken over. With no option layer at - * all (pure-PHP harnesses) callers are already single-process and the - * lock is a no-op success. + * Delegates to the shared routines option lock: one atomic `$wpdb` + * compare-and-set that inserts the lock row when absent and takes over a + * lock older than {@see SCHEDULE_LOCK_TTL} in the same query — never + * through the options API, whose cache could report a lock as held with + * no backing DB row (or hand a stale takeover to two processes at once, + * the flaw of the plain `update_option()` takeover this replaces). */ private function acquire_schedule_lock( string $routine_id ): bool { - if ( ! self::has_option_layer() ) { - return true; - } - - $name = self::SCHEDULE_LOCK_OPTION_PREFIX . $routine_id; - if ( add_option( $name, time(), '', false ) ) { - return true; - } - - $held_at = get_option( $name, 0 ); - if ( is_numeric( $held_at ) && ( time() - (int) $held_at ) < self::SCHEDULE_LOCK_TTL ) { - return false; // Freshly held by another process. - } - - // Stale (or unreadable): take over so a crashed holder cannot strand - // the routine unscheduled. - update_option( $name, time(), false ); - return true; + return WP_Agent_Routine_Option_Lock::acquire( + self::SCHEDULE_LOCK_OPTION_PREFIX . $routine_id, + self::SCHEDULE_LOCK_TTL + ); } private function release_schedule_lock( string $routine_id ): void { - if ( ! self::has_option_layer() ) { - return; - } - delete_option( self::SCHEDULE_LOCK_OPTION_PREFIX . $routine_id ); + WP_Agent_Routine_Option_Lock::release( self::SCHEDULE_LOCK_OPTION_PREFIX . $routine_id ); } /** diff --git a/src/Routines/class-wp-agent-routine-option-lock.php b/src/Routines/class-wp-agent-routine-option-lock.php new file mode 100644 index 0000000..727cc89 --- /dev/null +++ b/src/Routines/class-wp-agent-routine-option-lock.php @@ -0,0 +1,130 @@ +query()` returns the raw affected-rows count for a non-SELECT + * query, so `> 0` is "we now hold the lock" and `0` is "someone else + * holds it, and it is not stale". + * + * When there is no `$wpdb` layer at all (pure-PHP test harnesses) the + * caller is already single-process and the lock is a no-op success. + * + * @since 0.17.0 + * + * @param string $option_name wp_options row holding the lock timestamp. + * @param int $ttl Seconds after which a held lock is treated + * as stale and may be taken over. + * @return bool True when the lock is now held by this process. + */ + public static function acquire( string $option_name, int $ttl ): bool { + global $wpdb; + if ( ! $wpdb instanceof \wpdb ) { + return true; + } + + $now = time(); + $stale_before = $now - $ttl; + + $query = $wpdb->prepare( + 'INSERT INTO %i (option_name, option_value, autoload) + VALUES (%s, %s, %s) + ON DUPLICATE KEY UPDATE option_value = IF( CAST( option_value AS UNSIGNED ) <= %d, VALUES(option_value), option_value )', + $wpdb->options, + $option_name, + (string) $now, + 'no', + $stale_before + ); + if ( ! is_string( $query ) ) { + return false; + } + + // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- $query is built by $wpdb->prepare() above. + $affected = $wpdb->query( $query ); + + self::flush_cache( $option_name ); + + return is_int( $affected ) && $affected > 0; + } + + /** + * Release a named lock previously taken by {@see acquire()}: delete the + * backing row directly (bypassing the options cache) and evict any cache + * entry, so a later read through `get_option()` never observes a value + * the DB no longer has. + * + * @since 0.17.0 + * + * @param string $option_name wp_options row holding the lock timestamp. + */ + public static function release( string $option_name ): void { + global $wpdb; + if ( ! $wpdb instanceof \wpdb ) { + return; + } + + $query = $wpdb->prepare( 'DELETE FROM %i WHERE option_name = %s', $wpdb->options, $option_name ); + if ( is_string( $query ) ) { + // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- $query is built by $wpdb->prepare() above. + $wpdb->query( $query ); + } + + self::flush_cache( $option_name ); + } + + /** + * Explicitly evict the lock option from the object cache after every + * direct `$wpdb` write, so anything that does happen to read it via + * `get_option()` never observes a value the DB no longer has. + */ + private static function flush_cache( string $option_name ): void { + if ( function_exists( 'wp_cache_delete' ) ) { + wp_cache_delete( $option_name, 'options' ); + } + } +} diff --git a/src/Routines/class-wp-agent-routine-registry.php b/src/Routines/class-wp-agent-routine-registry.php index 7362d80..f582ba0 100644 --- a/src/Routines/class-wp-agent-routine-registry.php +++ b/src/Routines/class-wp-agent-routine-registry.php @@ -25,10 +25,11 @@ final class WP_Agent_Routine_Registry { /** * Option name of the reconcile compare-and-set lock. Written and read - * only through direct `$wpdb` queries (see {@see acquire_reconcile_lock()} - * / {@see release_reconcile_lock()}) — never through `get_option()` — - * so a stale or missing options-cache entry can never make the lock - * unavailable or un-releasable. + * only through direct `$wpdb` queries (see + * {@see WP_Agent_Routine_Option_Lock::acquire()} / + * {@see WP_Agent_Routine_Option_Lock::release()}) — never through + * `get_option()` — so a stale or missing options-cache entry can never + * make the lock unavailable or un-releasable. */ private const RECONCILE_LOCK_OPTION = 'agents_routine_reconcile_lock'; @@ -284,10 +285,10 @@ public static function current_generation( string $routine_id ): ?string { * * The whole run is serialized through a `$wpdb`-level compare-and-set * lock (`agents_routine_reconcile_lock`; see - * {@see acquire_reconcile_lock()}) that never trusts the options - * cache for the decision — a lock older than five minutes is treated - * as stale and taken over. Dry runs report the same shape without - * writing anything (and without taking the lock). + * {@see WP_Agent_Routine_Option_Lock::acquire()}) that never trusts the + * options cache for the decision — a lock older than five minutes is + * treated as stale and taken over. Dry runs report the same shape + * without writing anything (and without taking the lock). * * @param array $opts Recognised keys: `dry_run` (bool). * @return array{enqueued:string[],removed:string[],unchanged:string[],errors:array} @@ -308,7 +309,7 @@ public static function reconcile( array $opts = array() ): array { return self::reconcile_unlocked( true ); } - if ( ! self::acquire_reconcile_lock() ) { + if ( ! WP_Agent_Routine_Option_Lock::acquire( self::RECONCILE_LOCK_OPTION, self::RECONCILE_LOCK_TTL ) ) { return array( 'enqueued' => array(), 'removed' => array(), @@ -320,7 +321,7 @@ public static function reconcile( array $opts = array() ): array { try { return self::reconcile_unlocked( false ); } finally { - self::release_reconcile_lock(); + WP_Agent_Routine_Option_Lock::release( self::RECONCILE_LOCK_OPTION ); } } @@ -413,95 +414,6 @@ private static function reconcile_unlocked( bool $dry_run ): array { ); } - /** - * Acquire the reconcile lock via a single atomic `$wpdb` compare-and-set - * — `INSERT ... ON DUPLICATE KEY UPDATE` against `wp_options` directly, - * never through `get_option()`/`add_option()`. - * - * This closes a production failure mode: a killed request can leave - * `agents_routine_reconcile_lock` as a persistent object-cache entry - * (e.g. Redis) with no backing DB row. `add_option()` then always - * fails (the cache says the option exists), the old code's stale check - * read the same cached value, and `delete_option()` found nothing in - * the DB to delete — the lock stayed stuck "held" indefinitely. Reading - * and writing exclusively through `$wpdb` — and never consulting - * `get_option()`/the options cache for the decision — means a stale - * cache entry with no DB row cannot block acquisition: MySQL's - * `ON DUPLICATE KEY` path only ever sees the real row (or its absence). - * - * The single query handles all three cases atomically against - * `option_name`'s unique key: - * - no row exists: plain insert, 1 affected row; - * - row exists and is stale (older than {@see RECONCILE_LOCK_TTL}): - * `ON DUPLICATE KEY UPDATE` overwrites it, MySQL reports 2 affected - * rows for a changed row; - * - row exists and is fresh: the `IF()` leaves the value unchanged, - * MySQL reports 0 affected rows for a no-op update. - * - * `$wpdb->query()` returns the raw affected-rows count for a - * non-SELECT query, so `> 0` is "we now hold the lock" and `0` is - * "someone else holds it, and it is not stale". - * - * When there is no option layer at all (non-WordPress harness) callers - * are already single-process and the lock is a no-op success. - */ - private static function acquire_reconcile_lock(): bool { - global $wpdb; - if ( ! $wpdb instanceof \wpdb ) { - return true; - } - - $now = time(); - $stale_before = $now - self::RECONCILE_LOCK_TTL; - - $query = $wpdb->prepare( - 'INSERT INTO %i (option_name, option_value, autoload) - VALUES (%s, %s, %s) - ON DUPLICATE KEY UPDATE option_value = IF( CAST( option_value AS UNSIGNED ) <= %d, VALUES(option_value), option_value )', - $wpdb->options, - self::RECONCILE_LOCK_OPTION, - (string) $now, - 'no', - $stale_before - ); - if ( ! is_string( $query ) ) { - return false; - } - - // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- $query is built by $wpdb->prepare() above. - $affected = $wpdb->query( $query ); - - self::flush_reconcile_lock_cache(); - - return is_int( $affected ) && $affected > 0; - } - - private static function release_reconcile_lock(): void { - global $wpdb; - if ( ! $wpdb instanceof \wpdb ) { - return; - } - - $query = $wpdb->prepare( 'DELETE FROM %i WHERE option_name = %s', $wpdb->options, self::RECONCILE_LOCK_OPTION ); - if ( is_string( $query ) ) { - // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared -- $query is built by $wpdb->prepare() above. - $wpdb->query( $query ); - } - - self::flush_reconcile_lock_cache(); - } - - /** - * Explicitly evict the lock option from the object cache after every - * direct `$wpdb` write, so anything that does happen to read it via - * `get_option()` never observes a value the DB no longer has. - */ - private static function flush_reconcile_lock_cache(): void { - if ( function_exists( 'wp_cache_delete' ) ) { - wp_cache_delete( self::RECONCILE_LOCK_OPTION, 'options' ); - } - } - /** * @return WP_Agent_Routine[] */ diff --git a/src/Routines/register-routine-bridge-sync.php b/src/Routines/register-routine-bridge-sync.php index 4822d4b..4665788 100644 --- a/src/Routines/register-routine-bridge-sync.php +++ b/src/Routines/register-routine-bridge-sync.php @@ -11,6 +11,25 @@ * The same listeners handle unregister (cancelling the matching schedule), * pause, resume, and run-now requests. * + * This file also owns the hourly drift repair: it declares ONE WordPress + * cron event (`wp_agent_routines_reconcile`, hourly) on `init`, guarded by + * `wp_next_scheduled()` so repeat declarations schedule exactly one event. + * Its callback runs the authoritative + * {@see WP_Agent_Routine_Registry::reconcile()} (apply mode, the + * lock-protected path) and reports through + * `wp_agent_routines_reconcile_completed`. WP-Cron is deliberate: the guard + * reads the autoloaded `cron` option, so the per-request cost is a single + * array lookup in an already-loaded option, and the repair lives outside + * Action Scheduler — a lost Action Scheduler chain cannot also take out the + * mechanism that restores it. + * + * The event is deliberately not unscheduled on a deactivation hook: + * agents-api ships far more often as a Composer-embedded library inside a + * consumer plugin than as its own activatable plugin, so there is no + * deactivation path that would fire for most real installs. A leftover + * hourly event after removal is inert (the substrate is absent, nothing + * registers the callback) and self-heals on reinstall. + * * The filename keeps its historical `bridge-sync` name to avoid churn; it * syncs to whichever backend the `wp_agent_routine_backend` filter * resolves, not to any specific scheduler. @@ -23,6 +42,46 @@ defined( 'ABSPATH' ) || exit; +const WP_AGENT_ROUTINES_RECONCILE_HOOK = 'wp_agent_routines_reconcile'; + +add_action( + 'init', + static function (): void { + if ( ! function_exists( 'wp_next_scheduled' ) || ! function_exists( 'wp_schedule_event' ) ) { + return; + } + // One event per site, ever: the guard reads the autoloaded `cron` + // option, so this costs no queries per request once scheduled. + if ( false !== wp_next_scheduled( WP_AGENT_ROUTINES_RECONCILE_HOOK ) ) { + return; + } + wp_schedule_event( time(), 'hourly', WP_AGENT_ROUTINES_RECONCILE_HOOK ); + }, + 10, + 0 +); + +add_action( + WP_AGENT_ROUTINES_RECONCILE_HOOK, + static function (): void { + $report = WP_Agent_Routine_Registry::reconcile(); + + /** + * Fires after the scheduled hourly routine reconcile ran. Consumers + * can log or alert on the report (enqueued/removed/unchanged/errors + * — the same shape {@see WP_Agent_Routine_Registry::reconcile()} + * returns). + * + * @since 0.17.0 + * + * @param array{enqueued:string[],removed:string[],unchanged:string[],errors:array} $report The reconcile report. + */ + do_action( 'wp_agent_routines_reconcile_completed', $report ); + }, + 10, + 0 +); + add_action( 'wp_agent_routine_registered', static function ( WP_Agent_Routine $routine ): void { diff --git a/tests/routine-ability-target-smoke.php b/tests/routine-ability-target-smoke.php index 5d7b8e6..8ebd44b 100644 --- a/tests/routine-ability-target-smoke.php +++ b/tests/routine-ability-target-smoke.php @@ -313,6 +313,7 @@ function as_unschedule_all_actions( string $hook, array $args = array(), string // --------------------------------------------------------------------------- require_once __DIR__ . '/../src/Routines/class-wp-agent-routine.php'; +require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-option-lock.php'; require_once __DIR__ . '/../src/Routines/interface-wp-agent-routine-backend.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-registry.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-action-scheduler-bridge.php'; diff --git a/tests/routine-smoke.php b/tests/routine-smoke.php index f469d69..4021f65 100644 --- a/tests/routine-smoke.php +++ b/tests/routine-smoke.php @@ -111,6 +111,7 @@ function as_enqueue_async_action( string $hook, array $args = array(), string $g } require_once __DIR__ . '/../src/Routines/class-wp-agent-routine.php'; +require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-option-lock.php'; require_once __DIR__ . '/../src/Routines/interface-wp-agent-routine-backend.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-registry.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-action-scheduler-bridge.php'; diff --git a/tests/routines-backend-contract-smoke.php b/tests/routines-backend-contract-smoke.php index 2df02af..5a36517 100644 --- a/tests/routines-backend-contract-smoke.php +++ b/tests/routines-backend-contract-smoke.php @@ -107,6 +107,7 @@ function apply_filters( string $hook, $value, ...$args ) { // --------------------------------------------------------------------------- require_once __DIR__ . '/../src/Routines/class-wp-agent-routine.php'; +require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-option-lock.php'; require_once __DIR__ . '/../src/Routines/interface-wp-agent-routine-backend.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-registry.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-action-scheduler-bridge.php'; diff --git a/tests/routines-durability-smoke.php b/tests/routines-durability-smoke.php index 6cada68..aac86e0 100644 --- a/tests/routines-durability-smoke.php +++ b/tests/routines-durability-smoke.php @@ -160,6 +160,48 @@ function delete_option( string $name ): bool { } } +// Fake WP-Cron: the `cron` option equivalent, plus a call log so a test can +// assert the scheduling side (not just the stored events). +$GLOBALS['smoke_cron_events'] = array(); +$GLOBALS['smoke_cron_schedule_calls'] = array(); +if ( ! function_exists( 'wp_next_scheduled' ) ) { + /** + * @param array $args + * @return int|false + */ + function wp_next_scheduled( string $hook, array $args = array() ) { + unset( $args ); + $event = $GLOBALS['smoke_cron_events'][ $hook ] ?? null; + return is_array( $event ) ? (int) $event['timestamp'] : false; + } +} +if ( ! function_exists( 'wp_schedule_event' ) ) { + /** + * @param array $args + */ + function wp_schedule_event( int $timestamp, string $recurrence, string $hook, array $args = array(), bool $wp_error = false ): bool { + unset( $args, $wp_error ); + $GLOBALS['smoke_cron_schedule_calls'][] = $hook; + if ( isset( $GLOBALS['smoke_cron_events'][ $hook ] ) ) { + return false; + } + $GLOBALS['smoke_cron_events'][ $hook ] = array( + 'timestamp' => $timestamp, + 'recurrence' => $recurrence, + ); + return true; + } +} +if ( ! function_exists( 'wp_clear_scheduled_hook' ) ) { + /** + * @param array $args + */ + function wp_clear_scheduled_hook( string $hook, array $args = array() ): void { + unset( $args ); + unset( $GLOBALS['smoke_cron_events'][ $hook ] ); + } +} + // Fake chat ability invoked by the scheduled-run listener. class Smoke_Chat_Ability { /** @var array> */ @@ -525,6 +567,7 @@ private function purge_legacy(): int { require_once __DIR__ . '/../src/Routines/class-wp-agent-routine.php'; require_once __DIR__ . '/../src/Routines/interface-wp-agent-routine-backend.php'; +require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-option-lock.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-registry.php'; require_once __DIR__ . '/../src/Routines/class-wp-agent-routine-action-scheduler-bridge.php'; require_once __DIR__ . '/../src/Routines/register-routine-bridge-sync.php'; @@ -532,22 +575,25 @@ private function purge_legacy(): int { use AgentsAPI\AI\Routines\WP_Agent_Routine; use AgentsAPI\AI\Routines\WP_Agent_Routine_Action_Scheduler_Bridge; +use AgentsAPI\AI\Routines\WP_Agent_Routine_Option_Lock; use AgentsAPI\AI\Routines\WP_Agent_Routine_Registry; function smoke_reset_state(): void { - $GLOBALS['smoke_as_fetches'] = array(); - $GLOBALS['smoke_as_cancel_calls'] = array(); - $GLOBALS['smoke_as_query_count'] = 0; + $GLOBALS['smoke_as_fetches'] = array(); + $GLOBALS['smoke_as_cancel_calls'] = array(); + $GLOBALS['smoke_as_query_count'] = 0; WP_Agent_Routine_Registry::reset(); WP_Agent_Routine_Registry::reset_backend(); WP_Agent_Routine_Action_Scheduler_Bridge::instance()->reset_pending_cache_for_tests(); - $GLOBALS['smoke_as'] = array(); - $GLOBALS['smoke_as_next_id'] = 0; - $GLOBALS['smoke_options'] = array(); - $GLOBALS['smoke_object_cache'] = array(); - $GLOBALS['wpdb']->rows = array(); - $GLOBALS['smoke_fired_hooks'] = array(); - $GLOBALS['smoke_chat_ability'] = new Smoke_Chat_Ability(); + $GLOBALS['smoke_as'] = array(); + $GLOBALS['smoke_as_next_id'] = 0; + $GLOBALS['smoke_options'] = array(); + $GLOBALS['smoke_object_cache'] = array(); + $GLOBALS['wpdb']->rows = array(); + $GLOBALS['smoke_fired_hooks'] = array(); + $GLOBALS['smoke_chat_ability'] = new Smoke_Chat_Ability(); + $GLOBALS['smoke_cron_events'] = array(); + $GLOBALS['smoke_cron_schedule_calls'] = array(); } /** @return array> */ @@ -570,6 +616,21 @@ function smoke_hook_fired( string $hook ): bool { return false; } +/** + * The args each firing of a hook carried (first firing first). + * + * @return array> + */ +function smoke_hook_args( string $hook ): array { + $out = array(); + foreach ( $GLOBALS['smoke_fired_hooks'] as $entry ) { + if ( $entry[0] === $hook ) { + $out[] = $entry[1]; + } + } + return $out; +} + // --------------------------------------------------------------------------- // 1. Scheduled args stay logical; the routine's generation + watermark live // in ONE option row — never a per-action option. @@ -1036,23 +1097,45 @@ function smoke_hook_fired( string $hook ): bool { // --------------------------------------------------------------------------- // 12. Check-then-schedule is single-flight: a lock held by another process // schedules no second chain; a stale lock is taken over and released. +// Like the reconcile lock, the per-routine schedule lock now lives only +// in $wpdb (the shared routines option lock) — never in +// get_option()/the options cache — so it is seeded and inspected +// directly against the fake DB row store. // --------------------------------------------------------------------------- smoke_reset_state(); -$GLOBALS['smoke_options']['agents_routine_schedule_lock_conc'] = time(); // fresh lock, held by "another process" +$GLOBALS['wpdb']->rows['agents_routine_schedule_lock_conc'] = (string) time(); // fresh lock, held by "another process" WP_Agent_Routine_Registry::register( 'conc', array( 'agent' => 'commander', 'interval' => 600 ) ); durable_assert( 0, count( smoke_pending_rows() ), 'concurrent: a fresh lock held by another process schedules no chain' ); durable_assert( array(), get_option( 'agents_routine_schedules', array() ), 'concurrent: the losing caller records nothing' ); -$GLOBALS['smoke_options']['agents_routine_schedule_lock_conc'] = time() - 120; // stale: the holder crashed +$GLOBALS['wpdb']->rows['agents_routine_schedule_lock_conc'] = (string) ( time() - 120 ); // stale: the holder crashed WP_Agent_Routine_Registry::register( 'conc', array( 'agent' => 'commander', 'interval' => 600 ) ); durable_assert( 1, count( smoke_pending_rows() ), 'concurrent: a stale lock is taken over and the routine is scheduled' ); -durable_assert( false, array_key_exists( 'agents_routine_schedule_lock_conc', $GLOBALS['smoke_options'] ), 'concurrent: the schedule lock is released after the run' ); +durable_assert( false, array_key_exists( 'agents_routine_schedule_lock_conc', $GLOBALS['wpdb']->rows ), 'concurrent: the schedule lock is released after the run' ); +durable_assert( false, array_key_exists( 'agents_routine_schedule_lock_conc', $GLOBALS['smoke_options'] ), 'concurrent: the schedule lock never touches the options API' ); $conc_id = $GLOBALS['smoke_as_next_id']; WP_Agent_Routine_Registry::register( 'conc', array( 'agent' => 'commander', 'interval' => 600 ) ); durable_assert( $conc_id, $GLOBALS['smoke_as_next_id'], 'concurrent: with the schedule recorded, re-declaration schedules nothing' ); +// Atomic takeover: two acquirers that both observe the same stale lock row +// cannot both win — the takeover is the compare-and-set itself, so only the +// first write changes the row and the second sees a fresh value. +smoke_reset_state(); +$GLOBALS['wpdb']->rows['agents_routine_lock_race'] = (string) ( time() - 400 ); // stale: both processes observe this +$race_first = WP_Agent_Routine_Option_Lock::acquire( 'agents_routine_lock_race', 30 ); +$race_second = WP_Agent_Routine_Option_Lock::acquire( 'agents_routine_lock_race', 30 ); +durable_assert( true, $race_first, 'lock helper: the first taker of a stale lock wins' ); +durable_assert( false, $race_second, 'lock helper: the second taker of the same stale lock loses — exactly one acquirer wins' ); +WP_Agent_Routine_Option_Lock::release( 'agents_routine_lock_race' ); +durable_assert( false, array_key_exists( 'agents_routine_lock_race', $GLOBALS['wpdb']->rows ), 'lock helper: release deletes the row' ); + +// A fresh lock is never taken over, and an absent lock is acquired outright. +$GLOBALS['wpdb']->rows['agents_routine_lock_fresh'] = (string) time(); +durable_assert( false, WP_Agent_Routine_Option_Lock::acquire( 'agents_routine_lock_fresh', 30 ), 'lock helper: a fresh lock is not taken over' ); +durable_assert( true, WP_Agent_Routine_Option_Lock::acquire( 'agents_routine_lock_absent', 30 ), 'lock helper: an absent lock is acquired' ); + // --------------------------------------------------------------------------- // 13. A recorded signature whose action was deleted out-of-band: plain // declarations trust the record (zero I/O) and reconcile repairs it. @@ -1103,6 +1186,67 @@ function smoke_hook_fired( string $hook ): bool { WP_Agent_Routine_Registry::unregister( 'pr' ); durable_assert( array(), get_option( 'agents_routine_schedules', array() ), 'pause/resume: unregister drops the schedule record' ); +// --------------------------------------------------------------------------- +// 15. Hourly WP-Cron repair: one `wp_agent_routines_reconcile` event is +// declared on init (guarded by wp_next_scheduled, so repeat declarations +// schedule exactly one event), and firing its hook runs the +// lock-protected reconcile — restoring a chain deleted out of band and +// reporting through wp_agent_routines_reconcile_completed. +// --------------------------------------------------------------------------- + +smoke_reset_state(); +do_action( 'init' ); +do_action( 'init' ); // a second declaration on the same (or a later) request +durable_assert( array( 'wp_agent_routines_reconcile' ), array_keys( $GLOBALS['smoke_cron_events'] ), 'cron repair: declaring on init twice schedules exactly one cron event' ); +durable_assert( 'hourly', $GLOBALS['smoke_cron_events']['wp_agent_routines_reconcile']['recurrence'] ?? '', 'cron repair: the event recurs hourly' ); +durable_assert( array( 'wp_agent_routines_reconcile' ), $GLOBALS['smoke_cron_schedule_calls'], 'cron repair: exactly one wp_schedule_event() call happened' ); + +// A routine whose chain is deleted out of band (cancelled in the Action +// Scheduler admin UI, a pruned or restored table): plain declarations trust +// the record, so only the cron repair brings it back. +WP_Agent_Routine_Registry::register( 'cron-heal', array( 'agent' => 'commander', 'interval' => 600 ) ); +$cron_heal_old_id = $GLOBALS['smoke_as_next_id']; +unset( $GLOBALS['smoke_as'][ $cron_heal_old_id ] ); // the chain is lost out of band + +WP_Agent_Routine_Action_Scheduler_Bridge::instance()->reset_pending_cache_for_tests(); +do_action( \AgentsAPI\AI\Routines\WP_AGENT_ROUTINES_RECONCILE_HOOK ); // the hourly cron callback + +$cron_heal_rows = smoke_pending_rows(); +durable_assert( 1, count( $cron_heal_rows ), 'cron repair: exactly one matching chain is back after the cron run' ); +$cron_heal_new_id = (int) array_key_first( $cron_heal_rows ); +durable_assert( true, $cron_heal_new_id > $cron_heal_old_id, 'cron repair: the restored chain is a fresh one' ); +$durable_cron_record = get_option( 'agents_routine_schedules', array() )['cron-heal'] ?? array(); +durable_assert( 'interval:600', $durable_cron_record['signature'] ?? '', 'cron repair: the schedule record matches the restored recurrence' ); +durable_assert( $cron_heal_new_id, $durable_cron_record['action_id'] ?? 0, 'cron repair: the schedule record points at the restored chain' ); + +$cron_reports = smoke_hook_args( 'wp_agent_routines_reconcile_completed' ); +durable_assert( 1, count( $cron_reports ), 'cron repair: the completion action fired exactly once' ); +durable_assert( array( 'cron-heal' ), $cron_reports[0][0]['enqueued'] ?? array( 'no-report' ), 'cron repair: the completion action carries the reconcile report' ); +durable_assert( false, isset( $cron_reports[0][0]['errors']['_lock'] ), 'cron repair: the lock-protected reconcile ran to completion' ); + +// The repair is idempotent: a second cron run changes nothing. +WP_Agent_Routine_Action_Scheduler_Bridge::instance()->reset_pending_cache_for_tests(); +do_action( \AgentsAPI\AI\Routines\WP_AGENT_ROUTINES_RECONCILE_HOOK ); +durable_assert( 1, count( smoke_pending_rows() ), 'cron repair: a second cron run keeps exactly one chain' ); +durable_assert( array( 'wp_agent_routines_reconcile' ), $GLOBALS['smoke_cron_schedule_calls'], 'cron repair: firing the cron hook never reschedules the event itself' ); + +// A cron repair for a paused routine does not resurrect it. +WP_Agent_Routine_Registry::register( 'cron-paused', array( 'agent' => 'commander', 'interval' => 600 ) ); +WP_Agent_Routine_Registry::pause( 'cron-paused' ); +WP_Agent_Routine_Action_Scheduler_Bridge::instance()->reset_pending_cache_for_tests(); +$GLOBALS['smoke_fired_hooks'] = array(); +do_action( \AgentsAPI\AI\Routines\WP_AGENT_ROUTINES_RECONCILE_HOOK ); +durable_assert( 1, count( smoke_pending_rows() ), 'cron repair: a paused routine stays unscheduled' ); + +// An already-scheduled event (e.g. from a previous install) is not +// rescheduled: firing init again performs no second wp_schedule_event(). +smoke_reset_state(); +$preexisting_event_at = time() + 100; +$GLOBALS['smoke_cron_events']['wp_agent_routines_reconcile'] = array( 'timestamp' => $preexisting_event_at, 'recurrence' => 'hourly' ); +do_action( 'init' ); +durable_assert( array(), $GLOBALS['smoke_cron_schedule_calls'], 'cron repair: an already-scheduled event is not rescheduled' ); +durable_assert( $preexisting_event_at, $GLOBALS['smoke_cron_events']['wp_agent_routines_reconcile']['timestamp'] ?? 0, 'cron repair: the existing event is left untouched' ); + // --------------------------------------------------------------------------- if ( count( $failures ) > 0 ) {