fix(cache): cache rules never matched data.sec.gov, or a mirror's own host

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **`report.items` warned you about `chunked_document`, an attribute you never touched.** `TenK`, `TenQ`, `TwentyF` and `CurrentReport` try the new parser first and fall back to the legacy `ChunkedDocument`, and those fallbacks read the *public* deprecated property — so a plain `twentyf.items` emitted `chunked_document is deprecated` about a choice that was ours, not yours. 20-F got it on every call, because 20-F prefers the legacy path deliberately. If you run `-W error::DeprecationWarning`, that was not a warning but an exception. Internal paths now use a private accessor and say nothing; asking for `chunked_document` yourself still warns.
 
 - **Three report classes had silently lost that deprecation entirely.** `TenK`, `TenQ` and `CurrentReport` each overrode `chunked_document` to change how it was built, and an override that replaces the property also replaces the `warnings.warn` inside it — so their users got no notice that the attribute disappears in 6.0, which is the population the deprecation exists for. Construction now happens in `_chunked_document`, the warning lives in exactly one place, and a test asserts no subclass can take it away again.
+- **`Company.get_facts()` re-downloaded companyfacts on every call, and the 30s `/submissions` TTL never took effect.** Cache rules were keyed off `SEC_BASE_URL` alone, but httpxthrottlecache matches the request host against that key, and `re.match(r'.*www\.sec\.gov', 'data.sec.gov')` is `None` — so a fresh process logs `No patterns matched data.sec.gov` and pays full network cost every time. Keys now come from `httpx.URL(...).host`, one per host, matched exactly, which also restores caching for custom mirrors. Requires `httpxthrottlecache>=0.6.1`. (GH #989)
 
 ## [5.48.0] - 2026-08-12
 

edgar/httpclient.py

@@ -1,5 +1,6 @@
 import locale
 import os
+import re
 from contextlib import asynccontextmanager, contextmanager
 from typing import AsyncGenerator, Generator, Literal, Optional
 
@@ -25,7 +26,7 @@
 # test_verify_reaches_the_transport_params pins the behaviour the patch protected.
 from httpxthrottlecache import HttpxThrottleCache
 
-from edgar.core import get_identity, strtobool
+from edgar.core import get_identity, log, strtobool
 
 from .core import get_edgar_data_directory
 
@@ -42,30 +43,75 @@
 # Note that: revalidation consumes rate limit "hit", but will be served from cache if the data hasn't changed.
 
 
+def _host_key(url: str) -> str:
+    """Build a cache-rule key that matches exactly one host.
+
+    httpxthrottlecache keys its cache rules by a regex matched against
+    ``request.url.host`` (``controller.get_rules``), and returns the FIRST key
+    that matches. Two consequences shape this function:
+
+    * **The only regex we want here is equality.** An unanchored key leaks across
+      hosts: ``.*mirror\\.com`` also matches ``data.mirror.com``, so a base-host
+      rule set would answer for the data host and the data rules would never be
+      reached — the same shape as the bug this fixes, moved from sec.gov to
+      mirrors.
+    * **The host must come from the same parser that produces it at match time.**
+      ``httpx.URL(...).host`` lowercases, drops ``user@`` and the port, and
+      normalises IDNA; a hand-rolled ``https?://([^/]+)`` regex does none of
+      those, so ``EDGAR_DATA_URL=https://DATA.mirror.example.org`` (or a URL
+      carrying a port or credentials) yields a key no real request can match.
+
+    An unparseable URL yields a key that matches nothing: a misconfigured mirror
+    goes uncached, which is slow, rather than borrowing another host's rules,
+    which would be wrong.
+    """
+    host = httpx.URL(url).host
+    if not host:
+        log.warning("No host in %r; requests to it will not be cached.", url)
+        return r"(?!)"  # matches nothing
+    return f"{re.escape(host)}$"
+
+
 def _get_cache_rules() -> dict:
     """
-    Get cache rules based on configured SEC base URL.
+    Get cache rules based on configured SEC base and data URLs.
     This allows caching to work with custom SEC mirrors.
+
+    SEC serves two hosts: ``www.sec.gov`` (tickers, index, Archives) and
+    ``data.sec.gov`` (``/submissions``, ``/api/xbrl/companyfacts``).
+    httpxthrottlecache matches the request HOST against each top-level key
+    before it ever looks at the path, so a rule filed under the base-URL host
+    never applies to a request to the data host, whatever its path. Build one
+    key per host actually used by ``edgar.urls`` and file each rule under the
+    host that really serves it. A custom mirror pointing both at the same host
+    merges into a single key.
     """
-    import re
+    from edgar.config import SEC_BASE_URL, SEC_DATA_URL
 
-    from edgar.config import SEC_BASE_URL
+    base_domain = _host_key(SEC_BASE_URL)
+    data_domain = _host_key(SEC_DATA_URL)
 
-    # Extract domain pattern from base URL (e.g., "sec.gov" or "mysite.com")
-    domain_match = re.match(r'https?://([^/]+)', SEC_BASE_URL)
-    if domain_match:
-        domain = domain_match.group(1).replace('.', r'\.')
-    else:
-        domain = r'.*\.sec\.gov'  # Fallback to default
+    base_rules = {
+        r"/include/ticker\.txt.*": MAX_SUBMISSIONS_AGE_SECONDS,
+        r"/files/company_tickers\.json.*": MAX_SUBMISSIONS_AGE_SECONDS,
+        ".*index/.*": MAX_INDEX_AGE_SECONDS,
+        "/Archives/edgar/data": True,  # cache forever
+    }
+    data_rules = {
+        "/submissions.*": MAX_SUBMISSIONS_AGE_SECONDS,
+        # companyfacts is invalidated by the same event as /submissions (a new
+        # filing lands), so it takes the same freshness budget. Issue #471
+        # deliberately tuned that budget down to 30s because a longer TTL
+        # delayed visibility of same-day 8-Ks; a companyfacts-specific longer
+        # TTL would reintroduce that staleness for a sibling endpoint.
+        r"/api/xbrl/companyfacts/.*": MAX_SUBMISSIONS_AGE_SECONDS,
+    }
 
+    if base_domain == data_domain:
+        return {base_domain: {**base_rules, **data_rules}}
     return {
-        f".*{domain}": {
-            "/submissions.*": MAX_SUBMISSIONS_AGE_SECONDS,
-            r"/include/ticker\.txt.*": MAX_SUBMISSIONS_AGE_SECONDS,
-            r"/files/company_tickers\.json.*": MAX_SUBMISSIONS_AGE_SECONDS,
-            ".*index/.*": MAX_INDEX_AGE_SECONDS,
-            "/Archives/edgar/data": True,  # cache forever
-        }
+        base_domain: base_rules,
+        data_domain: data_rules,
     }
 
 # Cache rules evaluated at module load time

pyproject.toml

@@ -61,7 +61,11 @@ dependencies = [
   # 0.5.0 briefly hard-required the httpx2 fork — which is why this was capped
   # below it — and 0.6.0 reversed that, so the cap is no longer needed to stay on
   # plain httpx. Migrating to httpx2 is tracked separately (edgartools-q2iz).
-  "httpxthrottlecache[httpx]>=0.6.0",
+  # The 0.6.1 floor is load-bearing: caching data.sec.gov reaches an age check
+  # that raised uncaught on a machine whose clock trails the origin's Date header
+  # (paultiq/httpxthrottlecache#43), taking down the whole get_facts() call.
+  # 0.6.1 clamps it.
+  "httpxthrottlecache[httpx]>=0.6.1",
   "truststore>=0.9.0",
 ]
 dynamic = ["version"]