Feature/measurement parser

backend/apps/devices/migrations/0002_device_fe_language.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("devices", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="device",
+            name="fe_language",
+            field=models.CharField(blank=True, default="cs", max_length=5),
+        ),
+    ]

backend/apps/devices/models.py

@@ -14,6 +14,10 @@ class Device(models.Model):
     public_key = models.TextField()  # base64, raw Ed25519 public key (32 bytes)
     label = models.CharField(max_length=120, blank=True, default="")
 
+    # Preferred web-frontend language for this device (a per-device setting,
+    # stored server-side so it follows the device across browsers).
+    fe_language = models.CharField(max_length=5, blank=True, default="cs")
+
     created_at = models.DateTimeField(auto_now_add=True)
     last_seen_at = models.DateTimeField(null=True, blank=True)
     revoked_at = models.DateTimeField(null=True, blank=True)

backend/apps/ingest/management/commands/parse_pending.py

@@ -0,0 +1,44 @@
+"""Parse raw ingests into ParsedMeasurement rows (parse-later worker).
+
+Run on a schedule or after a deploy. Picks up rows that have never been parsed
+or whose parse_version is stale, so bumping PARSE_VERSION re-derives everything.
+
+    python manage.py parse_pending            # parse new / stale rows
+    python manage.py parse_pending --all      # re-parse every row
+    python manage.py parse_pending --limit 500
+"""
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from apps.ingest.models import RawIngest
+from apps.ingest.parser import PARSE_VERSION, ParseError, parse_and_store
+
+
+class Command(BaseCommand):
+    help = "Parse raw ingests into render-ready ParsedMeasurement rows."
+
+    def add_arguments(self, parser):
+        parser.add_argument("--all", action="store_true", help="Re-parse every row.")
+        parser.add_argument("--limit", type=int, default=1000, help="Max rows per run.")
+
+    def handle(self, *args, **options):
+        qs = RawIngest.objects.all().order_by("received_at")
+        if not options["all"]:
+            # Never parsed, or parsed by an older version.
+            qs = qs.filter(
+                Q(parsed__isnull=True) | ~Q(parsed__parse_version=PARSE_VERSION)
+            )
+        rows = list(qs[: options["limit"]])
+
+        ok = failed = 0
+        for raw in rows:
+            try:
+                parse_and_store(raw)
+                ok += 1
+            except ParseError as exc:
+                failed += 1
+                self.stderr.write(f"  failed {raw.id}: {exc}")
+
+        self.stdout.write(
+            self.style.SUCCESS(f"Parsed {ok} row(s), {failed} failed, {len(rows)} seen.")
+        )

backend/apps/ingest/migrations/0002_parsedmeasurement.py

@@ -0,0 +1,34 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ingest", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ParsedMeasurement",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("parse_version", models.PositiveIntegerField(default=0)),
+                ("event_time", models.DateTimeField(blank=True, null=True)),
+                ("event_tz", models.CharField(blank=True, default="", max_length=64)),
+                ("start_alt", models.FloatField(blank=True, null=True)),
+                ("start_az", models.FloatField(blank=True, null=True)),
+                ("end_alt", models.FloatField(blank=True, null=True)),
+                ("end_az", models.FloatField(blank=True, null=True)),
+                ("start_ra", models.FloatField(blank=True, null=True)),
+                ("start_dec", models.FloatField(blank=True, null=True)),
+                ("end_ra", models.FloatField(blank=True, null=True)),
+                ("end_dec", models.FloatField(blank=True, null=True)),
+                ("lat", models.FloatField(blank=True, null=True)),
+                ("lon", models.FloatField(blank=True, null=True)),
+                ("accuracy", models.FloatField(blank=True, null=True)),
+                ("quality", models.FloatField(blank=True, null=True)),
+                ("parsed_at", models.DateTimeField(auto_now=True)),
+                ("raw", models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name="parsed", to="ingest.rawingest")),
+            ],
+        ),
+    ]

backend/apps/ingest/models.py

@@ -41,3 +41,42 @@ class Meta:
         ]
         indexes = [models.Index(fields=["status", "received_at"])]
         ordering = ["-received_at"]
+
+
+class ParsedMeasurement(models.Model):
+    """Render-ready record derived from a RawIngest (the parse-later output).
+
+    One row per raw measurement. Holds the validated horizontal coordinates of
+    the trail's two aim points, the derived equatorial coordinates (RA/Dec) for
+    plotting on a star map, the absolute UTC event time, the observing site, and
+    the site's IANA time zone. ``parse_version`` lets stored rows be re-derived
+    when the parsing logic changes.
+    """
+
+    raw = models.OneToOneField(
+        RawIngest, on_delete=models.CASCADE, related_name="parsed"
+    )
+    parse_version = models.PositiveIntegerField(default=0)
+
+    event_time = models.DateTimeField(null=True, blank=True)  # absolute UTC
+    event_tz = models.CharField(max_length=64, blank=True, default="")  # IANA name
+
+    start_alt = models.FloatField(null=True, blank=True)
+    start_az = models.FloatField(null=True, blank=True)
+    end_alt = models.FloatField(null=True, blank=True)
+    end_az = models.FloatField(null=True, blank=True)
+
+    start_ra = models.FloatField(null=True, blank=True)
+    start_dec = models.FloatField(null=True, blank=True)
+    end_ra = models.FloatField(null=True, blank=True)
+    end_dec = models.FloatField(null=True, blank=True)
+
+    lat = models.FloatField(null=True, blank=True)
+    lon = models.FloatField(null=True, blank=True)
+    accuracy = models.FloatField(null=True, blank=True)
+    quality = models.FloatField(null=True, blank=True)
+
+    parsed_at = models.DateTimeField(auto_now=True)
+
+    def __str__(self) -> str:
+        return f"ParsedMeasurement(raw={self.raw_id}, v{self.parse_version})"

backend/apps/ingest/parser.py

@@ -0,0 +1,220 @@
+"""Parse raw measurement payloads into render-ready scientific records.
+
+Follows the ingest-first / parse-later contract: an upload only lands a
+``RawIngest`` row, and turning that into a ``ParsedMeasurement`` happens here,
+separately. A parse failure is recorded on the raw row (status=failed) and
+never touches or discards the raw payload, so it can be re-parsed later.
+
+The mobile app reports a meteor as two horizontal-coordinate aim points
+(altitude/azimuth) plus the observing site and the event time. For the sky
+view we also derive equatorial coordinates (RA/Dec) so any star map can place
+the trail among the stars. The phone's orientation accuracy is degree-level,
+so a compact closed-form alt/az -> RA/Dec conversion (arc-minute accuracy) is
+already far more precise than the input -- no heavy astrometry dependency is
+warranted.
+
+The event time arrives as ``Date.now()`` epoch milliseconds, which is an
+absolute UTC instant already (not a local wall-clock time). From the GPS site
+we additionally resolve the IANA time zone, so the UI can show the observer's
+*local* civil time at the observing site regardless of who is viewing it.
+"""
+from __future__ import annotations
+
+import math
+from datetime import UTC, datetime
+
+# Bump when the parsing logic changes so stored records can be re-derived.
+PARSE_VERSION = 1
+
+
+class ParseError(ValueError):
+    """Payload could not be parsed; recorded on the raw row, raw kept intact."""
+
+
+def _num(value):
+    if value is None:
+        return None
+    try:
+        f = float(value)
+    except (TypeError, ValueError):
+        return None
+    return f if math.isfinite(f) else None
+
+
+def _angle(value, lo, hi, name):
+    f = _num(value)
+    if f is None:
+        return None
+    if not (lo <= f <= hi):
+        raise ParseError(f"{name} out of range [{lo}, {hi}]: {f}")
+    return f
+
+
+def _parse_time(value):
+    if value in (None, ""):
+        return None
+    if isinstance(value, int | float):
+        # epoch seconds, or milliseconds when the value is implausibly large
+        secs = value / 1000.0 if value > 1e11 else float(value)
+        return datetime.fromtimestamp(secs, tz=UTC)
+    try:
+        dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError as exc:
+        raise ParseError(f"unparseable eventTimestamp: {value!r}") from exc
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=UTC)
+    return dt.astimezone(UTC)
+
+
+def parse_payload(payload: dict) -> dict:
+    """Extract and validate the render fields from a raw payload.
+
+    Raises ParseError on malformed data or a missing trail endpoint.
+    """
+    if not isinstance(payload, dict):
+        raise ParseError("payload is not an object")
+    start = payload.get("startPoint") or {}
+    end = payload.get("endPoint") or {}
+    site = payload.get("site") or {}
+
+    out = {
+        "event_time": _parse_time(payload.get("eventTimestamp")),
+        "start_alt": _angle(start.get("alt"), -90, 90, "start.alt"),
+        "start_az": _angle(start.get("az"), 0, 360, "start.az"),
+        "end_alt": _angle(end.get("alt"), -90, 90, "end.alt"),
+        "end_az": _angle(end.get("az"), 0, 360, "end.az"),
+        "lat": _angle(site.get("lat"), -90, 90, "site.lat"),
+        "lon": _angle(site.get("lon"), -180, 180, "site.lon"),
+        "accuracy": _num(site.get("accuracy")),
+        "quality": _num(payload.get("quality")),
+    }
+    # A trail needs both aim points; without them there is nothing to render.
+    for key in ("start_alt", "start_az", "end_alt", "end_az"):
+        if out[key] is None:
+            raise ParseError(f"missing required field: {key}")
+    return out
+
+
+def _julian_date(dt: datetime) -> float:
+    """Julian Date from a (tz-aware) datetime via the civil-calendar formula."""
+    dt = dt.astimezone(UTC)
+    year, month = dt.year, dt.month
+    if month <= 2:
+        year -= 1
+        month += 12
+    a = year // 100
+    b = 2 - a + a // 4
+    day = dt.day + (dt.hour + (dt.minute + (dt.second + dt.microsecond / 1e6) / 60) / 60) / 24
+    return math.floor(365.25 * (year + 4716)) + math.floor(30.6001 * (month + 1)) + day + b - 1524.5
+
+
+def altaz_to_radec(alt_deg, az_deg, lat_deg, lon_deg, when):
+    """Closed-form horizontal -> equatorial conversion, RA/Dec in degrees.
+
+    Azimuth is measured from North, increasing eastward (compass convention).
+    Uses mean sidereal time (IAU 1982); arc-minute accurate, which comfortably
+    exceeds the phone-orientation accuracy of the inputs. Returns ``(ra, dec)``
+    or ``None`` when location or time are unavailable.
+    """
+    if None in (alt_deg, az_deg, lat_deg, lon_deg) or when is None:
+        return None
+    alt = math.radians(alt_deg)
+    az = math.radians(az_deg)
+    lat = math.radians(lat_deg)
+
+    sin_dec = math.sin(alt) * math.sin(lat) + math.cos(alt) * math.cos(lat) * math.cos(az)
+    sin_dec = max(-1.0, min(1.0, sin_dec))
+    dec = math.asin(sin_dec)
+
+    # Hour angle via atan2 with both terms scaled by (cos dec * cos lat) >= 0,
+    # so there is no division and no singularity at the poles.
+    num = -math.sin(az) * math.cos(alt) * math.cos(lat)
+    den = math.sin(alt) - math.sin(lat) * sin_dec
+    hour_angle = math.degrees(math.atan2(num, den))
+
+    d = _julian_date(when) - 2451545.0
+    gmst = (280.46061837 + 360.98564736629 * d) % 360.0  # Greenwich mean sidereal time
+    lst = (gmst + lon_deg) % 360.0                        # local sidereal time
+    ra = (lst - hour_angle) % 360.0
+    return ra, math.degrees(dec)
+
+
+def site_timezone(lat, lon):
+    """IANA time-zone name for a GPS location, or None if unavailable.
+
+    Uses timezonefinder (offline polygon lookup); imported lazily so the pure
+    parsing/maths above stay dependency-free and unit-testable.
+    """
+    if lat is None or lon is None:
+        return None
+    finder = _tz_finder()
+    return finder.timezone_at(lat=lat, lng=lon) if finder else None
+
+
+_TZF = None
+
+
+def _tz_finder():
+    global _TZF
+    if _TZF is None:
+        try:
+            from timezonefinder import TimezoneFinder
+        except ImportError:
+            return None
+        _TZF = TimezoneFinder()  # loads bundled boundary data once
+    return _TZF
+
+
+def parse_and_store(raw):
+    """Parse a RawIngest, persist a ParsedMeasurement, and update raw.status.
+
+    On ParseError the raw row is marked failed (with the reason) and the error
+    re-raised; the raw payload is left untouched.
+    """
+    from django.utils import timezone as djtz
+
+    from .models import ParsedMeasurement, RawIngest
+
+    try:
+        fields = parse_payload(raw.payload or {})
+    except ParseError as exc:
+        raw.status = RawIngest.STATUS_FAILED
+        raw.error = str(exc)
+        raw.attempts = (raw.attempts or 0) + 1
+        raw.processed_at = djtz.now()
+        raw.save(update_fields=["status", "error", "attempts", "processed_at"])
+        raise
+
+    loc = (fields["lat"], fields["lon"], fields["event_time"])
+    start = altaz_to_radec(fields["start_alt"], fields["start_az"], *loc)
+    end = altaz_to_radec(fields["end_alt"], fields["end_az"], *loc)
+    data = dict(fields)
+    data["start_ra"], data["start_dec"] = start or (None, None)
+    data["end_ra"], data["end_dec"] = end or (None, None)
+    # UTC is already absolute (epoch ms); resolve the site's civil time zone so
+    # the observer's local time can be shown alongside it.
+    data["event_tz"] = site_timezone(fields["lat"], fields["lon"]) or ""
+
+    parsed, _ = ParsedMeasurement.objects.update_or_create(
+        raw=raw, defaults={**data, "parse_version": PARSE_VERSION}
+    )
+    raw.status = RawIngest.STATUS_PROCESSED
+    raw.error = ""
+    raw.attempts = (raw.attempts or 0) + 1
+    raw.processed_at = djtz.now()
+    raw.save(update_fields=["status", "error", "attempts", "processed_at"])
+    return parsed
+
+
+def ensure_parsed(raw):
+    """Return an up-to-date ParsedMeasurement, parsing lazily if needed.
+
+    Returns None if the payload cannot be parsed (raw is marked failed).
+    """
+    parsed = getattr(raw, "parsed", None)
+    if parsed is not None and parsed.parse_version == PARSE_VERSION:
+        return parsed
+    try:
+        return parse_and_store(raw)
+    except ParseError:
+        return None