use historical=True as a fallback if job not found

src/lclstream_api/v2/clients/iri.py

@@ -91,6 +91,17 @@ def modified_at(self) -> AwareDatetime | None:
         return self.stat.modified_at if self.stat else None
 
 
+# this is silly but we have to do it because amsc client
+# doesn't raise a 404 or a notfound error, it just returns this and assumes things
+# are complete
+_ASSUMED_COMPLETED_SNIPPET = "assumed completed"
+
+
+def _is_assumed_completed(job: Any) -> bool:
+    message = job.message
+    return message is not None and _ASSUMED_COMPLETED_SNIPPET in message.lower()
+
+
 # amscrot's status() returns AmSCROT JobState strings (upper case). Map them
 # back to the canonical amsc_iri JobState the core consumes; an unmapped/UNKNOWN
 # state means the job is gone (reads as None).
@@ -145,12 +156,16 @@ def _submit(self, jobspec: JobSpec, token: str) -> JobId:
         return job.id
 
     def _get(self, job_id: JobId, token: str) -> JobState | None:
-        # TODO: eventually we should use historical=True when that is
-        # supported by IRI. Right now it is unsupported in s3df IRI
-        # related to avoiding hammering slurmdb
+        # historical=False is the common case (cheap, avoids hammering
+        # slurmdb). Only fall back to historical=True when the job has
+        # fallen out of the active queue and amscrot merely *guessed*
+        # COMPLETED — a real reconciliation loop needs the actual terminal
+        # state (e.g. FAILED), not that guess.
         try:
             job = self._resource(token).job(job_id)
             state = job.refresh(historical=False)
+            if _is_assumed_completed(job):
+                state = job.refresh(historical=True)
         except Exception as exc:
             _raise_operation_error(exc, "job status")
         if state == "UNKNOWN" and job.message: