From 0a20988f7ed350b598db4c93247b7c5a51ea2b85 Mon Sep 17 00:00:00 2001 From: Sam Welborn Date: Fri, 17 Jul 2026 15:05:01 -0400 Subject: [PATCH] use historical=True as a fallback if job not found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normal way to get jobs uses squeue, and our IRI api didn’t have an implemented way of using `historical=True` now this is implemented as of https://github.com/slaclab/iri-facility-api-python/pull/15 we add this here as a fallback so jobs can resolve again --- src/lclstream_api/v2/clients/iri.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/lclstream_api/v2/clients/iri.py b/src/lclstream_api/v2/clients/iri.py index c5f0336..d797104 100644 --- a/src/lclstream_api/v2/clients/iri.py +++ b/src/lclstream_api/v2/clients/iri.py @@ -91,6 +91,17 @@ def modified_at(self) -> AwareDatetime | None: return self.stat.modified_at if self.stat else None +# this is silly but we have to do it because amsc client +# doesn't raise a 404 or a notfound error, it just returns this and assumes things +# are complete +_ASSUMED_COMPLETED_SNIPPET = "assumed completed" + + +def _is_assumed_completed(job: Any) -> bool: + message = job.message + return message is not None and _ASSUMED_COMPLETED_SNIPPET in message.lower() + + # amscrot's status() returns AmSCROT JobState strings (upper case). Map them # back to the canonical amsc_iri JobState the core consumes; an unmapped/UNKNOWN # state means the job is gone (reads as None). @@ -145,12 +156,16 @@ def _submit(self, jobspec: JobSpec, token: str) -> JobId: return job.id def _get(self, job_id: JobId, token: str) -> JobState | None: - # TODO: eventually we should use historical=True when that is - # supported by IRI. Right now it is unsupported in s3df IRI - # related to avoiding hammering slurmdb + # historical=False is the common case (cheap, avoids hammering + # slurmdb). Only fall back to historical=True when the job has + # fallen out of the active queue and amscrot merely *guessed* + # COMPLETED — a real reconciliation loop needs the actual terminal + # state (e.g. FAILED), not that guess. try: job = self._resource(token).job(job_id) state = job.refresh(historical=False) + if _is_assumed_completed(job): + state = job.refresh(historical=True) except Exception as exc: _raise_operation_error(exc, "job status") if state == "UNKNOWN" and job.message: