Repository navigation
Expand file tree
/
Copy pathslurm_queue.py
More file actions
155 lines (133 loc) · 5.51 KB
/
Copy pathslurm_queue.py
File metadata and controls
155 lines (133 loc) · 5.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
"""
Perlmutter SLURM queue status.
Polls `sacct` on the HPC cluster via the NERSC Superfacility API and
exposes it as JSON (/queue-data) plus a dashboard page (/queue).
"""
import json
import logging
import time
from datetime import date, datetime, timedelta
from zoneinfo import ZoneInfo
import jq
import litestar.status_codes as status_code
from authlib.integrations.requests_client import OAuth2Session
from authlib.oauth2.rfc7523 import PrivateKeyJWT
from litestar import Request, Router, Response, get
from litestar.datastructures import State
from litestar.response import Template
from auth import require_login, session_user_context
from config import config
from utils import read_file_content
def filter_sacct(data: dict) -> dict:
jq_filter = """
.jobs[] | {
jobid: .job_id,
jobname: .name,
account: .account,
user: .user,
state: .state.current[0],
start: .time.start,
elapsed: .time.elapsed,
timelimit: (.time.limit.number * 60)
}
"""
jobs = jq.all(jq_filter, data)
for job in jobs:
job["start"] = datetime.fromtimestamp(job["start"], tz=ZoneInfo(config.app.tzinfo)).strftime(
"%Y-%m-%d %H:%M:%S %Z"
)
job["elapsed"] = str(timedelta(seconds=job["elapsed"]))
job["timelimit"] = str(timedelta(seconds=job["timelimit"]))
total_jobs = len(jobs)
running_jobs = sum(1 for job in jobs if job["state"] == "RUNNING")
completed_jobs = sum(1 for job in jobs if job["state"] == "COMPLETED")
failed_jobs = sum(1 for job in jobs if job["state"] == "FAILED")
pending_jobs = sum(1 for job in jobs if job["state"] == "PENDING")
timeout_jobs = sum(1 for job in jobs if job["state"] == "TIMEOUT")
cancelled_jobs = sum(1 for job in jobs if job["state"] == "CANCELLED")
return {
"jobs": jobs,
"total_jobs": total_jobs,
"running_jobs": running_jobs,
"completed_jobs": completed_jobs,
"failed_jobs": failed_jobs,
"pending_jobs": pending_jobs,
"timeout_jobs": timeout_jobs,
"cancelled_jobs": cancelled_jobs,
}
# cache is in seconds; need to write a custom filter to only cache on successful responses
@get("/queue-data", cache=180, guards=[require_login])
async def get_queue_info(days: int = 1) -> Response:
wait_time = 15 # seconds to wait in between polling the SF API task
num_attempts = 4 # total tries to get the data before giving up
cluster = config.hpc.admission_conf["clusters"][config.hpc.cluster]
client_id = read_file_content(cluster["client_id"]).strip()
private_key = read_file_content(cluster["private_key"])
# Authenticate session for SF-API
logging.info(f"Running on {config.hpc.cluster} as {cluster['account']}")
logging.info(f"CLIENT_ID = {client_id}")
logging.info(f"TOKEN_URL = {config.hpc.token_url}")
session = OAuth2Session(
client_id,
private_key,
PrivateKeyJWT(config.hpc.token_url),
grant_type="client_credentials",
token_endpoint=config.hpc.token_url,
)
session.fetch_token()
# Build command to get SLURM queue via sacct
start_date = date.today() - timedelta(days=days)
cmd = f'bash -c "sacct -a -X -A dune,dune_g --json -S {start_date}"'
try:
# Run sacct on Perlmutter
logging.info(f"Running sacct on {config.hpc.cluster}")
r = session.post(
"https://api.nersc.gov/api/v1.2/utilities/command/perlmutter", data={"executable": cmd}
)
r.raise_for_status()
logging.info(f"Superfacility API status: {r.json()}")
post_output = r.json()
# It takes some time to actually run the command on Perlmutter and have the output available
# Delay the task retrieval and try a finite number of attempts before giving up
time.sleep(wait_time)
logging.info("Getting task output.")
data = {}
data["jobs"] = []
http_status_code = status_code.HTTP_504_GATEWAY_TIMEOUT
for i in range(num_attempts):
logging.info(f"Attempt {i}")
r = session.get(f"https://api.nersc.gov/api/v1.2/tasks/{post_output['task_id']}")
r.raise_for_status()
get_output = r.json()
# Extract `sacct` output from the SF API task payload
if get_output["result"] and get_output["status"] != "failed":
task_output = json.loads(get_output["result"])
task_output = json.loads(task_output["output"])
data["jobs"] = task_output["jobs"]
http_status_code = status_code.HTTP_200_OK
break
else:
time.sleep(wait_time)
filtered = filter_sacct(data)
return Response(
content=filtered,
status_code=http_status_code,
)
except Exception as e:
logging.error(f"An error occurred accessing SF API: {e}")
return Response(content={"error": str(e)}, status_code=status_code.HTTP_400_BAD_REQUEST)
@get("/queue", guards=[require_login])
async def display_queue(request: Request, state: State, days: int = 1) -> Template:
"""
Display Perlmutter job queue. Serves an initially empty webpage
with loading animation. A JS script inside the HTML performs an
API call to get the queue info and dynamically update the page.
"""
return Template(
template_name="slurm_queue.html",
context={
"request": request,
**session_user_context(request),
}
)
queue_router = Router(path="/", route_handlers=[display_queue, get_queue_info])