Repository navigation
Expand file tree
/
Copy pathwait_ready
More file actions
executable file
·249 lines (196 loc) · 9.75 KB
/
Copy pathwait_ready
File metadata and controls
executable file
·249 lines (196 loc) · 9.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
#!/usr/bin/env python3
"""Block until the lab worker is online and ready to execute a function block.
`docker compose up -d` returns once containers are *started*, not once the
worker inside them has connected to the engine and registered its function
blocks. Running a workflow before that races the worker and fails partway
through with:
failed_safe_ack — Function block with id fb.lab.neops.io/...:1.0.0 not found
(The sibling race — the *workflow definition* not yet being registered by
lab_bootstrap — surfaces earlier as "Workflow with ID null not found" and is
handled by `docker compose wait lab_bootstrap`.)
This polls the engine's per-function-block worker registry until at least one
`online` worker is registered for every requested function block, so callers
like `make local-lab-discover` can run deterministically instead of racing.
With `--zone`, a block is ready only once every named network zone has an
online worker serving it: one whose `networkZones` names that zone. Engines
from 0.46, the lab's pin, name `default` for a worker that registered no zones,
so an empty list serves nothing.
With `--workers`, the number of workers the lab starts, it stops at once when
the engine's license admits fewer: the engine refuses the rest, so waiting for
them could only time out.
Usage
-----
./wait_ready fb.base.neops.io/global_discover_network:0.2.0 [more fb ids ...]
./wait_ready --zone default --zone zone-a fb.base.neops.io/global_discover_network:0.2.0
./wait_ready --workers 3 --zone default --zone zone-a --zone zone-b fb.base.neops.io/global_discover_network:0.2.0
Exit codes
----------
0 every requested function block has an online worker
2 timed out before all function blocks became ready
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from urllib import error, request
import engine_http
def parse_fb_id(fb_id: str) -> tuple[str, str, str]:
"""Split `pkg/name:version` into (pkg, name, version) for the REST path."""
try:
pkg_name, version = fb_id.rsplit(":", 1)
pkg, name = pkg_name.rsplit("/", 1)
except ValueError as exc:
raise SystemExit(f"invalid function block id {fb_id!r}; expected 'pkg/name:version'") from exc
return pkg, name, version
def _get_json(url: str) -> object:
"""GET a JSON body from the engine; None while it is unreachable or answers with an error.
A 401 or 403 raises SystemExit, which ends the wait at once.
"""
req = request.Request(url, headers=engine_http.auth_headers())
try:
with request.urlopen(req, timeout=5) as resp:
return json.load(resp)
except error.HTTPError as exc:
if exc.code in engine_http.UNAUTHORIZED:
raise SystemExit(f"the engine refused the request {engine_http.refusal(exc)}") from exc
return None
except engine_http.REQUEST_ERRORS:
return None
def _get_list(url: str) -> list | None:
body = _get_json(url)
return body if isinstance(body, list) else None
def _get_dict(url: str) -> dict | None:
body = _get_json(url)
return body if isinstance(body, dict) else None
def block_workers(engine_url: str, fb_id: str) -> list | None:
"""Return every worker registered for a function block, with its state and network zones.
Returns None while the engine is unreachable or no worker has registered
the block; both keep the caller polling. (The engine learns about a block
when a worker registers it, so a 404 also reads as "no worker yet".)
"""
pkg, name, version = parse_fb_id(fb_id)
return _get_list(f"{engine_url}/function-blocks/{pkg}/{name}/{version}/workers")
def worker_states(engine_url: str, fb_id: str) -> list[str] | None:
"""Return the state of every worker registered for a function block (None while unknown)."""
workers = block_workers(engine_url, fb_id)
if workers is None:
return None
return [str(w.get("state", "?")) for w in workers]
def serves(worker: dict, zone: str) -> bool:
"""Whether a worker claims jobs in a zone."""
return zone in (worker.get("networkZones") or [])
def zones_ready(workers: list, zones: list) -> set:
"""The zones in which one of a block's workers is online."""
online = [w for w in workers if w.get("state") == "online"]
return {zone for zone in zones if any(serves(w, zone) for w in online)}
def online_worker_count(engine_url: str, fb_id: str) -> int | None:
"""Return the number of online workers for a function block (None while unknown)."""
states = worker_states(engine_url, fb_id)
if states is None:
return None
return sum(1 for s in states if s == "online")
WORKER_CAP_REFUSAL = """\
this lab starts {workers} workers, but the engine's {license} license admits {cap}, so it refuses the rest.
Run the licensed engine. A scenario whose scenario.json names "engine": "enterprise"
pulls quay.io/zebbra/neops-workflow-engine, which needs `docker login quay.io`,
unless NEOPS_WORKFLOW_ENGINE_IMAGE (in .env or the environment) names another
image: unset it, or point it at a licensed engine.\
"""
def worker_cap_refusal(engine_license: dict, workers: int) -> str | None:
"""Why an engine with this license cannot serve `workers` workers; None when it can."""
cap = (engine_license.get("limits") or {}).get("maxWorkers")
if not isinstance(cap, int) or cap >= workers:
return None
name = engine_license.get("displayName") or engine_license.get("flavor") or "unnamed"
return WORKER_CAP_REFUSAL.format(workers=workers, license=name, cap=cap)
def engine_license(engine_url: str) -> dict | None:
"""Return the engine's license (None while unknown)."""
return _get_dict(f"{engine_url}/license")
TIMEOUT_HINT = """\
Likely causes, most common first:
* the worker container is stopped or crash-looping: `make local-lab-logs`
shows its last words;
* the worker image lacks the block, most often because `.env` pins an older
worker than `.env.example` (`./check_env_pins` names it); see what the image
carries with `docker run --rm --entrypoint ls <NEOPS_WORKER_SDK_IMAGE> /app/neops/fb`;
* the worker is still starting: re-run, or raise WAIT_READY_TIMEOUT.\
"""
ZONE_HINT = """\
A zone that never becomes ready:
* its worker exited: `make local-lab-logs SCENARIO=<scenario>` shows every
worker's last words. It is refused while the CMS holds no such zone
(apply_cms_config seeds them), and a worker or engine image older than
network zones never serves one.\
"""
def ready_now(engine_url: str, targets: list) -> dict:
"""Map each `(fb_id, zone)` target that is ready now to how; a target without a zone needs any online worker."""
ready = {}
for fb_id, zone in targets:
if zone is None:
count = online_worker_count(engine_url, fb_id)
if count:
ready[(fb_id, None)] = f"{count} online worker(s)"
zoned = [target for target in targets if target[1] is not None]
for fb_id in dict.fromkeys(fb_id for fb_id, _ in zoned):
served = zones_ready(block_workers(engine_url, fb_id) or [], [zone for block, zone in zoned if block == fb_id])
ready.update({(fb_id, zone): f"zone {zone}" for zone in served})
return ready
def _label(target: tuple) -> str:
fb_id, zone = target
return fb_id if zone is None else f"{fb_id} in zone {zone}"
def _report_timeout(engine_url: str, pending: list, timeout: int) -> None:
print(f"timeout: no online worker for {', '.join(_label(t) for t in pending)} within {timeout}s", file=sys.stderr)
for fb_id in dict.fromkeys(fb_id for fb_id, _ in pending):
states = worker_states(engine_url, fb_id)
if states:
print(f" {fb_id}: registered worker states: {', '.join(states)}", file=sys.stderr)
print(TIMEOUT_HINT, file=sys.stderr)
if any(zone is not None for _, zone in pending):
print(ZONE_HINT, file=sys.stderr)
def main() -> int:
p = argparse.ArgumentParser(
prog="wait_ready",
description="Block until an online worker is registered for each function block.",
)
p.add_argument("function_blocks", nargs="+", help="Function block ids: pkg/name:version")
engine_http.add_engine_url_argument(p)
p.add_argument(
"--zone",
action="append",
dest="zones",
help="Network zone that needs an online worker for each block; repeat for several",
)
p.add_argument(
"--workers",
type=int,
help="Number of workers the lab starts; stop at once when the engine's license admits fewer",
)
p.add_argument("--timeout", type=int, default=180, help="Total wait budget in seconds (default: 180)")
p.add_argument("--interval", type=float, default=2.0, help="Poll interval in seconds (default: 2.0)")
args = p.parse_args()
zones = args.zones or [None]
pending = [(fb_id, zone) for fb_id in args.function_blocks for zone in zones]
deadline = time.monotonic() + args.timeout
started = time.monotonic()
license_checked = not args.workers
while pending:
if not license_checked:
known = engine_license(args.engine_url)
refusal = worker_cap_refusal(known, args.workers) if known is not None else None
if refusal:
raise SystemExit(refusal)
license_checked = known is not None
ready = ready_now(args.engine_url, pending)
for target, how in ready.items():
print(f" [{int(time.monotonic() - started):3d}s] ready: {target[0]} ({how})")
pending = [target for target in pending if target not in ready]
if not pending:
return 0
if time.monotonic() >= deadline:
_report_timeout(args.engine_url, pending, args.timeout)
return 2
time.sleep(args.interval)
return 0
if __name__ == "__main__":
sys.exit(main())