Repository navigation
Expand file tree
/
Copy pathconfig.py
More file actions
285 lines (227 loc) · 9.34 KB
/
Copy pathconfig.py
File metadata and controls
285 lines (227 loc) · 9.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
"""
Centralized application configuration.
Reads every environment variable the app depends on exactly once at
startup, validates what it can, and exposes a single frozen, typed
`Config` instance (`config`) for the rest of the application to import.
Usage:
from config import config
print(config.mongodb.url)
print(config.globus.client_id) # secret file already read & cached
"""
from __future__ import annotations
import logging
import os
from dataclasses import dataclass, field
from functools import lru_cache
import yaml
class ConfigError(RuntimeError):
"""Raised when required configuration is missing or invalid."""
def _read_secret_file(file_path: str) -> str:
"""Read a secret value from a file, stripped of surrounding whitespace.
Returns an empty string (and logs) if the path is unset or unreadable,
so callers/validators can decide whether that's fatal.
"""
if not file_path:
return ""
try:
with open(file_path, "r") as f:
return f.read().strip()
except OSError as e:
logging.error(f"Error reading secret file {file_path}: {e}")
return ""
def _read_yaml_file(file_path: str) -> dict:
"""Read and parse a YAML config file, e.g. the job-admission config.
Returns an empty dict (and logs) if the path is unset, unreadable, or
doesn't parse, so validate() can decide whether that's fatal.
"""
if not file_path:
return {}
try:
with open(file_path, "r") as f:
return yaml.safe_load(f) or {}
except (OSError, yaml.YAMLError) as e:
logging.error(f"Error reading YAML config {file_path}: {e}")
return {}
def _getenv_bool(name: str, default: bool = False) -> bool:
"""Parse a boolean-ish environment variable (1/true/yes/on)."""
raw = os.getenv(name)
if raw is None or raw == "":
return default
return raw.strip().lower() in ("1", "true", "yes", "on")
def _getenv_int(name: str, default: int) -> int:
raw = os.getenv(name)
if raw is None or raw == "":
return default
try:
return int(raw)
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}")
# --------------------------------------------------------------------------
# Sub-configs, grouped by concern
# --------------------------------------------------------------------------
@dataclass(frozen=True)
class MongoConfig:
url: str
db: str
user: str
password: str
@dataclass(frozen=True)
class GlobusConfig:
"""Globus Auth (OIDC) settings used by auth.py."""
client_id_file: str
client_secret_file: str
redirect_uri: str
auth_base: str
nersc_identity_provider_id: str
# Secrets are read once here (at Config construction time) rather than
# on every request, matching the "read once at startup" contract.
client_id: str = field(repr=False)
client_secret: str = field(repr=False)
@dataclass(frozen=True)
class SessionConfig:
"""Cookie/session policy shared by auth.py and the Litestar session middleware."""
secret_file: str
secret: bytes = field(repr=False)
max_age_seconds: int # hard cap on session lifetime regardless of activity
idle_timeout_seconds: int # session invalidated after this much inactivity
@dataclass(frozen=True)
class WebhookConfig:
secret_file: str
secret: str = field(repr=False)
verify_ip: bool
@dataclass(frozen=True)
class HPCConfig:
cluster: str
token_url: str
admission_conf_file: str
# Parsed contents of admission_conf_file (cluster credentials, the
# per-repo/branch admission policy, and authorized IP ranges). Read once
# here so every module that needs cluster credentials (webhook admission
# in app.py, the Slurm queue in queue.py) shares one copy.
admission_conf: dict = field(repr=False)
@dataclass(frozen=True)
class AppConfig:
tzinfo: str
# --------------------------------------------------------------------------
# Top-level config
# --------------------------------------------------------------------------
@dataclass(frozen=True)
class Config:
mongodb: MongoConfig
globus: GlobusConfig
session: SessionConfig
webhook: WebhookConfig
hpc: HPCConfig
app: AppConfig
@classmethod
def from_env(cls) -> "Config":
mongodb = MongoConfig(
url=os.getenv("MONGODB_URL", "mongodb://localhost:27017"),
db=os.getenv("MONGODB_DB", "github_webhooks"),
user=os.getenv("MONGODB_USER", ""),
password=os.getenv("MONGODB_PASS", ""),
)
globus_client_id_file = os.getenv("GLOBUS_CLIENT_ID_FILE", "")
globus_client_secret_file = os.getenv("GLOBUS_CLIENT_SECRET_FILE", "")
globus = GlobusConfig(
client_id_file=globus_client_id_file,
client_secret_file=globus_client_secret_file,
redirect_uri=os.getenv("GLOBUS_REDIRECT_URI", "https://localhost:8000/auth/callback"),
auth_base="https://auth.globus.org",
nersc_identity_provider_id=os.getenv("NERSC_IDENTITY_PROVIDER_ID", ""),
client_id=_read_secret_file(globus_client_id_file),
client_secret=_read_secret_file(globus_client_secret_file),
)
session_secret_file = os.getenv("SESSION_SECRET_FILE", "")
session_secret = _read_secret_file(session_secret_file)
session = SessionConfig(
secret_file=session_secret_file,
secret=session_secret.encode("utf-8"),
max_age_seconds=_getenv_int("MAX_SESSION_AGE_SECONDS", 8 * 60 * 60),
idle_timeout_seconds=_getenv_int("IDLE_TIMEOUT_SECONDS", 30 * 60),
)
webhook_secret_file = os.getenv("WEBHOOK_SECRET", "")
webhook = WebhookConfig(
secret_file=webhook_secret_file,
secret=_read_secret_file(webhook_secret_file),
verify_ip=_getenv_bool("VERIFY_IP", False),
)
admission_conf_file = os.getenv("ADMISSION_CONF_FILE", "configs/admission.yaml")
hpc = HPCConfig(
cluster=os.getenv("HPC_CLUSTER", "perlmutter"),
token_url=os.getenv("TOKEN_URL", "https://oidc.nersc.gov/c2id/token"),
admission_conf_file=admission_conf_file,
admission_conf=_read_yaml_file(admission_conf_file),
)
app = AppConfig(
tzinfo=os.getenv("TZINFO", "US/Pacific"),
)
config = cls(
mongodb=mongodb,
globus=globus,
session=session,
webhook=webhook,
hpc=hpc,
app=app,
)
config.validate()
return config
def validate(self) -> None:
"""Fail loudly at startup rather than surfacing confusing errors later."""
errors: list[str] = []
# Session secret is required for the cookie session backend to be
# usable/stable across restarts and workers.
if not self.session.secret:
errors.append(
"SESSION_SECRET_FILE is not configured or unreadable. "
"Set SESSION_SECRET_FILE to a file containing a random 32+ byte secret."
)
# Globus login can't function without a client id/secret.
if not self.globus.client_id:
errors.append(
"GLOBUS_CLIENT_ID_FILE is not configured or unreadable."
)
if not self.globus.client_secret:
errors.append(
"GLOBUS_CLIENT_SECRET_FILE is not configured or unreadable."
)
if self.session.max_age_seconds <= 0:
errors.append("MAX_SESSION_AGE_SECONDS must be a positive integer.")
if self.session.idle_timeout_seconds <= 0:
errors.append("IDLE_TIMEOUT_SECONDS must be a positive integer.")
if self.session.idle_timeout_seconds > self.session.max_age_seconds:
errors.append(
"IDLE_TIMEOUT_SECONDS should not exceed MAX_SESSION_AGE_SECONDS."
)
if not self.hpc.admission_conf:
errors.append(
f"Admission config file {self.hpc.admission_conf_file!r} is missing, "
"unreadable, or empty."
)
elif "clusters" not in self.hpc.admission_conf:
errors.append("Admission config is missing the required 'clusters' key.")
elif self.hpc.cluster not in self.hpc.admission_conf["clusters"]:
errors.append(
f"Admission config has no entry under 'clusters' for "
f"HPC_CLUSTER={self.hpc.cluster!r}."
)
if not self.webhook.secret:
logging.warning(
"WEBHOOK_SECRET is not configured or unreadable; incoming webhook "
"signatures will NOT be verified."
)
if not self.globus.nersc_identity_provider_id:
logging.warning(
"NERSC_IDENTITY_PROVIDER_ID is not set; any Globus identity "
"will be accepted for login."
)
if errors:
message = "Invalid configuration:\n" + "\n".join(f" - {e}" for e in errors)
raise ConfigError(message)
@lru_cache(maxsize=1)
def get_config() -> Config:
"""Return the process-wide Config singleton, building it on first use."""
return Config.from_env()
# Eagerly construct the singleton at import time so that misconfiguration
# fails fast at startup instead of on the first request that needs it.
config: Config = get_config()