Repository navigation
Expand file tree
/
Copy pathapp.py
More file actions
117 lines (98 loc) · 3.6 KB
/
Copy pathapp.py
File metadata and controls
117 lines (98 loc) · 3.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import logging
import os
import time
import asyncio
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from transformers import pipeline
from prometheus_fastapi_instrumentator import Instrumentator
from pythonjsonlogger import jsonlogger
logger = logging.getLogger("inframind-model")
logHandler = logging.StreamHandler()
formatter = jsonlogger.JsonFormatter("%(asctime)s %(levelname)s %(name)s %(message)s")
logHandler.setFormatter(formatter)
logger.addHandler(logHandler)
logger.setLevel(logging.INFO)
# 1. Pull dynamic configurations
inference_timeout = float(os.getenv("INFERENCE_TIMEOUT", "2.0"))
max_batch_size = int(os.getenv("MAX_BATCH_SIZE", "32"))
max_concurrent_workers = int(os.getenv("MAX_CONCURRENT_WORKERS", "4"))
max_text_length = int(os.getenv("MAX_TEXT_LENGTH", "2000"))
# 2. Create a semaphore to strictly limit concurrent model executions
concurrency_semaphore = asyncio.Semaphore(max_concurrent_workers)
logger.info("Loading DistilBERT model into memory...")
try:
# 3. Wire the batch size directly into the HuggingFace pipeline
classifier = pipeline(
"sentiment-analysis",
model="distilbert-base-uncased-finetuned-sst-2-english",
batch_size=max_batch_size
)
logger.info("DistilBERT model loaded successfully.")
except Exception as e:
logger.error("Failed to load model", extra={"error": str(e)})
raise RuntimeError("Model initialization failed")
app = FastAPI(title="InfraMind Mock ML API")
Instrumentator().instrument(app).expose(app)
class InferenceRequest(BaseModel):
text: str
class SentimentResponse(BaseModel):
label: str
score: float
@app.post("/predict", response_model=SentimentResponse)
async def predict(request: InferenceRequest):
if len(request.text) > max_text_length:
logger.warning(
"Rejected oversized inference request",
extra={"text_length": len(request.text), "max_allowed": max_text_length}
)
raise HTTPException(
status_code=413,
detail=f"Input text exceeds maximum length of {max_text_length} characters."
)
logger.info(
"Received inference request",
extra={"text_length": len(request.text)}
)
start = time.time()
try:
# 4. Force requests to wait in line if workers are maxed out
async with concurrency_semaphore:
# When inside the semaphore block, execute the model
result = classifier(request.text)[0]
elapsed = time.time() - start
if elapsed > inference_timeout:
logger.warning(
"Inference request exceeded configured timeout",
extra={
"elapsed_seconds": round(elapsed, 2),
"configured_timeout_seconds": inference_timeout,
}
)
logger.info(
"Prediction successful",
extra={
"predicted_label": result["label"],
"elapsed_seconds": round(elapsed, 2),
"configured_timeout_seconds": inference_timeout,
}
)
return SentimentResponse(
label=result["label"],
score=result["score"]
)
except Exception as e:
logger.error(
"Error during prediction",
extra={"error": str(e)}
)
raise HTTPException(
status_code=500,
detail="Internal server error during inference"
)
@app.get("/health")
def health_check():
return {"status": "healthy"}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)