diff --git a/README.md b/README.md index 32881b8..f3def88 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ AWS Lambda를 이용하여 학교 식당 메뉴를 스크랩하고 Slack으로 - AWS Lambda: 메뉴 스크랩 및 가공 로직 실행 - AWS IAM: 인증된 사용자와 역할에 Lambda 수동 호출 권한 부여 - AWS EventBridge: 스케줄링된 스크랩 작업 관리 -- AWS Step Functions: 기숙사식당 재시도 워크플로우 +- AWS Step Functions: 식당별 재시도 및 최종 실패 알림 워크플로우 - AWS SAM: 인프라 관리 및 배포 자동화 ## 런타임 아키텍처 @@ -69,29 +69,34 @@ AWS Lambda를 이용하여 학교 식당 메뉴를 스크랩하고 Slack으로 | `HaksikSchedulingFunction` | `schedule_haksik` | 학생식당 주간 스케줄 | | `FacultySchedulingFunction` | `schedule_faculty` | 교직원식당 주간 스케줄 | | `DormitorySchedulingFunction` | `schedule_dormitory` | 기숙사식당 주간 스케줄 (Step Functions 호출) | -| `NotifyFailureFunction` | `notify_final_failure` | 기숙사 최종 실패 Slack 알림 | +| `NotifyFailureFunction` | `notify_final_failure` | 식당별 최종 실패 Slack 알림 | 각 함수에 대응하는 CloudWatch 로그 그룹(`*LogGroup`)이 9개 추가로 존재하며, 로그는 30일간 보존됩니다. ### 자동 스케줄 -- **도담/학생/교직원식당 스케줄링**: 매주 일요일 오후 4시 KST (UTC 07:00) 자동 실행 -- **기숙사식당 Step Functions**: 매주 일요일 23:00 UTC (월요일 08:00 KST) 자동 실행, 내부에서 `DormitorySchedulingFunction`과 `NotifyFailureFunction` 호출 +- **도담/학생/교직원식당**: 매주 일요일 오후 4시 KST (UTC 07:00)에 다음 주 메뉴 실행 +- **기숙사식당**: 매주 월요일 오전 8시 KST (일요일 23:00 UTC)에 현재 주 메뉴 실행 +- 각 식당은 Step Functions를 통해 실행되며, 평일 복구 스케줄은 다음 날 메뉴 한 건만 다시 처리하고 성공 Slack 요약은 생략합니다. ### 직접 호출 vs 스케줄 실행 - **직접 호출(scrape_*)**: `is_dev=True` — dev Spring 클라이언트만 호출, dev 실패가 critical - **스케줄 실행(schedule_*)**: dev와 prod 모두 호출, prod 실패만 critical -### 기숙사 Step Functions 재시도 +### Step Functions 재시도 -- **도메인 재시도** (`RetryableEmptyMenuError`, `RetryableApiSendError`): 최대 5회, 7200초 간격, 백오프 1.0 +- **도메인 재시도** (`RetryableEmptyMenuError`, `RetryableApiSendError`, `RetryableMenuInterpretationError`): 최대 5회, 7200초 간격, 백오프 1.0 - **Lambda 일시 오류 재시도** (`Lambda.ServiceException` 등): 최대 3회, 2초 간격, 백오프 2.0 - 모든 재시도 소진 후 `NotifyFailureFunction`으로 최종 실패 알림 +- GPT 응답 계약 위반은 Lambda 내부에서 새 응답을 최대 3회 요청한 뒤 Step Functions 재시도로 승격합니다. +- 검증 실패 로그에는 날짜, 슬롯, 허용된 사유 코드만 기록하며 메뉴 원문, 모델 응답, API 키는 기록하지 않습니다. +- 상태머신 시작 시각을 날짜 기준점으로 전달하므로 자정을 넘겨 재시도해도 대상 주/날짜가 바뀌지 않습니다. +- Spring은 동일 날짜·식당·시간대와 동일 메뉴 집합을 기존 식사로 재사용하므로 워크플로우 재실행을 중복 레코드 없이 처리합니다. ### 최종 실패 알림 -`notify_final_failure`의 Slack 호출이 실패하면 예외가 그대로 전파됩니다. 보호할 Spring 쓰기가 없으므로 격리하지 않습니다. +`notify_final_failure`의 Slack 호출이 실패하면 예외가 그대로 전파됩니다. 보호할 Spring 쓰기가 없으므로 격리하지 않습니다. 최종 실패를 알린 상태머신 실행은 `FAILED`로 종료되어 식당별 CloudWatch Alarm에도 남습니다. ## 환경 설정 diff --git a/functions/clients.py b/functions/clients.py index 48fb4ea..0f0d069 100644 --- a/functions/clients.py +++ b/functions/clients.py @@ -215,6 +215,7 @@ def format_slack_text(notification: Mapping[str, object]) -> str: allowed_errors = { "RetryableEmptyMenuError": "메뉴 미게시", "RetryableApiSendError": "메뉴 저장 실패", + "RetryableMenuInterpretationError": "메뉴 파싱 실패", "Lambda.ServiceException": "Lambda 서비스 오류", "Lambda.AWSLambdaException": "Lambda 실행 오류", "Lambda.SdkClientException": "Lambda 호출 오류", diff --git a/functions/config.py b/functions/config.py index cb327d2..2e5aa3e 100644 --- a/functions/config.py +++ b/functions/config.py @@ -48,6 +48,16 @@ ) +def load_restaurant_config(restaurant: object) -> Mapping[str, Any] | None: + """Return public metadata only for an allowlisted restaurant name.""" + if not isinstance(restaurant, str): + return None + metadata = _RESTAURANTS.get(restaurant.upper()) + if metadata is None: + return None + return MappingProxyType({"restaurant": restaurant.upper(), **metadata}) + + def _required_environment(name: str) -> str: value = os.getenv(name) if not value: diff --git a/functions/handler.py b/functions/handler.py index 68970a6..71251e8 100644 --- a/functions/handler.py +++ b/functions/handler.py @@ -31,6 +31,7 @@ _DATE_PATTERN = re.compile(r"\d{8}") _TRIGGERS = frozenset({"direct", "eventbridge", "iam", "local", "step_functions"}) +_SCHEDULE_MODES = frozenset({"next_week", "current_week", "tomorrow"}) _CONTENT_HEADERS = {"Content-Type": "application/json; charset=utf-8"} @@ -58,6 +59,21 @@ def __init__( self.failed_days = failed_days +class RetryableMenuInterpretationError(Exception): + """Signal Step Functions that model output validation should be retried.""" + + def __init__( + self, + target_date: str, + restaurant: str = "DORMITORY", + reason_code: str = "VALIDATION_FAILED", + ) -> None: + super().__init__("retryable menu interpretation failure") + self.target_date = target_date + self.restaurant = restaurant + self.reason_code = reason_code + + def _mapping(value: object) -> Mapping[str, Any]: return value if isinstance(value, Mapping) else {} @@ -72,6 +88,16 @@ def _date(value: object) -> str | None: return value if isinstance(value, str) and _DATE_PATTERN.fullmatch(value) else None +def _schedule_anchor(value: object) -> str | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value if parsed.tzinfo is not None else None + + def _retry_count(value: object) -> int: if isinstance(value, bool) or not isinstance(value, (int, str)): return 0 @@ -87,6 +113,17 @@ def parse_event(event: object) -> dict[str, Any]: raw_trigger = payload.get("trigger") delayed = payload.get("delayed_schedule", query.get("delayed_schedule")) execution_id = payload.get("execution_id") + raw_schedule_mode = payload.get("schedule_mode", query.get("schedule_mode")) + schedule_mode = ( + raw_schedule_mode + if isinstance(raw_schedule_mode, str) and raw_schedule_mode in _SCHEDULE_MODES + else None + ) + if schedule_mode is None and _boolean(delayed): + schedule_mode = "current_week" + raw_notify_summary = payload.get( + "notify_summary", query.get("notify_summary", True) + ) return { "trigger": raw_trigger if raw_trigger in _TRIGGERS else "direct", "delayed_schedule": _boolean(delayed), @@ -97,6 +134,11 @@ def parse_event(event: object) -> dict[str, Any]: "target_date": _date( payload.get("target_date") or payload.get("date") or query.get("date") ), + "schedule_mode": schedule_mode, + "notify_summary": ( + raw_notify_summary if isinstance(raw_notify_summary, bool) else True + ), + "schedule_anchor": _schedule_anchor(payload.get("schedule_anchor")), } @@ -116,6 +158,13 @@ def load_operation_config(operation: str) -> Mapping[str, Any] | None: return _mapping(loaded) or None +def load_restaurant_config(restaurant: object) -> Mapping[str, Any] | None: + """Resolve allowlisted restaurant metadata without loading runtime secrets.""" + config_module = importlib.import_module("functions.config") + loaded = config_module.load_restaurant_config(restaurant) + return _mapping(loaded) or None + + async def scrape( config: Mapping[str, Any], target_date: str, @@ -199,11 +248,26 @@ def emit_event(level: str, event_name: str, stage: str, **fields: Any) -> None: ) -def _week_dates(day_count: int, *, next_week: bool) -> list[str]: - now = datetime.now(ZoneInfo("Asia/Seoul")).replace( +def _now_seoul() -> datetime: + return datetime.now(ZoneInfo("Asia/Seoul")) + + +def _request_now(request: Mapping[str, Any]) -> datetime: + anchor = request.get("schedule_anchor") + if isinstance(anchor, str): + return datetime.fromisoformat(anchor.replace("Z", "+00:00")).astimezone( + ZoneInfo("Asia/Seoul") + ) + return _now_seoul() + + +def _week_dates( + day_count: int, *, next_week: bool, now: datetime | None = None +) -> list[str]: + current = (now or _now_seoul()).replace( hour=0, minute=0, second=0, microsecond=0 ) - monday = now - timedelta(days=now.weekday()) + monday = current - timedelta(days=current.weekday()) if next_week: monday += timedelta(days=7) return [(monday + timedelta(days=index)).strftime("%Y%m%d") for index in range(day_count)] @@ -213,6 +277,16 @@ def _dates_for(config: Mapping[str, Any], request: Mapping[str, Any]) -> list[st target_date = request.get("target_date") if isinstance(target_date, str): return [target_date] + schedule_mode = request.get("schedule_mode") + now = _request_now(request) + if schedule_mode == "tomorrow": + return [(now + timedelta(days=1)).strftime("%Y%m%d")] + if schedule_mode in {"current_week", "next_week"}: + return _week_dates( + int(config["week_days"]), + next_week=schedule_mode == "next_week", + now=now, + ) if config["kind"] == "scrape": return _week_dates(7, next_week=False)[:1] if config["restaurant"] == "DORMITORY" or request["delayed_schedule"]: @@ -279,8 +353,9 @@ async def _process_source_date( *, scheduled: bool, requested_dates: Sequence[str] | None = None, + notify_summary: bool = True, ) -> list[dict[str, Any]]: - dormitory_retry = scheduled and config["restaurant"] == "DORMITORY" + workflow_retry = scheduled try: if requested_dates is None: raw_meals = list(await scrape(config, target_date)) @@ -291,8 +366,8 @@ async def _process_source_date( except Exception as error: outcome = getattr(error, "outcome", None) reason_code = getattr(error, "reason_code", "SOURCE_ERROR") - if dormitory_retry and outcome == "AMBIGUOUS_EMPTY": - raise RetryableEmptyMenuError(target_date) from None + if workflow_retry and outcome == "AMBIGUOUS_EMPTY": + raise RetryableEmptyMenuError(target_date, config["restaurant"]) from None if outcome not in {"EXPECTED_EMPTY", "AMBIGUOUS_EMPTY"}: raise raw_meals = [ @@ -305,14 +380,14 @@ async def _process_source_date( "reason_code": reason_code, } ] - if dormitory_retry and requested_dates is not None: + if workflow_retry and requested_dates is not None: represented_dates = { meal_date for raw_meal in raw_meals if (meal_date := _date(raw_meal.get("date"))) is not None } if set(requested_dates) - represented_dates: - raise RetryableEmptyMenuError(target_date) + raise RetryableEmptyMenuError(target_date, config["restaurant"]) summaries: dict[str, dict[str, Any]] = defaultdict( lambda: { @@ -348,8 +423,8 @@ async def _process_source_date( outcome = raw_meal.get("outcome", "SUCCESS") if outcome in {"EXPECTED_EMPTY", "AMBIGUOUS_EMPTY"}: reason_code = raw_meal.get("reason_code", "SOURCE_EMPTY") - if dormitory_retry and outcome == "AMBIGUOUS_EMPTY": - raise RetryableEmptyMenuError(meal_date) + if workflow_retry and outcome == "AMBIGUOUS_EMPTY": + raise RetryableEmptyMenuError(meal_date, config["restaurant"]) summary["menus"][source_slot] = [] summary["empty_reasons"][source_slot] = reason_code if outcome == "AMBIGUOUS_EMPTY": @@ -359,10 +434,48 @@ async def _process_source_date( continue try: interpreted = await interpret_menu(config, raw_meal) - except (RetryableEmptyMenuError, RetryableApiSendError): + except ( + RetryableEmptyMenuError, + RetryableApiSendError, + RetryableMenuInterpretationError, + ): raise except Exception as error: - logger.warning("menu interpretation failed: %s", type(error).__name__) + menu_ai_module = importlib.import_module("functions.menu_ai") + is_validation_error = isinstance( + error, menu_ai_module.MenuInterpretationError + ) + if is_validation_error: + reason_code = getattr(error, "reason_code", "VALIDATION_FAILED") + if reason_code not in menu_ai_module.MENU_INTERPRETATION_REASON_CODES: + reason_code = "VALIDATION_FAILED" + emit_event( + "WARNING", + "menu_ai.validation_failed", + "menu_ai", + date=meal_date, + slot=source_slot, + error_type="MenuInterpretationError", + reason_code=reason_code, + ) + if workflow_retry: + raise RetryableMenuInterpretationError( + meal_date, config["restaurant"], reason_code + ) from None + else: + emit_event( + "WARNING", + "menu_ai.request_failed", + "menu_ai", + date=meal_date, + slot=source_slot, + error_type="MenuProviderError", + reason_code="PROVIDER_FAILURE", + ) + if workflow_retry: + raise RetryableMenuInterpretationError( + meal_date, config["restaurant"], "PROVIDER_FAILURE" + ) from None summary["errors"].append( {"slot": source_slot, "stage": "menu_ai", "error_type": type(error).__name__} ) @@ -393,7 +506,11 @@ async def _process_source_date( for environment in environments: try: publication = await publish_menu(config, payload, environment) - except (RetryableEmptyMenuError, RetryableApiSendError): + except ( + RetryableEmptyMenuError, + RetryableApiSendError, + RetryableMenuInterpretationError, + ): raise except Exception as error: logger.warning("Spring publication failed: %s", type(error).__name__) @@ -433,8 +550,12 @@ async def _process_source_date( } ) - if dormitory_retry and critical_failures: - raise RetryableApiSendError(target_date, failed_days=len(critical_failures)) + if workflow_retry and critical_failures: + raise RetryableApiSendError( + target_date, + config["restaurant"], + failed_days=len(critical_failures), + ) if not summaries: summaries[target_date] @@ -446,24 +567,25 @@ async def _process_source_date( "restaurant": config["name_ko"], **summary, } - try: - await notify_slack(config, notification) - except Exception as error: - error_type = type(error).__name__ - summary["warnings"].append( - { - "stage": "notification", - "reason": "notification failed", - "error_type": error_type, - } - ) - emit_event( - "WARNING", - "notification.failed", - "notification", - date=meal_date, - error_type=error_type, - ) + if notify_summary: + try: + await notify_slack(config, notification) + except Exception as error: + error_type = type(error).__name__ + summary["warnings"].append( + { + "stage": "notification", + "reason": "notification failed", + "error_type": error_type, + } + ) + emit_event( + "WARNING", + "notification.failed", + "notification", + date=meal_date, + error_type=error_type, + ) results.append( { "date": meal_date, @@ -554,12 +676,23 @@ async def _run_schedule( if config["restaurant"] == "DORMITORY": results.extend( await _process_source_date( - config, dates[0], scheduled=True, requested_dates=dates + config, + dates[0], + scheduled=True, + requested_dates=dates, + notify_summary=request["notify_summary"], ) ) else: for target_date in dates: - results.extend(await _process_source_date(config, target_date, scheduled=True)) + results.extend( + await _process_source_date( + config, + target_date, + scheduled=True, + notify_summary=request["notify_summary"], + ) + ) return _response(200, results) @@ -571,19 +704,22 @@ async def _run_final_failure( known_errors = { "RetryableEmptyMenuError", "RetryableApiSendError", + "RetryableMenuInterpretationError", "Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException", "Lambda.TooManyRequestsException", } error_type = raw_error_type if raw_error_type in known_errors else "UnknownError" - target_date = request.get("target_date") or _week_dates(7, next_week=False)[0] + restaurant_metadata = load_restaurant_config(payload.get("restaurant")) or config + notification_config = {**config, **restaurant_metadata} + target_date = _dates_for(notification_config, request)[0] await notify_slack( - config, + notification_config, { "type": "final_failure", "date": target_date, - "restaurant": config["name_ko"], + "restaurant": notification_config["name_ko"], "error_type": error_type, "retry_count": request["retry_count"], }, @@ -628,6 +764,11 @@ async def orchestrate(event: object, context: object) -> dict[str, Any]: return _invalid_response("operation configuration mismatch") request = parse_event(event) payload = _mapping(event) + context_config = ( + load_restaurant_config(payload.get("restaurant")) or config + if operation == "notify_final_failure" + else config + ) invocation_id = getattr(context, "aws_request_id", "unknown") run_id = request.get("execution_id") or payload.get("id") or invocation_id token = _observation_context.set( @@ -636,7 +777,7 @@ async def orchestrate(event: object, context: object) -> dict[str, Any]: "faas.invocation_id": str(invocation_id), "run_id": str(run_id), "operation": operation, - "restaurant": config["restaurant"], + "restaurant": context_config["restaurant"], "trigger": request["trigger"], } ) @@ -645,7 +786,11 @@ async def orchestrate(event: object, context: object) -> dict[str, Any]: response = await dispatcher(config, request, event) emit_event("INFO", "handler.invocation.completed", "handler") return response - except (RetryableEmptyMenuError, RetryableApiSendError) as error: + except ( + RetryableEmptyMenuError, + RetryableApiSendError, + RetryableMenuInterpretationError, + ) as error: emit_event( "WARNING", "handler.invocation.retryable", diff --git a/functions/menu_ai.py b/functions/menu_ai.py index 4293c69..54f9971 100644 --- a/functions/menu_ai.py +++ b/functions/menu_ai.py @@ -12,6 +12,34 @@ TOOL_NAME = "extract_main_menus" SITE_ENGLISH_RESTAURANTS = frozenset({"HAKSIK", "DODAM", "FACULTY"}) DORMITORY = "DORMITORY" +MENU_INTERPRETATION_REASON_CODES = frozenset( + { + "INVALID_RESTAURANT", + "INVALID_TOOL_ARGUMENTS", + "INVALID_MENU_NAMES", + "DUPLICATE_MENU_NAMES", + "MENU_NAME_MISSING_HANGUL", + "MENU_NAME_SLOT_LABEL", + "MENU_NAME_BRACKETED_LABEL", + "INVALID_MAIN_CANDIDATES", + "INVALID_SOURCE_ENGLISH", + "INVALID_MAIN_CANDIDATE", + "INVALID_MENU_INDEX", + "MENU_INDEX_OUT_OF_RANGE", + "DUPLICATE_MENU_INDEX", + "INVALID_NAME_EN", + "NON_ENGLISH_NAME_EN", + "NAME_EN_NOT_VERBATIM", + "INVALID_CANDIDATE_COUNT", + "INVALID_RESPONSE_ENVELOPE", + "INVALID_TOOL_CALL", + "UNEXPECTED_TOOL_NAME", + "INVALID_TOOL_ARGUMENTS_JSON", + "INVALID_RAW_SOURCE", + "PROVIDER_FAILURE", + "VALIDATION_FAILED", + } +) MENU_TOOL: ChatCompletionToolParam = { "type": "function", @@ -67,6 +95,14 @@ class MenuInterpretationError(ValueError): """The model response does not satisfy the menu interpretation contract.""" + def __init__(self, message: str, reason_code: str = "VALIDATION_FAILED") -> None: + super().__init__(message) + self.reason_code = ( + reason_code + if reason_code in MENU_INTERPRETATION_REASON_CODES + else "VALIDATION_FAILED" + ) + class MainMenu(TypedDict): nameKo: str @@ -84,23 +120,34 @@ def _restaurant_name(restaurant: object) -> str: else: name = getattr(restaurant, "english_name", None) if not isinstance(name, str): - raise MenuInterpretationError("restaurant must identify a supported restaurant") + raise MenuInterpretationError( + "restaurant must identify a supported restaurant", "INVALID_RESTAURANT" + ) name = name.upper() if name not in SITE_ENGLISH_RESTAURANTS | {DORMITORY}: - raise MenuInterpretationError(f"unsupported restaurant: {name}") + raise MenuInterpretationError("unsupported restaurant", "INVALID_RESTAURANT") return name def _exact_fields(value: Mapping[str, Any], expected: set[str], label: str) -> None: if set(value) != expected: - raise MenuInterpretationError(f"{label} must contain exactly {sorted(expected)}") + reason_code = ( + "INVALID_MAIN_CANDIDATE" if label == "candidate" else "INVALID_TOOL_ARGUMENTS" + ) + raise MenuInterpretationError( + f"{label} must contain exactly {sorted(expected)}", reason_code + ) def _validate_name_en(name_en: Any) -> str: if not isinstance(name_en, str) or not name_en.strip(): - raise MenuInterpretationError("candidate nameEn must be a non-empty string") + raise MenuInterpretationError( + "candidate nameEn must be a non-empty string", "INVALID_NAME_EN" + ) if _HANGUL_RE.search(name_en) or not _LATIN_RE.search(name_en): - raise MenuInterpretationError("candidate nameEn must be English") + raise MenuInterpretationError( + "candidate nameEn must be English", "NON_ENGLISH_NAME_EN" + ) return name_en @@ -113,7 +160,9 @@ def validate_tool_arguments( """Validate parsed tool arguments and assemble canonical mainMenus.""" restaurant_name = _restaurant_name(restaurant) if not isinstance(arguments, Mapping): - raise MenuInterpretationError("tool arguments must be a JSON object") + raise MenuInterpretationError( + "tool arguments must be a JSON object", "INVALID_TOOL_ARGUMENTS" + ) _exact_fields(arguments, {"menuNames", "mainCandidates"}, "tool arguments") raw_menu_names = arguments["menuNames"] @@ -122,40 +171,63 @@ def validate_tool_arguments( or not raw_menu_names or not all(isinstance(menu, str) and bool(menu.strip()) for menu in raw_menu_names) ): - raise MenuInterpretationError("menuNames must be a non-empty list of strings") + raise MenuInterpretationError( + "menuNames must be a non-empty list of strings", "INVALID_MENU_NAMES" + ) menu_names = cast(list[str], raw_menu_names) if len(menu_names) != len(set(menu_names)): - raise MenuInterpretationError("menuNames must not contain duplicates") + raise MenuInterpretationError( + "menuNames must not contain duplicates", "DUPLICATE_MENU_NAMES" + ) for menu_name in menu_names: if not _HANGUL_RE.search(menu_name): - raise MenuInterpretationError("each menuNames item must contain Hangul") + raise MenuInterpretationError( + "each menuNames item must contain Hangul", "MENU_NAME_MISSING_HANGUL" + ) if _SLOT_LABEL_RE.fullmatch(menu_name.strip()): - raise MenuInterpretationError("menuNames must not contain meal-slot labels") + raise MenuInterpretationError( + "menuNames must not contain meal-slot labels", "MENU_NAME_SLOT_LABEL" + ) if _BRACKETED_LABEL_RE.fullmatch(menu_name): - raise MenuInterpretationError("menuNames must not contain bracketed labels") + raise MenuInterpretationError( + "menuNames must not contain bracketed labels", + "MENU_NAME_BRACKETED_LABEL", + ) raw_candidates = arguments["mainCandidates"] if not isinstance(raw_candidates, list): - raise MenuInterpretationError("mainCandidates must be a list") + raise MenuInterpretationError( + "mainCandidates must be a list", "INVALID_MAIN_CANDIDATES" + ) evidence = tuple(source_english) if not all(isinstance(item, str) for item in evidence): - raise MenuInterpretationError("source English evidence must contain only strings") + raise MenuInterpretationError( + "source English evidence must contain only strings", "INVALID_SOURCE_ENGLISH" + ) indexes: set[int] = set() main_menus: list[MainMenu] = [] for raw_candidate in raw_candidates: if not isinstance(raw_candidate, Mapping): - raise MenuInterpretationError("each candidate must be an object") + raise MenuInterpretationError( + "each candidate must be an object", "INVALID_MAIN_CANDIDATE" + ) _exact_fields(raw_candidate, {"menuIndex", "nameEn"}, "candidate") menu_index = raw_candidate["menuIndex"] if isinstance(menu_index, bool) or not isinstance(menu_index, int): - raise MenuInterpretationError("candidate menuIndex must be an integer") + raise MenuInterpretationError( + "candidate menuIndex must be an integer", "INVALID_MENU_INDEX" + ) if menu_index < 0 or menu_index >= len(menu_names): - raise MenuInterpretationError("candidate menuIndex is out of range") + raise MenuInterpretationError( + "candidate menuIndex is out of range", "MENU_INDEX_OUT_OF_RANGE" + ) if menu_index in indexes: - raise MenuInterpretationError("candidate menuIndex values must be unique") + raise MenuInterpretationError( + "candidate menuIndex values must be unique", "DUPLICATE_MENU_INDEX" + ) indexes.add(menu_index) name_en = _validate_name_en(raw_candidate["nameEn"]) @@ -163,18 +235,21 @@ def validate_tool_arguments( name_en in raw_source or name_en in evidence ): raise MenuInterpretationError( - "site restaurant nameEn must be verbatim source English" + "site restaurant nameEn must be verbatim source English", + "NAME_EN_NOT_VERBATIM", ) main_menus.append({"nameKo": menu_names[menu_index], "nameEn": name_en}) if restaurant_name in SITE_ENGLISH_RESTAURANTS: if not main_menus: raise MenuInterpretationError( - "site restaurants require at least one main candidate" + "site restaurants require at least one main candidate", + "INVALID_CANDIDATE_COUNT", ) elif len(main_menus) != min(3, len(menu_names)): raise MenuInterpretationError( - "DORMITORY requires exactly min(3, len(menuNames)) candidates" + "DORMITORY requires exactly min(3, len(menuNames)) candidates", + "INVALID_CANDIDATE_COUNT", ) return {"menuNames": menu_names, "mainMenus": main_menus} @@ -190,37 +265,46 @@ def parse_tool_response( try: choices = getattr(response, "choices") if not isinstance(choices, Sequence) or len(choices) != 1: - raise MenuInterpretationError("response must contain exactly one choice") + raise MenuInterpretationError( + "response must contain exactly one choice", "INVALID_RESPONSE_ENVELOPE" + ) message = getattr(choices[0], "message") tool_calls = getattr(message, "tool_calls") except MenuInterpretationError: raise except (AttributeError, IndexError, TypeError) as error: - raise MenuInterpretationError("malformed chat-completion response") from error + raise MenuInterpretationError( + "malformed chat-completion response", "INVALID_RESPONSE_ENVELOPE" + ) from error if not isinstance(tool_calls, Sequence) or len(tool_calls) != 1: - raise MenuInterpretationError("response must contain exactly one tool call") + raise MenuInterpretationError( + "response must contain exactly one tool call", "INVALID_TOOL_CALL" + ) tool_call = tool_calls[0] try: function = getattr(tool_call, "function") name = getattr(function, "name") raw_arguments = getattr(function, "arguments") except AttributeError as error: - raise MenuInterpretationError("malformed tool call") from error + raise MenuInterpretationError("malformed tool call", "INVALID_TOOL_CALL") from error if name != TOOL_NAME: - raise MenuInterpretationError(f"unexpected tool call: {name}") + raise MenuInterpretationError("unexpected tool call", "UNEXPECTED_TOOL_NAME") if not isinstance(raw_arguments, str): - raise MenuInterpretationError("tool arguments must be JSON text") + raise MenuInterpretationError( + "tool arguments must be JSON text", "INVALID_TOOL_ARGUMENTS_JSON" + ) try: arguments = json.loads(raw_arguments) except (json.JSONDecodeError, TypeError) as error: - raise MenuInterpretationError("tool arguments are not valid JSON") from error + raise MenuInterpretationError( + "tool arguments are not valid JSON", "INVALID_TOOL_ARGUMENTS_JSON" + ) from error return validate_tool_arguments( arguments, restaurant, raw_source, source_english ) -@retry(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True) async def _request_completion( client: AsyncOpenAI, restaurant: str, @@ -247,6 +331,19 @@ async def _request_completion( ) +@retry(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True) +async def _request_and_parse( + client: AsyncOpenAI, + restaurant: str, + raw_source: str, + source_english: tuple[str, ...], +) -> MenuInterpretation: + response = await _request_completion( + client, restaurant, raw_source, source_english + ) + return parse_tool_response(response, restaurant, raw_source, source_english) + + async def interpret_menu( api_key: str, restaurant: object, @@ -256,10 +353,9 @@ async def interpret_menu( """Interpret one meal and return menuNames plus canonical mainMenus.""" restaurant_name = _restaurant_name(restaurant) if not isinstance(raw_source, str) or not raw_source.strip(): - raise MenuInterpretationError("raw source must be a non-empty string") + raise MenuInterpretationError( + "raw source must be a non-empty string", "INVALID_RAW_SOURCE" + ) evidence = tuple(source_english) client = AsyncOpenAI(api_key=api_key) - response = await _request_completion( - client, restaurant_name, raw_source, evidence - ) - return parse_tool_response(response, restaurant_name, raw_source, evidence) + return await _request_and_parse(client, restaurant_name, raw_source, evidence) diff --git a/statemachine/dormitory-retry-workflow.asl.json b/statemachine/menu-retry-workflow.asl.json similarity index 59% rename from statemachine/dormitory-retry-workflow.asl.json rename to statemachine/menu-retry-workflow.asl.json index 8df266e..5ad9d37 100644 --- a/statemachine/dormitory-retry-workflow.asl.json +++ b/statemachine/menu-retry-workflow.asl.json @@ -1,17 +1,19 @@ { - "Comment": "기숙사 메뉴 스케줄 재시도 워크플로우 - 빈 결과(미게시) 시 2시간 간격 최대 6회 시도", - "StartAt": "InvokeDormitory", + "Comment": "식당 메뉴 스케줄 공통 재시도 및 최종 실패 알림 워크플로우", + "StartAt": "InvokeSchedule", "States": { - "InvokeDormitory": { + "InvokeSchedule": { "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { - "FunctionName": "${DormitorySchedulingFunctionArn}", + "FunctionName": "${SchedulingFunctionArn}", "Payload": { "trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "retry_count.$": "$$.State.RetryCount", - "delayed_schedule": false + "schedule_anchor.$": "$$.Execution.StartTime", + "schedule_mode.$": "$.schedule_mode", + "notify_summary.$": "$.notify_summary" } }, "Retry": [ @@ -27,7 +29,11 @@ "BackoffRate": 2.0 }, { - "ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError"], + "ErrorEquals": [ + "RetryableEmptyMenuError", + "RetryableApiSendError", + "RetryableMenuInterpretationError" + ], "IntervalSeconds": 7200, "MaxAttempts": 5, "BackoffRate": 1.0 @@ -50,11 +56,18 @@ "Payload": { "trigger": "step_functions", "execution_id.$": "$$.Execution.Id", - "restaurant": "DORMITORY", + "restaurant": "${Restaurant}", + "schedule_anchor.$": "$$.Execution.StartTime", + "schedule_mode.$": "$.schedule_mode", "error_type.$": "$.error.Error" } }, - "End": true + "Next": "MarkExecutionFailed" + }, + "MarkExecutionFailed": { + "Type": "Fail", + "Error": "MenuWorkflowFailed", + "Cause": "Retries exhausted; final failure notification sent" } } } diff --git a/template.yml b/template.yml index 2ce8926..466f59b 100644 --- a/template.yml +++ b/template.yml @@ -101,17 +101,6 @@ Resources: OPERATION: schedule_dodam Layers: - !Ref PythonRequirementsLayer - Events: - # 매주 일요일 오후 4시에 자동 실행 (KST 기준으로는 일요일 07시 UTC) - WeeklySchedule: - Type: Schedule - Properties: - Schedule: cron(0 7 ? * SUN *) - Input: | - { - "trigger": "eventbridge", - "delayed_schedule": false - } HaksikSchedulingFunction: Type: AWS::Serverless::Function @@ -123,17 +112,6 @@ Resources: OPERATION: schedule_haksik Layers: - !Ref PythonRequirementsLayer - Events: - # 매주 일요일 오후 4시에 자동 실행 - WeeklySchedule: - Type: Schedule - Properties: - Schedule: cron(0 7 ? * SUN *) - Input: | - { - "trigger": "eventbridge", - "delayed_schedule": false - } FacultySchedulingFunction: Type: AWS::Serverless::Function @@ -145,17 +123,6 @@ Resources: OPERATION: schedule_faculty Layers: - !Ref PythonRequirementsLayer - Events: - # 매주 일요일 오후 4시에 자동 실행 - WeeklySchedule: - Type: Schedule - Properties: - Schedule: cron(0 7 ? * SUN *) - Input: | - { - "trigger": "eventbridge", - "delayed_schedule": false - } DormitorySchedulingFunction: Type: AWS::Serverless::Function @@ -297,13 +264,89 @@ Resources: LogGroupName: !Sub "/aws/lambda/${NotifyFailureFunction}" RetentionInDays: 30 + DodamRetryStateMachine: + Type: AWS::Serverless::StateMachine + Properties: + DefinitionUri: statemachine/menu-retry-workflow.asl.json + DefinitionSubstitutions: + SchedulingFunctionArn: !GetAtt DodamSchedulingFunction.Arn + NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn + Restaurant: DODAM + Policies: + - LambdaInvokePolicy: + FunctionName: !Ref DodamSchedulingFunction + - LambdaInvokePolicy: + FunctionName: !Ref NotifyFailureFunction + Events: + WeeklySchedule: + Type: Schedule + Properties: + Schedule: cron(0 7 ? * SUN *) + Input: '{"schedule_mode":"next_week","notify_summary":true}' + RecoverySchedule: + Type: Schedule + Properties: + Schedule: cron(5 7 ? * MON-THU *) + Input: '{"schedule_mode":"tomorrow","notify_summary":false}' + + HaksikRetryStateMachine: + Type: AWS::Serverless::StateMachine + Properties: + DefinitionUri: statemachine/menu-retry-workflow.asl.json + DefinitionSubstitutions: + SchedulingFunctionArn: !GetAtt HaksikSchedulingFunction.Arn + NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn + Restaurant: HAKSIK + Policies: + - LambdaInvokePolicy: + FunctionName: !Ref HaksikSchedulingFunction + - LambdaInvokePolicy: + FunctionName: !Ref NotifyFailureFunction + Events: + WeeklySchedule: + Type: Schedule + Properties: + Schedule: cron(0 7 ? * SUN *) + Input: '{"schedule_mode":"next_week","notify_summary":true}' + RecoverySchedule: + Type: Schedule + Properties: + Schedule: cron(5 7 ? * MON-THU *) + Input: '{"schedule_mode":"tomorrow","notify_summary":false}' + + FacultyRetryStateMachine: + Type: AWS::Serverless::StateMachine + Properties: + DefinitionUri: statemachine/menu-retry-workflow.asl.json + DefinitionSubstitutions: + SchedulingFunctionArn: !GetAtt FacultySchedulingFunction.Arn + NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn + Restaurant: FACULTY + Policies: + - LambdaInvokePolicy: + FunctionName: !Ref FacultySchedulingFunction + - LambdaInvokePolicy: + FunctionName: !Ref NotifyFailureFunction + Events: + WeeklySchedule: + Type: Schedule + Properties: + Schedule: cron(0 7 ? * SUN *) + Input: '{"schedule_mode":"next_week","notify_summary":true}' + RecoverySchedule: + Type: Schedule + Properties: + Schedule: cron(5 7 ? * MON-THU *) + Input: '{"schedule_mode":"tomorrow","notify_summary":false}' + DormitoryRetryStateMachine: Type: AWS::Serverless::StateMachine Properties: - DefinitionUri: statemachine/dormitory-retry-workflow.asl.json + DefinitionUri: statemachine/menu-retry-workflow.asl.json DefinitionSubstitutions: - DormitorySchedulingFunctionArn: !GetAtt DormitorySchedulingFunction.Arn + SchedulingFunctionArn: !GetAtt DormitorySchedulingFunction.Arn NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn + Restaurant: DORMITORY Policies: - LambdaInvokePolicy: FunctionName: !Ref DormitorySchedulingFunction @@ -314,3 +357,73 @@ Resources: Type: Schedule Properties: Schedule: cron(0 23 ? * SUN *) + Input: '{"schedule_mode":"current_week","notify_summary":true}' + RecoverySchedule: + Type: Schedule + Properties: + Schedule: cron(5 23 ? * MON-FRI *) + Input: '{"schedule_mode":"tomorrow","notify_summary":false}' + + DodamRetryStateMachineAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmDescription: Dodam menu workflow exhausted all retries + Namespace: AWS/States + MetricName: ExecutionsFailed + Dimensions: + - Name: StateMachineArn + Value: !Ref DodamRetryStateMachine + Statistic: Sum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + + HaksikRetryStateMachineAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmDescription: Haksik menu workflow exhausted all retries + Namespace: AWS/States + MetricName: ExecutionsFailed + Dimensions: + - Name: StateMachineArn + Value: !Ref HaksikRetryStateMachine + Statistic: Sum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + + FacultyRetryStateMachineAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmDescription: Faculty menu workflow exhausted all retries + Namespace: AWS/States + MetricName: ExecutionsFailed + Dimensions: + - Name: StateMachineArn + Value: !Ref FacultyRetryStateMachine + Statistic: Sum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + + DormitoryRetryStateMachineAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmDescription: Dormitory menu workflow exhausted all retries + Namespace: AWS/States + MetricName: ExecutionsFailed + Dimensions: + - Name: StateMachineArn + Value: !Ref DormitoryRetryStateMachine + Statistic: Sum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching diff --git a/tests/fixtures/characterization/infrastructure.json b/tests/fixtures/characterization/infrastructure.json index a183518..49edf50 100644 --- a/tests/fixtures/characterization/infrastructure.json +++ b/tests/fixtures/characterization/infrastructure.json @@ -1,5 +1,5 @@ { - "authoritative_sha": "59d021dc5ad8a5d4a7449f4e807760dff4184d0e", + "authoritative_sha": "3719e46781cba3915ba8d2dd421726a1694e3e61", "functions": { "DodamScrapingFunction": "functions.handler.lambda_handler", "HaksikScrapingFunction": "functions.handler.lambda_handler", @@ -34,23 +34,24 @@ "function_policy_count": 0 }, "general_schedule": { - "logical_resources": ["DodamSchedulingFunction", "HaksikSchedulingFunction", "FacultySchedulingFunction"], + "state_machines": ["DodamRetryStateMachine", "HaksikRetryStateMachine", "FacultyRetryStateMachine"], "expression": "cron(0 7 ? * SUN *)", - "input": {"trigger": "eventbridge", "delayed_schedule": false}, + "weekly_input": "{\"schedule_mode\":\"next_week\",\"notify_summary\":true}", + "recovery_input": "{\"schedule_mode\":\"tomorrow\",\"notify_summary\":false}", "event_type": "Schedule" }, "dormitory_schedule": {"expression": "cron(0 23 ? * SUN *)"}, "state_machine": { "logical_resource": "DormitoryRetryStateMachine", - "definition": "statemachine/dormitory-retry-workflow.asl.json", + "definition": "statemachine/menu-retry-workflow.asl.json", "substitutions": { - "DormitorySchedulingFunctionArn": "DormitorySchedulingFunction.Arn", + "SchedulingFunctionArn": "DormitorySchedulingFunction.Arn", "NotifyFailureFunctionArn": "NotifyFailureFunction.Arn" }, "policies": ["DormitorySchedulingFunction", "NotifyFailureFunction"], - "invoke_payload": {"trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "retry_count.$": "$$.State.RetryCount", "delayed_schedule": false}, - "failure_payload": {"trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "restaurant": "DORMITORY", "error_type.$": "$.error.Error"}, + "invoke_payload": {"trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "retry_count.$": "$$.State.RetryCount", "schedule_anchor.$": "$$.Execution.StartTime", "schedule_mode.$": "$.schedule_mode", "notify_summary.$": "$.notify_summary"}, + "failure_payload": {"trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "restaurant": "${Restaurant}", "schedule_anchor.$": "$$.Execution.StartTime", "schedule_mode.$": "$.schedule_mode", "error_type.$": "$.error.Error"}, "local_retry": {"ErrorEquals": ["Lambda.ServiceException", "Lambda.AWSLambdaException", "Lambda.SdkClientException", "Lambda.TooManyRequestsException"], "IntervalSeconds": 2, "MaxAttempts": 3, "BackoffRate": 2.0}, - "domain_retry": {"ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError"], "IntervalSeconds": 7200, "MaxAttempts": 5, "BackoffRate": 1.0} + "domain_retry": {"ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError", "RetryableMenuInterpretationError"], "IntervalSeconds": 7200, "MaxAttempts": 5, "BackoffRate": 1.0} } } diff --git a/tests/test_characterization_infrastructure.py b/tests/test_characterization_infrastructure.py index 19a67db..b4e4751 100644 --- a/tests/test_characterization_infrastructure.py +++ b/tests/test_characterization_infrastructure.py @@ -5,7 +5,7 @@ ROOT = Path(__file__).resolve().parents[1] TEMPLATE = (ROOT / "template.yml").read_text(encoding="utf-8") ASL = json.loads( - (ROOT / "statemachine/dormitory-retry-workflow.asl.json").read_text( + (ROOT / "statemachine/menu-retry-workflow.asl.json").read_text( encoding="utf-8" ) ) @@ -38,7 +38,7 @@ def test_authoritative_snapshot_has_exactly_nine_lambda_resources_and_handlers() if "Type: AWS::Serverless::Function" in block } - assert CONTRACT["authoritative_sha"] == "59d021dc5ad8a5d4a7449f4e807760dff4184d0e" + assert CONTRACT["authoritative_sha"] == "3719e46781cba3915ba8d2dd421726a1694e3e61" assert set(functions) == set(CONTRACT["functions"]) assert len(functions) == CONTRACT["function_contract"]["count"] == 9 for logical_id, handler in CONTRACT["functions"].items(): @@ -86,27 +86,27 @@ def test_lambda_global_configuration_and_function_policy_absence_are_frozen(): assert function_policy_count == contract["function_policy_count"] == 0 -def test_three_eventbridge_inputs_and_state_machine_schedule_are_exact(): +def test_state_machine_weekly_and_recovery_schedules_are_exact(): blocks = _resource_blocks() schedule = CONTRACT["general_schedule"] - for logical_id in schedule["logical_resources"]: + for logical_id in schedule["state_machines"]: block = blocks[logical_id] - assert f"Type: {schedule['event_type']}" in block + assert block.count(f"Type: {schedule['event_type']}") == 2 assert f"Schedule: {schedule['expression']}" in block - assert '"trigger": "eventbridge"' in block - assert '"delayed_schedule": false' in block + assert schedule["weekly_input"] in block + assert schedule["recovery_input"] in block assert "Type: Schedule" not in blocks["DormitorySchedulingFunction"] assert ( f"Schedule: {CONTRACT['dormitory_schedule']['expression']}" in blocks[CONTRACT["state_machine"]["logical_resource"]] ) - assert TEMPLATE.count("Type: Schedule") == 4 + assert TEMPLATE.count("Type: Schedule") == 8 -def test_dormitory_asl_payloads_substitutions_policies_and_retries_are_exact(): +def test_common_asl_payloads_substitutions_policies_and_retries_are_exact(): state_machine = CONTRACT["state_machine"] - invoke = ASL["States"]["InvokeDormitory"] + invoke = ASL["States"]["InvokeSchedule"] notify = ASL["States"]["NotifyFinalFailure"] block = _resource_blocks()[state_machine["logical_resource"]] diff --git a/tests/test_characterization_invocations.py b/tests/test_characterization_invocations.py index 91eb440..df9ecc8 100644 --- a/tests/test_characterization_invocations.py +++ b/tests/test_characterization_invocations.py @@ -52,4 +52,7 @@ def test_null_query_map_and_malformed_date_have_safe_defaults(): "execution_id": None, "retry_count": 0, "target_date": None, + "schedule_mode": None, + "notify_summary": True, + "schedule_anchor": None, } diff --git a/tests/test_characterization_pipeline.py b/tests/test_characterization_pipeline.py index ed36910..c02a8ea 100644 --- a/tests/test_characterization_pipeline.py +++ b/tests/test_characterization_pipeline.py @@ -43,7 +43,7 @@ def test_restaurant_slot_time_and_price_mappings_are_frozen(restaurant): def test_client_retry_attempts_and_fixed_waits_are_frozen(): retry_functions = { - "gpt": menu_ai._request_completion, + "gpt": menu_ai._request_and_parse, "spring": clients.publish_spring_meal, "slack": clients.send_slack_text, } diff --git a/tests/test_exception_sanitization.py b/tests/test_exception_sanitization.py index bb7157f..0af5fbb 100644 --- a/tests/test_exception_sanitization.py +++ b/tests/test_exception_sanitization.py @@ -19,6 +19,19 @@ def test_final_failure_slack_uses_only_allowlisted_error_text(): assert "provider.invalid" not in text +def test_retryable_menu_interpretation_failure_has_safe_slack_mapping(): + text = format_slack_text( + { + "type": "final_failure", + "date": "20260918", + "restaurant": "학생식당", + "error_type": "RetryableMenuInterpretationError", + } + ) + + assert text == "🍽️ 학생식당 (20260918)\n⚠️ 최종 처리 실패: 메뉴 파싱 실패" + + def test_production_like_partial_failure_uses_only_allowlisted_slot_reason(): text = format_slack_text( { diff --git a/tests/test_menu_ai.py b/tests/test_menu_ai.py index 80006b3..fcf3aad 100644 --- a/tests/test_menu_ai.py +++ b/tests/test_menu_ai.py @@ -334,29 +334,132 @@ async def test_transient_provider_failure_retries_three_times_without_test_sleep completion = AsyncMock(side_effect=RuntimeError("transient")) client = MagicMock() client.chat.completions.create = completion - request_completion = cast(Any, menu_ai._request_completion) - original_wait = request_completion.retry.wait - request_completion.retry.wait = wait_none() + request_and_parse = cast(Any, menu_ai._request_and_parse) + original_wait = request_and_parse.retry.wait + request_and_parse.retry.wait = wait_none() try: with patch("functions.menu_ai.AsyncOpenAI", return_value=client): with pytest.raises(RuntimeError, match="transient"): await menu_ai.interpret_menu("secret", "DORMITORY", "밥") finally: - request_completion.retry.wait = original_wait + request_and_parse.retry.wait = original_wait assert completion.await_count == 3 - assert request_completion.retry.stop.max_attempt_number == 3 + assert request_and_parse.retry.stop.max_attempt_number == 3 assert original_wait.wait_fixed == 5 @pytest.mark.asyncio -async def test_malformed_completion_is_not_retried(): - completion = AsyncMock(return_value=_response("{")) +async def test_malformed_completion_retries_fresh_completions_three_times(): + completion = AsyncMock( + side_effect=[_response("{"), _response("{"), _response(_valid_dormitory())] + ) + client = MagicMock() + client.chat.completions.create = completion + request_and_parse = cast(Any, menu_ai._request_and_parse) + original_wait = request_and_parse.retry.wait + request_and_parse.retry.wait = wait_none() + try: + with patch("functions.menu_ai.AsyncOpenAI", return_value=client): + result = await menu_ai.interpret_menu( + "secret", "DORMITORY", "김치찌개 쌀밥 계란말이" + ) + finally: + request_and_parse.retry.wait = original_wait + + assert completion.await_count == 3 + assert result["menuNames"] == ["김치찌개", "쌀밥", "계란말이"] + + +@pytest.mark.asyncio +async def test_exhausted_provider_retry_is_not_multiplied_by_validation_retry(): + completion = AsyncMock(side_effect=RuntimeError("provider secret")) + client = MagicMock() + client.chat.completions.create = completion + request_and_parse = cast(Any, menu_ai._request_and_parse) + original_wait = request_and_parse.retry.wait + request_and_parse.retry.wait = wait_none() + try: + with patch("functions.menu_ai.AsyncOpenAI", return_value=client): + with pytest.raises(RuntimeError, match="provider secret"): + await menu_ai.interpret_menu("secret", "DORMITORY", "밥") + finally: + request_and_parse.retry.wait = original_wait + + assert completion.await_count == 3 + + +@pytest.mark.asyncio +async def test_transport_and_validation_failures_share_one_three_call_budget(): + completion = AsyncMock( + side_effect=[RuntimeError("transient"), _response("{"), _response(_valid_dormitory())] + ) client = MagicMock() client.chat.completions.create = completion + request_and_parse = cast(Any, menu_ai._request_and_parse) + original_wait = request_and_parse.retry.wait + request_and_parse.retry.wait = wait_none() + try: + with patch("functions.menu_ai.AsyncOpenAI", return_value=client): + result = await menu_ai.interpret_menu( + "secret", "DORMITORY", "김치찌개 쌀밥 계란말이" + ) + finally: + request_and_parse.retry.wait = original_wait + + assert completion.await_count == 3 + assert result["menuNames"] == ["김치찌개", "쌀밥", "계란말이"] + + +@pytest.mark.parametrize( + ("call", "expected_reason"), + [ + ( + lambda: menu_ai.validate_tool_arguments([], "DORMITORY", "밥"), + "INVALID_TOOL_ARGUMENTS", + ), + ( + lambda: menu_ai.validate_tool_arguments( + {"menuNames": ["밥", "밥"], "mainCandidates": []}, + "DORMITORY", + "밥", + ), + "DUPLICATE_MENU_NAMES", + ), + ( + lambda: menu_ai.parse_tool_response(_response("{"), "DORMITORY", "밥"), + "INVALID_TOOL_ARGUMENTS_JSON", + ), + ( + lambda: menu_ai.parse_tool_response( + _response({}, name="https://provider.invalid/secret"), + "DORMITORY", + "밥", + ), + "UNEXPECTED_TOOL_NAME", + ), + ( + lambda: menu_ai.validate_tool_arguments( + { + "menuNames": ["밥"], + "mainCandidates": [{"menuIndex": 0, "nameEn": "쌀밥"}], + }, + "DORMITORY", + "밥", + ), + "NON_ENGLISH_NAME_EN", + ), + ], +) +def test_validation_errors_expose_allowlisted_reason_codes(call, expected_reason): + with pytest.raises(menu_ai.MenuInterpretationError) as raised: + call() + + assert raised.value.reason_code == expected_reason + assert raised.value.reason_code in menu_ai.MENU_INTERPRETATION_REASON_CODES + - with patch("functions.menu_ai.AsyncOpenAI", return_value=client): - with pytest.raises(menu_ai.MenuInterpretationError, match="valid JSON"): - await menu_ai.interpret_menu("secret", "DORMITORY", "밥") +def test_unknown_reason_code_collapses_to_generic_allowlisted_reason(): + error = menu_ai.MenuInterpretationError("untrusted detail", "PROVIDER_SECRET") - completion.assert_awaited_once() + assert error.reason_code == "VALIDATION_FAILED" diff --git a/tests/test_observability.py b/tests/test_observability.py index a406b37..7828764 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock, patch from functions import handler +from functions import menu_ai class _Context: @@ -55,3 +56,51 @@ def test_structured_context_is_reset_between_invocations(): assert second assert all(event["restaurant"] == "HAKSIK" for event in second) assert all(event["run_id"] == "request-456" for event in second) + + +def test_menu_validation_event_contains_only_safe_structured_fields(): + sentinel = "SECRET_TOKEN https://provider.invalid Cause traceback" + stream, original_streams = _capture_observation_stream() + try: + with ( + patch.object( + handler, + "scrape", + AsyncMock( + return_value=[ + { + "date": "20260918", + "restaurant": "HAKSIK", + "source_slot": "중식2", + "raw_text": sentinel, + } + ] + ), + ), + patch.object( + handler, + "interpret_menu", + AsyncMock( + side_effect=menu_ai.MenuInterpretationError( + sentinel, "INVALID_TOOL_ARGUMENTS_JSON" + ) + ), + ), + patch.object(handler, "notify_slack", AsyncMock()), + ): + handler.lambda_handler( + {"operation": "scrape_haksik", "target_date": "20260918"}, + _Context(), + ) + finally: + for log_handler, original_stream in original_streams: + log_handler.setStream(original_stream) + + events = [json.loads(line) for line in stream.getvalue().splitlines()] + validation = next(event for event in events if event["event.name"] == "menu_ai.validation_failed") + assert validation["date"] == "20260918" + assert validation["slot"] == "중식2" + assert validation["error_type"] == "MenuInterpretationError" + assert validation["reason_code"] == "INVALID_TOOL_ARGUMENTS_JSON" + assert sentinel not in stream.getvalue() + assert "provider.invalid" not in stream.getvalue() diff --git a/tests/test_state_machine_contract.py b/tests/test_state_machine_contract.py index 150428c..90ef10f 100644 --- a/tests/test_state_machine_contract.py +++ b/tests/test_state_machine_contract.py @@ -3,7 +3,7 @@ ROOT = Path(__file__).resolve().parents[1] -ASL_PATH = ROOT / "statemachine" / "dormitory-retry-workflow.asl.json" +ASL_PATH = ROOT / "statemachine" / "menu-retry-workflow.asl.json" TEMPLATE_PATH = ROOT / "template.yml" EXPECTED_RETRIES = [ @@ -19,7 +19,11 @@ "BackoffRate": 2.0, }, { - "ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError"], + "ErrorEquals": [ + "RetryableEmptyMenuError", + "RetryableApiSendError", + "RetryableMenuInterpretationError", + ], "IntervalSeconds": 7200, "MaxAttempts": 5, "BackoffRate": 1.0, @@ -31,15 +35,17 @@ def _workflow(): return json.loads(ASL_PATH.read_text(encoding="utf-8")) -def test_dormitory_invoke_payload_has_correlation_without_changing_retry_policy(): - invoke = _workflow()["States"]["InvokeDormitory"] +def test_common_invoke_payload_has_correlation_and_schedule_controls(): + invoke = _workflow()["States"]["InvokeSchedule"] assert invoke["Retry"] == EXPECTED_RETRIES assert invoke["Parameters"]["Payload"] == { "trigger": "step_functions", "execution_id.$": "$$.Execution.Id", "retry_count.$": "$$.State.RetryCount", - "delayed_schedule": False, + "schedule_anchor.$": "$$.Execution.StartTime", + "schedule_mode.$": "$.schedule_mode", + "notify_summary.$": "$.notify_summary", } @@ -50,17 +56,19 @@ def test_final_notifier_payload_is_allowlisted_and_never_forwards_cause_or_state assert notify["Payload"] == { "trigger": "step_functions", "execution_id.$": "$$.Execution.Id", - "restaurant": "DORMITORY", + "restaurant": "${Restaurant}", + "schedule_anchor.$": "$$.Execution.StartTime", + "schedule_mode.$": "$.schedule_mode", "error_type.$": "$.error.Error", } assert "Payload.$" not in notify - assert "Cause" not in ASL_PATH.read_text(encoding="utf-8") + assert "$.error.Cause" not in ASL_PATH.read_text(encoding="utf-8") def test_eventbridge_inputs_use_internal_shape_instead_of_api_gateway_shape(): template = TEMPLATE_PATH.read_text(encoding="utf-8") - assert template.count('"trigger": "eventbridge"') == 3 - assert template.count('"delayed_schedule": false') == 3 + assert template.count('"schedule_mode":"next_week"') == 3 + assert template.count('"schedule_mode":"tomorrow"') == 4 assert '"httpMethod"' not in template assert '"queryStringParameters"' not in template diff --git a/tests/test_template_contract.py b/tests/test_template_contract.py index 1fad100..f446114 100644 --- a/tests/test_template_contract.py +++ b/tests/test_template_contract.py @@ -6,7 +6,7 @@ ROOT = Path(__file__).resolve().parents[1] TEMPLATE_PATH = ROOT / "template.yml" README_PATH = ROOT / "README.md" -DORMITORY_ASL_PATH = ROOT / "statemachine" / "dormitory-retry-workflow.asl.json" +MENU_ASL_PATH = ROOT / "statemachine" / "menu-retry-workflow.asl.json" FUNCTION_HANDLERS = { "DodamScrapingFunction": "functions.handler.lambda_handler", @@ -80,7 +80,6 @@ def test_no_public_api_events_outputs_or_prohibited_resources(): "AWS::SNS::Topic", "DeadLetterQueue", "DestinationConfig", - "AWS::CloudWatch::Alarm", "AWS::CloudWatch::Dashboard", "AWS::Logs::MetricFilter", "Tracing: Active", @@ -120,48 +119,44 @@ def test_preserves_all_nine_functions_and_global_configuration(): assert "!Ref PythonRequirementsLayer" in block -def test_preserves_three_direct_schedules_and_dormitory_state_machine_schedule(): +def test_all_schedules_run_through_per_restaurant_state_machines(): template = _template_text() resources = _resource_blocks(template) - assert template.count("Type: Schedule") == 4 + assert template.count("Type: Schedule") == 8 for function_id in DIRECT_SCHEDULE_FUNCTIONS: - block = resources[function_id] - assert block.count("Type: Schedule") == 1 - assert "WeeklySchedule:" in block - assert "Schedule: cron(0 7 ? * SUN *)" in block + assert "Type: Schedule" not in resources[function_id] assert "Type: Schedule" not in resources["DormitorySchedulingFunction"] - state_machine = resources["DormitoryRetryStateMachine"] - assert "Type: AWS::Serverless::StateMachine" in state_machine - assert "DefinitionUri: statemachine/dormitory-retry-workflow.asl.json" in state_machine - assert "DormitorySchedulingFunctionArn: !GetAtt DormitorySchedulingFunction.Arn" in state_machine - assert "NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn" in state_machine - assert state_machine.count("LambdaInvokePolicy:") == 2 - assert state_machine.count("Type: Schedule") == 1 - assert "Schedule: cron(0 23 ? * SUN *)" in state_machine - assert "Logging:" not in state_machine - assert "MenuRetryStateMachine" not in resources + for restaurant in ("Dodam", "Haksik", "Faculty", "Dormitory"): + state_machine = resources[f"{restaurant}RetryStateMachine"] + assert "Type: AWS::Serverless::StateMachine" in state_machine + assert "DefinitionUri: statemachine/menu-retry-workflow.asl.json" in state_machine + assert "NotifyFailureFunctionArn: !GetAtt NotifyFailureFunction.Arn" in state_machine + assert state_machine.count("LambdaInvokePolicy:") == 2 + assert state_machine.count("Type: Schedule") == 2 + assert "RecoverySchedule:" in state_machine + assert resources[f"{restaurant}RetryStateMachineAlarm"].count("AWS::CloudWatch::Alarm") == 1 -def test_preserves_dormitory_retry_workflow(): +def test_preserves_common_retry_workflow(): workflow = cast( dict[str, object], - json.loads(DORMITORY_ASL_PATH.read_text(encoding="utf-8")), + json.loads(MENU_ASL_PATH.read_text(encoding="utf-8")), ) states = cast(dict[str, object], workflow["States"]) - invoke = cast(dict[str, object], states["InvokeDormitory"]) + invoke = cast(dict[str, object], states["InvokeSchedule"]) invoke_parameters = cast(dict[str, object], invoke["Parameters"]) retries = cast(list[object], invoke["Retry"]) notify = cast(dict[str, object], states["NotifyFinalFailure"]) notify_parameters = cast(dict[str, object], notify["Parameters"]) - assert workflow["StartAt"] == "InvokeDormitory" - assert set(states) == {"InvokeDormitory", "NotifyFinalFailure"} - assert invoke_parameters["FunctionName"] == "${DormitorySchedulingFunctionArn}" + assert workflow["StartAt"] == "InvokeSchedule" + assert set(states) == {"InvokeSchedule", "NotifyFinalFailure", "MarkExecutionFailed"} + assert invoke_parameters["FunctionName"] == "${SchedulingFunctionArn}" assert retries[1] == { - "ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError"], + "ErrorEquals": ["RetryableEmptyMenuError", "RetryableApiSendError", "RetryableMenuInterpretationError"], "IntervalSeconds": 7200, "MaxAttempts": 5, "BackoffRate": 1.0, diff --git a/tests/test_unified_handler.py b/tests/test_unified_handler.py index 7a93308..6f18e2e 100644 --- a/tests/test_unified_handler.py +++ b/tests/test_unified_handler.py @@ -1,12 +1,14 @@ import asyncio import json +from datetime import datetime from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, patch +from zoneinfo import ZoneInfo import pytest -from functions import handler # pyright: ignore[reportAttributeAccessIssue] +from functions import handler, menu_ai # pyright: ignore[reportAttributeAccessIssue] ROOT = Path(__file__).resolve().parents[1] @@ -575,3 +577,197 @@ def test_direct_dormitory_fetches_seven_dates_once_and_aggregates_weekly_respons assert body["date"] == "20260713_weekly" assert body["message"] == "기숙사식당 주간 메뉴 처리 완료 (7일치)" assert set(body["menus"]) == {f"{date}_중식1" for date in dates} + + +def test_parse_event_allowlists_schedule_mode_and_defaults_notify_summary_true(): + assert handler.parse_event({"schedule_mode": "tomorrow"})["schedule_mode"] == "tomorrow" + assert handler.parse_event({"schedule_mode": "unsafe"})["schedule_mode"] is None + assert handler.parse_event({"schedule_mode": ["tomorrow"]})["schedule_mode"] is None + assert handler.parse_event({})["notify_summary"] is True + assert handler.parse_event({"notify_summary": False})["notify_summary"] is False + assert handler.parse_event({"notify_summary": "unsafe"})["notify_summary"] is True + + +def test_tomorrow_schedule_uses_asia_seoul_date(): + config = handler.load_operation_config("schedule_haksik") + assert config is not None + request = handler.parse_event({"schedule_mode": "tomorrow"}) + fixed_now = datetime(2026, 9, 17, 23, 30, tzinfo=ZoneInfo("Asia/Seoul")) + + with patch.object(handler, "_now_seoul", return_value=fixed_now): + assert handler._dates_for(config, request) == ["20260918"] + + +def test_schedule_anchor_keeps_next_week_stable_after_retry_crosses_monday(): + config = handler.load_operation_config("schedule_haksik") + assert config is not None + request = handler.parse_event( + { + "schedule_mode": "next_week", + "schedule_anchor": "2026-09-20T07:00:00Z", + } + ) + + assert handler._dates_for(config, request) == [ + "20260921", "20260922", "20260923", "20260924", "20260925" + ] + + +def test_quiet_schedule_suppresses_only_date_summary_slack(): + scrape = AsyncMock(return_value=[_raw("20260918", "HAKSIK")]) + slack = AsyncMock() + with ( + patch.object(handler, "scrape", scrape), + patch.object( + handler, + "interpret_menu", + AsyncMock(return_value={"menuNames": ["밥"], "mainMenus": []}), + ), + patch.object(handler, "publish_menu", AsyncMock(return_value=_accepted())), + patch.object(handler, "notify_slack", slack), + ): + response = handler.lambda_handler( + { + "operation": "schedule_haksik", + "target_date": "20260918", + "notify_summary": False, + }, + _Context(), + ) + + assert response["statusCode"] == 200 + slack.assert_not_awaited() + + +def test_scheduled_menu_validation_failure_is_retryable_for_any_restaurant(): + error = menu_ai.MenuInterpretationError("unsafe model output", "INVALID_TOOL_CALL") + with ( + patch.object(handler, "scrape", AsyncMock(return_value=[_raw("20260918", "HAKSIK")])), + patch.object(handler, "interpret_menu", AsyncMock(side_effect=error)), + patch.object(handler, "publish_menu", AsyncMock()), + patch.object(handler, "notify_slack", AsyncMock()) as slack, + ): + with pytest.raises(handler.RetryableMenuInterpretationError) as raised: + handler.lambda_handler( + {"operation": "schedule_haksik", "target_date": "20260918"}, + _Context(), + ) + + assert raised.value.target_date == "20260918" + assert raised.value.restaurant == "HAKSIK" + slack.assert_not_awaited() + + +def test_direct_menu_validation_failure_remains_400_summary(): + error = menu_ai.MenuInterpretationError("unsafe model output", "INVALID_TOOL_CALL") + with ( + patch.object(handler, "scrape", AsyncMock(return_value=[_raw("20260918", "HAKSIK")])), + patch.object(handler, "interpret_menu", AsyncMock(side_effect=error)), + patch.object(handler, "publish_menu", AsyncMock()) as publish, + patch.object(handler, "notify_slack", AsyncMock()), + ): + response = handler.lambda_handler( + {"operation": "scrape_haksik", "target_date": "20260918"}, _Context() + ) + + assert response["statusCode"] == 400 + publish.assert_not_awaited() + + +def test_scheduled_provider_failure_is_retryable_for_step_functions(): + with ( + patch.object(handler, "scrape", AsyncMock(return_value=[_raw("20260918", "HAKSIK")])), + patch.object(handler, "interpret_menu", AsyncMock(side_effect=RuntimeError("secret"))), + patch.object(handler, "notify_slack", AsyncMock()), + ): + with pytest.raises(handler.RetryableMenuInterpretationError) as raised: + handler.lambda_handler( + {"operation": "schedule_haksik", "target_date": "20260918"}, + _Context(), + ) + + assert raised.value.restaurant == "HAKSIK" + assert raised.value.reason_code == "PROVIDER_FAILURE" + + +def test_generic_final_failure_resolves_allowlisted_restaurant_and_schedule_date(): + slack = AsyncMock() + with ( + patch.object(handler, "notify_slack", slack), + patch.object(handler, "_week_dates", return_value=["20260921"]), + ): + response = handler.lambda_handler( + { + "operation": "notify_final_failure", + "restaurant": "HAKSIK", + "schedule_mode": "next_week", + "error_type": "RetryableMenuInterpretationError", + }, + _Context(), + ) + + assert response["statusCode"] == 200 + assert slack.await_args is not None + assert slack.await_args.args[0]["slack_webhook_url"] == "https://hooks.slack.test/webhook" + notification = slack.await_args.args[1] + assert notification["restaurant"] == "학생식당" + assert notification["date"] == "20260921" + assert notification["error_type"] == "RetryableMenuInterpretationError" + + +def test_scheduled_empty_failure_uses_actual_restaurant_name(): + class SourceError(RuntimeError): + outcome = "AMBIGUOUS_EMPTY" + + source_error = SourceError("unsafe source detail") + with ( + patch.object(handler, "scrape", AsyncMock(side_effect=source_error)), + patch.object(handler, "notify_slack", AsyncMock()), + ): + with pytest.raises(handler.RetryableEmptyMenuError) as raised: + handler.lambda_handler( + {"operation": "schedule_haksik", "target_date": "20260918"}, + _Context(), + ) + + assert raised.value.restaurant == "HAKSIK" + + +def test_scheduled_api_failure_uses_actual_restaurant_name(): + with ( + patch.object(handler, "scrape", AsyncMock(return_value=[_raw("20260918", "HAKSIK")])), + patch.object( + handler, + "interpret_menu", + AsyncMock(return_value={"menuNames": ["밥"], "mainMenus": []}), + ), + patch.object( + handler, + "publish_menu", + AsyncMock(side_effect=[_accepted(), RuntimeError("unsafe API detail")]), + ), + patch.object(handler, "notify_slack", AsyncMock()), + ): + with pytest.raises(handler.RetryableApiSendError) as raised: + handler.lambda_handler( + {"operation": "schedule_haksik", "target_date": "20260918"}, + _Context(), + ) + + assert raised.value.restaurant == "HAKSIK" + + +def test_notify_summary_false_does_not_suppress_final_failure_slack(): + slack = AsyncMock() + with patch.object(handler, "notify_slack", slack): + handler.lambda_handler( + { + "operation": "notify_final_failure", + "target_date": "20260918", + "notify_summary": False, + "error_type": "RetryableMenuInterpretationError", + }, + _Context(), + ) + + slack.assert_awaited_once()