diff --git a/docs/architecture/adr-0003-independent-realtime-pose-detection.md b/docs/architecture/adr-0003-independent-realtime-pose-detection.md index 94aedf55..065980b0 100644 --- a/docs/architecture/adr-0003-independent-realtime-pose-detection.md +++ b/docs/architecture/adr-0003-independent-realtime-pose-detection.md @@ -10,6 +10,13 @@ VisionCamera를 유일한 Camera owner로 유지하고 RTC와 Pose를 각각 Pose Landmarker Lite는 `LIVE_STREAM`, 최대 4명, 최대 10 FPS, 입력 긴 변 640px, segmentation mask 비활성 상태로 시작한다. +Feature model의 `useLivePoseDetection`이 AppState, 활성 조건, +검출 상태와 React lifecycle을 소유한다. `lib`의 native Pose runtime +adapter는 callback 등록, FrameSink 제어와 native 결과 변환만 담당한다. +따라서 model은 native package를 직접 호출하지 않고 runtime port를 통해 +검출 lifecycle을 조율한다. Native 모듈이 전역 callback과 FrameSink를 +하나만 제공하므로 runtime은 동시에 하나의 owner만 허용한다. + ## Context RTC 송출과 Pose 추론은 같은 카메라 프레임이 필요하지만 처리 시간, @@ -34,6 +41,10 @@ lifetime을 침범하지 않는다. latest-only 전달은 JS가 느릴 때 과거 자세 결과가 누적되는 것을 막고 현재 자세에 가까운 결과를 유지한다. +React hook을 단순 계산 유틸과 구분해 model에 배치하고 native runtime을 +adapter로 격리하면 화면 lifecycle 정책과 MediaPipe/VisionCamera 연동 +구현을 독립적으로 변경할 수 있다. + ## Trade-off Pose가 수락한 프레임에는 플랫폼별 YUV→RGB/BGRA 복사 비용이 @@ -54,6 +65,10 @@ pose detection 단위 테스트를 통과했다. Android 공식 AAR의 API 시그니처는 검증했지만 작업 환경에 Android SDK가 없어 Android 타깃 컴파일과 실제 기기 추론 검증은 남아 있다. +Feature model의 검출 hook과 native Pose runtime adapter를 분리하되, +기존 configure/start/stop/release 순서와 latest-only callback 정책은 +유지했다. + 실제 기기에서는 DEV debug 로그의 `poseCount`와 `landmarkCounts`로 1명, 2명, 3명 이상 결과를 확인하고, 평균 추론 시간, 수락/드롭 프레임, 발열과 배터리 사용량을 함께 기록한다. 이 측정 결과에 따라 diff --git a/src/features/camera/guide-feed/lib/feed-camera-aspect-ratio.ts b/src/features/camera/guide-feed/lib/feed-camera-aspect-ratio.ts index 0a2532e3..f405b263 100644 --- a/src/features/camera/guide-feed/lib/feed-camera-aspect-ratio.ts +++ b/src/features/camera/guide-feed/lib/feed-camera-aspect-ratio.ts @@ -1,5 +1,5 @@ +import type { CoordinateSize } from "../model/pose-types"; import type { CameraGuideAspectRatio } from "../model/types"; -import type { CoordinateSize } from "./pose-matching"; const SUPPORTED_ASPECT_RATIOS = { "4:3": 4 / 3, diff --git a/src/features/camera/guide-feed/lib/guide-contour-projection.ts b/src/features/camera/guide-feed/lib/guide-contour-projection.ts index e9b6be09..dbe6dbb1 100644 --- a/src/features/camera/guide-feed/lib/guide-contour-projection.ts +++ b/src/features/camera/guide-feed/lib/guide-contour-projection.ts @@ -1,8 +1,5 @@ -import { - calculateCanvasRenderRect, - type CoordinateSize, - type PreviewPoint, -} from "./pose-matching"; +import { calculateCanvasRenderRect, type PreviewPoint } from "./pose-matching"; +import type { CoordinateSize } from "../model/pose-types"; import type { CameraGuideContour, CameraGuideOutline } from "../model/types"; export interface ProjectedCameraGuideContour extends Omit< diff --git a/src/features/camera/guide-feed/lib/pose-detection/index.ts b/src/features/camera/guide-feed/lib/pose-detection/index.ts index b568f31c..2d6e4253 100644 --- a/src/features/camera/guide-feed/lib/pose-detection/index.ts +++ b/src/features/camera/guide-feed/lib/pose-detection/index.ts @@ -1,4 +1,2 @@ export * from "./native-pose-result-adapter"; export * from "./pose-detection-config"; -export * from "./types"; -export * from "./use-live-pose-detection"; diff --git a/src/features/camera/guide-feed/lib/pose-detection/native-pose-detection-runtime.ts b/src/features/camera/guide-feed/lib/pose-detection/native-pose-detection-runtime.ts new file mode 100644 index 00000000..1235ae7a --- /dev/null +++ b/src/features/camera/guide-feed/lib/pose-detection/native-pose-detection-runtime.ts @@ -0,0 +1,54 @@ +import { + setVisionCameraPoseErrorCallback, + setVisionCameraPoseResultCallback, + visionCameraPoseFrameSink, +} from "@newpic/vision-camera-pose"; +import type { PoseDetectionRuntime } from "../../model/pose-detection-types"; +import { adaptNativeDetectedPoseFrame } from "./native-pose-result-adapter"; + +let activeOwner: symbol | null = null; + +export const nativePoseDetectionRuntime = { + frameSink: visionCameraPoseFrameSink, + + acquire({ shouldAcceptResult, onFrame, onError }) { + if (activeOwner) { + throw new Error("Native pose detection runtime already has an owner."); + } + + const owner = Symbol("native-pose-detection-owner"); + activeOwner = owner; + setVisionCameraPoseResultCallback((nativeFrame) => { + if (!shouldAcceptResult()) return; + onFrame(adaptNativeDetectedPoseFrame(nativeFrame)); + }); + setVisionCameraPoseErrorCallback((error) => { + if (!shouldAcceptResult()) return; + onError(error); + }); + + return () => { + if (activeOwner !== owner) return; + + activeOwner = null; + setVisionCameraPoseResultCallback(undefined); + setVisionCameraPoseErrorCallback(undefined); + }; + }, + + configure(config) { + visionCameraPoseFrameSink.configure(config); + }, + + start() { + visionCameraPoseFrameSink.startAcceptingFrames(); + }, + + stop() { + visionCameraPoseFrameSink.stopAcceptingFrames(); + }, + + release() { + visionCameraPoseFrameSink.releaseDetector(); + }, +} satisfies PoseDetectionRuntime; diff --git a/src/features/camera/guide-feed/lib/pose-detection/native-pose-result-adapter.ts b/src/features/camera/guide-feed/lib/pose-detection/native-pose-result-adapter.ts index 1a5f5677..8eda6687 100644 --- a/src/features/camera/guide-feed/lib/pose-detection/native-pose-result-adapter.ts +++ b/src/features/camera/guide-feed/lib/pose-detection/native-pose-result-adapter.ts @@ -1,5 +1,5 @@ import type { NativeDetectedPoseFrame } from "@newpic/vision-camera-pose"; -import type { DetectedPoseFrame } from "./types"; +import type { DetectedPoseFrame } from "../../model/pose-types"; const QUARTER_TURNS = [0, 90, 180, 270] as const; diff --git a/src/features/camera/guide-feed/lib/pose-detection/pose-detection-config.ts b/src/features/camera/guide-feed/lib/pose-detection/pose-detection-config.ts index 6387e395..3bf37796 100644 --- a/src/features/camera/guide-feed/lib/pose-detection/pose-detection-config.ts +++ b/src/features/camera/guide-feed/lib/pose-detection/pose-detection-config.ts @@ -1,3 +1,8 @@ +import type { + PoseDetectionConfig, + ResolvedPoseDetectionConfig, +} from "../../model/pose-detection-types"; + export const MAX_POSE_COUNT = 4; export const MAX_POSE_INFERENCE_FPS = 10; @@ -14,22 +19,15 @@ export const DEFAULT_POSE_DETECTION_CONFIG = { minTrackingConfidence: 0.5, } as const; -export type PoseDetectionConfig = { - targetPersonCount?: number; - maxInferenceFps?: number; - maxInputLongEdge?: number; - minPoseDetectionConfidence?: number; - minPosePresenceConfidence?: number; - minTrackingConfidence?: number; -}; - export function resolvePoseCount(targetPersonCount?: number) { const requested = targetPersonCount ?? DEFAULT_POSE_DETECTION_CONFIG.defaultPoseCount; return Math.min(MAX_POSE_COUNT, Math.max(1, Math.trunc(requested))); } -export function resolvePoseDetectionConfig(config: PoseDetectionConfig = {}) { +export function resolvePoseDetectionConfig( + config: PoseDetectionConfig = {}, +): ResolvedPoseDetectionConfig { return { numPoses: resolvePoseCount(config.targetPersonCount), maxInferenceFps: Math.min( diff --git a/src/features/camera/guide-feed/lib/pose-detection/types.ts b/src/features/camera/guide-feed/lib/pose-detection/types.ts deleted file mode 100644 index 2180fab1..00000000 --- a/src/features/camera/guide-feed/lib/pose-detection/types.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { VisionCameraPoseFrameSink } from "@newpic/vision-camera-pose"; - -export type PosePoint = { - x: number; - y: number; - z?: number; - confidence?: number; -}; - -export type DetectedPersonPose = { - landmarks: PosePoint[]; -}; - -export type DetectedPoseFrame = { - timestamp: number; - poses: DetectedPersonPose[]; - inputSize: { - width: number; - height: number; - }; - sourceFrame: { - width: number; - height: number; - rotationDegrees: 0 | 90 | 180 | 270; - isMirrored: boolean; - }; -}; - -export type PoseDetectionStatus = "idle" | "initializing" | "running" | "error"; - -export type PoseDetectionError = { - code: string; - message: string; -}; - -export type LivePoseDetection = { - frame: DetectedPoseFrame | null; - error: PoseDetectionError | null; - status: PoseDetectionStatus; - frameSink: VisionCameraPoseFrameSink; -}; diff --git a/src/features/camera/guide-feed/lib/pose-detection/use-live-pose-detection.ts b/src/features/camera/guide-feed/lib/pose-detection/use-live-pose-detection.ts deleted file mode 100644 index 3dbe6e0b..00000000 --- a/src/features/camera/guide-feed/lib/pose-detection/use-live-pose-detection.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { - setVisionCameraPoseErrorCallback, - setVisionCameraPoseResultCallback, - visionCameraPoseFrameSink, -} from "@newpic/vision-camera-pose"; -import { useEffect, useMemo, useRef, useState } from "react"; -import { AppState } from "react-native"; -import { adaptNativeDetectedPoseFrame } from "./native-pose-result-adapter"; -import { - type PoseDetectionConfig, - resolvePoseDetectionConfig, -} from "./pose-detection-config"; -import type { - DetectedPoseFrame, - LivePoseDetection, - PoseDetectionError, - PoseDetectionStatus, -} from "./types"; - -export type UseLivePoseDetectionOptions = PoseDetectionConfig & { - /** - * Set this only while the VisionCamera session is active and a target - * overlay is available and visible. - */ - enabled: boolean; - debug?: boolean; - /** - * Receives latest-only native results without requiring React state to - * update at inference FPS. - */ - onFrame?: (frame: DetectedPoseFrame) => void; - exposeFrame?: boolean; -}; - -export function useLivePoseDetection({ - enabled, - debug = false, - onFrame, - exposeFrame = true, - ...config -}: UseLivePoseDetectionOptions): LivePoseDetection { - const [appState, setAppState] = useState(AppState.currentState); - const [frame, setFrame] = useState(null); - const [error, setError] = useState(null); - const [status, setStatus] = useState("idle"); - const shouldAcceptResultsRef = useRef(false); - const debugRef = useRef(debug); - const onFrameRef = useRef(onFrame); - const exposeFrameRef = useRef(exposeFrame); - debugRef.current = debug; - onFrameRef.current = onFrame; - exposeFrameRef.current = exposeFrame; - const resolvedConfig = useMemo( - () => resolvePoseDetectionConfig(config), - [ - config.maxInferenceFps, - config.maxInputLongEdge, - config.minPoseDetectionConfidence, - config.minPosePresenceConfidence, - config.minTrackingConfidence, - config.targetPersonCount, - ], - ); - const isForeground = appState === "active"; - const shouldRun = enabled && isForeground; - - useEffect(() => { - const subscription = AppState.addEventListener("change", setAppState); - return () => subscription.remove(); - }, []); - - useEffect(() => { - setVisionCameraPoseResultCallback((nativeFrame) => { - if (!shouldAcceptResultsRef.current) return; - - const detectedFrame = adaptNativeDetectedPoseFrame(nativeFrame); - onFrameRef.current?.(detectedFrame); - if (exposeFrameRef.current) { - setFrame(detectedFrame); - } - setError(null); - setStatus("running"); - - if (__DEV__ && debugRef.current) { - console.debug("[PoseDetection]", { - timestamp: detectedFrame.timestamp, - poseCount: detectedFrame.poses.length, - landmarkCounts: detectedFrame.poses.map( - (pose) => pose.landmarks.length, - ), - inputSize: detectedFrame.inputSize, - }); - } - }); - setVisionCameraPoseErrorCallback((nativeError) => { - if (!shouldAcceptResultsRef.current) return; - - setError(nativeError); - setStatus("error"); - if (__DEV__ && debugRef.current) { - console.warn("[PoseDetection]", nativeError); - } - }); - - return () => { - shouldAcceptResultsRef.current = false; - setVisionCameraPoseResultCallback(undefined); - setVisionCameraPoseErrorCallback(undefined); - visionCameraPoseFrameSink.stopAcceptingFrames(); - visionCameraPoseFrameSink.releaseDetector(); - }; - }, []); - - useEffect(() => { - visionCameraPoseFrameSink.configure(resolvedConfig); - - if (shouldRun) { - shouldAcceptResultsRef.current = true; - setStatus("initializing"); - setError(null); - visionCameraPoseFrameSink.startAcceptingFrames(); - return; - } - - shouldAcceptResultsRef.current = false; - visionCameraPoseFrameSink.stopAcceptingFrames(); - setFrame(null); - setError(null); - setStatus("idle"); - }, [resolvedConfig, shouldRun]); - - return { - frame, - error, - status, - frameSink: visionCameraPoseFrameSink, - }; -} diff --git a/src/features/camera/guide-feed/lib/pose-matching/coordinate-transform.ts b/src/features/camera/guide-feed/lib/pose-matching/coordinate-transform.ts index 772d4a89..a4f0e4a1 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/coordinate-transform.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/coordinate-transform.ts @@ -1,14 +1,16 @@ import type { - CaptureAspectRatio, - CaptureToPreviewTransform, - CanvasRenderRect, CommonPose, CommonPosePoint, CoordinateSize, DWPoseSourcePose, MediaPipeInputPose, - MediaPipePoseToCaptureTransform, PoseJointMap, +} from "../../model/pose-types"; +import type { + CaptureAspectRatio, + CaptureToPreviewTransform, + CanvasRenderRect, + MediaPipePoseToCaptureTransform, PreviewPoint, QuarterTurn, ResizeMode, diff --git a/src/features/camera/guide-feed/lib/pose-matching/dwpose-pose-adapter.ts b/src/features/camera/guide-feed/lib/pose-matching/dwpose-pose-adapter.ts index 0261139a..d8dbaf11 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/dwpose-pose-adapter.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/dwpose-pose-adapter.ts @@ -3,7 +3,11 @@ import type { NormalizedPosePerson, NormalizedPoseResult, } from "@entities/feed"; -import type { CommonJoint, CoordinateSize, DWPoseSourcePose } from "./types"; +import type { + CommonJoint, + CoordinateSize, + DWPoseSourcePose, +} from "../../model/pose-types"; export const DWPOSE_KEYPOINT_FORMAT = "dwpose_xy_score" as const; diff --git a/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target-preparer.ts b/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target-preparer.ts index ad9d1668..f41ad0e9 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target-preparer.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target-preparer.ts @@ -1,7 +1,7 @@ import type { FeedPoseResponse } from "@entities/feed"; +import type { CoordinateSize, DWPoseSourcePose } from "../../model/pose-types"; import { resolveFeedCameraAspectRatio } from "../feed-camera-aspect-ratio"; import { adaptDWPoseResult } from "./dwpose-pose-adapter"; -import type { CoordinateSize, DWPoseSourcePose } from "./types"; export interface FeedPoseTargetSelection { feedId: string; diff --git a/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target.ts b/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target.ts index a3e08fb3..032cadad 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/feed-pose-target.ts @@ -1,7 +1,8 @@ import type { FeedPoseResponse, NormalizedPoseResult } from "@entities/feed"; -import type { CommonPose, SourcePoseToCaptureTransform } from "./types"; +import type { CommonPose } from "../../model/pose-types"; import { projectDWPosePoseToCapture } from "./coordinate-transform"; import { adaptDWPoseResult } from "./dwpose-pose-adapter"; +import type { SourcePoseToCaptureTransform } from "./types"; export function getNormalizedPoseResult( response: FeedPoseResponse, diff --git a/src/features/camera/guide-feed/lib/pose-matching/live-pose-frame.ts b/src/features/camera/guide-feed/lib/pose-matching/live-pose-frame.ts index 6f9f578e..c1f1060d 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/live-pose-frame.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/live-pose-frame.ts @@ -1,6 +1,11 @@ -import type { CommonPose, CoordinateSize, ResizeMode } from "./types"; +import type { + CommonPose, + CoordinateSize, + DetectedPoseFrame, +} from "../../model/pose-types"; import { projectMediaPipePoseToCapture } from "./coordinate-transform"; import { adaptMediaPipePoses } from "./mediapipe-pose-adapter"; +import type { ResizeMode } from "./types"; export interface PrepareLivePoseFrameOptions { captureSize: CoordinateSize; @@ -8,20 +13,8 @@ export interface PrepareLivePoseFrameOptions { captureResizeMode?: ResizeMode; } -export interface LiveDetectedPoseFrame { - poses: { - landmarks: { - x: number; - y: number; - z?: number; - confidence?: number; - }[]; - }[]; - inputSize: CoordinateSize; -} - export function prepareLivePoses( - frame: LiveDetectedPoseFrame, + frame: DetectedPoseFrame, { captureSize, mirrorX, diff --git a/src/features/camera/guide-feed/lib/pose-matching/mediapipe-pose-adapter.ts b/src/features/camera/guide-feed/lib/pose-matching/mediapipe-pose-adapter.ts index 62fd223c..cd89de0b 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/mediapipe-pose-adapter.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/mediapipe-pose-adapter.ts @@ -1,11 +1,8 @@ -import type { CommonJoint, MediaPipeInputPose } from "./types"; - -export interface MediaPipePoseLandmark { - x: number; - y: number; - z?: number; - confidence?: number; -} +import type { + CommonJoint, + MediaPipeInputPose, + MediaPipePoseLandmark, +} from "../../model/pose-types"; const MEDIAPIPE_POSE_INDEX: Record = { NOSE: 0, diff --git a/src/features/camera/guide-feed/lib/pose-matching/pose-assignment.ts b/src/features/camera/guide-feed/lib/pose-matching/pose-assignment.ts index 660b7065..26da31db 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/pose-assignment.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/pose-assignment.ts @@ -1,4 +1,8 @@ -import type { CommonPose, PoseAssignment, PosePairMatch } from "./types"; +import type { + CommonPose, + PoseAssignment, + PosePairMatch, +} from "../../model/pose-types"; import type { PoseMatchConfig } from "./pose-match-config"; import { matchPosePair } from "./pose-geometry"; diff --git a/src/features/camera/guide-feed/lib/pose-matching/pose-geometry.ts b/src/features/camera/guide-feed/lib/pose-matching/pose-geometry.ts index b9d348a2..f2386621 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/pose-geometry.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/pose-geometry.ts @@ -6,7 +6,7 @@ import { type PoseBoundingBox, type PoseJointGroup, type PosePairMatch, -} from "./types"; +} from "../../model/pose-types"; import type { PoseMatchConfig } from "./pose-match-config"; const EMPTY_SCORE = { diff --git a/src/features/camera/guide-feed/lib/pose-matching/pose-match-config.ts b/src/features/camera/guide-feed/lib/pose-matching/pose-match-config.ts index b8459553..d2e03e22 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/pose-match-config.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/pose-match-config.ts @@ -1,4 +1,4 @@ -import type { CommonJoint, PoseJointGroup } from "./types"; +import type { CommonJoint, PoseJointGroup } from "../../model/pose-types"; export interface PoseMatchConfig { minConfidence: number; diff --git a/src/features/camera/guide-feed/lib/pose-matching/pose-matcher.ts b/src/features/camera/guide-feed/lib/pose-matching/pose-matcher.ts index d3eb5609..9b5342ec 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/pose-matcher.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/pose-matcher.ts @@ -6,7 +6,7 @@ import { type PoseMismatchCause, type PosePairMatch, type PoseSceneMatchResult, -} from "./types"; +} from "../../model/pose-types"; import { DEFAULT_POSE_MATCH_CONFIG, type PoseMatchConfig, diff --git a/src/features/camera/guide-feed/lib/pose-matching/types.ts b/src/features/camera/guide-feed/lib/pose-matching/types.ts index 6c730721..7608e2bd 100644 --- a/src/features/camera/guide-feed/lib/pose-matching/types.ts +++ b/src/features/camera/guide-feed/lib/pose-matching/types.ts @@ -1,155 +1,9 @@ -export const COMMON_JOINTS = [ - "NOSE", - "LEFT_SHOULDER", - "RIGHT_SHOULDER", - "LEFT_ELBOW", - "RIGHT_ELBOW", - "LEFT_WRIST", - "RIGHT_WRIST", - "LEFT_HIP", - "RIGHT_HIP", - "LEFT_KNEE", - "RIGHT_KNEE", - "LEFT_ANKLE", - "RIGHT_ANKLE", -] as const; - -export type CommonJoint = (typeof COMMON_JOINTS)[number]; - -/** - * CaptureNormalizedCoordinate contract: - * - origin: final capture canvas top-left - * - x: right, y: down - * - x/y: normalized by final capture width/height - * - values outside 0...1 are retained when cover-cropped - */ -export interface CommonPosePoint { - x: number; - y: number; - confidence: number; -} - -export interface PoseJointMap { - joints: Partial>; -} - -export interface CommonPose { - joints: Partial>; -} - -export interface DWPoseSourcePose extends PoseJointMap { - coordinateSpace: "dwpose_source_normalized"; - sourcePersonIndex: number; -} - -export interface MediaPipeInputPose extends PoseJointMap { - coordinateSpace: "mediapipe_input_normalized"; -} - -export interface PoseBoundingBox { - minX: number; - minY: number; - maxX: number; - maxY: number; - width: number; - height: number; - area: number; - center: { x: number; y: number }; -} - -export type PoseJointGroup = - "TORSO" | "LEFT_ARM" | "RIGHT_ARM" | "LEFT_LEG" | "RIGHT_LEG"; - -export type PoseFeedback = - | "NO_PERSON" - | "PERSON_COUNT_MISMATCH" - | "MOVE_LEFT" - | "MOVE_RIGHT" - | "MOVE_UP" - | "MOVE_DOWN" - | "MOVE_CLOSER" - | "MOVE_FARTHER" - | "ADJUST_LEFT_ARM" - | "ADJUST_RIGHT_ARM" - | "ADJUST_LEFT_LEG" - | "ADJUST_RIGHT_LEG" - | "ADJUST_TORSO" - | "LOW_CONFIDENCE" - | "ALIGNED"; - -export interface PoseMatchScore { - overall: number; - position: number; - scale: number; - pose: number; -} - -export interface PosePairMetrics { - targetCenter: { x: number; y: number }; - liveCenter: { x: number; y: number }; - centerDelta: { x: number; y: number }; - centerDistance: number; - targetBoundingBox: PoseBoundingBox; - liveBoundingBox: PoseBoundingBox; - scaleRatio: number; - comparableJointCount: number; - jointErrors: Partial>; - jointGroupScores: Partial>; -} - -export interface PosePairMatch { - score: PoseMatchScore; - metrics: PosePairMetrics | null; - isComparable: boolean; - isAligned: boolean; -} - -export interface PoseAssignment { - targetIndex: number; - liveIndex: number; - score: number; - match: PosePairMatch; -} - -export type PoseMismatchCause = - | "NO_PERSON" - | "PERSON_COUNT" - | "POSITION" - | "SCALE" - | "TORSO" - | "LEFT_ARM" - | "RIGHT_ARM" - | "LEFT_LEG" - | "RIGHT_LEG" - | "CONFIDENCE"; - -export interface WorstPoseMatch { - targetIndex: number; - liveIndex: number; - score: number; - cause: PoseMismatchCause; -} - -export interface PoseSceneMatchResult { - aligned: boolean; - sceneScore: number; - feedback: PoseFeedback; - assignments: PoseAssignment[]; - unmatchedTargetIndices: number[]; - unmatchedLiveIndices: number[]; - worstMatch: WorstPoseMatch | null; - largestMismatch: PoseMismatchCause | null; -} +import type { CoordinateSize } from "../../model/pose-types"; export type QuarterTurn = 0 | 90 | 180 | 270; export type ResizeMode = "cover" | "contain"; export type CaptureAspectRatio = "4:3" | "16:9"; -export interface CoordinateSize { - width: number; - height: number; -} - export interface SourcePoseToCaptureTransform { sourceSize: CoordinateSize; captureSize: CoordinateSize; diff --git a/src/features/camera/guide-feed/model/guide-state.ts b/src/features/camera/guide-feed/model/guide-state.ts index 7ab5a59c..7dfcfcde 100644 --- a/src/features/camera/guide-feed/model/guide-state.ts +++ b/src/features/camera/guide-feed/model/guide-state.ts @@ -1,5 +1,5 @@ import { resolveFeedCameraAspectRatio } from "../lib/feed-camera-aspect-ratio"; -import type { CoordinateSize, DWPoseSourcePose } from "../lib/pose-matching"; +import type { CoordinateSize, DWPoseSourcePose } from "./pose-types"; import type { ActiveCameraGuide, CameraGuideOutline, diff --git a/src/features/camera/guide-feed/model/index.ts b/src/features/camera/guide-feed/model/index.ts index 5094d11f..2987fd3c 100644 --- a/src/features/camera/guide-feed/model/index.ts +++ b/src/features/camera/guide-feed/model/index.ts @@ -1,5 +1,6 @@ export * from "./camera-guide-navigation"; export * from "./guide-state"; +export * from "./pose-types"; export * from "./pose-guide-alignment-policy"; export * from "./pose-guide-feedback-config"; export * from "./types"; diff --git a/src/features/camera/guide-feed/model/pose-detection-types.ts b/src/features/camera/guide-feed/model/pose-detection-types.ts new file mode 100644 index 00000000..74747fc4 --- /dev/null +++ b/src/features/camera/guide-feed/model/pose-detection-types.ts @@ -0,0 +1,63 @@ +import type { DetectedPoseFrame } from "./pose-types"; + +export interface PoseDetectionConfig { + targetPersonCount?: number; + maxInferenceFps?: number; + maxInputLongEdge?: number; + minPoseDetectionConfidence?: number; + minPosePresenceConfidence?: number; + minTrackingConfidence?: number; +} + +export interface ResolvedPoseDetectionConfig { + numPoses: number; + maxInferenceFps: number; + maxInputLongEdge: number; + minPoseDetectionConfidence: number; + minPosePresenceConfidence: number; + minTrackingConfidence: number; +} + +export type PoseDetectionStatus = "idle" | "initializing" | "running" | "error"; + +export interface PoseDetectionError { + code: string; + message: string; +} + +export interface PoseDetectionRuntimeListener { + shouldAcceptResult: () => boolean; + onFrame: (frame: DetectedPoseFrame) => void; + onError: (error: PoseDetectionError) => void; +} + +export interface PoseDetectionRuntime { + frameSink: TFrameSink; + acquire: (listener: PoseDetectionRuntimeListener) => () => void; + configure: (config: ResolvedPoseDetectionConfig) => void; + start: () => void; + stop: () => void; + release: () => void; +} + +export interface LivePoseDetection { + frame: DetectedPoseFrame | null; + error: PoseDetectionError | null; + status: PoseDetectionStatus; + frameSink: TFrameSink; +} + +export interface UseLivePoseDetectionOptions extends PoseDetectionConfig { + /** + * Set this only while the VisionCamera session is active and a target + * overlay is available and visible. + */ + enabled: boolean; + debug?: boolean; + /** + * Receives latest-only native results without requiring React state to + * update at inference FPS. + */ + onFrame?: (frame: DetectedPoseFrame) => void; + exposeFrame?: boolean; +} diff --git a/src/features/camera/guide-feed/model/pose-guide-alignment-policy.ts b/src/features/camera/guide-feed/model/pose-guide-alignment-policy.ts index 4db9a386..c3343988 100644 --- a/src/features/camera/guide-feed/model/pose-guide-alignment-policy.ts +++ b/src/features/camera/guide-feed/model/pose-guide-alignment-policy.ts @@ -2,7 +2,7 @@ import type { PoseAssignment, PoseFeedback, PoseSceneMatchResult, -} from "../lib/pose-matching"; +} from "./pose-types"; import { DEFAULT_POSE_GUIDE_FEEDBACK_CONFIG, type PoseGuideFeedbackConfig, diff --git a/src/features/camera/guide-feed/model/pose-types.ts b/src/features/camera/guide-feed/model/pose-types.ts new file mode 100644 index 00000000..2d83e259 --- /dev/null +++ b/src/features/camera/guide-feed/model/pose-types.ts @@ -0,0 +1,170 @@ +export const COMMON_JOINTS = [ + "NOSE", + "LEFT_SHOULDER", + "RIGHT_SHOULDER", + "LEFT_ELBOW", + "RIGHT_ELBOW", + "LEFT_WRIST", + "RIGHT_WRIST", + "LEFT_HIP", + "RIGHT_HIP", + "LEFT_KNEE", + "RIGHT_KNEE", + "LEFT_ANKLE", + "RIGHT_ANKLE", +] as const; + +export type CommonJoint = (typeof COMMON_JOINTS)[number]; + +export interface CoordinateSize { + width: number; + height: number; +} + +/** + * CaptureNormalizedCoordinate contract: + * - origin: final capture canvas top-left + * - x: right, y: down + * - x/y: normalized by final capture width/height + * - values outside 0...1 are retained when cover-cropped + */ +export interface CommonPosePoint { + x: number; + y: number; + confidence: number; +} + +export interface PoseJointMap { + joints: Partial>; +} + +export interface CommonPose { + joints: Partial>; +} + +export interface DWPoseSourcePose extends PoseJointMap { + coordinateSpace: "dwpose_source_normalized"; + sourcePersonIndex: number; +} + +export interface MediaPipeInputPose extends PoseJointMap { + coordinateSpace: "mediapipe_input_normalized"; +} + +export interface MediaPipePoseLandmark { + x: number; + y: number; + z?: number; + confidence?: number; +} + +export interface DetectedPersonPose { + landmarks: MediaPipePoseLandmark[]; +} + +export interface DetectedPoseFrame { + timestamp: number; + poses: DetectedPersonPose[]; + inputSize: CoordinateSize; + sourceFrame: { + width: number; + height: number; + rotationDegrees: 0 | 90 | 180 | 270; + isMirrored: boolean; + }; +} + +export interface PoseBoundingBox { + minX: number; + minY: number; + maxX: number; + maxY: number; + width: number; + height: number; + area: number; + center: { x: number; y: number }; +} + +export type PoseJointGroup = + "TORSO" | "LEFT_ARM" | "RIGHT_ARM" | "LEFT_LEG" | "RIGHT_LEG"; + +export type PoseFeedback = + | "NO_PERSON" + | "PERSON_COUNT_MISMATCH" + | "MOVE_LEFT" + | "MOVE_RIGHT" + | "MOVE_UP" + | "MOVE_DOWN" + | "MOVE_CLOSER" + | "MOVE_FARTHER" + | "ADJUST_LEFT_ARM" + | "ADJUST_RIGHT_ARM" + | "ADJUST_LEFT_LEG" + | "ADJUST_RIGHT_LEG" + | "ADJUST_TORSO" + | "LOW_CONFIDENCE" + | "ALIGNED"; + +export interface PoseMatchScore { + overall: number; + position: number; + scale: number; + pose: number; +} + +export interface PosePairMetrics { + targetCenter: { x: number; y: number }; + liveCenter: { x: number; y: number }; + centerDelta: { x: number; y: number }; + centerDistance: number; + targetBoundingBox: PoseBoundingBox; + liveBoundingBox: PoseBoundingBox; + scaleRatio: number; + comparableJointCount: number; + jointErrors: Partial>; + jointGroupScores: Partial>; +} + +export interface PosePairMatch { + score: PoseMatchScore; + metrics: PosePairMetrics | null; + isComparable: boolean; + isAligned: boolean; +} + +export interface PoseAssignment { + targetIndex: number; + liveIndex: number; + score: number; + match: PosePairMatch; +} + +export type PoseMismatchCause = + | "NO_PERSON" + | "PERSON_COUNT" + | "POSITION" + | "SCALE" + | "TORSO" + | "LEFT_ARM" + | "RIGHT_ARM" + | "LEFT_LEG" + | "RIGHT_LEG" + | "CONFIDENCE"; + +export interface WorstPoseMatch { + targetIndex: number; + liveIndex: number; + score: number; + cause: PoseMismatchCause; +} + +export interface PoseSceneMatchResult { + aligned: boolean; + sceneScore: number; + feedback: PoseFeedback; + assignments: PoseAssignment[]; + unmatchedTargetIndices: number[]; + unmatchedLiveIndices: number[]; + worstMatch: WorstPoseMatch | null; + largestMismatch: PoseMismatchCause | null; +} diff --git a/src/features/camera/guide-feed/model/types.ts b/src/features/camera/guide-feed/model/types.ts index 541eedb6..daf24ae4 100644 --- a/src/features/camera/guide-feed/model/types.ts +++ b/src/features/camera/guide-feed/model/types.ts @@ -3,7 +3,7 @@ import type { CoordinateSize, DWPoseSourcePose, PoseSceneMatchResult, -} from "../lib/pose-matching"; +} from "./pose-types"; export type CameraGuideAspectRatio = "4:3" | "16:9"; diff --git a/src/features/camera/guide-feed/model/use-camera-guide-controller.ts b/src/features/camera/guide-feed/model/use-camera-guide-controller.ts index fff42991..f0d173c1 100644 --- a/src/features/camera/guide-feed/model/use-camera-guide-controller.ts +++ b/src/features/camera/guide-feed/model/use-camera-guide-controller.ts @@ -6,10 +6,6 @@ import { useRef, useState, } from "react"; -import { - type DetectedPoseFrame, - useLivePoseDetection, -} from "../lib/pose-detection"; import { adaptDWPoseResult, DWPoseContractError, @@ -20,7 +16,9 @@ import { } from "../lib/pose-matching"; import { adaptFeedBackgroundRemoval } from "../lib/feed-guide-contour-adapter"; import { cameraGuideReducer, INITIAL_CAMERA_GUIDE_STATE } from "./guide-state"; +import type { DetectedPoseFrame } from "./pose-types"; import { usePoseGuideAlignment } from "./use-pose-guide-alignment"; +import { useLivePoseDetection } from "./use-live-pose-detection"; import type { CameraGuideErrors, CameraGuideGeometry, diff --git a/src/features/camera/guide-feed/model/use-live-pose-detection.ts b/src/features/camera/guide-feed/model/use-live-pose-detection.ts new file mode 100644 index 00000000..fbdd2e60 --- /dev/null +++ b/src/features/camera/guide-feed/model/use-live-pose-detection.ts @@ -0,0 +1,118 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { AppState } from "react-native"; +import { nativePoseDetectionRuntime } from "../lib/pose-detection/native-pose-detection-runtime"; +import { resolvePoseDetectionConfig } from "../lib/pose-detection/pose-detection-config"; +import type { + LivePoseDetection, + PoseDetectionError, + PoseDetectionStatus, + UseLivePoseDetectionOptions, +} from "./pose-detection-types"; +import type { DetectedPoseFrame } from "./pose-types"; + +export function useLivePoseDetection({ + enabled, + debug = false, + onFrame, + exposeFrame = true, + ...config +}: UseLivePoseDetectionOptions): LivePoseDetection< + typeof nativePoseDetectionRuntime.frameSink +> { + const [appState, setAppState] = useState(AppState.currentState); + const [frame, setFrame] = useState(null); + const [error, setError] = useState(null); + const [status, setStatus] = useState("idle"); + const shouldAcceptResultsRef = useRef(false); + const debugRef = useRef(debug); + const onFrameRef = useRef(onFrame); + const exposeFrameRef = useRef(exposeFrame); + const resolvedConfig = useMemo( + () => resolvePoseDetectionConfig(config), + [ + config.maxInferenceFps, + config.maxInputLongEdge, + config.minPoseDetectionConfidence, + config.minPosePresenceConfidence, + config.minTrackingConfidence, + config.targetPersonCount, + ], + ); + const isForeground = appState === "active"; + const shouldRun = enabled && isForeground; + + useEffect(() => { + debugRef.current = debug; + onFrameRef.current = onFrame; + exposeFrameRef.current = exposeFrame; + }, [debug, exposeFrame, onFrame]); + + useEffect(() => { + const subscription = AppState.addEventListener("change", setAppState); + return () => subscription.remove(); + }, []); + + useEffect(() => { + const releaseOwnership = nativePoseDetectionRuntime.acquire({ + shouldAcceptResult: () => shouldAcceptResultsRef.current, + onFrame: (detectedFrame) => { + onFrameRef.current?.(detectedFrame); + if (exposeFrameRef.current) { + setFrame(detectedFrame); + } + setError(null); + setStatus("running"); + + if (__DEV__ && debugRef.current) { + console.debug("[PoseDetection]", { + timestamp: detectedFrame.timestamp, + poseCount: detectedFrame.poses.length, + landmarkCounts: detectedFrame.poses.map( + (pose) => pose.landmarks.length, + ), + inputSize: detectedFrame.inputSize, + }); + } + }, + onError: (nativeError) => { + setError(nativeError); + setStatus("error"); + if (__DEV__ && debugRef.current) { + console.warn("[PoseDetection]", nativeError); + } + }, + }); + + return () => { + shouldAcceptResultsRef.current = false; + releaseOwnership(); + nativePoseDetectionRuntime.stop(); + nativePoseDetectionRuntime.release(); + }; + }, []); + + useEffect(() => { + nativePoseDetectionRuntime.configure(resolvedConfig); + + if (shouldRun) { + shouldAcceptResultsRef.current = true; + setStatus("initializing"); + setError(null); + nativePoseDetectionRuntime.start(); + return; + } + + shouldAcceptResultsRef.current = false; + nativePoseDetectionRuntime.stop(); + setFrame(null); + setError(null); + setStatus("idle"); + }, [resolvedConfig, shouldRun]); + + return { + frame, + error, + status, + frameSink: nativePoseDetectionRuntime.frameSink, + }; +} diff --git a/src/features/camera/guide-feed/model/use-pose-guide-alignment.ts b/src/features/camera/guide-feed/model/use-pose-guide-alignment.ts index e85405c7..a5e55911 100644 --- a/src/features/camera/guide-feed/model/use-pose-guide-alignment.ts +++ b/src/features/camera/guide-feed/model/use-pose-guide-alignment.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import type { PoseSceneMatchResult } from "../lib/pose-matching"; +import type { PoseSceneMatchResult } from "./pose-types"; import { advancePoseGuideAlignmentPolicy, createPoseGuideAlignmentPolicyState, diff --git a/src/features/camera/guide-feed/ui/camera-guide-overlay.tsx b/src/features/camera/guide-feed/ui/camera-guide-overlay.tsx index 2bf7cba2..371e4944 100644 --- a/src/features/camera/guide-feed/ui/camera-guide-overlay.tsx +++ b/src/features/camera/guide-feed/ui/camera-guide-overlay.tsx @@ -1,12 +1,11 @@ import { memo, useMemo } from "react"; import { StyleSheet, View } from "react-native"; import Svg, { Path } from "react-native-svg"; -import type { CoordinateSize } from "../lib/pose-matching"; import { createGuideContourPath, projectGuideOutlineToPreview, } from "../lib/guide-contour-projection"; -import type { CameraGuideOutline } from "../model"; +import type { CameraGuideOutline, CoordinateSize } from "../model"; interface CameraGuideOverlayProps { previewSize: CoordinateSize; diff --git a/src/features/camera/guide-feed/ui/camera-guide-reference-overlay.tsx b/src/features/camera/guide-feed/ui/camera-guide-reference-overlay.tsx index 77ca1e27..548880f1 100644 --- a/src/features/camera/guide-feed/ui/camera-guide-reference-overlay.tsx +++ b/src/features/camera/guide-feed/ui/camera-guide-reference-overlay.tsx @@ -1,7 +1,7 @@ import { Image } from "@shared/ui"; import { memo, useMemo } from "react"; import { View } from "react-native"; -import type { CoordinateSize } from "../lib/pose-matching"; +import type { CoordinateSize } from "../model"; import { projectGuideSourceToPreviewRect } from "../lib/guide-contour-projection"; interface CameraGuideReferenceOverlayProps {