Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions docs/architecture/adr-0003-independent-realtime-pose-detection.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ VisionCamera를 유일한 Camera owner로 유지하고 RTC와 Pose를 각각
Pose Landmarker Lite는 `LIVE_STREAM`, 최대 4명, 최대 10 FPS,
입력 긴 변 640px, segmentation mask 비활성 상태로 시작한다.

Feature model의 `useLivePoseDetection`이 AppState, 활성 조건,
검출 상태와 React lifecycle을 소유한다. `lib`의 native Pose runtime
adapter는 callback 등록, FrameSink 제어와 native 결과 변환만 담당한다.
따라서 model은 native package를 직접 호출하지 않고 runtime port를 통해
검출 lifecycle을 조율한다. Native 모듈이 전역 callback과 FrameSink를
하나만 제공하므로 runtime은 동시에 하나의 owner만 허용한다.

## Context

RTC 송출과 Pose 추론은 같은 카메라 프레임이 필요하지만 처리 시간,
Expand All @@ -34,6 +41,10 @@ lifetime을 침범하지 않는다.
latest-only 전달은 JS가 느릴 때 과거 자세 결과가 누적되는 것을
막고 현재 자세에 가까운 결과를 유지한다.

React hook을 단순 계산 유틸과 구분해 model에 배치하고 native runtime을
adapter로 격리하면 화면 lifecycle 정책과 MediaPipe/VisionCamera 연동
구현을 독립적으로 변경할 수 있다.

## Trade-off

Pose가 수락한 프레임에는 플랫폼별 YUV→RGB/BGRA 복사 비용이
Expand All @@ -54,6 +65,10 @@ pose detection 단위 테스트를 통과했다. Android 공식 AAR의 API
시그니처는 검증했지만 작업 환경에 Android SDK가 없어 Android
타깃 컴파일과 실제 기기 추론 검증은 남아 있다.

Feature model의 검출 hook과 native Pose runtime adapter를 분리하되,
기존 configure/start/stop/release 순서와 latest-only callback 정책은
유지했다.

실제 기기에서는 DEV debug 로그의 `poseCount`와 `landmarkCounts`로
1명, 2명, 3명 이상 결과를 확인하고, 평균 추론 시간, 수락/드롭
프레임, 발열과 배터리 사용량을 함께 기록한다. 이 측정 결과에 따라
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { CoordinateSize } from "../model/pose-types";
import type { CameraGuideAspectRatio } from "../model/types";
import type { CoordinateSize } from "./pose-matching";

const SUPPORTED_ASPECT_RATIOS = {
"4:3": 4 / 3,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import {
calculateCanvasRenderRect,
type CoordinateSize,
type PreviewPoint,
} from "./pose-matching";
import { calculateCanvasRenderRect, type PreviewPoint } from "./pose-matching";
import type { CoordinateSize } from "../model/pose-types";
import type { CameraGuideContour, CameraGuideOutline } from "../model/types";

export interface ProjectedCameraGuideContour extends Omit<
Expand Down
2 changes: 0 additions & 2 deletions src/features/camera/guide-feed/lib/pose-detection/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,2 @@
export * from "./native-pose-result-adapter";
export * from "./pose-detection-config";
export * from "./types";
export * from "./use-live-pose-detection";
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import {
setVisionCameraPoseErrorCallback,
setVisionCameraPoseResultCallback,
visionCameraPoseFrameSink,
} from "@newpic/vision-camera-pose";
import type { PoseDetectionRuntime } from "../../model/pose-detection-types";
import { adaptNativeDetectedPoseFrame } from "./native-pose-result-adapter";

let activeOwner: symbol | null = null;

export const nativePoseDetectionRuntime = {
frameSink: visionCameraPoseFrameSink,

acquire({ shouldAcceptResult, onFrame, onError }) {
if (activeOwner) {
throw new Error("Native pose detection runtime already has an owner.");
}

const owner = Symbol("native-pose-detection-owner");
activeOwner = owner;
setVisionCameraPoseResultCallback((nativeFrame) => {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if (!shouldAcceptResult()) return;
onFrame(adaptNativeDetectedPoseFrame(nativeFrame));
});
setVisionCameraPoseErrorCallback((error) => {
if (!shouldAcceptResult()) return;
onError(error);
});

return () => {
if (activeOwner !== owner) return;

activeOwner = null;
setVisionCameraPoseResultCallback(undefined);
setVisionCameraPoseErrorCallback(undefined);
};
},

configure(config) {
visionCameraPoseFrameSink.configure(config);
},

start() {
visionCameraPoseFrameSink.startAcceptingFrames();
},

stop() {
visionCameraPoseFrameSink.stopAcceptingFrames();
},

release() {
visionCameraPoseFrameSink.releaseDetector();
},
} satisfies PoseDetectionRuntime<typeof visionCameraPoseFrameSink>;
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { NativeDetectedPoseFrame } from "@newpic/vision-camera-pose";
import type { DetectedPoseFrame } from "./types";
import type { DetectedPoseFrame } from "../../model/pose-types";

const QUARTER_TURNS = [0, 90, 180, 270] as const;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
import type {
PoseDetectionConfig,
ResolvedPoseDetectionConfig,
} from "../../model/pose-detection-types";

export const MAX_POSE_COUNT = 4;
export const MAX_POSE_INFERENCE_FPS = 10;

Expand All @@ -14,22 +19,15 @@ export const DEFAULT_POSE_DETECTION_CONFIG = {
minTrackingConfidence: 0.5,
} as const;

export type PoseDetectionConfig = {
targetPersonCount?: number;
maxInferenceFps?: number;
maxInputLongEdge?: number;
minPoseDetectionConfidence?: number;
minPosePresenceConfidence?: number;
minTrackingConfidence?: number;
};

export function resolvePoseCount(targetPersonCount?: number) {
const requested =
targetPersonCount ?? DEFAULT_POSE_DETECTION_CONFIG.defaultPoseCount;
return Math.min(MAX_POSE_COUNT, Math.max(1, Math.trunc(requested)));
}

export function resolvePoseDetectionConfig(config: PoseDetectionConfig = {}) {
export function resolvePoseDetectionConfig(
config: PoseDetectionConfig = {},
): ResolvedPoseDetectionConfig {
return {
numPoses: resolvePoseCount(config.targetPersonCount),
maxInferenceFps: Math.min(
Expand Down
41 changes: 0 additions & 41 deletions src/features/camera/guide-feed/lib/pose-detection/types.ts

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import type {
CaptureAspectRatio,
CaptureToPreviewTransform,
CanvasRenderRect,
CommonPose,
CommonPosePoint,
CoordinateSize,
DWPoseSourcePose,
MediaPipeInputPose,
MediaPipePoseToCaptureTransform,
PoseJointMap,
} from "../../model/pose-types";
import type {
CaptureAspectRatio,
CaptureToPreviewTransform,
CanvasRenderRect,
MediaPipePoseToCaptureTransform,
PreviewPoint,
QuarterTurn,
ResizeMode,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ import type {
NormalizedPosePerson,
NormalizedPoseResult,
} from "@entities/feed";
import type { CommonJoint, CoordinateSize, DWPoseSourcePose } from "./types";
import type {
CommonJoint,
CoordinateSize,
DWPoseSourcePose,
} from "../../model/pose-types";

export const DWPOSE_KEYPOINT_FORMAT = "dwpose_xy_score" as const;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { FeedPoseResponse } from "@entities/feed";
import type { CoordinateSize, DWPoseSourcePose } from "../../model/pose-types";
import { resolveFeedCameraAspectRatio } from "../feed-camera-aspect-ratio";
import { adaptDWPoseResult } from "./dwpose-pose-adapter";
import type { CoordinateSize, DWPoseSourcePose } from "./types";

export interface FeedPoseTargetSelection {
feedId: string;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import type { FeedPoseResponse, NormalizedPoseResult } from "@entities/feed";
import type { CommonPose, SourcePoseToCaptureTransform } from "./types";
import type { CommonPose } from "../../model/pose-types";
import { projectDWPosePoseToCapture } from "./coordinate-transform";
import { adaptDWPoseResult } from "./dwpose-pose-adapter";
import type { SourcePoseToCaptureTransform } from "./types";

export function getNormalizedPoseResult(
response: FeedPoseResponse,
Expand Down
Loading
Loading