diff --git a/README.md b/README.md index 9201678..f96e2fe 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,10 @@ blanks the source. - **Up to 4K at 60 fps**, in **H.264 or HEVC** (HEVC looks the same at ~40% less data). The app only offers resolution/frame-rate combinations your specific camera supports. +- **HDR streaming (beta).** On cameras that support it, an **HDR (HLG)** + toggle captures and streams 10-bit HLG over HEVC; OBS renders it + correctly on both SDR and HDR canvases. Off by default — SDR remains + the standard path. - **Pick any lens** — Main, Ultra Wide, Telephoto, or Front — switchable live while you stream. - **Hold it however.** The app's UI follows the phone's rotation (unless diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 2fd73ed..4a8aa43 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -210,9 +210,12 @@ Deliberate divergences (don't "fix" these without reading this): - **Video stabilization stays off** (the AVCaptureVideoDataOutput default). Every stabilization mode adds frames of latency; this is a latency-first product. Revisit only as an opt-in. -- **The wire is 8-bit 4:2:0 video-range** (`420v`); HDR formats are not - selected. The whole chain — encoder profile, wire protocol, plugin - decode, OBS frame — assumes 8-bit; HDR would need end-to-end work. +- **The wire defaults to 8-bit 4:2:0 video-range** (`420v`). With the + opt-in HDR toggle the camera path switches to 10-bit HLG (`x420` + capture, HEVC Main10) end-to-end — but 10-bit only ever enters the + pipeline through that toggle. SDR capture, the screen mirror, and the + H.264 path stay 8-bit; keep the 8-bit paths free of 10-bit branches + (the decoder maps formats per-frame, so SDR costs nothing extra). - **Rotation is sensor-native** (`.landscapeRight`, an effective 0°). The AVCaptureConnection docs warn per-frame rotation costs; we never rotate the stream, only the on-phone preview. diff --git a/docs/PROTOCOL.md b/docs/PROTOCOL.md index ba41768..954bbb4 100644 --- a/docs/PROTOCOL.md +++ b/docs/PROTOCOL.md @@ -59,6 +59,16 @@ change mid-stream resets the decoder (the next keyframe re-initializes it). `kind` mirrors the HELLO field. Dimensions/fps are informational; the authoritative values come from the bitstream parameter sets. +An optional `"color"` field names the stream's colour mode: `"hlg"` +(10-bit HEVC, BT.2020 primaries, HLG transfer) today, with `"log"` +(Apple Log) reserved for a future mode. Absent = 8-bit SDR BT.709, which +every stream was before the field existed. Like dimensions, it is +**informational** — the decoder takes the authoritative colour +description from the bitstream's VUI, so a receiver that ignores the +field still renders correctly; it exists so UIs can label the stream +without decoding it. HDR streams are HEVC-only (the iPhone encoder has +no 10-bit H.264) and camera-only (screen broadcast stays 8-bit). + ### 3 — VIDEO Payload: one H.264 or HEVC **access unit in Annex B format** (start-code delimited NAL units), per the codec announced in VIDEO_CONFIG. Keyframe diff --git a/ios-app/Sources/CameraManager.swift b/ios-app/Sources/CameraManager.swift index 24315fb..49945b4 100644 --- a/ios-app/Sources/CameraManager.swift +++ b/ios-app/Sources/CameraManager.swift @@ -20,7 +20,8 @@ final class CameraManager: NSObject { } } - func bitrate(for codec: VideoCodec) -> Int { + func bitrate(for codec: VideoCodec, + color: StreamColor = .sdr) -> Int { let h264: Int switch self { case .hd720: h264 = 4_000_000 @@ -28,7 +29,14 @@ final class CameraManager: NSObject { case .uhd4k: h264 = 30_000_000 } // HEVC reaches comparable quality at roughly 60% of the bits. - return codec == .hevc ? h264 * 6 / 10 : h264 + let base = codec == .hevc ? h264 * 6 / 10 : h264 + switch color { + case .sdr: + return base + case .hlg: + // 10-bit carries more data per pixel; give it headroom. + return base * 5 / 4 + } } } @@ -159,15 +167,29 @@ final class CameraManager: NSObject { private static func format(for device: AVCaptureDevice, resolution: Resolution, - fps: Int32) -> AVCaptureDevice.Format? { + fps: Int32, + color: StreamColor) -> AVCaptureDevice.Format? { let target = resolution.size // Require exact dimensions and a frame-rate range covering the // requested rate; earlier formats (unbinned, video-range) win ties. + // HLG additionally requires a 10-bit (x420) format that can + // capture BT.2020 HLG — nil (unsupported) if none exists. SDR + // considers every format, exactly as before colour existed. let candidates = device.formats.filter { format in let dims = CMVideoFormatDescriptionGetDimensions(format.formatDescription) guard dims.width == target.width, dims.height == target.height else { return false } + switch color { + case .sdr: + break + case .hlg: + guard CMFormatDescriptionGetMediaSubType(format.formatDescription) + == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, + format.supportedColorSpaces.contains(.HLG_BT2020) else { + return false + } + } return format.videoSupportedFrameRateRanges .contains { $0.maxFrameRate >= Double(fps) } } @@ -218,7 +240,7 @@ final class CameraManager: NSObject { static func formatReport() -> String { var out = ["\(UIDevice.current.model) — iOS " + UIDevice.current.systemVersion, - "flags: binned / CS / P / SL / R", ""] + "flags: binned / CS / P / SL / R / colour", ""] for lens in availableLenses() { guard let device = device(for: lens) else { continue } out.append("== \(lens.label) ==") @@ -240,6 +262,7 @@ final class CameraManager: NSObject { } else { flags.append("?") } + flags.append(colourFlags(format)) out.append(String(format: "%5dx%-5d fps<=%-3.0f %@", dims.width, dims.height, maxFps, flags.joined(separator: " "))) @@ -249,6 +272,26 @@ final class CameraManager: NSObject { return out.joined(separator: "\n") } + /// Colour column of the diagnostics table: the format's bit depth + /// plus the HDR colour spaces it can capture. This is the per-device + /// truth behind the HDR toggle — and, once Apple Log lands, behind + /// that too — the same way the effect flags are for Video Effects. + private static func colourFlags(_ format: AVCaptureDevice.Format) -> String { + let subtype = CMFormatDescriptionGetMediaSubType( + format.formatDescription) + let tenBit = subtype == kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + || subtype == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange + var flags = [tenBit ? "10bit" : "8bit "] + if format.supportedColorSpaces.contains(.HLG_BT2020) { + flags.append("HLG") + } + if #available(iOS 17.0, *), + format.supportedColorSpaces.contains(.appleLog) { + flags.append("Log") + } + return flags.joined(separator: " ") + } + /// How many of the system video effects this format can run. The /// effects themselves are user-toggled in Control Center — apps can't /// switch them on, only pick a format that permits them. @@ -265,12 +308,21 @@ final class CameraManager: NSObject { return score } - /// Whether this lens can capture the resolution at the frame rate. - /// Drives the app's pickers so unsupported combos are never offered. + /// Whether this lens can capture the resolution at the frame rate + /// (in the given colour pipeline). Drives the app's pickers so + /// unsupported combos are never offered. static func supports(resolution: Resolution, fps: Int32, - lens: Lens) -> Bool { + lens: Lens, color: StreamColor = .sdr) -> Bool { guard let device = device(for: lens) else { return false } - return format(for: device, resolution: resolution, fps: fps) != nil + return format(for: device, resolution: resolution, fps: fps, + color: color) != nil + } + + /// Whether this combo has a 10-bit HLG capture format — the UI's + /// gate for what the HDR toggle can actually deliver. + static func hdrAvailable(lens: Lens, resolution: Resolution, + fps: Int32) -> Bool { + supports(resolution: resolution, fps: fps, lens: lens, color: .hlg) } /// Fires on system-pressure (thermal/power) level changes, on the @@ -328,17 +380,20 @@ final class CameraManager: NSObject { func configure(lens: Lens, resolution: Resolution, fps: Int32, - lockFrameRate: Bool = true) throws { + lockFrameRate: Bool = true, + color: StreamColor = .sdr) throws { try sessionQueue.sync { try configureOnQueue(lens: lens, resolution: resolution, - fps: fps, lockFrameRate: lockFrameRate) + fps: fps, lockFrameRate: lockFrameRate, + color: color) } } private func configureOnQueue(lens: Lens, resolution: Resolution, fps: Int32, - lockFrameRate: Bool) throws { + lockFrameRate: Bool, + color: StreamColor) throws { let position = lens.position session.beginConfiguration() defer { session.commitConfiguration() } @@ -349,11 +404,23 @@ final class CameraManager: NSObject { // Format is chosen manually below; presets can't express 4K60. session.sessionPreset = .inputPriority + // For HLG we set the device's colour space ourselves below, so + // the session must not manage it (it would override our choice + // on input changes). For SDR, automatic is the iOS default — + // keeping it preserves the pre-HDR behaviour exactly. + switch color { + case .sdr: + session.automaticallyConfiguresCaptureDeviceForWideColor = true + case .hlg: + session.automaticallyConfiguresCaptureDeviceForWideColor = false + } + guard let device = Self.device(for: lens) else { throw NSError(domain: "CameraManager", code: 1, userInfo: [NSLocalizedDescriptionKey: "Camera not available"]) } - guard let format = Self.format(for: device, resolution: resolution, fps: fps) else { + guard let format = Self.format(for: device, resolution: resolution, + fps: fps, color: color) else { throw NSError(domain: "CameraManager", code: 4, userInfo: [NSLocalizedDescriptionKey: "\(resolution.rawValue) at \(fps) fps is not supported by the \(lens.label) camera"]) @@ -368,6 +435,13 @@ final class CameraManager: NSObject { try device.lockForConfiguration() device.activeFormat = format + switch color { + case .sdr: + // The session manages colour (automatic wide colour, above). + break + case .hlg: + device.activeColorSpace = .HLG_BT2020 + } // Min duration always caps the rate at the user's choice. The max // (the cadence lock, see PERFORMANCE.md) is normally pinned too — // but the "Allow system video effects" experiment leaves it at the @@ -398,9 +472,16 @@ final class CameraManager: NSObject { } let output = AVCaptureVideoDataOutput() + // Video-range in both pipelines; only the bit depth differs. + let pixelFormat: OSType + switch color { + case .sdr: + pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + case .hlg: + pixelFormat = kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange + } output.videoSettings = [ - kCVPixelBufferPixelFormatTypeKey as String: - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange + kCVPixelBufferPixelFormatTypeKey as String: pixelFormat ] output.alwaysDiscardsLateVideoFrames = true output.setSampleBufferDelegate(self, queue: videoQueue) diff --git a/ios-app/Sources/ContentView.swift b/ios-app/Sources/ContentView.swift index 15300a8..9c2dec7 100644 --- a/ios-app/Sources/ContentView.swift +++ b/ios-app/Sources/ContentView.swift @@ -261,7 +261,11 @@ struct ContentView: View { } Picker("Codec", selection: $streamer.codec) { - Text(VideoCodec.h264.label).tag(VideoCodec.h264) + // HDR is HEVC-only; offering H.264 would let the picker + // pick a value the didSet immediately reverts. + if !streamer.hdrEnabled { + Text(VideoCodec.h264.label).tag(VideoCodec.h264) + } if VideoEncoder.isSupported(.hevc) { Text(VideoCodec.hevc.label).tag(VideoCodec.hevc) } diff --git a/ios-app/Sources/OptionsView.swift b/ios-app/Sources/OptionsView.swift index 5b28d2c..95976b8 100644 --- a/ios-app/Sources/OptionsView.swift +++ b/ios-app/Sources/OptionsView.swift @@ -45,6 +45,17 @@ struct OptionsView: View { Text("Experimental: lets iOS lower the frame rate on its own, which the Control Center video effects (Portrait, Studio Light) may require. Takes effect when the camera next starts.") } + // Hidden entirely on devices that can't encode Main10 — + // a toggle that can never work is worse than none. + if VideoEncoder.hdrSupported { + Section { + Toggle("HDR (10-bit HEVC)", + isOn: $streamer.hdrEnabled) + } footer: { + Text("Streams 10-bit HLG color — OBS tone-maps it for SDR scenes, and HDR canvases get the real thing. HEVC only; takes effect when the camera next starts.") + } + } + Section { NavigationLink(destination: TallyLightOptionsView()) { Text("Tally light") diff --git a/ios-app/Sources/StreamClient.swift b/ios-app/Sources/StreamClient.swift index 0418e96..c017d99 100644 --- a/ios-app/Sources/StreamClient.swift +++ b/ios-app/Sources/StreamClient.swift @@ -485,14 +485,20 @@ final class StreamClient { send(OBSCProtocol.packet(type: .hello, payload: payload)) } - func sendVideoConfig(codec: VideoCodec, width: Int32, height: Int32, fps: Int32) { - let config: [String: Any] = [ + func sendVideoConfig(codec: VideoCodec, width: Int32, height: Int32, fps: Int32, + color: StreamColor = .sdr) { + var config: [String: Any] = [ "codec": codec.rawValue, "width": Int(width), "height": Int(height), "fps": Int(fps), "kind": sourceKind.rawValue, ] + // Absent = SDR: older plugins ignore unknown keys, and remote + // UIs key off the key's absence. + if color != .sdr { + config["color"] = color.rawValue + } guard let payload = try? JSONSerialization.data(withJSONObject: config) else { return } send(OBSCProtocol.packet(type: .videoConfig, payload: payload)) } diff --git a/ios-app/Sources/Streamer.swift b/ios-app/Sources/Streamer.swift index 5729b3f..3eb3b45 100644 --- a/ios-app/Sources/Streamer.swift +++ b/ios-app/Sources/Streamer.swift @@ -83,11 +83,13 @@ final class Streamer: ObservableObject { } private func reconfigureLiveCapture(formatChanged: Bool) { + let color = activeColor do { try camera.configure(lens: selectedLens, resolution: resolution, fps: Int32(fps), - lockFrameRate: !allowVideoEffects) + lockFrameRate: !allowVideoEffects, + color: color) } catch { // Never swallow this: a failed reconfigure leaves the session // without input/output — a black stream labelled "Live". @@ -102,8 +104,9 @@ final class Streamer: ObservableObject { let activeCodec = (codec == .hevc && !VideoEncoder.isSupported(.hevc)) ? VideoCodec.h264 : codec if let oldEncoder = encoder, - formatChanged || oldEncoder.codec != activeCodec { - rebuildEncoder(codec: activeCodec) + formatChanged || oldEncoder.codec != activeCodec + || oldEncoder.color != color { + rebuildEncoder(codec: activeCodec, color: color) } encoder?.requestKeyframe() @@ -113,7 +116,8 @@ final class Streamer: ObservableObject { /// Tears down the running encoder and builds a fresh one for the /// current capture size (fixed-dimension encoders can't follow a /// resolution/orientation change), re-announcing the format to OBS. - private func rebuildEncoder(codec activeCodec: VideoCodec) { + private func rebuildEncoder(codec activeCodec: VideoCodec, + color: StreamColor) { guard let oldEncoder = encoder else { return } oldEncoder.stop() let size = resolution.size @@ -121,7 +125,8 @@ final class Streamer: ObservableObject { codec: activeCodec, width: size.width, height: size.height, fps: Int32(fps), - bitrate: resolution.bitrate(for: activeCodec)) + bitrate: resolution.bitrate(for: activeCodec, color: color), + color: color) do { try newEncoder.start() } catch { @@ -138,12 +143,55 @@ final class Streamer: ObservableObject { encoder = newEncoder client.sendVideoConfig(codec: activeCodec, width: size.width, height: size.height, - fps: Int32(fps)) + fps: Int32(fps), + color: color) startAdaptiveBitrate( - target: resolution.bitrate(for: activeCodec)) + target: resolution.bitrate(for: activeCodec, color: color)) } @Published var codec: VideoCodec { - didSet { UserDefaults.standard.set(codec.rawValue, forKey: "videoCodec") } + didSet { + UserDefaults.standard.set(codec.rawValue, forKey: "videoCodec") + // HDR is 10-bit HEVC Main10 only; switching to H.264 + // switches it off (mirrors the mic toggles' exclusivity). + if codec == .h264 && hdrEnabled { + hdrEnabled = false + } + } + } + /// Stream 10-bit HLG (HDR) instead of 8-bit BT.709. HEVC only — + /// enabling it forces the codec — and it degrades to SDR (never a + /// dead stream) when the device or the current lens/format can't + /// capture HLG; see `activeColor`. + @Published var hdrEnabled: Bool { + didSet { + UserDefaults.standard.set(hdrEnabled, forKey: "hdrEnabled") + if hdrEnabled && codec != .hevc { + codec = .hevc + } + // The capture pixel format and the encoder profile both + // follow the colour, so a live stream rebuilds like a + // format change. + if isStreaming && hdrEnabled != oldValue { + reconfigureLiveCapture(formatChanged: true) + } + } + } + /// The colour pipeline the next capture/encode will use — the single + /// source of truth every configure/encoder/config-send site reads. + var activeColor: StreamColor { color(for: resolution, fps: fps) } + + /// `activeColor`'s degrade rule, parameterised so remote format + /// requests can be validated for a combo before it's applied: HDR + /// falls back to SDR — never to a black stream — when the encoder + /// can't do Main10 or the combo has no HLG capture format. + private func color(for resolution: CameraManager.Resolution, + fps: Int) -> StreamColor { + guard hdrEnabled, VideoEncoder.hdrSupported, + CameraManager.supports(resolution: resolution, fps: Int32(fps), + lens: selectedLens, color: .hlg) else { + return .sdr + } + return .hlg } /// "Dim screen to save battery": governs both the Live screen's 10 s /// dim and the Setup screen's standby dim. The name and stored key @@ -351,6 +399,19 @@ final class Streamer: ObservableObject { private func controlStateSnapshot() -> [String: Any] { let (resolutions, frameRates) = formatCapabilities() + let color = activeColor + // While HLG is active the only valid codec is HEVC; advertising + // just it makes remote UIs refuse h264 switches through the + // existing set_format validation machinery. + let codecs: [String] + switch color { + case .sdr: + codecs = VideoCodec.allCases + .filter { VideoEncoder.isSupported($0) } + .map { $0.rawValue } + case .hlg: + codecs = [VideoCodec.hevc.rawValue] + } var state: [String: Any] = [ "zoom": Double(zoom), "maxZoom": Double(camera.maxZoomFactor), @@ -382,10 +443,13 @@ final class Streamer: ObservableObject { "codec": (encoder?.codec ?? codec).rawValue, "resolutions": resolutions, "frameRates": frameRates, - "codecs": VideoCodec.allCases - .filter { VideoEncoder.isSupported($0) } - .map { $0.rawValue }, + "codecs": codecs, ] + // Absent = SDR — remote UIs key off the key's absence, and an + // SDR snapshot must look exactly as it did before HDR existed. + if color == .hlg { + state["hdr"] = true + } // Mic picker (docs/PROTOCOL.md §8): only while the phone mic is // live as the source's audio — remote UIs key their row off // micEnabled, and the list is only meaningful with capture up. @@ -404,16 +468,21 @@ final class Streamer: ObservableObject { private var capabilityCache: (key: String, resolutions: [String], frameRates: [Int])? private func formatCapabilities() -> ([String], [Int]) { - let key = "\(selectedLens.id)|\(resolution.rawValue)" + // The colour is part of the key: flipping the HDR toggle changes + // which combos are supported, and a stale list would let remote + // UIs offer combos the HLG pipeline can't capture. + let color = activeColor + let key = "\(selectedLens.id)|\(resolution.rawValue)|\(color.rawValue)" if let cached = capabilityCache, cached.key == key { return (cached.resolutions, cached.frameRates) } let resolutions = CameraManager.Resolution.allCases.filter { - CameraManager.supports(resolution: $0, fps: 30, lens: selectedLens) + CameraManager.supports(resolution: $0, fps: 30, lens: selectedLens, + color: color) }.map { $0.rawValue } let frameRates = [30, 60].filter { CameraManager.supports(resolution: resolution, fps: Int32($0), - lens: selectedLens) + lens: selectedLens, color: color) } capabilityCache = (key, resolutions, frameRates) return (resolutions, frameRates) @@ -504,9 +573,13 @@ final class Streamer: ObservableObject { /// Keeps resolution/fps within what the selected lens supports. func clampCaptureSettings() { + // Colour-aware: if the current combo can't do HLG, activeColor + // has already degraded to .sdr and the checks match today's — + // clamping never changes resolution just to preserve HDR. + let color = activeColor let supported = { (r: CameraManager.Resolution, f: Int) in CameraManager.supports(resolution: r, fps: Int32(f), - lens: self.selectedLens) + lens: self.selectedLens, color: color) } if supported(resolution, fps) { return } if supported(resolution, 30) { @@ -542,8 +615,12 @@ final class Streamer: ObservableObject { let savedLensID = defaults.string(forKey: "selectedLens") selectedLens = availableLenses.first { $0.id == savedLensID } ?? availableLenses[0] - codec = VideoCodec(rawValue: defaults.string(forKey: "videoCodec") ?? "") + let storedCodec = VideoCodec(rawValue: defaults.string(forKey: "videoCodec") ?? "") ?? (VideoEncoder.isSupported(.hevc) ? .hevc : .h264) + codec = storedCodec + // didSet doesn't run during init; enforce the HEVC-only rule here + // (stored defaults could disagree after an edit or migration). + hdrEnabled = defaults.bool(forKey: "hdrEnabled") && storedCodec == .hevc dimWhileStreaming = defaults.object(forKey: "dimWhileStreaming") as? Bool ?? true allowVideoEffects = defaults.bool(forKey: "allowVideoEffects") sendAudioReference = defaults.bool(forKey: "sendAudioReference") @@ -828,11 +905,17 @@ final class Streamer: ObservableObject { if let raw = command["codec"] as? String { guard let parsed = VideoCodec(rawValue: raw), VideoEncoder.isSupported(parsed) else { return } + // Double-guard: while HDR is on the advertised codec list is + // HEVC-only, but a stale remote UI could still ask — refuse + // rather than let the didSet chain silently drop HDR. + guard !(hdrEnabled && parsed != .hevc) else { return } newCodec = parsed } guard CameraManager.supports(resolution: newResolution, fps: Int32(newFps), - lens: selectedLens) else { return } + lens: selectedLens, + color: color(for: newResolution, + fps: newFps)) else { return } let formatChanged = newResolution != resolution || newFps != fps let codecChanged = newCodec != codec @@ -880,17 +963,21 @@ final class Streamer: ObservableObject { // Fall back to H.264 automatically if this device can't encode HEVC. let activeCodec = (codec == .hevc && !VideoEncoder.isSupported(.hevc)) ? VideoCodec.h264 : codec + let color = activeColor let size = resolution.size let encoder = VideoEncoder(codec: activeCodec, width: size.width, height: size.height, fps: Int32(fps), - bitrate: resolution.bitrate(for: activeCodec)) + bitrate: resolution.bitrate(for: activeCodec, + color: color), + color: color) do { try camera.configure(lens: selectedLens, resolution: resolution, fps: Int32(fps), - lockFrameRate: !allowVideoEffects) + lockFrameRate: !allowVideoEffects, + color: color) try encoder.start() } catch { status = .error(error.localizedDescription) @@ -912,7 +999,8 @@ final class Streamer: ObservableObject { camera.start() resetCameraControls() healthDroppedBaseline = client.statsSnapshot().framesDropped - startAdaptiveBitrate(target: resolution.bitrate(for: activeCodec)) + startAdaptiveBitrate(target: resolution.bitrate(for: activeCodec, + color: color)) client.setStandby(false) if standbyActive { // Remote start: reuse the standby transport. If OBS is already @@ -1096,9 +1184,12 @@ final class Streamer: ObservableObject { case .connected: status = .streaming let size = resolution.size + // The running encoder is the ground truth for what's on the + // wire; fall back to the settings only before it exists. client.sendVideoConfig(codec: encoder?.codec ?? codec, width: size.width, height: size.height, - fps: Int32(fps)) + fps: Int32(fps), + color: encoder?.color ?? activeColor) // Fresh connection: make sure OBS gets a decodable frame ASAP, // and seed the remote UI with the current control state. encoder?.requestKeyframe() diff --git a/ios-app/Sources/VideoEncoder.swift b/ios-app/Sources/VideoEncoder.swift index 21b99a2..4991144 100644 --- a/ios-app/Sources/VideoEncoder.swift +++ b/ios-app/Sources/VideoEncoder.swift @@ -14,6 +14,18 @@ enum VideoCodec: String, CaseIterable, Identifiable { } } +/// The stream's colour pipeline. Apple Log will slot in as another case +/// (capture colorSpace .appleLog, iOS 17+); everything downstream switches +/// on this rather than on booleans. +/// +/// Lives here (not CameraManager.swift) because the broadcast extension +/// compiles VideoEncoder.swift and StreamClient.swift — both of which +/// take a colour — but not CameraManager.swift. +enum StreamColor: String { + case sdr // 8-bit 420v, BT.709 — every stream before 1.9 + case hlg // 10-bit x420, BT.2020 + HLG, HEVC Main10 only +} + /// Hardware video encoder (VideoToolbox). Emits Annex B access units; /// keyframes are self-contained (parameter sets prepended: SPS/PPS for /// H.264, VPS/SPS/PPS for HEVC) so the OBS plugin can join at any keyframe. @@ -27,6 +39,7 @@ final class VideoEncoder { var onEncodedFrame: ((EncodedFrame) -> Void)? let codec: VideoCodec + let color: StreamColor /// Guards `session` and `forceNextKeyframe`: `encode()` runs on the /// capture queue while `stop()`/`setBitrate()`/`requestKeyframe()` @@ -41,12 +54,14 @@ final class VideoEncoder { private static let startCode = Data([0x00, 0x00, 0x00, 0x01]) - init(codec: VideoCodec, width: Int32, height: Int32, fps: Int32, bitrate: Int) { + init(codec: VideoCodec, width: Int32, height: Int32, fps: Int32, bitrate: Int, + color: StreamColor = .sdr) { self.codec = codec self.width = width self.height = height self.fps = fps self.bitrate = bitrate + self.color = color } /// Whether this device can hardware-encode the codec (HEVC needs A10+). @@ -73,6 +88,27 @@ final class VideoEncoder { return status == noErr }() + /// Whether this device can hardware-encode 10-bit HEVC (Main10) — the + /// gate for the HDR toggle. Cached for the same reason as + /// `hevcSupported`; any error means unsupported. + static let hdrSupported: Bool = { + var session: VTCompressionSession? + let status = VTCompressionSessionCreate( + allocator: kCFAllocatorDefault, + width: 1280, height: 720, + codecType: VideoCodec.hevc.vtCodecType, + encoderSpecification: nil, + imageBufferAttributes: nil, + compressedDataAllocator: nil, + outputCallback: nil, refcon: nil, + compressionSessionOut: &session) + guard status == noErr, let session else { return false } + defer { VTCompressionSessionInvalidate(session) } + return VTSessionSetProperty( + session, key: kVTCompressionPropertyKey_ProfileLevel, + value: kVTProfileLevel_HEVC_Main10_AutoLevel) == noErr + }() + func start() throws { var session: VTCompressionSession? let status = VTCompressionSessionCreate( @@ -94,9 +130,27 @@ final class VideoEncoder { VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue) - VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, - value: codec == .h264 ? kVTProfileLevel_H264_Main_AutoLevel - : kVTProfileLevel_HEVC_Main_AutoLevel) + switch color { + case .sdr: + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, + value: codec == .h264 ? kVTProfileLevel_H264_Main_AutoLevel + : kVTProfileLevel_HEVC_Main_AutoLevel) + case .hlg: + // Streamer never pairs HLG with H.264 (its didSets force HEVC + // and `activeColor` requires `hdrSupported`). + assert(codec == .hevc, "HLG requires HEVC Main10") + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, + value: kVTProfileLevel_HEVC_Main10_AutoLevel) + // Tag the bitstream itself (VUI colour description): pixel + // buffer attachments can be lost between capture and decode, + // and downstream tone mapping trusts these. + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ColorPrimaries, + value: kCVImageBufferColorPrimaries_ITU_R_2020) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_TransferFunction, + value: kCVImageBufferTransferFunction_ITU_R_2100_HLG) + VTSessionSetProperty(session, key: kVTCompressionPropertyKey_YCbCrMatrix, + value: kCVImageBufferYCbCrMatrix_ITU_R_2020) + } VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse) // Shave per-frame encode time; quality difference is negligible diff --git a/obs-plugin/data/nv12.effect b/obs-plugin/data/nv12.effect index 1830f84..56975d5 100644 --- a/obs-plugin/data/nv12.effect +++ b/obs-plugin/data/nv12.effect @@ -69,3 +69,105 @@ technique DrawI420 pixel_shader = PSI420(vert_in); } } + +// --------------------------------------------------------------------- +// 10-bit HDR (HLG): HEVC Main10 from the phone — BT.2020 primaries, +// ARIB STD-B67 (HLG) transfer, limited range. The planes arrive as +// normalized 16-bit textures (GS_R16 / GS_RG16); sample_scale converts a +// sample to 10-bit code / 1023 (P010 packs the 10 bits MSB-aligned: +// scale 65535/65472 ~= 1; I010 packs them LSB-aligned: scale 65535/1023 +// ~= 64). The chain: limited-range BT.2020 Y'CbCr -> R'G'B', BT.2100 +// HLG inverse OETF, the BT.2100 reference OOTF, BT.2020 -> BT.709 +// primaries, then scale so 1.0 = SDR white on the extended-range +// (GS_CS_709_EXTENDED, linear) canvas. Deliberately NO saturate(): +// values above SDR white and outside 709 gamut are the point of HDR. + +uniform float sample_scale = 1.0; +uniform float hdr_scale = 3.3333333; // 1000-nit HLG peak / 300-nit SDR + // white default; set per draw from + // obs_get_video_sdr_white_level(). + +float3 yuv2020_to_rgb(float y, float u, float v) +{ + // BT.2020 NCL, 10-bit limited (64-940 / 64-960) -> full, unclamped. + float yl = (y - 0.0625611) * 1.1678082; + float uc = u * 1.1417411; + float vc = v * 1.1417411; + float3 rgb; + rgb.r = yl + 1.4746 * vc; + rgb.g = yl - 0.1645531 * uc - 0.5713531 * vc; + rgb.b = yl + 1.8814 * uc; + return rgb; +} + +float hlg_inv_oetf(float e) +{ + // BT.2100 HLG inverse OETF: E' in [0,1] -> scene light E in [0,1]. + float a = 0.17883277; + float b = 0.28466892; // 1 - 4a + float c = 0.55991073; // 0.5 - a*ln(4a) + return (e <= 0.5) ? ((e * e) / 3.0) : ((exp((e - c) / a) + b) / 12.0); +} + +float3 hlg2020_to_canvas(float3 rgbp) +{ + float3 scene; + scene.r = hlg_inv_oetf(rgbp.r); + scene.g = hlg_inv_oetf(rgbp.g); + scene.b = hlg_inv_oetf(rgbp.b); + + // BT.2100 OOTF: Fd = Ys^(gamma-1) * Es per channel, Ys = BT.2020 + // scene luminance. Chosen gamma: the reference system gamma 1.2, + // i.e. the nominal 1000-nit HLG display (BT.2100's gamma = + // 1.2 + 0.42*log10(Lw/1000) at Lw = 1000) — the same assumption + // hdr_scale's 1000-nit numerator makes. + float ys = dot(scene, float3(0.2627, 0.6780, 0.0593)); + float3 display = scene * pow(max(ys, 0.000001), 0.2); + + // Linear BT.2020 -> BT.709 primaries (BT.2087), unclamped: + // out-of-gamut 709 values are valid on the extended canvas. + float3 c709; + c709.r = dot(display, float3(1.660491, -0.587641, -0.072850)); + c709.g = dot(display, float3(-0.124550, 1.132900, -0.008349)); + c709.b = dot(display, float3(-0.018151, -0.100579, 1.118730)); + + return c709 * hdr_scale; // 1.0 = SDR white, peak at hdr_scale +} + +float4 PSP010HLG(VertInOut vert_in) : TARGET +{ + // 512/1023 = 0.5004888: the 10-bit chroma midpoint. + float y = y_tex.Sample(tex_sampler, vert_in.uv).r * sample_scale; + float2 uv = uv_tex.Sample(tex_sampler, vert_in.uv).rg * sample_scale; + return float4(hlg2020_to_canvas(yuv2020_to_rgb(y, uv.r - 0.5004888, + uv.g - 0.5004888)), + 1.0); +} + +float4 PSI010HLG(VertInOut vert_in) : TARGET +{ + float y = y_tex.Sample(tex_sampler, vert_in.uv).r * sample_scale; + float u = uv_tex.Sample(tex_sampler, vert_in.uv).r * sample_scale; + float v = v_tex.Sample(tex_sampler, vert_in.uv).r * sample_scale; + return float4(hlg2020_to_canvas(yuv2020_to_rgb(y, u - 0.5004888, + v - 0.5004888)), + 1.0); +} + +technique DrawP010HLG +{ + pass + { + vertex_shader = VSDefault(vert_in); + pixel_shader = PSP010HLG(vert_in); + } +} + +technique DrawI010HLG +{ + pass + { + vertex_shader = VSDefault(vert_in); + pixel_shader = PSI010HLG(vert_in); + } +} diff --git a/obs-plugin/src/gpu-frame.c b/obs-plugin/src/gpu-frame.c index 6ba5f72..00f3547 100644 --- a/obs-plugin/src/gpu-frame.c +++ b/obs-plugin/src/gpu-frame.c @@ -55,7 +55,16 @@ void gpu_frame_ctx_destroy(struct gpu_frame_ctx *ctx) bool gpu_frame_supported(const AVFrame *frame) { - return frame && frame->format == AV_PIX_FMT_VAAPI; + if (!frame || frame->format != AV_PIX_FMT_VAAPI || + !frame->hw_frames_ctx) + return false; + /* The dmabuf import below hardcodes 8-bit NV12 (R8 + GR88); a P010 + * surface (10-bit HDR hardware decode) would import as garbage. + * Refuse it so those frames take the CPU path — zero-copy 10-bit + * (R16/GR1616) is stage 2. */ + const AVHWFramesContext *fctx = + (const AVHWFramesContext *)frame->hw_frames_ctx->data; + return fctx->sw_format == AV_PIX_FMT_NV12; } bool gpu_frame_map(struct gpu_frame_ctx *ctx, AVFrame *frame, @@ -144,7 +153,7 @@ bool gpu_frame_map(struct gpu_frame_ctx *ctx, AVFrame *frame, out->tex[0] = y; out->tex[1] = uv; - out->rgba = false; + out->format = GPU_FRAME_NV12; out->width = w; out->height = h; return true; @@ -252,7 +261,7 @@ bool gpu_frame_map(struct gpu_frame_ctx *ctx, AVFrame *frame, out->tex[0] = ctx->tex; out->tex[1] = NULL; - out->rgba = true; + out->format = GPU_FRAME_RGBA; out->width = (uint32_t)frame->width; out->height = (uint32_t)frame->height; return true; @@ -339,7 +348,16 @@ bool gpu_frame_supported(const AVFrame *frame) { /* DXVA2 frames are D3D9 surfaces — no D3D11 interop; the decoder's * hardware priority list tries D3D11VA first anyway. */ - return frame && frame->format == AV_PIX_FMT_D3D11; + if (!frame || frame->format != AV_PIX_FMT_D3D11 || + !frame->hw_frames_ctx) + return false; + /* The shared texture below is NV12 (8-bit); copying a P010 decode + * surface (10-bit HDR) into it fails, leaving a black source. + * Refuse so those frames take the CPU path — a shared P010 texture + * (gs_texture_create_p010) is stage 2. */ + const AVHWFramesContext *fctx = + (const AVHWFramesContext *)frame->hw_frames_ctx->data; + return fctx->sw_format == AV_PIX_FMT_NV12; } static bool win_ensure_textures(struct gpu_frame_ctx *ctx, uint32_t w, @@ -470,7 +488,7 @@ bool gpu_frame_map(struct gpu_frame_ctx *ctx, AVFrame *frame, out->tex[0] = ctx->tex_y; out->tex[1] = ctx->tex_uv; - out->rgba = false; + out->format = GPU_FRAME_NV12; out->width = ctx->width; out->height = ctx->height; return true; diff --git a/obs-plugin/src/gpu-frame.h b/obs-plugin/src/gpu-frame.h index b0cdb88..135dbc0 100644 --- a/obs-plugin/src/gpu-frame.h +++ b/obs-plugin/src/gpu-frame.h @@ -31,11 +31,22 @@ extern "C" { struct gpu_frame_ctx; +/* Pixel layout of a mapped/uploaded frame; tells the draw site which + * technique (and which plane textures) to use. */ +enum gpu_frame_format { + GPU_FRAME_NV12, /* tex[0] = luma (R8), tex[1] = chroma (R8G8, half) */ + GPU_FRAME_I420, /* CPU upload only: three R8 planes (the caller owns + * the third texture) */ + GPU_FRAME_P010, /* tex[0] = luma (R16), tex[1] = chroma (RG16, half); + * 10 bits MSB-aligned in each 16-bit word */ + GPU_FRAME_I010, /* CPU upload only: three R16 planes; 10 bits + * LSB-aligned (samples read 1/64 of intended) */ + GPU_FRAME_RGBA, /* tex[0] is a full-color texture (macOS) */ +}; + struct gpu_frame_map_result { - /* rgba: tex[0] is a full-color texture (macOS). Otherwise NV12: - * tex[0] = luma (R8), tex[1] = chroma (R8G8, half size). */ gs_texture_t *tex[2]; - bool rgba; + enum gpu_frame_format format; uint32_t width, height; }; diff --git a/obs-plugin/src/h264-decoder.c b/obs-plugin/src/h264-decoder.c index 7b64fc5..e477faf 100644 --- a/obs-plugin/src/h264-decoder.c +++ b/obs-plugin/src/h264-decoder.c @@ -279,6 +279,10 @@ static bool avframe_to_obs(const AVFrame *frame, struct obs_source_frame *out) case AV_PIX_FMT_P010: out->format = VIDEO_FORMAT_P010; break; + case AV_PIX_FMT_YUV420P10: + /* Software HEVC Main10 decode (HDR without a GPU decoder). */ + out->format = VIDEO_FORMAT_I010; + break; default: return false; } @@ -309,10 +313,28 @@ static bool avframe_to_obs(const AVFrame *frame, struct obs_source_frame *out) ? VIDEO_CS_2100_HLG : VIDEO_CS_2100_PQ; + /* The transfer function rides along with the frame: without it, + * libobs treats 10-bit HDR video as SDR 709 (washed out / grey). + * VIDEO_TRC_DEFAULT keeps today's behaviour for SDR streams. */ + switch (frame->color_trc) { + case AVCOL_TRC_ARIB_STD_B67: + out->trc = (uint8_t)VIDEO_TRC_HLG; + break; + case AVCOL_TRC_SMPTE2084: + out->trc = (uint8_t)VIDEO_TRC_PQ; + break; + default: + out->trc = (uint8_t)VIDEO_TRC_DEFAULT; + break; + } + out->full_range = range == VIDEO_RANGE_FULL; - video_format_get_parameters(cs, range, out->color_matrix, - out->color_range_min, - out->color_range_max); + /* _for_format: the plain variant assumes 8-bit code ranges, which + * are wrong for the 10-bit formats (I010/P010). */ + video_format_get_parameters_for_format(cs, range, out->format, + out->color_matrix, + out->color_range_min, + out->color_range_max); return true; } @@ -407,7 +429,8 @@ bool h264_decoder_decode(struct h264_decoder *dec, obs_source_t *source, size_t frame_bytes = px * 3 / 2; if (out.format == VIDEO_FORMAT_I422) frame_bytes = px * 2; - else if (out.format == VIDEO_FORMAT_P010) + else if (out.format == VIDEO_FORMAT_P010 || + out.format == VIDEO_FORMAT_I010) frame_bytes = px * 3; lenslink_bench_frame( os_gettime_ns() - bench_start, diff --git a/obs-plugin/src/ios-camera-source.c b/obs-plugin/src/ios-camera-source.c index 577c24c..77840d5 100644 --- a/obs-plugin/src/ios-camera-source.c +++ b/obs-plugin/src/ios-camera-source.c @@ -236,6 +236,9 @@ struct ios_camera_source { * graphics thread frees current_frame for it */ int render_path; /* 0 unknown, 1 zero-copy, 2 cpu fallback (logged * on transition so a black source is explicable) */ + bool render_hdr; /* current frame is HDR (HLG/PQ transfer); set on + * the graphics thread when a frame is adopted and + * read there by video_get_color_space */ }; @@ -3024,6 +3027,8 @@ static bool g_yuv_effect_tried = false; static gs_eparam_t *g_yuv_y = NULL; static gs_eparam_t *g_yuv_uv = NULL; static gs_eparam_t *g_yuv_v = NULL; +static gs_eparam_t *g_yuv_scale = NULL; /* 16-bit sample -> code/1023 */ +static gs_eparam_t *g_yuv_hdr = NULL; /* HLG peak over SDR white */ static gs_effect_t *yuv_effect(void) { @@ -3045,6 +3050,10 @@ static gs_effect_t *yuv_effect(void) "uv_tex"); g_yuv_v = gs_effect_get_param_by_name(g_yuv_effect, "v_tex"); + g_yuv_scale = gs_effect_get_param_by_name( + g_yuv_effect, "sample_scale"); + g_yuv_hdr = gs_effect_get_param_by_name(g_yuv_effect, + "hdr_scale"); } } return g_yuv_effect; @@ -3057,9 +3066,28 @@ static bool upload_sw_frame(struct ios_camera_source *s, const AVFrame *f, struct gpu_frame_map_result *out) { int fmt = f->format; - if (fmt != AV_PIX_FMT_NV12 && fmt != AV_PIX_FMT_YUV420P && - fmt != AV_PIX_FMT_YUVJ420P) + enum gpu_frame_format map_fmt; + switch (fmt) { + case AV_PIX_FMT_NV12: + map_fmt = GPU_FRAME_NV12; + break; + case AV_PIX_FMT_YUV420P: + case AV_PIX_FMT_YUVJ420P: + map_fmt = GPU_FRAME_I420; + break; + case AV_PIX_FMT_P010: /* HDR hardware decode, downloaded */ + map_fmt = GPU_FRAME_P010; + break; + case AV_PIX_FMT_YUV420P10: /* HDR software decode (HEVC Main10) */ + map_fmt = GPU_FRAME_I010; + break; + default: return false; + } + /* 10-bit planes are 16-bit words; linesize (bytes) already covers + * the ×2 — pass it through, never width×bpp. */ + bool two_plane = map_fmt == GPU_FRAME_NV12 || map_fmt == GPU_FRAME_P010; + bool ten_bit = map_fmt == GPU_FRAME_P010 || map_fmt == GPU_FRAME_I010; uint32_t w = (uint32_t)f->width, h = (uint32_t)f->height; if (!s->cpu_tex[0] || s->cpu_tex_w != w || s->cpu_tex_h != h || @@ -3070,19 +3098,21 @@ static bool upload_sw_frame(struct ios_camera_source *s, const AVFrame *f, s->cpu_tex[i] = NULL; } } + enum gs_color_format one = ten_bit ? GS_R16 : GS_R8; + enum gs_color_format two = ten_bit ? GS_RG16 : GS_R8G8; s->cpu_tex[0] = - gs_texture_create(w, h, GS_R8, 1, NULL, GS_DYNAMIC); - if (fmt == AV_PIX_FMT_NV12) { - s->cpu_tex[1] = gs_texture_create(w / 2, h / 2, GS_R8G8, + gs_texture_create(w, h, one, 1, NULL, GS_DYNAMIC); + if (two_plane) { + s->cpu_tex[1] = gs_texture_create(w / 2, h / 2, two, 1, NULL, GS_DYNAMIC); } else { - s->cpu_tex[1] = gs_texture_create(w / 2, h / 2, GS_R8, + s->cpu_tex[1] = gs_texture_create(w / 2, h / 2, one, 1, NULL, GS_DYNAMIC); - s->cpu_tex[2] = gs_texture_create(w / 2, h / 2, GS_R8, + s->cpu_tex[2] = gs_texture_create(w / 2, h / 2, one, 1, NULL, GS_DYNAMIC); } if (!s->cpu_tex[0] || !s->cpu_tex[1] || - (fmt != AV_PIX_FMT_NV12 && !s->cpu_tex[2])) + (!two_plane && !s->cpu_tex[2])) return false; s->cpu_tex_w = w; s->cpu_tex_h = h; @@ -3093,13 +3123,13 @@ static bool upload_sw_frame(struct ios_camera_source *s, const AVFrame *f, (uint32_t)f->linesize[0], false); gs_texture_set_image(s->cpu_tex[1], f->data[1], (uint32_t)f->linesize[1], false); - if (fmt != AV_PIX_FMT_NV12) + if (!two_plane) gs_texture_set_image(s->cpu_tex[2], f->data[2], (uint32_t)f->linesize[2], false); out->tex[0] = s->cpu_tex[0]; out->tex[1] = s->cpu_tex[1]; - out->rgba = false; + out->format = map_fmt; out->width = w; out->height = h; return true; @@ -3123,6 +3153,7 @@ static void ios_camera_video_render(void *data, gs_effect_t *unused) if (clear && s->current_frame) { av_frame_free(&s->current_frame); s->gpu_mapped = false; + s->render_hdr = false; } if (fresh) { @@ -3173,10 +3204,18 @@ static void ios_camera_video_render(void *data, gs_effect_t *unused) } copied = (size_t)s->gpu_map.width * s->gpu_map.height * 3 / 2; + if (s->gpu_map.format == GPU_FRAME_P010 || + s->gpu_map.format == GPU_FRAME_I010) + copied *= 2; /* 16-bit words per sample */ if (up != fresh) copied *= 2; /* download + upload */ } + /* HDR (HLG/PQ) frames render extended-range linear values; + * video_get_color_space reports the matching canvas space. */ + s->render_hdr = fresh->color_trc == AVCOL_TRC_ARIB_STD_B67 || + fresh->color_trc == AVCOL_TRC_SMPTE2084; + int path = s->gpu_mapped ? 1 : 2; if (path != s->render_path) { s->render_path = path; @@ -3200,7 +3239,7 @@ static void ios_camera_video_render(void *data, gs_effect_t *unused) if (!locked) return; - if (s->gpu_map.rgba) { + if (s->gpu_map.format == GPU_FRAME_RGBA) { gs_effect_t *eff = obs_get_base_effect(OBS_EFFECT_DEFAULT); /* OBS owns the base effect; cache its "image" handle against * the effect we resolved it from, so a different (or reloaded) @@ -3219,13 +3258,48 @@ static void ios_camera_video_render(void *data, gs_effect_t *unused) } else { gs_effect_t *eff = yuv_effect(); if (eff) { - bool i420 = !s->gpu_mapped && - s->cpu_tex_fmt != AV_PIX_FMT_NV12; + /* 10-bit frames from the phone are HLG (the app + * signals BT.2020 + ARIB STD-B67); the HLG techniques + * emit extended-range linear 709 to match the + * GS_CS_709_EXTENDED report from + * video_get_color_space. */ + const char *tech = "Draw"; + bool three_plane = false; + float scale = 0.0f; + switch (s->gpu_map.format) { + case GPU_FRAME_I420: + tech = "DrawI420"; + three_plane = true; + break; + case GPU_FRAME_P010: + /* MSB-aligned: sample ~= code/1023 already; + * the exact ratio is 65535/(1023*64). */ + tech = "DrawP010HLG"; + scale = 65535.0f / 65472.0f; + break; + case GPU_FRAME_I010: + /* LSB-aligned: samples read 1/64 of the + * intended value. */ + tech = "DrawI010HLG"; + three_plane = true; + scale = 65535.0f / 1023.0f; + break; + default: + break; + } + if (scale != 0.0f) { + float white = obs_get_video_sdr_white_level(); + gs_effect_set_float(g_yuv_scale, scale); + gs_effect_set_float(g_yuv_hdr, + white > 0.0f + ? 1000.0f / white + : 1000.0f / 300.0f); + } gs_effect_set_texture(g_yuv_y, s->gpu_map.tex[0]); gs_effect_set_texture(g_yuv_uv, s->gpu_map.tex[1]); - if (i420) + if (three_plane) gs_effect_set_texture(g_yuv_v, s->cpu_tex[2]); - while (gs_effect_loop(eff, i420 ? "DrawI420" : "Draw")) + while (gs_effect_loop(eff, tech)) gs_draw_sprite(s->gpu_map.tex[0], 0, s->gpu_map.width, s->gpu_map.height); @@ -3256,6 +3330,25 @@ static void gpu_pipeline_free(struct ios_camera_source *s) av_frame_free(&s->xfer); } +/* HDR frames need the extended-range canvas: the HLG techniques output + * linear light with 1.0 = SDR white and highlights above it, which + * GS_CS_SRGB would clip. SDR keeps today's space untouched. Graphics + * thread, like video_render (which is where render_hdr is written). */ +static enum gs_color_space +ios_camera_video_get_color_space(void *data, size_t count, + const enum gs_color_space *preferred_spaces) +{ + struct ios_camera_source *s = data; + enum gs_color_space space = s->render_hdr ? GS_CS_709_EXTENDED + : GS_CS_SRGB; + for (size_t i = 0; i < count; i++) { + if (preferred_spaces[i] == space) + return space; + } + /* Not preferred: return it anyway; libobs converts. */ + return space; +} + /* Called from plugin-main.c at module load, BEFORE registration, when * the plugin-wide "GPU pipeline (beta)" setting is on: the sources * become self-rendering sync sources instead of async pixel-pushers. */ @@ -3267,6 +3360,7 @@ void lenslink_sources_use_gpu_pipeline(struct obs_source_info *info) info->video_render = ios_camera_video_render; info->get_width = ios_camera_get_width; info->get_height = ios_camera_get_height; + info->video_get_color_space = ios_camera_video_get_color_space; } struct obs_source_info ios_camera_source_info = {