Respira.py

import argparse
import os
import numpy as np
import cv2
import librosa
from moviepy import VideoFileClip, AudioFileClip, VideoClip

# -----------------------------
# Utility: smoothing
# -----------------------------
def smooth_1d(x, win=9):
    if win <= 1:
        return x
    win = int(win)
    if win % 2 == 0:
        win += 1
    k = np.ones(win, dtype=np.float32) / win
    return np.convolve(x, k, mode="same")

def clamp01(x):
    return float(max(0.0, min(1.0, x)))

# -----------------------------
# Audio analysis -> per-frame features (low/mid/high + rms)
# -----------------------------
def analyze_audio(audio_path, fps, hop_length=512, n_fft=2048):
    # Load in mono for stability
    y, sr = librosa.load(audio_path, sr=None, mono=True)
    duration = librosa.get_duration(y=y, sr=sr)

    # STFT magnitude
    S = np.abs(librosa.stft(y, n_fft=n_fft, hop_length=hop_length))
    freqs = librosa.fft_frequencies(sr=sr, n_fft=n_fft)
    times = librosa.frames_to_time(np.arange(S.shape[1]), sr=sr, hop_length=hop_length)

    # Band masks (ambient-friendly bands)
    low_mask = (freqs >= 20) & (freqs < 180)
    mid_mask = (freqs >= 180) & (freqs < 2000)
    high_mask = (freqs >= 2000) & (freqs < 8000)

    # Energy per time frame
    low = S[low_mask].mean(axis=0) if low_mask.any() else S.mean(axis=0)
    mid = S[mid_mask].mean(axis=0) if mid_mask.any() else S.mean(axis=0)
    high = S[high_mask].mean(axis=0) if high_mask.any() else S.mean(axis=0)

    # RMS as overall envelope
    rms = librosa.feature.rms(y=y, frame_length=n_fft, hop_length=hop_length)[0]

    # Normalize robustly (percentile-based)
    def norm(v):
        v = np.asarray(v, dtype=np.float32)
        lo = np.percentile(v, 10)
        hi = np.percentile(v, 95)
        if hi - lo < 1e-9:
            return np.zeros_like(v)
        out = (v - lo) / (hi - lo)
        return np.clip(out, 0, 1)

    low_n = norm(low)
    mid_n = norm(mid)
    high_n = norm(high)
    rms_n = norm(rms)

    # Smooth for ambient feel (slower motion)
    low_s  = smooth_1d(low_n, 11)
    mid_s  = smooth_1d(mid_n, 11)
    high_s = smooth_1d(high_n, 11)
    rms_s  = smooth_1d(rms_n, 21)

    # Build per-video-frame arrays (sample audio features at each frame time)
    n_frames = int(np.ceil(duration * fps))
    frame_t = np.arange(n_frames) / fps

    def sample_at(t, series, series_t):
        return np.interp(t, series_t, series).astype(np.float32)

    low_f  = sample_at(frame_t, low_s, times)
    mid_f  = sample_at(frame_t, mid_s, times)
    high_f = sample_at(frame_t, high_s, times)
    rms_f  = sample_at(frame_t, rms_s, librosa.frames_to_time(np.arange(len(rms_s)), sr=sr, hop_length=hop_length))

    return duration, frame_t, low_f, mid_f, high_f, rms_f

# -----------------------------
# Image extraction: from image or first frame of video
# -----------------------------
def load_base_image(image_path=None, video_path=None):
    if image_path:
        img = cv2.imread(image_path, cv2.IMREAD_COLOR)
        if img is None:
            raise FileNotFoundError(f"Impossibile leggere immagine: {image_path}")
        return img
    if video_path:
        cap = cv2.VideoCapture(video_path)
        ok, frame = cap.read()
        cap.release()
        if not ok or frame is None:
            raise RuntimeError(f"Impossibile estrarre frame dal video: {video_path}")
        return frame
    raise ValueError("Devi fornire --image oppure --video")

# -----------------------------
# Visual effects: zoom/pan + glow + subtle liquid warp
# -----------------------------
def apply_effects(img_bgr, out_w, out_h, zoom, pan_x, pan_y, glow, warp_amt, vignette=0.12):
    # Resize to cover output after zoom (we'll crop center with pan)
    h, w = img_bgr.shape[:2]

    # Target scaled size
    scale = zoom
    new_w = int(np.ceil(w * scale))
    new_h = int(np.ceil(h * scale))

    img = cv2.resize(img_bgr, (new_w, new_h), interpolation=cv2.INTER_CUBIC)

    # Liquid warp (subtle): create displacement maps
    if warp_amt > 1e-6:
        yy, xx = np.mgrid[0:new_h, 0:new_w].astype(np.float32)

        # Low-frequency sin displacement
        fx = 2 * np.pi / max(new_w, 1)
        fy = 2 * np.pi / max(new_h, 1)

        disp_x = (np.sin(yy * fy * 1.3) + np.sin(xx * fx * 0.9)) * (warp_amt * 6.0)
        disp_y = (np.sin(xx * fx * 1.1) + np.sin(yy * fy * 0.7)) * (warp_amt * 6.0)

        map_x = (xx + disp_x).astype(np.float32)
        map_y = (yy + disp_y).astype(np.float32)
        img = cv2.remap(img, map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)

    # Crop to output with pan (pan_x/pan_y in [-1..1], moves crop window)
    ch, cw = out_h, out_w
    if new_w < cw or new_h < ch:
        # Ensure coverage
        pad_w = max(0, cw - new_w)
        pad_h = max(0, ch - new_h)
        img = cv2.copyMakeBorder(img, pad_h//2, pad_h - pad_h//2, pad_w//2, pad_w - pad_w//2, cv2.BORDER_REFLECT)
        new_h, new_w = img.shape[:2]

    # Center crop coordinates
    cx = new_w // 2 + int(pan_x * (new_w - cw) * 0.15)
    cy = new_h // 2 + int(pan_y * (new_h - ch) * 0.15)

    x0 = int(np.clip(cx - cw//2, 0, new_w - cw))
    y0 = int(np.clip(cy - ch//2, 0, new_h - ch))
    crop = img[y0:y0+ch, x0:x0+cw].copy()

    # Glow (soft): blur and screen-like blend
    if glow > 1e-6:
        blur = cv2.GaussianBlur(crop, (0, 0), sigmaX=10, sigmaY=10)
        # Convert to float for blending
        a = np.clip(glow * 0.65, 0, 1)
        crop_f = crop.astype(np.float32) / 255.0
        blur_f = blur.astype(np.float32) / 255.0
        # Screen blend: 1 - (1-A)(1-B)
        screen = 1.0 - (1.0 - crop_f) * (1.0 - blur_f)
        out = (1.0 - a) * crop_f + a * screen
        crop = np.clip(out * 255.0, 0, 255).astype(np.uint8)

    # Subtle vignette to feel cinematic
    if vignette > 0:
        y = np.linspace(-1, 1, out_h).astype(np.float32)
        x = np.linspace(-1, 1, out_w).astype(np.float32)
        xv, yv = np.meshgrid(x, y)
        r = np.sqrt(xv*xv + yv*yv)
        mask = np.clip(1.0 - vignette * (r**1.6), 0.75, 1.0)
        crop = (crop.astype(np.float32) * mask[..., None]).astype(np.uint8)

    return crop

# -----------------------------
# Main render
# -----------------------------
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--image", type=str, default=None, help="Immagine (jpg/png)")
    ap.add_argument("--video", type=str, default=None, help="Video sorgente (estrae primo frame)")
    ap.add_argument("--audio", type=str, required=True, help="Audio (mp3/wav/m4a)")
    ap.add_argument("--out", type=str, default="output_reactive.mp4")
    ap.add_argument("--w", type=int, default=1920)
    ap.add_argument("--h", type=int, default=1080)
    ap.add_argument("--fps", type=int, default=30)
    ap.add_argument("--base_zoom", type=float, default=1.02, help="Zoom base (>=1)")
    ap.add_argument("--zoom_range", type=float, default=0.06, help="Quanto zoom reagisce (ambient: 0.03-0.08)")
    ap.add_argument("--pan", type=float, default=0.35, help="Ampiezza pan (0-1)")
    ap.add_argument("--warp", type=float, default=0.35, help="Ampiezza warp (0-1)")
    ap.add_argument("--glow", type=float, default=0.45, help="Ampiezza glow (0-1)")
    args = ap.parse_args()

    base_img = load_base_image(args.image, args.video)

    # Analyze audio -> features per frame
    duration, frame_t, low_f, mid_f, high_f, rms_f = analyze_audio(args.audio, fps=args.fps)

    # Make motion more "ambient": slow drift component
    # slow oscillators for camera drift
    drift1 = np.sin(2*np.pi*frame_t * 0.03)   # very slow
    drift2 = np.sin(2*np.pi*frame_t * 0.017)  # even slower

    # Prepare audio clip
    #audio_clip = AudioFileClip(args.audio).subclip(0, duration)
    #audio_clip = AudioFileClip(args.audio).subclipped(0, duration)
    audio_clip_full = AudioFileClip(args.audio)
    audio_dur = audio_clip_full.duration  # durata reale
    duration = min(duration, audio_dur)   # evita sforamenti
    audio_clip = audio_clip_full.subclipped(0, duration)

    # Frame generator
    n_frames = len(frame_t)

    def make_frame(t):
        # Convert t to frame index
        idx = int(np.clip(round(t * args.fps), 0, n_frames - 1))

        low  = float(low_f[idx])
        mid  = float(mid_f[idx])
        high = float(high_f[idx])
        env  = float(rms_f[idx])

        # Map features -> effects (tuned for ambient)
        zoom = args.base_zoom + args.zoom_range * (0.55*env + 0.25*mid + 0.20*high)

        # Pan is slow drift + slight audio influence (mid)
        #pan_x = args.pan * (0.70*drift1 + 0.30*(mid - 0.5))
        #pan_y = args.pan * (0.70*drift2 + 0.30*(high - 0.5))
        pan_x = float(args.pan * (0.70*drift1[idx] + 0.30*(mid - 0.5)))
        pan_y = float(args.pan * (0.70*drift2[idx] + 0.30*(high - 0.5)))
        # Warp follows low band softly
        warp_amt = args.warp * (0.15 + 0.85*low)  # always subtle

        # Glow follows high band
        glow = args.glow * (0.10 + 0.90*high)

        # out_bgr = apply_effects(
        #     base_img,
        #     out_w=args.w,
        #     out_h=args.h,
        #     zoom=zoom,
        #     pan_x=pan_x,
        #     pan_y=pan_y,
        #     glow=glow,
        #     warp_amt=warp_amt,
        #     vignette=0.12
        # )
        out_bgr = apply_effects(
        base_img,
        out_w=args.w,
        out_h=args.h,
        zoom=float(zoom),
        pan_x=float(pan_x),
        pan_y=float(pan_y),
        glow=float(glow),
        warp_amt=float(warp_amt),
        vignette=0.12
         )
        # Convert BGR -> RGB for moviepy
        return cv2.cvtColor(out_bgr, cv2.COLOR_BGR2RGB)

    #clip = VideoClip(make_frame=make_frame, duration=duration).set_fps(args.fps)
    clip = VideoClip(frame_function=make_frame, duration=duration).with_fps(args.fps)
    #clip = clip.set_audio(audio_clip)
    clip = clip.with_audio(audio_clip)

    # Render
    # Use H.264 + AAC for compatibility
    clip.write_videofile(
        args.out,
        codec="libx264",
        audio_codec="aac",
        fps=args.fps,
        preset="medium",
        threads=os.cpu_count() or 4
    )

if __name__ == "__main__":
    main()