Realtime Mobile App

components/camera/
  hooks/useWebRTC.ts      — Core Decart SDK integration
  ui/ModelSelector.tsx     — Model toggle UI
  ui/VideoRenderer.tsx     — Displays local + remote streams
  ui/style-carousel/       — Swipeable style picker

lib/realtime/
  media-streams.ts         — Camera stream setup with model constraints
  webrtc-utils.ts          — VP8 codec forcing for React Native

lib/skins/
  lucy-skin-list.ts        — 21 text prompts for Lucy editing presets
  mirage-skin-list.ts      — Style presets with optional audio

import { useCallback, useRef, useState } from "react";
import { createDecartClient, type ModelDefinition, type RealTimeClient } from "@decartai/sdk";
import type { MediaStream } from "react-native-webrtc";
import { getMediaStream } from "@/lib/realtime/media-streams";
import { forceCodecInSDP } from "@/lib/realtime/webrtc-utils";

export function useWebRTC({ model, facingMode }: { model: ModelDefinition; facingMode: string }) {
  const [localMediaStream, setLocalMediaStream] = useState<MediaStream | null>(null);
  const [remoteMediaStream, setRemoteMediaStream] = useState<MediaStream | null>(null);
  const [connectionState, setConnectionState] = useState<string>("disconnected");
  const realtimeClientRef = useRef<RealTimeClient | null>(null);

  const initializeWebRTC = useCallback(
    async ({ model, facingMode }: { model: ModelDefinition; facingMode: string }) => {
      try {
        // 1. Create the Decart client
        const decartClient = createDecartClient({
          apiKey: process.env.EXPO_PUBLIC_DECART_API_KEY as string,
        });

        // 2. Capture camera with model-specific constraints
        const stream = await getMediaStream(facingMode, {
          fps: model.fps,
          width: model.width,
          height: model.height,
        });
        setLocalMediaStream(stream);
        setConnectionState("connecting");

        // 3. Connect to the realtime model
        realtimeClientRef.current = await decartClient.realtime.connect(
          stream as unknown as globalThis.MediaStream,
          {
            model,
            onRemoteStream: (transformedStream) => {
              setRemoteMediaStream(transformedStream as unknown as MediaStream);
              setConnectionState("connected");
            },
            customizeOffer: (offer: RTCSessionDescriptionInit) => {
              forceCodecInSDP(offer, "VP8");
            },
          },
        );
      } catch (error) {
        console.error("Connection failed:", error);
        setConnectionState("failed");
      }
    },
    [],
  );

  const cleanupWebRTC = useCallback(() => {
    realtimeClientRef.current?.disconnect();
    setRemoteMediaStream(null);
    setConnectionState("disconnected");
  }, []);

  return {
    localMediaStream,
    remoteMediaStream,
    connectionState,
    realtimeClientRef,
    initializeWebRTC,
    cleanupWebRTC,
  };
}

import { models, type ModelDefinition } from "@decartai/sdk";

// Select a model
const mirage = models.realtime("mirage_v2");
const lucy1 = models.realtime("lucy_v2v_720p_rt");
const lucy2 = models.realtime("lucy_2_rt");

// To switch: disconnect, update state, reconnect
async function switchModel(newModel: ModelDefinition) {
  cleanupWebRTC();
  await new Promise((resolve) => setTimeout(resolve, 100)); // allow cleanup to complete
  await initializeWebRTC({ model: newModel, facingMode });
}

// Apply a style preset to the current model
if (realtimeClientRef.current) {
  realtimeClientRef.current.setPrompt(skin.prompt, {
    enhance: skin.enrich ?? false,
  });
}

const presets = [
  {
    title: "Anime Character",
    prompt:
      "Transform the person into a 2D anime character with smooth cel-shaded lines, " +
      "soft pastel highlights, large expressive eyes, clean contours, even lighting, " +
      "simplified textures, and a bright studio-style background for a polished anime look.",
  },
  {
    title: "Black Tuxedo",
    prompt:
      "Change the outfit to a sharp black tuxedo with satin lapels, crisp white shirt, " +
      "black bow tie, tailored fit, polished shoes, and soft indoor lighting reflecting " +
      "off a marble floor.",
  },
  {
    title: "Sunglasses",
    prompt:
      "Add a pair of dark tinted sunglasses resting naturally on the person's face, " +
      "smooth acetate frames, subtle reflections on the lenses, accurate nose " +
      "placement, and soft shadows across the cheeks.",
  },
];

// Lucy 2: transform into a character using a reference image
await realtimeClientRef.current.set({
  prompt: "Transform into this character",
  image: characterImage, // File, Blob, or URL
  enhance: true,
});

import { AppState, type AppStateStatus } from "react-native";

useEffect(() => {
  const handleAppStateChange = (nextAppState: AppStateStatus) => {
    if (appStateRef.current.match(/inactive|background/) && nextAppState === "active") {
      // App returned to foreground — reconnect
      initializeWebRTC({ model, facingMode });
    } else if (appStateRef.current === "active" && nextAppState.match(/inactive|background/)) {
      // App going to background — disconnect to free resources
      cleanupWebRTC();
    }
    appStateRef.current = nextAppState;
  };

  const subscription = AppState.addEventListener("change", handleAppStateChange);
  return () => subscription?.remove();
}, [cleanupWebRTC, initializeWebRTC]);

function switchCamera(newFacingMode: "user" | "environment") {
  if (realtimeClientRef.current) {
    realtimeClientRef.current.setMirror(newFacingMode === "user");
  }
}

import { models } from "@decartai/sdk";

// Add lucy_2_rt alongside the existing models
const handleTabPress = (modelName: "mirage_v2" | "lucy_v2v_720p_rt" | "lucy_2_rt") => {
  const newModel = models.realtime(modelName);
  onModelChange(newModel);
};

// For Lucy 2 with a reference image
await realtimeClientRef.current.set({
  prompt: "Transform into this character",
  image: selectedImage,
  enhance: true,
});

// For text-only editing (works with all models)
realtimeClientRef.current.setPrompt("Add sunglasses", { enhance: true });