Realtime API

The Realtime API enables you to transform live video streams with minimal latency using WebRTC. Perfect for building iOS camera effects, video conferencing filters, AR applications, and interactive live streaming.

Quick Start

import DecartSDK
import WebRTC

let model = Models.realtime(.lucy_v2v_720p_rt)

// Get device camera stream
let stream = try await captureLocalStream(
    fps: model.fps,
    width: model.width,
    height: model.height
)

// Create client and connect
let config = try DecartConfiguration(
    apiKey: "your-api-key-here"
)

let client = try createDecartClient(configuration: config)

let realtimeClient = try await client.realtime.connect(
    stream: stream,
    options: RealtimeConnectOptions(
        model: model,
        onRemoteStream: { transformedStream in
            videoView.srcObject = transformedStream
        },
        initialState: ModelState(
            prompt: Prompt(text: "Anime", enrich: true)
        )
    )
)

// Change style on the fly
try await realtimeClient.setPrompt("Cyberpunk city")

// Disconnect when done
await realtimeClient.disconnect()

Connecting

Getting Camera Access

Request access to the device’s camera using iOS’s AVFoundation and WebRTC’s camera capturer:

import WebRTC

func captureLocalStream(fps: Int, width: Int, height: Int) async throws -> RTCMediaStream {
    // Initialize WebRTC
    RTCInitializeSSL()
    
    // Create peer connection factory
    let factory = RTCPeerConnectionFactory()
    let videoSource = factory.videoSource()
    
    // Create camera capturer
    let capturer = RTCCameraVideoCapturer(delegate: videoSource)
    
    // Get front camera
    let devices = RTCCameraVideoCapturer.captureDevices()
    guard let frontCamera = devices.first(where: { $0.position == .front }) else {
        throw DecartError.invalidInput("No front camera found")
    }
    
    // Select optimal format
    let formats = RTCCameraVideoCapturer.supportedFormats(for: frontCamera)
    guard let format = formats.first(where: { format in
        let dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription)
        return dimensions.width >= width && dimensions.height >= height
    }) else {
        throw DecartError.invalidInput("No suitable camera format")
    }
    
    // Start capture
    try await withCheckedThrowingContinuation { continuation in
        capturer.startCapture(with: frontCamera, format: format, fps: fps) { error in
            if let error = error {
                continuation.resume(throwing: error)
            } else {
                continuation.resume()
            }
        }
    }
    
    // Create media stream
    let videoTrack = factory.videoTrack(with: videoSource, trackId: "video0")
    let stream = factory.mediaStream(withStreamId: "stream0")
    stream.addVideoTrack(videoTrack)
    
    return stream
}

Use the model’s fps, width, and height properties to ensure optimal performance.

Camera capture requires a real iOS device. The simulator does not support camera access for WebRTC.

Establishing Connection

Connect to the Realtime API with your media stream:

let realtimeClient = try await client.realtime.connect(
    stream: stream,
    options: RealtimeConnectOptions(
        model: Models.realtime(.lucy_v2v_720p_rt),
        onRemoteStream: { transformedStream in
            // Display the transformed video
            videoView.srcObject = transformedStream
        },
        initialState: ModelState(
            prompt: Prompt(
                text: "Lego World",
                enrich: true  // Let Decart enhance the prompt (recommended)
            ),
            mirror: false  // Set to true for front-facing cameras
        )
    )
)

Parameters:

stream (required) - RTCMediaStream from camera capture
model (required) - Realtime model from Models.realtime()
onRemoteStream (required) - Closure that receives the transformed video stream
initialState.prompt (required) - Initial style prompt
- text - Style description
- enrich - Whether to auto-enhance the prompt (default: true)
initialState.mirror (optional) - Enable mirror mode for front-facing cameras

Managing Prompts

Change the transformation style dynamically without reconnecting:

// Simple prompt with automatic enhancement
try await realtimeClient.setPrompt("Anime style")

// Custom detailed prompt without enhancement
try await realtimeClient.setPrompt(
    "A detailed artistic style with vibrant colors and dramatic lighting",
    enrich: false
)

Parameters:

prompt (required) - Text description of desired style
enrich (optional) - Whether to enhance the prompt (default: true)

Prompt enhancement uses Decart’s AI to expand simple prompts for better results. Disable it if you want full control over the exact prompt.

Camera Mirroring

Toggle horizontal mirroring, useful for front-facing cameras:

// Enable mirror mode
await realtimeClient.setMirror(true)

// Disable mirror mode
await realtimeClient.setMirror(false)

This flips the video horizontally, making front-facing camera views feel more natural to users.

Connection State

Monitor and react to connection state changes using Combine publishers:

import Combine

var cancellables = Set<AnyCancellable>()

// Check state synchronously
let isConnected = await realtimeClient.isConnected()  // Bool
let state = await realtimeClient.getConnectionState()  // ConnectionState

// Listen to state changes
realtimeClient.connectionStatePublisher
    .sink { state in
        print("Connection state: \(state)")
        
        switch state {
        case .disconnected:
            // Handle disconnection
            showReconnectButton()
        case .connected:
            // Handle successful connection
            hideReconnectButton()
        case .connecting:
            // Show loading state
            showLoadingIndicator()
        }
    }
    .store(in: &cancellables)

Connection States:

.connecting - Establishing connection
.connected - Successfully connected
.disconnected - Not connected

Use this to update your UI and handle reconnection logic.

Error Handling

Handle errors using the error publisher:

realtimeClient.errorPublisher
    .sink { error in
        print("SDK error: \(error.errorCode) - \(error.localizedDescription)")
        
        switch error {
        case .invalidAPIKey:
            showError("Invalid API key. Please check your credentials.")
        case .webRTCError(let underlyingError):
            showError("Connection error: \(underlyingError.localizedDescription)")
        case .modelNotFound(let model):
            showError("Model '\(model)' not found. Please check the model name.")
        default:
            showError(error.localizedDescription)
        }
    }
    .store(in: &cancellables)

Error Cases:

.invalidAPIKey - API key is invalid or missing
.webRTCError(Error) - WebRTC connection failed
.modelNotFound(String) - Specified model doesn’t exist
.invalidInput(String) - Invalid input parameters
.connectionTimeout - Connection timed out
.websocketError(String) - WebSocket connection error

Session Management

Access the current session ID:

let sessionId = await realtimeClient.sessionId
print("Current session: \(sessionId)")

This can be useful for logging, analytics, or debugging.

Cleanup

Always disconnect when done to free up resources:

// Disconnect from the service
await realtimeClient.disconnect()

// Stop camera capture
if let capturer = videoCapturer {
    await withCheckedContinuation { continuation in
        capturer.stopCapture {
            continuation.resume()
        }
    }
}

// Remove video tracks
localStream?.videoTracks.first?.remove(localVideoView)

// Cleanup WebRTC
RTCCleanupSSL()

// Cancel subscriptions
cancellables.removeAll()

Failing to disconnect can leave WebRTC connections open and waste resources.

Complete SwiftUI Example

Here’s a full SwiftUI application with all features:

import SwiftUI
import DecartSDK
import WebRTC
import Combine

@main
struct RealtimeApp: App {
    var body: some Scene {
        WindowGroup {
            ContentView()
        }
    }
}

struct ContentView: View {
    @StateObject private var viewModel = RealtimeViewModel()
    
    var body: some View {
        ZStack {
            // Remote video background
            VideoView(videoView: viewModel.remoteVideoView)
                .edgesIgnoringSafeArea(.all)
            
            // UI overlay
            VStack(spacing: 16) {
                // Status bar
                HStack {
                    VStack(alignment: .leading, spacing: 4) {
                        Text("Decart Realtime")
                            .font(.headline)
                            .foregroundColor(.white)
                        Text(viewModel.connectionState)
                            .font(.caption)
                            .foregroundColor(viewModel.isConnected ? .green : .white)
                    }
                    Spacer()
                }
                .padding()
                .background(Color.black.opacity(0.6))
                
                Spacer()
                
                // Local video preview
                if viewModel.showLocalPreview {
                    HStack {
                        Spacer()
                        VideoView(videoView: viewModel.localVideoView)
                            .frame(width: 120, height: 160)
                            .cornerRadius(12)
                            .overlay(
                                RoundedRectangle(cornerRadius: 12)
                                    .stroke(Color.white, lineWidth: 2)
                            )
                            .padding()
                    }
                }
                
                // Controls
                VStack(spacing: 12) {
                    if let error = viewModel.lastError {
                        Text(error)
                            .foregroundColor(.red)
                            .font(.caption)
                            .padding(8)
                            .background(Color.black.opacity(0.8))
                            .cornerRadius(8)
                    }
                    
                    HStack(spacing: 12) {
                        TextField("Enter style prompt", text: $viewModel.promptText)
                            .textFieldStyle(RoundedBorderTextFieldStyle())
                        
                        Button {
                            Task { await viewModel.setPrompt() }
                        } label: {
                            Image(systemName: "paperplane.fill")
                                .foregroundColor(.white)
                                .padding(12)
                                .background(
                                    viewModel.isConnected ? Color.blue : Color.gray
                                )
                                .cornerRadius(8)
                        }
                        .disabled(!viewModel.isConnected)
                    }
                    
                    HStack(spacing: 12) {
                        Toggle("Mirror", isOn: $viewModel.mirror)
                            .toggleStyle(SwitchToggleStyle(tint: .blue))
                            .disabled(!viewModel.isConnected)
                        
                        Spacer()
                        
                        Button {
                            Task { await viewModel.toggleConnection() }
                        } label: {
                            Text(viewModel.isConnected ? "Disconnect" : "Connect")
                                .fontWeight(.semibold)
                                .foregroundColor(.white)
                                .frame(maxWidth: .infinity)
                                .padding()
                                .background(
                                    viewModel.isConnected ? Color.red : Color.green
                                )
                                .cornerRadius(12)
                        }
                    }
                }
                .padding()
                .background(Color.black.opacity(0.8))
                .cornerRadius(16)
                .padding()
            }
        }
    }
}

// UIKit wrapper for WebRTC video view
struct VideoView: UIViewRepresentable {
    let videoView: RTCMTLVideoView
    
    func makeUIView(context: Context) -> RTCMTLVideoView {
        videoView.contentMode = .scaleAspectFill
        return videoView
    }
    
    func updateUIView(_ uiView: RTCMTLVideoView, context: Context) {}
}

@MainActor
class RealtimeViewModel: ObservableObject {
    @Published var connectionState: String = "Disconnected"
    @Published var promptText: String = "Turn into a fantasy figure"
    @Published var mirror: Bool = false {
        didSet {
            if mirror != oldValue {
                Task { await setMirror(mirror) }
            }
        }
    }
    @Published var lastError: String?
    @Published var showLocalPreview = false
    
    var isConnected: Bool {
        connectionState == "Connected"
    }
    
    private var client: RealtimeClient?
    private var cancellables = Set<AnyCancellable>()
    private var localStream: RTCMediaStream?
    private var videoCapturer: RTCCameraVideoCapturer?
    private var peerConnectionFactory: RTCPeerConnectionFactory?
    
    let remoteVideoView = RTCMTLVideoView()
    let localVideoView = RTCMTLVideoView()
    
    func toggleConnection() async {
        if isConnected {
            await disconnect()
        } else {
            await connect()
        }
    }
    
    func connect() async {
        connectionState = "Connecting"
        lastError = nil
        
        do {
            let config = try DecartConfiguration(
                apiKey: ProcessInfo.processInfo.environment["DECART_API_KEY"] ?? ""
            )
            
            let decartClient = try createDecartClient(configuration: config)
            let model = Models.realtime(.lucy_v2v_720p_rt)
            
            // Capture camera
            localStream = try await captureLocalStream(
                fps: model.fps,
                width: model.width,
                height: model.height
            )
            
            guard let stream = localStream else {
                throw DecartError.invalidInput("Failed to get camera stream")
            }
            
            // Attach local preview
            if let localVideoTrack = stream.videoTracks.first {
                localVideoTrack.add(localVideoView)
                showLocalPreview = true
            }
            
            // Connect
            let realtimeClient = try await decartClient.realtime.connect(
                stream: stream,
                options: RealtimeConnectOptions(
                    model: model,
                    onRemoteStream: { [weak self] mediaStream in
                        Task { @MainActor in
                            guard let self = self,
                                  let videoTrack = mediaStream.videoTracks.first else {
                                return
                            }
                            videoTrack.add(self.remoteVideoView)
                        }
                    },
                    initialState: ModelState(
                        prompt: Prompt(text: promptText, enrich: true),
                        mirror: mirror
                    )
                )
            )
            
            self.client = realtimeClient
            
            // Subscribe to events
            realtimeClient.connectionStatePublisher
                .receive(on: DispatchQueue.main)
                .sink { [weak self] state in
                    self?.handleConnectionState(state)
                }
                .store(in: &cancellables)
            
            realtimeClient.errorPublisher
                .receive(on: DispatchQueue.main)
                .sink { [weak self] error in
                    self?.lastError = error.localizedDescription
                }
                .store(in: &cancellables)
                
        } catch {
            lastError = error.localizedDescription
            connectionState = "Disconnected"
        }
    }
    
    func disconnect() async {
        // Stop camera
        if let capturer = videoCapturer {
            await withCheckedContinuation { continuation in
                capturer.stopCapture { continuation.resume() }
            }
        }
        
        // Remove tracks
        localStream?.videoTracks.first?.remove(localVideoView)
        
        // Cleanup
        await client?.disconnect()
        client = nil
        connectionState = "Disconnected"
        showLocalPreview = false
        videoCapturer = nil
        localStream = nil
        peerConnectionFactory = nil
        
        RTCCleanupSSL()
    }
    
    func setPrompt() async {
        guard let client = client else { return }
        
        do {
            try await client.setPrompt(promptText, enrich: true)
        } catch {
            lastError = error.localizedDescription
        }
    }
    
    func setMirror(_ enabled: Bool) async {
        guard let client = client else { return }
        await client.setMirror(enabled)
    }
    
    private func handleConnectionState(_ state: ConnectionState) {
        switch state {
        case .connecting:
            connectionState = "Connecting"
        case .connected:
            connectionState = "Connected"
        case .disconnected:
            connectionState = "Disconnected"
        }
    }
    
    private func captureLocalStream(
        fps: Int,
        width: Int,
        height: Int
    ) async throws -> RTCMediaStream {
        RTCInitializeSSL()
        
        let factory = RTCPeerConnectionFactory()
        self.peerConnectionFactory = factory
        
        let videoSource = factory.videoSource()
        let capturer = RTCCameraVideoCapturer(delegate: videoSource)
        self.videoCapturer = capturer
        
        let devices = RTCCameraVideoCapturer.captureDevices()
        guard let frontCamera = devices.first(where: { $0.position == .front }) else {
            throw DecartError.invalidInput("No front camera found")
        }
        
        let formats = RTCCameraVideoCapturer.supportedFormats(for: frontCamera)
        guard let format = formats.first(where: { format in
            let dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription)
            return dimensions.width >= width && dimensions.height >= height
        }) ?? formats.first else {
            throw DecartError.invalidInput("No suitable camera format")
        }
        
        try await withCheckedThrowingContinuation { continuation in
            capturer.startCapture(with: frontCamera, format: format, fps: fps) { error in
                if let error = error {
                    continuation.resume(throwing: error)
                } else {
                    continuation.resume()
                }
            }
        }
        
        let videoTrack = factory.videoTrack(with: videoSource, trackId: "video0")
        let stream = factory.mediaStream(withStreamId: "stream0")
        stream.addVideoTrack(videoTrack)
        
        return stream
    }
}

Best Practices

Use model properties for video constraints

Always use the model’s fps, width, and height properties when configuring camera capture to ensure optimal performance and compatibility.

let model = Models.realtime(.lucy_v2v_720p_rt)
let stream = try await captureLocalStream(
    fps: model.fps,
    width: model.width,
    height: model.height
)

Enable prompt enrichment

For best results, keep enrich: true (default) to let Decart’s AI enhance your prompts. Only disable it if you need exact prompt control.

Handle connection state changes

Always subscribe to connectionStatePublisher to update your UI and handle reconnection logic gracefully.

Clean up properly

Always call disconnect(), stop camera capture, and cleanup WebRTC when done to avoid memory leaks and unnecessary resource usage.

Use mirror mode for front cameras

Enable mirror mode when using front-facing cameras to match user expectations from selfie/video call experiences.

Test on real devices

Always test camera features on real iOS devices, as the simulator does not support WebRTC camera access.

Request permissions properly

Add camera and microphone usage descriptions to your Info.plist and handle permission denials gracefully in your UI.

API Reference

`client.realtime.connect(stream:options:)`

Connects to the realtime transformation service. Parameters:

stream: RTCMediaStream - MediaStream from camera capture
options: RealtimeConnectOptions - Connection options
- model: ModelDefinition - Realtime model from Models.realtime()
- onRemoteStream: (RTCMediaStream) -> Void - Closure for transformed video stream
- initialState: ModelState? - Initial state configuration
  - prompt: Prompt? - Initial transformation prompt
    - text: String - Style description (required)
    - enrich: Bool - Whether to enhance the prompt (default: true)
  - mirror: Bool - Enable mirror mode (default: false)
- customizeOffer: ((RTCSessionDescription) async -> Void)? - Optional WebRTC offer customization

Returns: RealtimeClient - Connected realtime client instance Throws: DecartError if connection fails

`realtimeClient.setPrompt(_:enrich:)`

Changes the transformation style. Parameters:

prompt: String - Text description of desired style
enrich: Bool - Whether to enhance the prompt (default: true)

Throws: DecartError if prompt is invalid or client not connected

`realtimeClient.setMirror(_:)`

Toggles video mirroring. Parameters:

enabled: Bool - Whether to enable mirror mode

`realtimeClient.isConnected()`

Check if currently connected. Returns: Bool

`realtimeClient.getConnectionState()`

Get current connection state. Returns: ConnectionState - .connected, .connecting, or .disconnected

`realtimeClient.sessionId`

The ID of the current realtime inference session. Type: UUID

`realtimeClient.disconnect()`

Closes the connection and cleans up resources.

Publishers

`connectionStatePublisher`

Publishes connection state changes. Type: AnyPublisher<ConnectionState, Never> Values:

.connecting - Establishing connection
.connected - Successfully connected
.disconnected - Not connected

`errorPublisher`

Publishes errors that occur during the session. Type: AnyPublisher<DecartError, Never>

Getting started

Image

Video

Realtime

Libraries & SDKs

Resources

Quick Start

Connecting

Getting Camera Access

Establishing Connection

Managing Prompts

Camera Mirroring

Connection State

Error Handling

Session Management

Cleanup

Complete SwiftUI Example

Best Practices

API Reference

`client.realtime.connect(stream:options:)`

`realtimeClient.setPrompt(_:enrich:)`

`realtimeClient.setMirror(_:)`

`realtimeClient.isConnected()`

`realtimeClient.getConnectionState()`

`realtimeClient.sessionId`

`realtimeClient.disconnect()`

Publishers

`connectionStatePublisher`

`errorPublisher`

Next Steps

SDK Overview

Example App

Getting started

Image

Video

Realtime

Libraries & SDKs

Resources

​Quick Start

​Connecting

​Getting Camera Access

​Establishing Connection

​Managing Prompts

​Camera Mirroring

​Connection State

​Error Handling

​Session Management

​Cleanup

​Complete SwiftUI Example

​Best Practices

​API Reference

​client.realtime.connect(stream:options:)

​realtimeClient.setPrompt(_:enrich:)

​realtimeClient.setMirror(_:)

​realtimeClient.isConnected()

​realtimeClient.getConnectionState()

​realtimeClient.sessionId

​realtimeClient.disconnect()

​Publishers

​connectionStatePublisher

​errorPublisher

​Next Steps

SDK Overview

Example App

Quick Start

Connecting

Getting Camera Access

Establishing Connection

Managing Prompts

Camera Mirroring

Connection State

Error Handling

Session Management

Cleanup

Complete SwiftUI Example

Best Practices

API Reference

`client.realtime.connect(stream:options:)`

`realtimeClient.setPrompt(_:enrich:)`

`realtimeClient.setMirror(_:)`

`realtimeClient.isConnected()`

`realtimeClient.getConnectionState()`

`realtimeClient.sessionId`

`realtimeClient.disconnect()`

Publishers

`connectionStatePublisher`

`errorPublisher`

Next Steps