Multi-Camera System Implementation Guide

Overview

In many robotic applications, a single camera is insufficient for full environment perception. For example:

• 🚗 Autonomous Driving: requires 360° environment perception
• 🏠 Indoor Navigation: needs front + rear obstacle avoidance
• 🎯 Object Tracking: benefits from stereo vision

This article details how to implement a multi-camera system on Jetson Orin Nano, covering:

• ✅ Hardware connection strategies
• ✅ Software synchronization mechanisms
• ✅ Data fusion approaches
• ✅ Performance optimization techniques

Hardware Connection Strategies

Option 1: Front-Rear Dual Cameras

Jetson Orin Nano
├── USB 3.0 Port 0 ───→ Front Camera (RGB)
├── USB 3.0 Port 1 ───→ Rear Camera (RGB)
├── USB 2.0 Port 0 ───→ URT-1 Debug Board (Servo control)
└── CSI Port 0 ───────→ IMX477 (High-end vision, optional)

Option 2: Three-Camera Surround (Front + Left + Right)

         Front Camera
            ↑
            │
    Left Camera ───┼───  Right Camera
            │
           Orin Nano

Jetson Orin Nano
├── USB 3.0 Hub ───┬──→ Front Camera
│                  ├──→ Left Camera
│                  └──→ Right Camera
└── USB 2.0 ────────→ URT-1 Debug Board

Software Synchronization

Software Triggered Synchronization

Software triggering captures from all cameras simultaneously with accuracy of approximately ±10-50ms:

import cv2
import numpy as np
import time
from dataclasses import dataclass
from typing import Dict

@dataclass
class SyncedFrame:
    timestamp: float
    frames: Dict[str, np.ndarray]

class SoftwareSyncedCameras:
    def __init__(self, max_sync_error_ms=50.0):
        self.cameras = {}
        self.max_sync_error = max_sync_error_ms / 1000.0

    def add_camera(self, name, device_index, width=640, height=480, fps=30):
        cap = cv2.VideoCapture(device_index)
        cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
        cap.set(cv2.CAP_PROP_FPS, fps)

        # Clear buffer
        for _ in range(5):
            cap.read()

        self.cameras[name] = cap

    def capture_synced(self, timeout_ms=100):
        start_time = time.time()
        timeout_sec = timeout_ms / 1000.0

        while (time.time() - start_time) < timeout_sec:
            frames = {}
            timestamps = []

            for name, cap in self.cameras.items():
                t0 = time.time()
                ret, frame = cap.read()
                t1 = time.time()

                if ret:
                    frames[name] = frame
                    timestamps.append(t1)

            if len(frames) != len(self.cameras):
                continue

            time_diff = max(timestamps) - min(timestamps)

            if time_diff <= self.max_sync_error:
                return SyncedFrame(
                    timestamp=np.mean(timestamps),
                    frames=frames
                )

        raise TimeoutError("Synchronized capture timed out")

# Example usage
sync_cams = SoftwareSyncedCameras(max_sync_error_ms=30.0)
sync_cams.add_camera("front", 0, 640, 480, 30)
sync_cams.add_camera("rear", 1, 640, 480, 30)

while True:
    synced = sync_cams.capture_synced(timeout_ms=200)
    print(f"Synced: front={synced.frames['front'].shape}, rear={synced.frames['rear'].shape}")

    # Display
    display = np.hstack([synced.frames['front'], synced.frames['rear']])
    cv2.imshow('Synced Cameras', display)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cv2.destroyAllWindows()

Data Fusion Strategy

BEV (Bird's Eye View) Fusion

Transform multiple camera images into the BEV perspective and then fuse them:

import cv2
import numpy as np

def create_bev_projection(image, camera_matrix, dist_coeffs, src_points, dst_size=(400, 600)):
    """
    Create bird's-eye-view projection

    Args:
        image: input image
        camera_matrix: camera intrinsics
        dist_coeffs: distortion coefficients
        src_points: four ground corner points in original image
        dst_size: output image size

    Returns:
        bev_image: bird's-eye-view image
    """
    # Undistort
    undistorted = cv2.undistort(image, camera_matrix, dist_coeffs)

    # Define destination points (BEV coordinates)
    dst_points = np.array([
        [0, 0],
        [dst_size[0], 0],
        [dst_size[0], dst_size[1]],
        [0, dst_size[1]]
    ], dtype=np.float32)

    # Calculate perspective transform matrix
    M = cv2.getPerspectiveTransform(src_points, dst_points)

    # Apply perspective transformation
    bev_image = cv2.warpPerspective(undistorted, M, dst_size)

    return bev_image


# Multi-camera BEV fusion
def fuse_multi_camera_bev(front_image, left_image, right_image,
                          camera_params, vehicle_pose=None):
    """
    Fuse multi-camera BEV images

    Args:
        front_image: front camera image
        left_image: left camera image
        right_image: right camera image
        camera_params: camera parameters for each camera
        vehicle_pose: vehicle pose (for dynamic fusion)

    Returns:
        fused_bev: fused BEV image
    """
    # Generate BEV for each camera
    front_bev = create_bev_projection(
        front_image,
        camera_params['front']['K'],
        camera_params['front']['D'],
        camera_params['front']['src_points']
    )

    left_bev = create_bev_projection(
        left_image,
        camera_params['left']['K'],
        camera_params['left']['D'],
        camera_params['left']['src_points']
    )

    right_bev = create_bev_projection(
        right_image,
        camera_params['right']['K'],
        camera_params['right']['D'],
        camera_params['right']['src_points']
    )

    # Fusion (simple weighted average, more complex algorithms can be used in practice)
    # Assumes all BEV images are already aligned to the same coordinate frame
    fused_bev = np.zeros_like(front_bev)

    # Define fusion weights
    weights = {
        'front': 0.5,
        'left': 0.25,
        'right': 0.25
    }

    # Weighted fusion
    fused_bev = (weights['front'] * front_bev +
                 weights['left'] * left_bev +
                 weights['right'] * right_bev).astype(np.uint8)

    return fused_bev

Performance Optimization

GPU Acceleration

Utilize the Jetson Orin Nano GPU for image preprocessing:

import cv2
import numpy as np

# Use CUDA acceleration
# Make sure OpenCV is compiled with CUDA support

def preprocess_image_gpu(image, target_size=(224, 224)):
    """
    GPU-accelerated image preprocessing

    Args:
        image: input image (CPU memory)
        target_size: target size

    Returns:
        processed: preprocessed image (CPU memory)
    """
    # Upload to GPU
    gpu_image = cv2.cuda_GpuMat()
    gpu_image.upload(image)

    # Resize on GPU
    gpu_resized = cv2.cuda.resize(gpu_image, target_size)

    # Normalization on GPU (optional)
    # Note: OpenCV CUDA module doesn't directly support normalization, requires custom kernel

    # Download back to CPU
    processed = gpu_resized.download()

    return processed


# Batch preprocessing
class GPUPreprocessor:
    """GPU batch preprocessor"""

    def __init__(self, batch_size=4, target_size=(224, 224)):
        self.batch_size = batch_size
        self.target_size = target_size
        self.stream = cv2.cuda_Stream()

    def preprocess_batch(self, images):
        """
        Batch preprocessing

        Args:
            images: list of images

        Returns:
            batch: preprocessed batch
        """
        batch = np.zeros((len(images), *self.target_size, 3), dtype=np.float32)

        for i, image in enumerate(images):
            # Async upload and preprocessing
            gpu_image = cv2.cuda_GpuMat()
            gpu_image.upload(image, self.stream)

            gpu_resized = cv2.cuda.resize(gpu_image, self.target_size, stream=self.stream)

            # Download
            batch[i] = gpu_resized.download() / 255.0  # normalize

        return batch

Summary

A multi-camera system provides robots with 360° environmental awareness, which is key to advanced navigation and obstacle avoidance. Following this guide, you can:

• ✅ Set up a multi-camera hardware system
• ✅ Implement software synchronization
• ✅ Perform BEV data fusion
• ✅ Optimize system performance