camera: Added more accurate timestamps.

icculus · Dec 16, 2023 · 8f71217 · 8f71217
1 parent c107664
commit 8f71217
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 4 deletions.
diff --git a/src/camera/SDL_camera.c b/src/camera/SDL_camera.c
@@ -109,6 +109,9 @@ static void ClosePhysicalCameraDevice(SDL_CameraDevice *device)
     device->filled_output_surfaces.next = NULL;
     device->empty_output_surfaces.next = NULL;
     device->app_held_output_surfaces.next = NULL;
+
+    device->base_timestamp = 0;
+    device->adjust_timestamp = 0;
 }
 
 // this must not be called while `device` is still in a device list, or while a device's camera thread is still running.
@@ -536,7 +539,12 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device)
         SDL_Log("CAMERA: New frame available!");
         #endif
 
-        if (device->empty_output_surfaces.next == NULL) {
+        if (device->drop_frames > 0) {
+            device->drop_frames--;
+            camera_driver.impl.ReleaseFrame(device, device->acquire_surface);
+            device->acquire_surface->pixels = NULL;
+            device->acquire_surface->pitch = 0;
+        } else if (device->empty_output_surfaces.next == NULL) {
             // uhoh, no output frames available! Either the app is slow, or it forgot to release frames when done with them. Drop this new frame.
             #if DEBUG_CAMERA
             SDL_Log("CAMERA: No empty output surfaces! Dropping frame!");
@@ -545,6 +553,12 @@ SDL_bool SDL_CameraThreadIterate(SDL_CameraDevice *device)
             device->acquire_surface->pixels = NULL;
             device->acquire_surface->pitch = 0;
         } else {
+            if (!device->adjust_timestamp) {
+                device->adjust_timestamp = SDL_GetTicksNS();
+                device->base_timestamp = timestampNS;
+            }
+            timestampNS = (timestampNS - device->base_timestamp) + device->adjust_timestamp;
+
             slist = device->empty_output_surfaces.next;
             output_surface = slist->surface;
             device->empty_output_surfaces.next = slist->next;
@@ -829,6 +843,8 @@ SDL_Camera *SDL_OpenCameraDevice(SDL_CameraDeviceID instance_id, const SDL_Camer
         device->output_surfaces[i].surface = surf;
     }
 
+    device->drop_frames = 1;
+
     // Start the camera thread if necessary
     if (!camera_driver.impl.ProvidesOwnCallbackThread) {
         char threadname[64];

diff --git a/src/camera/SDL_syscamera.h b/src/camera/SDL_syscamera.h
@@ -25,7 +25,7 @@
 
 #include "../SDL_hashtable.h"
 
-#define DEBUG_CAMERA 1
+#define DEBUG_CAMERA 0
 
 
 // !!! FIXME: update these drivers!
@@ -92,6 +92,15 @@ struct SDL_CameraDevice
     // Driver-specific hardware data on how to open device (`hidden` is driver-specific data _when opened_).
     void *handle;
 
+    // Dropping the first frame(s) after open seems to help timing on some platforms.
+    int drop_frames;
+
+    // Backend timestamp of first acquired frame, so we can keep these meaningful regardless of epoch.
+    Uint64 base_timestamp;
+
+    // SDL timestamp of first acquired frame, so we can roughly convert to SDL ticks.
+    Uint64 adjust_timestamp;
+
     // Pixel data flows from the driver into these, then gets converted for the app if necessary.
     SDL_Surface *acquire_surface;
 

diff --git a/src/camera/v4l2/SDL_camera_v4l2.c b/src/camera/v4l2/SDL_camera_v4l2.c
@@ -67,7 +67,6 @@ struct SDL_PrivateCameraData
     io_method io;
     int nb_buffers;
     struct buffer *buffers;
-    int first_start;
     int driver_pitch;
 };
 
@@ -129,6 +128,7 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6
                 }
             }
 
+            *timestampNS = SDL_GetTicksNS();  // oh well, close enough.
             frame->pixels = device->hidden->buffers[0].start;
             frame->pitch = device->hidden->driver_pitch;
             break;
@@ -161,6 +161,8 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6
             frame->pitch = device->hidden->driver_pitch;
             device->hidden->buffers[buf.index].available = 1;
 
+            *timestampNS = (((Uint64) buf.timestamp.tv_sec) * SDL_NS_PER_SECOND) + SDL_US_TO_NS(buf.timestamp.tv_usec);
+
             #if DEBUG_CAMERA
             SDL_Log("CAMERA: debug mmap: image %d/%d  data[0]=%p", buf.index, device->hidden->nb_buffers, (void*)frame->pixels);
             #endif
@@ -202,6 +204,8 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6
             frame->pitch = device->hidden->driver_pitch;
             device->hidden->buffers[i].available = 1;
 
+            *timestampNS = (((Uint64) buf.timestamp.tv_sec) * SDL_NS_PER_SECOND) + SDL_US_TO_NS(buf.timestamp.tv_usec);
+
             #if DEBUG_CAMERA
             SDL_Log("CAMERA: debug userptr: image %d/%d  data[0]=%p", buf.index, device->hidden->nb_buffers, (void*)frame->pixels);
             #endif
@@ -212,7 +216,6 @@ static int V4L2_AcquireFrame(SDL_CameraDevice *device, SDL_Surface *frame, Uint6
             break;
     }
 
-    *timestampNS = SDL_GetTicksNS();  // !!! FIXME: can we get this info more accurately from v4l2?
     return 1;
 }