Feature/signal real timestamp (#969)

src/urh/controller/MainController.py

@@ -391,7 +391,9 @@ def add_simulator_profile(self, filename):
         self.ui.tabWidget.setCurrentIndex(3)
         self.simulator_tab_controller.load_simulator_file(filename)
 
-    def add_signalfile(self, filename: str, group_id=0, enforce_sample_rate=None):
+    def add_signalfile(
+        self, filename: str, group_id=0, enforce_sample_rate=None, signal_timestamp=0
+    ):
         if not os.path.exists(filename):
             QMessageBox.critical(
                 self,
@@ -411,7 +413,9 @@ def add_signalfile(self, filename: str, group_id=0, enforce_sample_rate=None):
         else:
             sample_rate = self.project_manager.device_conf["sample_rate"]
 
-        signal = Signal(filename, sig_name, sample_rate=sample_rate)
+        signal = Signal(
+            filename, sig_name, sample_rate=sample_rate, timestamp=signal_timestamp
+        )
 
         self.file_proxy_model.open_files.add(filename)
         self.add_signal(signal, group_id)
@@ -944,11 +948,15 @@ def on_show_spectrum_dialog_action_triggered(self):
         r.device_parameters_changed.connect(pm.set_device_parameters)
         r.show()
 
-    @pyqtSlot(list, float)
-    def on_signals_recorded(self, file_names: list, sample_rate: float):
+    @pyqtSlot(list)
+    def on_signals_recorded(self, recorded_files: list):
         QApplication.instance().setOverrideCursor(Qt.WaitCursor)
-        for filename in file_names:
-            self.add_signalfile(filename, enforce_sample_rate=sample_rate)
+        for recorded_file in recorded_files:
+            self.add_signalfile(
+                recorded_file.filename,
+                enforce_sample_rate=recorded_file.sample_rate,
+                signal_timestamp=recorded_file.timestamp,
+            )
         QApplication.instance().restoreOverrideCursor()
 
     @pyqtSlot()

src/urh/controller/dialogs/ReceiveDialog.py

@@ -9,10 +9,11 @@
 from urh.util import FileOperator
 from urh.util.Formatter import Formatter
 from datetime import datetime
+from urh.signalprocessing.RecordedFile import RecordedFile
 
 
 class ReceiveDialog(SendRecvDialog):
-    files_recorded = pyqtSignal(list, float)
+    files_recorded = pyqtSignal(list)
 
     def __init__(self, project_manager, parent=None, testing_mode=False):
         try:
@@ -53,12 +54,7 @@ def save_before_close(self):
             elif reply == QMessageBox.Abort:
                 return False
 
-        try:
-            sample_rate = self.device.sample_rate
-        except:
-            sample_rate = 1e6
-
-        self.files_recorded.emit(self.recorded_files, sample_rate)
+        self.files_recorded.emit(self.recorded_files)
         return True
 
     def update_view(self):
@@ -109,9 +105,11 @@ def on_save_clicked(self):
 
         dev = self.device
         big_val = Formatter.big_value_with_suffix
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        timestamp_str = datetime.fromtimestamp(dev.data_timestamp).strftime(
+            "%Y%m%d_%H%M%S"
+        )
         initial_name = "{0}-{1}-{2}Hz-{3}Sps".format(
-            dev.name, timestamp, big_val(dev.frequency), big_val(dev.sample_rate)
+            dev.name, timestamp_str, big_val(dev.frequency), big_val(dev.sample_rate)
         )
 
         if dev.bandwidth_is_adjustable:
@@ -125,5 +123,9 @@ def on_save_clicked(self):
             initial_name, data, sample_rate=dev.sample_rate, parent=self
         )
         self.already_saved = True
-        if filename is not None and filename not in self.recorded_files:
-            self.recorded_files.append(filename)
+        if filename is not None and filename not in (
+            x.filename for x in self.recorded_files
+        ):
+            self.recorded_files.append(
+                RecordedFile(filename, dev.sample_rate, dev.data_timestamp)
+            )

src/urh/controller/dialogs/SendRecvDialog.py

@@ -157,6 +157,7 @@ def _create_device_connects(self):
     def reset(self):
         self.device.current_index = 0
         self.device.current_iteration = 0
+        self.device.reset_data_timestamp()
         self.ui.lSamplesCaptured.setText("0")
         self.ui.lSignalSize.setText("0")
         self.ui.lTime.setText("0")

src/urh/controller/widgets/SignalFrame.py

@@ -1574,7 +1574,9 @@ def on_bandpass_filter_triggered(self, f_low: float, f_high: float):
             time.sleep(0.1)
 
         filtered = np.frombuffer(filtered.get_obj(), dtype=np.complex64)
-        signal = self.signal.create_new(new_data=filtered.astype(np.complex64))
+        signal = self.signal.create_new(
+            new_data=filtered.astype(np.complex64), new_timestamp=self.signal.timestamp
+        )
         signal.name = (
             self.signal.name
             + " filtered with f_low={0:.4n} f_high={1:.4n} bw={2:.4n}".format(

src/urh/dev/VirtualDevice.py

@@ -60,6 +60,7 @@ def __init__(
         self.name = name
         self.mode = mode
         self.backend_handler = backend_handler
+        self.__data_timestamp = 0
 
         freq = config.DEFAULT_FREQUENCY if freq is None else freq
         sample_rate = config.DEFAULT_SAMPLE_RATE if sample_rate is None else sample_rate
@@ -493,7 +494,10 @@ def baseband_gain(self, value):
 
     @property
     def sample_rate(self):
-        return self.__dev.sample_rate
+        try:
+            return self.__dev.sample_rate
+        except:
+            return 1e6
 
     @sample_rate.setter
     def sample_rate(self, value):
@@ -631,6 +635,23 @@ def data(self, value):
                 "{}:{} has no data".format(self.__class__.__name__, self.backend.name)
             )
 
+    @property
+    def data_timestamp(self):
+        if self.backend == Backends.native:
+            try:
+                self.__data_timestamp = (
+                    self.__dev.first_data_timestamp
+                )  # more accurate timestamp
+            except:
+                pass
+        return self.__data_timestamp
+
+    def reset_data_timestamp(self):
+        if self.backend == Backends.native:
+            self.__dev.reset_first_data_timestamp()
+        else:
+            self.__data_timestamp = time.time()
+
     def free_data(self):
         if self.backend == Backends.grc:
             self.__dev.data = None
@@ -740,6 +761,7 @@ def spectrum(self):
             raise ValueError("Spectrum x only available in spectrum mode")
 
     def start(self):
+        self.__data_timestamp = time.time()
         if self.backend == Backends.grc:
             self.__dev.setTerminationEnabled(True)
             self.__dev.terminate()

src/urh/dev/native/Device.py

@@ -304,6 +304,7 @@ def __init__(
         self.error_codes = {}
         self.device_messages = []
 
+        self.__first_data_timestamp = 0
         self.receive_process_function = self.device_receive
         self.send_process_function = self.device_send
 
@@ -654,7 +655,15 @@ def set_device_direct_sampling_mode(self, value):
         except (BrokenPipeError, OSError):
             pass
 
+    @property
+    def first_data_timestamp(self):
+        return self.__first_data_timestamp
+
+    def reset_first_data_timestamp(self):
+        self.__first_data_timestamp = 0
+
     def start_rx_mode(self):
+        self.__first_data_timestamp = 0
         self.init_recv_buffer()
         self.parent_data_conn, self.child_data_conn = Pipe(duplex=False)
         self.parent_ctrl_conn, self.child_ctrl_conn = Pipe()
@@ -783,8 +792,20 @@ def read_receiving_queue(self):
         while self.is_receiving:
             try:
                 byte_buffer = self.parent_data_conn.recv_bytes()
+
+                if self.__first_data_timestamp == 0:
+                    self.__first_data_timestamp = time.time()
+                    calculating_timestamp = True
+                else:
+                    calculating_timestamp = False
+
                 samples = self.bytes_to_iq(byte_buffer)
                 n_samples = len(samples)
+
+                if calculating_timestamp:
+                    # Timestamp accurate correction
+                    self.__first_data_timestamp -= n_samples / self.sample_rate
+
                 if n_samples == 0:
                     continue
 

src/urh/signalprocessing/Message.py

@@ -56,6 +56,7 @@ def __init__(
         samples_per_symbol=100,
         participant=None,
         bits_per_symbol=1,
+        timestamp=0,
     ):
         """
 
@@ -75,7 +76,12 @@ def __init__(
         self.participant = participant  # type: Participant
         self.message_type = message_type  # type: MessageType
 
-        self.timestamp = time.time()
+        if timestamp == 0:
+            self.timestamp = time.time()
+        else:
+            # Caller passed specific timestamp for this message
+            self.timestamp = timestamp
+
         self.absolute_time = 0  # set in Compare Frame
         self.relative_time = 0  # set in Compare Frame
 

src/urh/signalprocessing/ProtocolAnalyzer.py

@@ -267,6 +267,9 @@ def get_protocol_from_signal(self):
             middle_bit_pos = bit_sample_pos[i][int(len(bits) / 2)]
             start, end = middle_bit_pos, middle_bit_pos + samples_per_symbol
             rssi = np.mean(signal.iq_array.subarray(start, end).magnitudes_normalized)
+            message_timestamp = signal.timestamp + (
+                bit_sample_pos[i][0] / signal.sample_rate
+            )
             message = Message(
                 bits,
                 pause,
@@ -276,6 +279,7 @@ def get_protocol_from_signal(self):
                 decoder=self.decoder,
                 bit_sample_pos=bit_sample_pos[i],
                 bits_per_symbol=signal.bits_per_symbol,
+                timestamp=message_timestamp,
             )
             self.messages.append(message)
             i += 1

src/urh/signalprocessing/ProtocolSniffer.py

@@ -236,6 +236,9 @@ def __demodulate_data(self, data):
 
         # clear cache and start a new message
         self.signal.iq_array = IQArray(self.__buffer[0 : self.__current_buffer_index])
+        self.signal.timestamp = time.time() - (
+            len(self.signal.iq_array) / self.rcv_device.sample_rate
+        )  # timestamp of the first sample in our buffer
         self.__clear_buffer()
         self.signal._qad = None
 
@@ -259,19 +262,25 @@ def __demodulate_data(self, data):
             ppseq,
             samples_per_symbol,
             self.signal.bits_per_symbol,
-            write_bit_sample_pos=False,
+            write_bit_sample_pos=True,
         )
 
+        i = 0
         for bits, pause in zip(bit_data, pauses):
+            message_timestamp = self.signal.timestamp + (
+                bit_sample_pos[i][0] / self.rcv_device.sample_rate
+            )
             message = Message(
                 bits,
                 pause,
                 samples_per_symbol=samples_per_symbol,
                 message_type=self.default_message_type,
                 decoder=self.decoder,
+                timestamp=message_timestamp,
             )
             self.messages.append(message)
             self.message_sniffed.emit(len(self.messages) - 1)
+            i += 1
 
     def stop(self):
         self.is_running = False

src/urh/signalprocessing/RecordedFile.py

@@ -0,0 +1,5 @@
+class RecordedFile:
+    def __init__(self, filename, sample_rate, timestamp):
+        self.filename = filename
+        self.sample_rate = sample_rate
+        self.timestamp = timestamp

src/urh/signalprocessing/Signal.py

@@ -45,6 +45,7 @@ def __init__(
         name="Signal",
         modulation: str = None,
         sample_rate: float = 1e6,
+        timestamp: float = 0,
         parent=None,
     ):
         super().__init__(parent)
@@ -58,6 +59,7 @@ def __init__(
         self.__center = 0
         self._noise_threshold = 0
         self.__sample_rate = sample_rate
+        self.__timestamp = timestamp
         self.noise_min_plot = 0
         self.noise_max_plot = 0
         self.block_protocol_update = False
@@ -224,6 +226,14 @@ def sample_rate(self, val):
             self.__sample_rate = val
             self.sample_rate_changed.emit(val)
 
+    @property
+    def timestamp(self):
+        return self.__timestamp
+
+    @timestamp.setter
+    def timestamp(self, val):
+        self.__timestamp = val
+
     @property
     def parameter_cache(self) -> dict:
         """
@@ -495,13 +505,15 @@ def calc_relative_noise_threshold_from_range(
             )
             return self.noise_threshold_relative
 
-    def create_new(self, start=0, end=0, new_data=None):
+    def create_new(self, start=0, end=0, new_data=None, new_timestamp=0):
         new_signal = Signal("", "New " + self.name)
 
         if new_data is None:
             new_signal.iq_array = IQArray(self.iq_array[start:end])
+            new_signal.__timestamp = self.timestamp + (start / self.sample_rate)
         else:
             new_signal.iq_array = IQArray(new_data)
+            new_signal.__timestamp = new_timestamp
 
         new_signal._noise_threshold = self.noise_threshold
         new_signal.noise_min_plot = self.noise_min_plot

tests/test_protocol_sniffer.py

@@ -65,6 +65,12 @@ def test_protocol_sniffer(self):
         for d in data:
             packages.append(modulator.modulate(list(map(int, d)), pause))
 
+        next_msg_timestamp = (
+            time.time()
+        )  # Due to simulated transmission method used in testing,
+        # we won't be quite precise on the first timestamp on which data will be received
+        # But at least we'll validate it is received near this time
+
         # verify modulation was correct
         pa = ProtocolAnalyzer(None)
         signal = Signal("", "", sample_rate=sample_rate)
@@ -91,3 +97,20 @@ def test_protocol_sniffer(self):
 
         sniffer.stop()
         self.assertEqual(sniffer.plain_bits_str, data)
+        # Validate timestamps:
+        for i in range(len(sniffer.messages)):
+            msg = sniffer.messages[i]
+            if i == 0:
+                # For the first message, we can't have much accuracy due to the simulated mechanism used to deliver data.
+                # Let's just verify the timestamp makes sense with following condition:
+                # (next_msg_timestamp < msg.timestamp  &&  msg.timestamp < next_msg_timestamp + SLEEP_TIME_DURING_PROCESS)
+                self.assertLess(next_msg_timestamp, msg.timestamp)
+                self.assertLess(msg.timestamp, next_msg_timestamp + 2)
+            else:
+                # For each message, verify the timestamp according to the theoretical calculation:
+                # (next_msg_timestamp - 0.0001 < msg.timestamp  &&  msg.timestamp < next_msg_timestamp + 0.0001)
+                self.assertLess(next_msg_timestamp - 0.0001, msg.timestamp)
+                self.assertLess(msg.timestamp, next_msg_timestamp + 0.0001)
+            next_msg_timestamp = (
+                msg.timestamp + len(packages[i]) / modulator.sample_rate
+            )