fix Linux flakiness

.github/workflows/tests.yml

@@ -105,11 +105,17 @@ jobs:
           pip install -r test/requirements.txt
           deactivate
 
-      - name: Run tests
+      - name: Run tests (with sudo)
         run: |
           ulimit -c unlimited
           source .venv/bin/activate
           sudo -E env PATH="$PATH" .venv/bin/pytest --pastebin=failed -svr a
+          deactivate
+
+      - name: Run tests (without sudo)
+        run: |
+          ulimit -c unlimited
+          source .venv/bin/activate
           .venv/bin/pytest --pastebin=failed -svr a
           deactivate
 
@@ -338,7 +344,7 @@ jobs:
       - name: Run tests
         run: |
           venv\Scripts\Activate.ps1
-          python -m pytest --ignore=test\cunit --pastebin=failed -svr a
+          python -m pytest --full-trace --ignore=test\cunit --pastebin=failed -svr a
           deactivate
 
   wheels-win:

src/linux/futils.h

@@ -0,0 +1,62 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <sys/stat.h>
+
+#define BUF_SIZE 1024
+
+typedef uint32_t crc32_t;
+
+// ----------------------------------------------------------------------------
+static inline crc32_t
+crc32(const uint8_t *data, size_t length)
+{
+    crc32_t crc = 0xFFFFFFFF;
+    for (size_t i = 0; i < length; i++)
+    {
+        crc ^= data[i];
+        for (size_t j = 0; j < 8; j++)
+        {
+            crc = (crc >> 1) ^ (0xEDB88320 & (-(int32_t)(crc & 1))); // cppcheck-suppress [integerOverflow]
+        }
+    }
+    return ~crc;
+}
+
+// ----------------------------------------------------------------------------
+static inline crc32_t
+fhash(FILE *fp)
+{
+
+    uint8_t buf[BUF_SIZE];
+    size_t bytes_read;
+    crc32_t crc = 0xFFFFFFFF;
+
+    // Save the current position
+    long int current_offset = ftell(fp);
+
+    // Move to the beginning of the file
+    fseek(fp, 0, SEEK_SET);
+
+    while ((bytes_read = fread(buf, 1, BUF_SIZE, fp)) != 0)
+    {
+        crc = crc32(buf, bytes_read);
+    }
+
+    // Restore the position
+    fseek(fp, current_offset, SEEK_SET);
+
+    return ~crc;
+}
+
+// ----------------------------------------------------------------------------
+static inline long
+fmtime_ns(FILE *fp)
+{
+    struct stat st;
+    if (fstat(fileno(fp), &st) != 0)
+    {
+        return -1;
+    }
+    return st.st_mtim.tv_nsec;
+}

src/linux/py_proc.h

@@ -36,6 +36,7 @@
 #include <unistd.h>
 
 #include "common.h"
+#include "futils.h"
 #include "../mem.h"
 #include "../resources.h"
 
@@ -354,6 +355,11 @@ _py_proc__parse_maps_file(py_proc_t * self) {
     FAIL;
   }
 
+  // Save the file hash and the modified time. We'll use them to detect if the
+  // content has changed since the last time we read it.
+  crc32_t file_hash = fhash(fp);
+  long modified_time = fmtime_ns(fp);
+
   self->min_raddr = (void *) -1;
   self->max_raddr = NULL;
 
@@ -402,7 +408,6 @@ _py_proc__parse_maps_file(py_proc_t * self) {
       size_t page_size = getpagesize();
       map->bss_base = (void *) lower - page_size;
       map->bss_size = upper - lower + page_size;
-      maps_flag |= BSS_MAP;
       log_d("Inferred BSS for %s: %lx-%lx", map->path, lower, upper);
     }
 
@@ -539,10 +544,28 @@ _py_proc__parse_maps_file(py_proc_t * self) {
   self->map.bss.base = pd->maps[map_index].bss_base;
   self->map.bss.size = pd->maps[map_index].bss_size;
 
+  if (!isvalid(self->map.bss.base)) {
+    log_e("Cannot find valid BSS map");
+    set_error(EPROCVM);
+    FAIL;
+  }
+
+  if (!(maps_flag & (BIN_MAP))) {
+    log_e("No usable Python binary found");
+    set_error(EPROC);
+    FAIL;
+  }
+
+  if (!(modified_time == fmtime_ns(fp) && file_hash == fhash(fp))) {
+    log_e("VM maps file has changed since last read");
+    set_error(EPROCVM);
+    FAIL;
+  }
+
   log_d("BSS map %d from %s @ %p", map_index, pd->maps[map_index].path, self->map.bss.base);
   log_d("VM maps parsing result: bin=%s lib=%s flags=%d", self->bin_path, self->lib_path, maps_flag);
 
-  return !(maps_flag & (BIN_MAP | BSS_MAP));
+  SUCCESS;
 } /* _py_proc__parse_maps_file */
 
 

src/py_proc.c

@@ -541,80 +541,65 @@ _py_proc__get_current_thread_state_raddr(py_proc_t * self) {
   return (void *) -1;
 }
 
-
 // ----------------------------------------------------------------------------
 static int
-_py_proc__wait_for_interp_state(py_proc_t * self) {
-  #ifdef DEBUG
-  register int attempts = 0;
-  #endif
+_py_proc__find_interpreter_state(py_proc_t * self) {
+  if (!isvalid(self)) {
+    set_error(EPROC);
+    FAIL;
+  }
 
-  self->is_raddr = NULL;
+  if (fail(_py_proc__init(self)))
+    FAIL;
 
-  TIMER_START(pargs.timeout)
-    if (!py_proc__is_running(self)) {
-      log_e("Process %d is not running.", self->pid);
-      set_error(EPROCNPID);
-      FAIL;
-    }
+  // Determine and set version
+  if (fail(_py_proc__infer_python_version(self)))
+    FAIL;
 
-    #ifdef DEBUG
-    attempts++;
-    #endif
+  if (self->sym_loaded) {
+    // Try to resolve the symbols if we have them
 
-    if (success(_py_proc__deref_interp_head(self))) {
-      log_d("✨ Interpreter head de-referenced from symbols ✨ ");
-      TIMER_STOP
+    self->is_raddr = NULL;
+
+    if (fail(_py_proc__deref_interp_head(self))) {
+      log_d("Cannot dereference PyInterpreterState head from symbols (pid: %d)", self->pid);
+      FAIL;
     }
 
-    log_d("Cannot dereference PyInterpreterState head from symbols (pid: %d)", self->pid);
-
-    if (austin_errno == EPROCNPID) {
-      log_d(
-        "Stop waiting for interpreter state: process %ld is not running",
-        self->pid
-      );
-      self->is_raddr = NULL;
+    log_d("✨ Interpreter head de-referenced from symbols ✨ ");
+  } else {
+    // Attempt a BSS scan if we don't have symbols
+    if (fail(_py_proc__scan_bss(self))) {
+      log_d("BSS scan failed (no symbols available)");
       FAIL;
     }
-
-    // Try once for child processes.
-    if (self->child)
-      TIMER_STOP
-
-  TIMER_END
-
-  if (isvalid(self->is_raddr)) {
-    log_d("Interpreter State de-referenced @ raddr: %p after %d attempts",
-      self->is_raddr,
-      attempts
-    );
-    SUCCESS;
-  }
-
-  if (self->child) {
-    set_error(EPROC);
-    FAIL;
+
+    log_d("Interpreter state located from BSS scan (no symbols available)");
   }
 
-  set_error(EPROCISTIMEOUT);
-  FAIL;
-} // _py_proc__wait_for_interp_state
-
+  SUCCESS;
+}
 
 // ----------------------------------------------------------------------------
 static int
 _py_proc__run(py_proc_t * self) {
   int try_once = self->child;
   int init = FALSE;
+  int attempts = 0;
+
   #ifdef DEBUG
-  if (try_once == FALSE)
+  if (!try_once)
     log_d("Start up timeout: %d ms", pargs.timeout / 1000);
   else
     log_d("Single attempt to attach to process %d", self->pid);
   #endif
 
   TIMER_START(pargs.timeout)
+    if (try_once && ++attempts > 1) {
+      log_d("Cannot attach to process %d with a single attempt.", self->pid);
+      FAIL;
+    }
+
     if (!py_proc__is_running(self)) {
       log_e("Process %d is not running.", self->pid);
       set_error(EPROCNPID);
@@ -625,31 +610,36 @@ _py_proc__run(py_proc_t * self) {
     sfree(self->lib_path);
     self->sym_loaded = FALSE;
 
-    if (success(_py_proc__init(self))) {
-      log_d("Process is ready");
+    if (success(_py_proc__find_interpreter_state(self))) {
       init = TRUE;
+
+      log_d("Interpreter State de-referenced @ raddr: %p after %d attempts",
+        self->is_raddr,
+        attempts
+      );
+
       TIMER_STOP;
     }
 
-    log_d("Process is not ready");
-
-    if (try_once)
-      TIMER_STOP
   TIMER_END
+
   log_d("_py_proc__init timer loop terminated");
 
   if (!init) {
-    if (try_once)
-      log_d("Cannot attach to process %d with a single attempt.", self->pid);
-    FAIL;
+    log_d("Interpreter state search timed out");
+
+    // Scan the BSS section as a last resort
+    if (fail(_py_proc__scan_bss(self))) {
+      log_d("BSS scan failed");
+      FAIL;
+    }
+
+    log_d("Interpreter state located from BSS scan");
   }
 
   if (!(isvalid(self->bin_path) || isvalid(self->lib_path)))
     log_w("No Python binary files detected");
 
-  if (self->map.bss.size == 0 || self->map.bss.base == NULL)
-    log_e("Unable to fully locate the BSS section.");
-
   if (self->min_raddr > self->max_raddr)
     log_w("Invalid remote VM maximal bounds.");
 
@@ -665,33 +655,6 @@ _py_proc__run(py_proc_t * self) {
   log_d("Maximal VM address space: %p-%p", self->min_raddr, self->max_raddr);
   #endif
 
-  // Determine and set version
-  if (fail(_py_proc__infer_python_version(self)))
-    FAIL;
-
-  if (self->sym_loaded) {
-    if (fail(_py_proc__wait_for_interp_state(self))) {
-      if (fail(_py_proc__scan_bss(self))) {
-        log_d("BSS scan failed");
-        FAIL;
-      }
-      log_d("Interpreter state located from BSS scan");
-    }
-  } else {
-    int found = FALSE;
-    TIMER_START(pargs.timeout)
-      if (success(_py_proc__scan_bss(self))) {
-        log_d("Interpreter state located from BSS scan (no symbols available)");
-        found = TRUE;
-        TIMER_STOP
-      }
-    TIMER_END
-    if (!found) {
-      log_d("BSS scan failed (no symbols available)");
-      FAIL;
-    }
-  }
-
   self->timestamp = gettime();
 
   #ifdef NATIVE

test/cunit/__init__.py

@@ -28,11 +28,6 @@
 SRC = ROOT / "src"
 
 
-# Remove any existing shared objects
-for so in SRC.glob("*.so"):
-    so.unlink(missing_ok=True)
-
-
 restrict_re = re.compile(r"__restrict \w+")
 
 _header_head = r"""

test/requirements.txt

@@ -1,4 +1,5 @@
 austin-python~=1.5
+flaky
 pytest
 pytest-xdist