sanitizer thread CI

.github/workflows/daily.yml

@@ -561,101 +561,46 @@ jobs:
         valgrind --track-origins=yes --suppressions=./src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full --log-file=err.txt ./src/redis-server test all --valgrind
         if grep -q 0x err.txt; then cat err.txt; exit 1; fi
 
-  test-sanitizer-address:
+  test-sanitizer-thread:
     runs-on: ubuntu-latest
     if: |
       (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
       !contains(github.event.inputs.skipjobs, 'sanitizer')
     timeout-minutes: 14400
     strategy:
       matrix:
-        compiler: [ gcc, clang ]
+        compiler: [ gcc-13, clang-15 ]
     env:
       CC: ${{ matrix.compiler }}
     steps:
-      - name: prep
-        if: github.event_name == 'workflow_dispatch'
-        run: |
-          echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
-          echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
-          echo "skipjobs: ${{github.event.inputs.skipjobs}}"
-          echo "skiptests: ${{github.event.inputs.skiptests}}"
-          echo "test_args: ${{github.event.inputs.test_args}}"
-          echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}"
-      - uses: actions/checkout@v3
-        with:
-          repository: ${{ env.GITHUB_REPOSITORY }}
-          ref: ${{ env.GITHUB_HEAD_REF }}
-      - name: make
-        run: make SANITIZER=address REDIS_CFLAGS='-DREDIS_TEST -Werror -DDEBUG_ASSERTIONS'
-      - name: testprep
-        # Work around ASAN issue, see https://github.com/google/sanitizers/issues/1716
-        run: |
-          sudo apt-get update
-          sudo apt-get install tcl8.6 tclx -y
-          sudo sysctl vm.mmap_rnd_bits=28
-      - name: test
-        if: true && !contains(github.event.inputs.skiptests, 'redis')
-        run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
-      - name: module api test
-        if: true && !contains(github.event.inputs.skiptests, 'modules')
-        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
-      - name: sentinel tests
-        if: true && !contains(github.event.inputs.skiptests, 'sentinel')
-        run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
-      - name: cluster tests
-        if: true && !contains(github.event.inputs.skiptests, 'cluster')
-        run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
-      - name: unittest
-        if: true && !contains(github.event.inputs.skiptests, 'unittest')
-        run: ./src/redis-server test all
-
-  test-sanitizer-undefined:
-    runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'workflow_dispatch' || (github.event_name != 'workflow_dispatch' && github.repository == 'redis/redis')) &&
-      !contains(github.event.inputs.skipjobs, 'sanitizer')
-    timeout-minutes: 14400
-    strategy:
-      matrix:
-        compiler: [ gcc, clang ]
-    env:
-      CC: ${{ matrix.compiler }}
-    steps:
-      - name: prep
-        if: github.event_name == 'workflow_dispatch'
-        run: |
-          echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
-          echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
-          echo "skipjobs: ${{github.event.inputs.skipjobs}}"
-          echo "skiptests: ${{github.event.inputs.skiptests}}"
-          echo "test_args: ${{github.event.inputs.test_args}}"
-          echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}"
-      - uses: actions/checkout@v3
-        with:
-          repository: ${{ env.GITHUB_REPOSITORY }}
-          ref: ${{ env.GITHUB_HEAD_REF }}
-      - name: make
-        run: make SANITIZER=undefined REDIS_CFLAGS='-DREDIS_TEST -Werror' LUA_DEBUG=yes # we (ab)use this flow to also check Lua C API violations
-      - name: testprep
-        run: |
-          sudo apt-get update
-          sudo apt-get install tcl8.6 tclx -y
-      - name: test
-        if: true && !contains(github.event.inputs.skiptests, 'redis')
-        run: ./runtest --accurate --verbose --dump-logs ${{github.event.inputs.test_args}}
-      - name: module api test
-        if: true && !contains(github.event.inputs.skiptests, 'modules')
-        run: CFLAGS='-Werror' ./runtest-moduleapi --verbose --dump-logs ${{github.event.inputs.test_args}}
-      - name: sentinel tests
-        if: true && !contains(github.event.inputs.skiptests, 'sentinel')
-        run: ./runtest-sentinel ${{github.event.inputs.cluster_test_args}}
-      - name: cluster tests
-        if: true && !contains(github.event.inputs.skiptests, 'cluster')
-        run: ./runtest-cluster ${{github.event.inputs.cluster_test_args}}
-      - name: unittest
-        if: true && !contains(github.event.inputs.skiptests, 'unittest')
-        run: ./src/redis-server test all --accurate
+    - name: prep
+      if: github.event_name == 'workflow_dispatch'
+      run: |
+        echo "GITHUB_REPOSITORY=${{github.event.inputs.use_repo}}" >> $GITHUB_ENV
+        echo "GITHUB_HEAD_REF=${{github.event.inputs.use_git_ref}}" >> $GITHUB_ENV
+        echo "skipjobs: ${{github.event.inputs.skipjobs}}"
+        echo "skiptests: ${{github.event.inputs.skiptests}}"
+        echo "test_args: ${{github.event.inputs.test_args}}"
+        echo "cluster_test_args: ${{github.event.inputs.cluster_test_args}}"
+    - uses: actions/checkout@v3
+      with:
+        repository: ${{ env.GITHUB_REPOSITORY }}
+        ref: ${{ env.GITHUB_HEAD_REF }}
+    - name: make
+      run: |
+        sudo apt-get update
+        gcc -v
+        make SANITIZER=thread REDIS_CFLAGS='-DREDIS_TEST -Werror -DDEBUG_ASSERTIONS'
+    - name: testprep
+      run: |
+        sudo apt-get install tcl8.6 tclx -y
+        sudo sysctl vm.mmap_rnd_bits=28
+    - name: test
+      if: true && !contains(github.event.inputs.skiptests, 'redis')
+      run: TSAN_OPTIONS="suppressions=./asan.sup:allocator_may_return_null=1:detect_deadlocks=0" ./runtest --config io-threads 4 --config io-threads-do-reads yes --accurate --verbose --tags -signal --dump-logs ${{github.event.inputs.test_args}}
+    - name: module api test
+      if: true && !contains(github.event.inputs.skiptests, 'modules')
+      run: TSAN_OPTIONS="suppressions=./asan.sup:allocator_may_return_null=1:detect_deadlocks=0" CFLAGS='-Werror' ./runtest-moduleapi --config io-threads 4 --config io-threads-do-reads yes --verbose --tags -signal --dump-logs ${{github.event.inputs.test_args}}
 
   test-centos7-jemalloc:
     runs-on: ubuntu-latest

src/Makefile

@@ -110,6 +110,7 @@ ifeq ($(SANITIZER),undefined)
 	LDFLAGS+=-fsanitize=undefined
 else
 ifeq ($(SANITIZER),thread)
+	MALLOC=libc
 	CFLAGS+=-fsanitize=thread -fno-sanitize-recover=all -fno-omit-frame-pointer
 	LDFLAGS+=-fsanitize=thread
 else

src/asan.sup

@@ -0,0 +1,7 @@
+signal:collect_stacktrace_data
+signal:invoke_callback
+signal:printCrashReport
+signal:sigalrmSignalHandler
+signal:sigsegvHandler
+race:g_callback
+race:sigShutdownHandler

src/module.c

@@ -2445,7 +2445,7 @@ void RM_Yield(RedisModuleCtx *ctx, int flags, const char *busy_reply) {
                     /* Release the GIL, yielding CPU to give the main thread an opportunity to start
                      * event processing, and then acquire the GIL again until the main thread releases it. */
                     moduleReleaseGIL();
-                    sched_yield();
+                    usleep(0);
                     moduleAcquireGIL();
                 }
             } else {

src/server.c

@@ -130,11 +130,13 @@ void serverLogRaw(int level, const char *msg) {
         int off;
         struct timeval tv;
         int role_char;
+        int daylight_active;
         pid_t pid = getpid();
 
         gettimeofday(&tv,NULL);
         struct tm tm;
-        nolocks_localtime(&tm,tv.tv_sec,server.timezone,server.daylight_active);
+        atomicGet(server.daylight_active, daylight_active);
+        nolocks_localtime(&tm,tv.tv_sec,server.timezone,daylight_active);
         off = strftime(buf,sizeof(buf),"%d %b %Y %H:%M:%S.",&tm);
         snprintf(buf+off,sizeof(buf)-off,"%03d",(int)tv.tv_usec/1000);
         if (server.sentinel_mode) {
@@ -1120,7 +1122,7 @@ static inline void updateCachedTimeWithUs(int update_daylight_info, const long l
         struct tm tm;
         time_t ut = server.unixtime;
         localtime_r(&ut,&tm);
-        server.daylight_active = tm.tm_isdst;
+        atomicSet(server.daylight_active, tm.tm_isdst);
     }
 }
 

src/server.h

@@ -1975,7 +1975,7 @@ struct redisServer {
     /* time cache */
     redisAtomic time_t unixtime; /* Unix time sampled every cron cycle. */
     time_t timezone;            /* Cached timezone. As set by tzset(). */
-    int daylight_active;        /* Currently in daylight saving time. */
+    redisAtomic int daylight_active;        /* Currently in daylight saving time. */
     mstime_t mstime;            /* 'unixtime' in milliseconds. */
     ustime_t ustime;            /* 'unixtime' in microseconds. */
     mstime_t cmd_time_snapshot; /* Time snapshot of the root execution nesting. */

tests/integration/rdb.tcl

@@ -158,7 +158,7 @@ start_server {} {
         assert_lessthan 999 [s rdb_changes_since_last_save]
         # make sure the server is still writable
         r set x xx
-    }
+    } {OK} {signal}
 
     test {bgsave resets the change counter} {
         r config set rdb-key-save-delay 0
@@ -169,7 +169,7 @@ start_server {} {
             fail "bgsave not done"
         }
         assert_equal [s rdb_changes_since_last_save] 0
-    }
+    } {} {signal}
 }
 
 test {client freed during loading} {

tests/integration/replication.tcl

@@ -813,7 +813,7 @@ test {diskless loading short read} {
             $master config set rdb-key-save-delay 0
         }
     }
-} {} {external:skip}
+} {} {external:skip signal}
 
 # get current stime and utime metrics for a thread (since it's creation)
 proc get_cpu_metrics { statfile } {
@@ -992,7 +992,7 @@ start_server {tags {"repl external:skip"} overrides {save ""}} {
                     }
                 }
             }
-        }
+        } {} {signal}
     }
 }
 
@@ -1117,7 +1117,7 @@ test "diskless replication read pipe cleanup" {
             $master ping
         }
     }
-} {} {external:skip}
+} {} {external:skip signal}
 
 test {replicaof right after disconnection} {
     # this is a rare race condition that was reproduced sporadically by the psync2 unit.

tests/modules/auth.c

@@ -209,6 +209,7 @@ int blocking_auth_cb(RedisModuleCtx *ctx, RedisModuleString *username, RedisModu
     if (pthread_create(&tid, NULL, AuthBlock_ThreadMain, targ) != 0) {
         RedisModule_AbortBlock(bc);
     }
+    pthread_detach(tid);
     return REDISMODULE_AUTH_HANDLED;
 }
 

tests/modules/blockedclient.c

@@ -90,6 +90,7 @@ int acquire_gil(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     pthread_t tid;
     int res = pthread_create(&tid, NULL, worker, bc);
     assert(res == 0);
+    pthread_detach(tid);
 
     return REDISMODULE_OK;
 }
@@ -195,6 +196,7 @@ int do_bg_rm_call(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     pthread_t tid;
     int res = pthread_create(&tid, NULL, bg_call_worker, bg);
     assert(res == 0);
+    pthread_detach(tid);
 
     return REDISMODULE_OK;
 }
@@ -344,6 +346,7 @@ static void rm_call_async_reply_on_thread(RedisModuleCtx *ctx, RedisModuleCallRe
     pthread_t tid;
     int res = pthread_create(&tid, NULL, send_async_reply, ta_rm_call_ctx);
     assert(res == 0);
+    pthread_detach(tid);
 }
 
 /*

tests/modules/blockonbackground.c

@@ -159,6 +159,7 @@ int HelloBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int a
         RedisModule_AbortBlock(bc);
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
     }
+    pthread_detach(tid);
     return REDISMODULE_OK;
 }
 
@@ -201,6 +202,7 @@ int HelloBlockNoTracking_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **a
         RedisModule_AbortBlock(bc);
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
     }
+    pthread_detach(tid);
     return REDISMODULE_OK;
 }
 
@@ -231,6 +233,7 @@ int HelloDoubleBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv,
         RedisModule_AbortBlock(bc);
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
     }
+    pthread_detach(tid);
     return REDISMODULE_OK;
 }
 

tests/modules/postnotifications.c

@@ -191,6 +191,7 @@ static int KeySpace_PostNotificationsAsyncSet(RedisModuleCtx *ctx, RedisModuleSt
         RedisModule_AbortBlock(bc);
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
     }
+    pthread_detach(tid);
     return REDISMODULE_OK;
 }
 

tests/modules/propagate.c

@@ -206,7 +206,7 @@ int propagateTestThreadCommand(RedisModuleCtx *ctx, RedisModuleString **argv, in
     pthread_t tid;
     if (pthread_create(&tid,NULL,threadMain,NULL) != 0)
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
-    REDISMODULE_NOT_USED(tid);
+    pthread_detach(tid);
 
     RedisModule_ReplyWithSimpleString(ctx,"OK");
     return REDISMODULE_OK;
@@ -237,7 +237,7 @@ int propagateTestDetachedThreadCommand(RedisModuleCtx *ctx, RedisModuleString **
     pthread_t tid;
     if (pthread_create(&tid,NULL,threadDetachedMain,NULL) != 0)
         return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread");
-    REDISMODULE_NOT_USED(tid);
+    pthread_detach(tid);
 
     RedisModule_ReplyWithSimpleString(ctx,"OK");
     return REDISMODULE_OK;

tests/modules/usercall.c

@@ -196,6 +196,7 @@ int call_with_user_bg(RedisModuleCtx *ctx, RedisModuleString **argv, int argc)
     pthread_t tid;
     int res = pthread_create(&tid, NULL, bg_call_worker, bg);
     assert(res == 0);
+    pthread_detach(tid);
 
     return REDISMODULE_OK;
 }

tests/unit/moduleapi/crash.tcl

@@ -58,6 +58,6 @@ if {!$::valgrind} {
                 assert_equal 3 [count_log_message 0 "modulesCollectInfo"]
             }
             assert_equal 1 [count_log_message 0 "=== REDIS BUG REPORT START: Cut & paste starting from here ==="]
-        }
+        } {} {signal}
     }
 }

tests/unit/moduleapi/testrdb.tcl

@@ -182,7 +182,7 @@ tags "modules" {
                     $master config set rdb-key-save-delay 0
                 }
             }
-        }
+        } {} {signal}
 
         # Module events for diskless load swapdb when async_loading (matching master replid)
         foreach test_case {6 7} {