ST: Support thread-local for multiple threads.

README.md

@@ -90,7 +90,7 @@ The branch [srs](https://github.com/ossrs/state-threads/tree/srs) will be patche
 - [x] Windows: Support Windows 64bits. [#20](https://github.com/ossrs/state-threads/issues/20).
 - [x] MIPS: Support Linux/MIPS for OpenWRT, [#21](https://github.com/ossrs/state-threads/issues/21).
 - [x] LOONGARCH: Support loongarch for loongson CPU, [#24](https://github.com/ossrs/state-threads/issues/24). 
-- [ ] System: Support Multiple Threads for Linux and Darwin. [#19](https://github.com/ossrs/state-threads/issues/19), [srs#2188](https://github.com/ossrs/srs/issues/2188).
+- [x] System: Support Multiple Threads for Linux and Darwin. [#19](https://github.com/ossrs/state-threads/issues/19), [srs#2188](https://github.com/ossrs/srs/issues/2188).
 - [ ] System: Support sendmmsg for UDP, [#12](https://github.com/ossrs/state-threads/issues/12).
 
 ## GDB Tools

common.h

@@ -266,9 +266,9 @@ typedef struct _st_netfd {
  * Current vp, thread, and event system
  */
 
-extern _st_vp_t        _st_this_vp;
-extern _st_thread_t *_st_this_thread;
-extern _st_eventsys_t *_st_eventsys;
+extern __thread _st_vp_t        _st_this_vp;
+extern __thread _st_thread_t *_st_this_thread;
+extern __thread _st_eventsys_t *_st_eventsys;
 
 #define _ST_CURRENT_THREAD()            (_st_this_thread)
 #define _ST_SET_CURRENT_THREAD(_thread) (_st_this_thread = (_thread))

event.c

@@ -51,10 +51,10 @@
 
 // Global stat.
 #if defined(DEBUG) && defined(DEBUG_STATS)
-unsigned long long _st_stat_epoll = 0;
-unsigned long long _st_stat_epoll_zero = 0;
-unsigned long long _st_stat_epoll_shake = 0;
-unsigned long long _st_stat_epoll_spin = 0;
+__thread unsigned long long _st_stat_epoll = 0;
+__thread unsigned long long _st_stat_epoll_zero = 0;
+__thread unsigned long long _st_stat_epoll_shake = 0;
+__thread unsigned long long _st_stat_epoll_spin = 0;
 #endif
 
 #if !defined(MD_HAVE_KQUEUE) && !defined(MD_HAVE_EPOLL) && !defined(MD_HAVE_SELECT)
@@ -86,7 +86,7 @@ typedef struct _kq_fd_data {
     int revents;
 } _kq_fd_data_t;
 
-static struct _st_kqdata {
+static __thread struct _st_kqdata {
     _kq_fd_data_t *fd_data;
     struct kevent *evtlist;
     struct kevent *addlist;
@@ -119,7 +119,7 @@ typedef struct _epoll_fd_data {
     int revents;
 } _epoll_fd_data_t;
 
-static struct _st_epolldata {
+static __thread struct _st_epolldata {
     _epoll_fd_data_t *fd_data;
     struct epoll_event *evtlist;
     int fd_data_size;
@@ -147,7 +147,7 @@ static struct _st_epolldata {
 
 #endif  /* MD_HAVE_EPOLL */
 
-_st_eventsys_t *_st_eventsys = NULL;
+__thread _st_eventsys_t *_st_eventsys = NULL;
 
 
 #ifdef MD_HAVE_SELECT

io.c

@@ -56,20 +56,20 @@
 
 // Global stat.
 #if defined(DEBUG) && defined(DEBUG_STATS)
-unsigned long long _st_stat_recvfrom = 0;
-unsigned long long _st_stat_recvfrom_eagain = 0;
-unsigned long long _st_stat_sendto = 0;
-unsigned long long _st_stat_sendto_eagain = 0;
-unsigned long long _st_stat_read = 0;
-unsigned long long _st_stat_read_eagain = 0;
-unsigned long long _st_stat_readv = 0;
-unsigned long long _st_stat_readv_eagain = 0;
-unsigned long long _st_stat_writev = 0;
-unsigned long long _st_stat_writev_eagain = 0;
-unsigned long long _st_stat_recvmsg = 0;
-unsigned long long _st_stat_recvmsg_eagain = 0;
-unsigned long long _st_stat_sendmsg = 0;
-unsigned long long _st_stat_sendmsg_eagain = 0;
+__thread unsigned long long _st_stat_recvfrom = 0;
+__thread unsigned long long _st_stat_recvfrom_eagain = 0;
+__thread unsigned long long _st_stat_sendto = 0;
+__thread unsigned long long _st_stat_sendto_eagain = 0;
+__thread unsigned long long _st_stat_read = 0;
+__thread unsigned long long _st_stat_read_eagain = 0;
+__thread unsigned long long _st_stat_readv = 0;
+__thread unsigned long long _st_stat_readv_eagain = 0;
+__thread unsigned long long _st_stat_writev = 0;
+__thread unsigned long long _st_stat_writev_eagain = 0;
+__thread unsigned long long _st_stat_recvmsg = 0;
+__thread unsigned long long _st_stat_recvmsg_eagain = 0;
+__thread unsigned long long _st_stat_sendmsg = 0;
+__thread unsigned long long _st_stat_sendmsg_eagain = 0;
 #endif
 
 #if EAGAIN != EWOULDBLOCK
@@ -81,7 +81,7 @@ unsigned long long _st_stat_sendmsg_eagain = 0;
 #define _LOCAL_MAXIOV  16
 
 /* File descriptor object free list */
-static _st_netfd_t *_st_netfd_freelist = NULL;
+static __thread _st_netfd_t *_st_netfd_freelist = NULL;
 /* Maximum number of file descriptors that the process can open */
 static int _st_osfd_limit = -1;
 

sched.c

@@ -56,31 +56,33 @@
 
 // Global stat.
 #if defined(DEBUG) && defined(DEBUG_STATS)
-unsigned long long _st_stat_sched_15ms = 0;
-unsigned long long _st_stat_sched_20ms = 0;
-unsigned long long _st_stat_sched_25ms = 0;
-unsigned long long _st_stat_sched_30ms = 0;
-unsigned long long _st_stat_sched_35ms = 0;
-unsigned long long _st_stat_sched_40ms = 0;
-unsigned long long _st_stat_sched_80ms = 0;
-unsigned long long _st_stat_sched_160ms = 0;
-unsigned long long _st_stat_sched_s = 0;
-
-unsigned long long _st_stat_thread_run = 0;
-unsigned long long _st_stat_thread_idle = 0;
-unsigned long long _st_stat_thread_yield = 0;
-unsigned long long _st_stat_thread_yield2 = 0;
+__thread unsigned long long _st_stat_sched_15ms = 0;
+__thread unsigned long long _st_stat_sched_20ms = 0;
+__thread unsigned long long _st_stat_sched_25ms = 0;
+__thread unsigned long long _st_stat_sched_30ms = 0;
+__thread unsigned long long _st_stat_sched_35ms = 0;
+__thread unsigned long long _st_stat_sched_40ms = 0;
+__thread unsigned long long _st_stat_sched_80ms = 0;
+__thread unsigned long long _st_stat_sched_160ms = 0;
+__thread unsigned long long _st_stat_sched_s = 0;
+
+__thread unsigned long long _st_stat_thread_run = 0;
+__thread unsigned long long _st_stat_thread_idle = 0;
+__thread unsigned long long _st_stat_thread_yield = 0;
+__thread unsigned long long _st_stat_thread_yield2 = 0;
 #endif
 
 
 /* Global data */
-_st_vp_t _st_this_vp;           /* This VP */
-_st_thread_t *_st_this_thread;  /* Current thread */
-int _st_active_count = 0;       /* Active thread count */
+__thread _st_vp_t _st_this_vp;           /* This VP */
+__thread _st_thread_t *_st_this_thread;  /* Current thread */
+__thread int _st_active_count = 0;       /* Active thread count */
 
-time_t _st_curr_time = 0;       /* Current time as returned by time(2) */
-st_utime_t _st_last_tset;       /* Last time it was fetched */
+__thread time_t _st_curr_time = 0;       /* Current time as returned by time(2) */
+__thread st_utime_t _st_last_tset;       /* Last time it was fetched */
 
+// We should initialize the thread-local variable in st_init().
+extern __thread _st_clist_t _st_free_stacks;
 
 int st_poll(struct pollfd *pds, int npds, st_utime_t timeout)
 {
@@ -167,7 +169,7 @@ void _st_vp_schedule(void)
 int st_init(void)
 {
     _st_thread_t *thread;
-    
+
     if (_st_active_count) {
         /* Already initialized */
         return 0;
@@ -178,7 +180,11 @@ int st_init(void)
 
     if (_st_io_init() < 0)
         return -1;
-
+
+    // Initialize the thread-local variables.
+    ST_INIT_CLIST(&_st_free_stacks);
+
+    // Initialize ST.
     memset(&_st_this_vp, 0, sizeof(_st_vp_t));
 
     ST_INIT_CLIST(&_ST_RUNQ);
@@ -713,8 +719,8 @@ int _st_iterate_threads_flag = 0;
 
 void _st_iterate_threads(void)
 {
-    static _st_thread_t *thread = NULL;
-    static jmp_buf orig_jb, save_jb;
+    static __thread _st_thread_t *thread = NULL;
+    static __thread jmp_buf orig_jb, save_jb;
     _st_clist_t *q;
 
     if (!_st_iterate_threads_flag) {

stk.c

@@ -52,9 +52,9 @@
 /* How much space to leave between the stacks, at each end */
 #define REDZONE	_ST_PAGE_SIZE
 
-_st_clist_t _st_free_stacks = ST_INIT_STATIC_CLIST(&_st_free_stacks);
-int _st_num_free_stacks = 0;
-int _st_randomize_stacks = 0;
+__thread _st_clist_t _st_free_stacks;
+__thread int _st_num_free_stacks = 0;
+__thread int _st_randomize_stacks = 0;
 
 static char *_st_new_stk_segment(int size);
 
@@ -89,8 +89,9 @@ _st_stack_t *_st_stack_new(int stack_size)
     ts->stk_size = stack_size;
     ts->stk_bottom = ts->vaddr + REDZONE;
     ts->stk_top = ts->stk_bottom + stack_size;
-
-#ifdef DEBUG
+
+    /* For example, in OpenWRT, the memory at the begin minus 16B by mprotect is read-only. */
+#if defined(DEBUG) && !defined(MD_NO_PROTECT)
     mprotect(ts->vaddr, REDZONE, PROT_NONE);
     mprotect(ts->stk_top + extra, REDZONE, PROT_NONE);
 #endif

sync.c

@@ -47,9 +47,9 @@
 #include "common.h"
 
 
-extern time_t _st_curr_time;
-extern st_utime_t _st_last_tset;
-extern int _st_active_count;
+extern __thread time_t _st_curr_time;
+extern __thread st_utime_t _st_last_tset;
+extern __thread int _st_active_count;
 
 static st_utime_t (*_st_utime)(void) = NULL;