From 7059e96c3b73d0c97bd0988374854417e27a4f19 Mon Sep 17 00:00:00 2001 From: Narayanan Iyer Date: Fri, 22 Sep 2017 17:05:37 -0400 Subject: [PATCH] [YDB#30] [NARS] [estess] Fix SIG-11 from source server if started with an external filter followed by a deactivate and shutdown Release Note ------------- The replication source server terminates normally in case it was started with an external filter and later transitioned from an active to a passive source server before being asked to shut down. Previously, the source server would terminate abnormally with a SIG-11. (YDB#30) Test ----- * New r110/srcsrv_extfilter_sig11 subtest fails reliably without the fixes and passes reliably with the fix. * E_ALL run many times to ensure no regressions. README ------- When an active replication source server is deactivated, it changes mode from ACTIVE to PASSIVE mode. This also causes it to close external filters (if it had them open) by a call to repl_stop_filter() (in gtmsource.c). 517 if (gtmsource_filter & EXTERNAL_FILTER) 518 repl_stop_filter(); While it is in passive mode (and waiting to be activated again or asked to shutdown), if it encounters a shutdown signal, it goes to gtmsource_end() which in turn ends up invoking replstop_filter(). But this function cannot be invoked more than once. That is because it sends a message to the other side of the filter (the receiver side) to stop and then closes the source side of the filter and frees up and nullifies the associated buffers (including the global variable "extract_buff"). The second invocation of this function ends up with a SIG-11 when trying to send a message to the other side because "extract_buff" is NULL. Below is the stack trace. (gdb) where #0 0x000000000082200a in rel_lock (reg=0x3044958) at /Distrib/GT.M/V63002/sr_unix/rel_lock.c:85 #1 0x0000000000474602 in gtmsource_end1 (auto_shutdown=1) at /Distrib/GT.M/V63002/sr_unix/gtmsource_end.c:80 #2 0x000000000049d514 in gtmsource_stop (exit=0) at /Distrib/GT.M/V63002/sr_unix/gtmsource_shutdown.c:323 #3 0x000000000049d55b in gtmsource_sigstop () at /Distrib/GT.M/V63002/sr_unix/gtmsource_shutdown.c:333 #4 0x00000000006d5377 in generic_signal_handler (sig=11, info=0x7ffd3122d6b0, context=0x7ffd3122d580) at /Distrib/GT.M/V63002/sr_unix/generic_signal_handler.c:374 #5 #6 0x000000000057b776 in repl_filter_send (tr_num=0, tr=0x0, tr_len=0, first_send=1) at /Distrib/GT.M/V63002/sr_port/repl_filter.c:660 #7 0x0000000000581342 in repl_stop_filter () at /Distrib/GT.M/V63002/sr_port/repl_filter.c:1128 #8 0x000000000047504f in gtmsource_end1 (auto_shutdown=0) at /Distrib/GT.M/V63002/sr_unix/gtmsource_end.c:136 #9 0x000000000047509e in gtmsource_end () at /Distrib/GT.M/V63002/sr_unix/gtmsource_end.c:148 #10 0x000000000046d968 in gtmsource () at /Distrib/GT.M/V63002/sr_unix/gtmsource.c:520 #11 0x000000000044a022 in main (argc=11, argv=0x7ffd3122e7d8) at /Distrib/GT.M/V63002/sr_unix/mupip.c:123 The fix is to turn off the EXTERNAL_FILTER bit in "gtmsource_filter" right after a call to repl_stop_filter(). Since repl_stop_filter() is called from various places in the source server a macro STOP_EXTERNAL_FILTER_IF_NEEDED was introduced to take care of this. Although the receiver server does not suffer from this exact issue, it also invokes repl_stop_filter() in various places and might have a similar issue. So all those callers too were fixed to use this new macro. But they manipulate the global variable "gtmrecv_filter" (instead of "gtmsource_filter"). --- sr_port/gtmsource_poll_actions.c | 9 ++++----- sr_port/mdef.h | 3 +++ sr_port/repl_filter.c | 14 ++++++++++++-- sr_port/repl_filter.h | 19 +++++++++++++++++++ sr_unix/gtmrecv_end.c | 6 ++++-- sr_unix/gtmsource.c | 6 ++++-- sr_unix/gtmsource_end.c | 6 ++++-- sr_unix/gtmsource_process_ops.c | 10 +++++----- 8 files changed, 55 insertions(+), 18 deletions(-) diff --git a/sr_port/gtmsource_poll_actions.c b/sr_port/gtmsource_poll_actions.c index 955f008c1..64210db39 100755 --- a/sr_port/gtmsource_poll_actions.c +++ b/sr_port/gtmsource_poll_actions.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -198,10 +201,6 @@ int gtmsource_poll_actions(boolean_t poll_secondary) repl_log(gtmsource_log_fp, TRUE, TRUE, "End statistics logging\n"); } if ((gtmsource_filter & EXTERNAL_FILTER) && ('\0' == gtmsource_local->filter_cmd[0])) - { - repl_log(gtmsource_log_fp, TRUE, TRUE, "Stopping filter\n"); - repl_stop_filter(); - gtmsource_filter &= ~EXTERNAL_FILTER; - } + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_log_fp, "GTMSOURCE_POLL_ACTIONS"); return (SS_NORMAL); } diff --git a/sr_port/mdef.h b/sr_port/mdef.h index db1c5374d..e82a468bc 100644 --- a/sr_port/mdef.h +++ b/sr_port/mdef.h @@ -3,6 +3,9 @@ * Copyright (c) 2001-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * diff --git a/sr_port/repl_filter.c b/sr_port/repl_filter.c index e3d7356fb..07fe0f896 100644 --- a/sr_port/repl_filter.c +++ b/sr_port/repl_filter.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -442,6 +445,10 @@ GBLREF repl_conn_info_t *this_side, *remote_side; GBLREF uint4 process_id; GBLREF boolean_t err_same_as_out; GBLREF volatile boolean_t timer_in_handler; +GBLREF FILE *gtmsource_log_fp; +GBLREF FILE *gtmrecv_log_fp; +GBLREF int gtmsource_filter; +GBLREF int gtmrecv_filter; LITREF char *trigger_subs[]; @@ -1140,8 +1147,11 @@ int repl_stop_filter(void) void repl_filter_error(seq_num filter_seqno, int why) { - repl_log(stderr, TRUE, TRUE, "Stopping filter due to error\n"); - repl_stop_filter(); + assert(is_src_server || is_rcvr_server); + if (is_src_server) + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_log_fp, "REPL_FILTER_ERROR"); + else + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmrecv_filter, gtmrecv_log_fp, "REPL_FILTER_ERROR"); switch (repl_errno) { case EREPL_FILTERNOTALIVE : diff --git a/sr_port/repl_filter.h b/sr_port/repl_filter.h index 9a94e8966..55f50b325 100644 --- a/sr_port/repl_filter.h +++ b/sr_port/repl_filter.h @@ -3,6 +3,9 @@ * Copyright (c) 2001-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -273,4 +276,20 @@ error_def(ERR_REPLNOHASHTREC); else /* (EREPL_INTLFILTER_INCMPLREC == REPL_ERRNO) */ \ assertpro(FALSE); \ } + +/* This macro is called by the source or receiver server when they need to stop an external filter. + * CALLER_FILTER : is gtmsource_filter (if caller is source server) and gtmrecv_filter (if caller is receiver server) + * CALLER_FP : is gtmsource_log_fp (if caller is source server) and gtmrecv_log_fp (if caller is receiver server) + * CALLER_DETAIL : is string that provides caller context and is printed in the corresponding log file. + */ +#define STOP_EXTERNAL_FILTER_IF_NEEDED(CALLER_FILTER, CALLER_FP, CALLER_DETAIL) \ +MBSTART { \ + if (CALLER_FILTER & EXTERNAL_FILTER) \ + { \ + repl_log(CALLER_FP, TRUE, TRUE, "Stopping filter : " CALLER_DETAIL "\n"); \ + repl_stop_filter(); \ + CALLER_FILTER &= ~EXTERNAL_FILTER; \ + } \ +} MBEND + #endif diff --git a/sr_unix/gtmrecv_end.c b/sr_unix/gtmrecv_end.c index 3fe279686..33da82de8 100644 --- a/sr_unix/gtmrecv_end.c +++ b/sr_unix/gtmrecv_end.c @@ -3,6 +3,9 @@ * Copyright (c) 2006-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -205,8 +208,7 @@ int gtmrecv_end1(boolean_t auto_shutdown) log_seqno, repl_recv_data_processed, repl_recv_data_recvd); repl_log(gtmrecv_log_fp, TRUE, TRUE, "REPL INFO - Last Seqno processed by update process : %llu\n", log_seqno1); gtm_event_log_close(); - if (gtmrecv_filter & EXTERNAL_FILTER) - repl_stop_filter(); + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmrecv_filter, gtmrecv_log_fp, "GTMRECV_END"); if (auto_shutdown) return (exit_status); else diff --git a/sr_unix/gtmsource.c b/sr_unix/gtmsource.c index 8525aa360..aa3cc0e7b 100644 --- a/sr_unix/gtmsource.c +++ b/sr_unix/gtmsource.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -514,8 +517,7 @@ int gtmsource() gtmsource_stop_heartbeat(); if (FD_INVALID != gtmsource_sock_fd) repl_close(>msource_sock_fd); - if (gtmsource_filter & EXTERNAL_FILTER) - repl_stop_filter(); + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_log_fp, "GTMSOURCE_CHANGING_MODE"); } while (TRUE); gtmsource_end(); return(SS_NORMAL); diff --git a/sr_unix/gtmsource_end.c b/sr_unix/gtmsource_end.c index c9f6ae811..b12179464 100644 --- a/sr_unix/gtmsource_end.c +++ b/sr_unix/gtmsource_end.c @@ -3,6 +3,9 @@ * Copyright (c) 2006-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -132,8 +135,7 @@ int gtmsource_end1(boolean_t auto_shutdown) repl_log(gtmsource_log_fp, FALSE, TRUE, " Number of unsent Seqno : %llu\n", 0 < diff_seqno ? diff_seqno : 0); repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Jnl Total : %llu Msg Total : %llu CmpMsg Total : %llu\n", repl_source_data_sent, repl_source_msg_sent, repl_source_cmp_sent); - if (gtmsource_filter & EXTERNAL_FILTER) - repl_stop_filter(); + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_log_fp, "GTMSOURCE_END"); gtm_event_log_close(); if (auto_shutdown) return (exit_status); diff --git a/sr_unix/gtmsource_process_ops.c b/sr_unix/gtmsource_process_ops.c index 913605c0e..3683ec2a3 100644 --- a/sr_unix/gtmsource_process_ops.c +++ b/sr_unix/gtmsource_process_ops.c @@ -3,6 +3,9 @@ * Copyright (c) 2006-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2017 YottaDB LLC. and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -553,13 +556,10 @@ int gtmsource_recv_restart(seq_num *recvd_jnl_seqno, int *msg_type, int *start_f { repl_log(gtmsource_log_fp, TRUE, TRUE, "Start JNL_SEQNO msg tagged with STOP SOURCE FILTER\n"); - if (gtmsource_filter & EXTERNAL_FILTER) - { - repl_stop_filter(); - gtmsource_filter &= ~EXTERNAL_FILTER; - } else + if (!(gtmsource_filter & EXTERNAL_FILTER)) repl_log(gtmsource_log_fp, TRUE, TRUE, "Filter is not active, ignoring STOP SOURCE FILTER msg\n"); + STOP_EXTERNAL_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_log_fp, "GTMSOURCE_RECV_RESTART"); } /* Determine the protocol version of the receiver side. That information is encoded in the * "proto_ver" field of the message from V51 onwards but to differentiate V50 vs V51 we need