CRASH when libc tries to load libnss in a client

[ghost]

The following client produces a segmentation fault when run, due to some loading issue:

#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <unistd.h>
#include <netdb.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include "dr_api.h"
#include "drmgr.h"
#include "drreg.h"

int pid;

char portno[] = "87878";
char host[] = "localhost";
char *message_fmt =
    "POSt /event HTTP/1.1\r\n"
    "Host: localhost:87878\r\n"
    "User-Agent: DynamoRIO\r\n"
    "Accept: */*\r\n"
    "Content-Length: %d\r\n"
    "Content-Type: application/json\r\n\r\n"
    "%s\r\n";


void error(const char* msg) {
    dr_printf("ERROR: %s\n", msg);
}

/* For each app instr, we insert checks, conditionally */
dr_emit_flags_t
event_app_instruction(void *drcontext, void *tag, instrlist_t *bb, instr_t *instr,
                      bool for_trace, bool translating, void *user_data)
{
    int host_result;
    struct addrinfo hints;
    struct addrinfo *lookup_results;
    struct addrinfo *result;
    struct sockaddr_in serv_addr;
    int sockfd, bytes, sent, received, total;
    char message[1024],response[4096];

    /* we don't want to auto-predicate anything */
    drmgr_disable_auto_predication(drcontext, bb);

    if (!instr_is_app(instr))
        return DR_EMIT_DEFAULT;

    if (drmgr_is_first_instr(drcontext, instr)) {
        app_pc pc = instr_get_app_pc(instr);

        char buffer[100];
        dr_snprintf(buffer, 100, "{\"pid\" : %d, \"addr\" : \"%X\"}", pid, pc);

        /* fill in the parameters */
        dr_snprintf(message, 1024, message_fmt, strlen(buffer), buffer);
        dr_printf("Request:\n%s\n",message);

        /* create the socket */
        dr_printf("Making socket.\n\n");
        sockfd = socket(AF_INET, SOCK_STREAM, 0);
        if (sockfd < 0) error("ERROR opening socket");

        /* lookup the ip address */
        memset(&hints, 0, sizeof(struct addrinfo));
        hints.ai_family = AF_INET;    /* Allow IPv4 or IPv6 */
        hints.ai_socktype = SOCK_STREAM; /* socket stream */
        hints.ai_flags = 0;
        hints.ai_protocol = 0;          /* Any protocol */

        dr_printf("Looking up host.\n\n");

        /* SEGFAULT HERE */
        host_result = getaddrinfo(host, portno, &hints, &result);
        dr_printf("Checking server.\n\n");
        if (host_result != 0) error("ERROR, no such host");
        
        /* [Do host reporting ... ] */
    }
}

DR_EXPORT void
dr_client_main(client_id_t id, int argc, const char *argv[])
{
    pid = getpid();
    /* We need 2 reg slots beyond drreg's eflags slots => 3 slots */
    drreg_options_t ops = { sizeof(ops), 3, false };
    dr_set_client_name("DynamoRIO Sample Client REST",
                       "http://dynamorio.org/issues");
    if (!drmgr_init() || drreg_init(&ops) != DRREG_SUCCESS)
        DR_ASSERT(false);

    /* register events */
    //main event handler, for reporting instrumentation addresses
    if (!drmgr_register_bb_instrumentation_event(NULL, event_app_instruction, NULL))
        DR_ASSERT(false);

    dr_log(NULL, DR_LOG_ALL, 1, "Client 'rest report' initializing\n");
}

Here is the trace:

~/dynamo-rio > gdb -x gdb.txt -args ./bin64/drrun -debug -loglevel 3 -c build/bin/libsock_rest.so -- ls
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from ./bin64/drrun...Reading symbols from /home/cswords/dynamo-rio/bin64/drrun.debug...done.
done.
process 31762 is executing new program: /home/cswords/dynamo-rio/lib64/debug/libdynamorio.so
Loading gdb scripts for debugging DynamoRIO...
Traceback (most recent call last):
  File "/home/cswords/dynamo-rio/lib64/debug/libdynamorio.so-gdb.py", line 185, in <module>
    PrivloadBP()
  File "/home/cswords/dynamo-rio/lib64/debug/libdynamorio.so-gdb.py", line 151, in __init__
    internal=not self.DEBUG)
RuntimeError: Warning:
Cannot insert breakpoint -13.
Cannot access memory at address 0x712c1b64

<log dir=/home/cswords/dynamo-rio/bin64/../logs/ls.31762.00000000>
Executing gdb command: add-symbol-file '/home/cswords/dynamo-rio/build/bin/libsock_rest.so' 0x7fba10c0ad70
<Starting application /bin/ls (31762)>

Program received signal SIGILL, Illegal instruction.
<Initial options = -no_dynamic_options -loglevel 3 -client_lib '/home/cswords/dynamo-rio/build/bin/libsock_rest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct >
Executing gdb command: add-symbol-file '/home/cswords/dynamo-rio/lib64/debug/libdynamorio.so' 0x7fba94c3a6a0
Executing gdb command: add-symbol-file '/home/cswords/dynamo-rio/ext/lib64/debug/libdrreg.so' 0x7fba10e10b30
Executing gdb command: add-symbol-file '/home/cswords/dynamo-rio/ext/lib64/debug/libdrmgr.so' 0x7fba1101e590
Executing gdb command: add-symbol-file '/lib/x86_64-linux-gnu/libc.so.6' 0x7fba9460f2d0
Executing gdb command: add-symbol-file '/lib64/ld-linux-x86-64.so.2' 0x7fba943c3f10
<Paste into GDB to debug DynamoRIO clients:
set confirm off
add-symbol-file '/home/cswords/dynamo-rio/build/bin/libsock_rest.so' 0x00007fba10c0ad70
add-symbol-file '/home/cswords/dynamo-rio/lib64/debug/libdynamorio.so' 0x00007fba94c3a6a0
add-symbol-file '/home/cswords/dynamo-rio/ext/lib64/debug/libdrreg.so' 0x00007fba10e10b30
add-symbol-file '/home/cswords/dynamo-rio/ext/lib64/debug/libdrmgr.so' 0x00007fba1101e590
add-symbol-file '/lib/x86_64-linux-gnu/libc.so.6' 0x00007fba9460f2d0
add-symbol-file '/lib64/ld-linux-x86-64.so.2' 0x00007fba943c3f10
>
Request:
POSt /event HTTP/1.1
Host: localhost:87878
User-Agent: DynamoRIO
Accept: */*
Content-Length: 36
Content-Type: application/json

{"pid" : 31762, "addr" : "949E1090"}

Making socket.

Looking up host.


Program received signal SIGSEGV, Segmentation fault.
0x00007fba9475448f in __GI___libc_dlopen_mode (name=name@entry=0x7fba80a9a1d0 "libnss_files.so.2", mode=mode@entry=-2147483647)
    at dl-libc.c:194
194	dl-libc.c: No such file or directory.
(gdb) bt
#0  0x00007fba9475448f in __GI___libc_dlopen_mode (name=name@entry=0x7fba80a9a1d0 "libnss_files.so.2", mode=mode@entry=-2147483647)
    at dl-libc.c:194
#1  0x00007fba94736886 in nss_load_library (ni=0x7fba80a40620) at nsswitch.c:369
#2  0x00007fba94737088 in __GI___nss_lookup_function (ni=0x7fba80a40620, fct_name=<optimized out>) at nsswitch.c:477
#3  0x00007fba9473719d in __GI___nss_lookup (ni=ni@entry=0x7fba80a9a388, fct_name=fct_name@entry=0x7fba947a36cb "gethostbyname2_r", 
    fct2_name=fct2_name@entry=0x0, fctp=fctp@entry=0x7fba80a9a390) at nsswitch.c:198
#4  0x00007fba94739360 in __GI___nss_hosts_lookup2 (ni=ni@entry=0x7fba80a9a388, 
    fct_name=fct_name@entry=0x7fba947a36cb "gethostbyname2_r", fct2_name=fct2_name@entry=0x0, fctp=fctp@entry=0x7fba80a9a390)
    at XXX-lookup.c:66
#5  0x00007fba94724138 in __gethostbyname2_r (name=name@entry=0x7fba10e0c0a8 "localhost", af=af@entry=2, 
    resbuf=resbuf@entry=0x7fba80a9a510, buffer=<optimized out>, buflen=1024, result=result@entry=0x7fba80a9a508, 
    h_errnop=0x7fba80a8b2e4) at ../nss/getXXbyYY_r.c:269
#6  0x00007fba946f4054 in gaih_inet (name=name@entry=0x7fba10e0c0a8 "localhost", service=<optimized out>, 
    req=req@entry=0x7fba80a9abe0, pai=pai@entry=0x7fba80a9a708, naddrs=naddrs@entry=0x7fba80a9a704, 
    tmpbuf=tmpbuf@entry=0x7fba80a9a770) at ../sysdeps/posix/getaddrinfo.c:591
#7  0x00007fba946f5ce4 in __GI_getaddrinfo (name=<optimized out>, name@entry=0x7fba10e0c0a8 "localhost", service=<optimized out>, 
    service@entry=0x7fba10e0c0b2 "87878", hints=hints@entry=0x7fba80a9abe0, pai=pai@entry=0x7fba80a9abd8)
    at ../sysdeps/posix/getaddrinfo.c:2300
#8  0x00007fba10c0afae in event_app_instruction (drcontext=<optimized out>, instr=<optimized out>, user_data=<optimized out>, 
    translating=<optimized out>, for_trace=<optimized out>, bb=<optimized out>, tag=<optimized out>)
    at /home/cswords/dynamo-rio/samples/sock_rest_client.c:76
#9  0x00007fba1101f250 in drmgr_bb_event (drcontext=0x7fba80a2f940, tag=0x7fba949e1090, bb=0x7fba80a9f0a0, for_trace=0 '\000', 
    translating=0 '\000') at /home/travis/build/DynamoRIO/dynamorio/ext/drmgr/drmgr.c:712
#10 0x00007fba94dd69c5 in instrument_basic_block (dcontext=0x7fba80a2f940, tag=0x7fba949e1090 "H\211\347\350\b\016", 
    bb=0x7fba80a9f0a0, for_trace=false, translating=false, emitflags=0x7fba80a9ba14)
    at /home/travis/build/DynamoRIO/dynamorio/core/lib/instrument.c:1614
#11 0x00007fba94e49d3f in client_process_bb (dcontext=0x7fba80a2f940, bb=0x7fba80a9bd70)
    at /home/travis/build/DynamoRIO/dynamorio/core/arch/interp.c:2780
#12 0x00007fba94e4f578 in build_bb_ilist (dcontext=0x7fba80a2f940, bb=0x7fba80a9bd70)
    at /home/travis/build/DynamoRIO/dynamorio/core/arch/interp.c:4134
#13 0x00007fba94e53ffe in build_basic_block_fragment (dcontext=0x7fba80a2f940, start=0x7fba949e1090 "H\211\347\350\b\016", 
    initial_flags=0, link=true, visible=true, for_trace=false, unmangled_ilist=0x0)
    at /home/travis/build/DynamoRIO/dynamorio/core/arch/interp.c:5137
#14 0x00007fba94cbe539 in dispatch (dcontext=0x7fba80a2f940) at /home/travis/build/DynamoRIO/dynamorio/core/dispatch.c:215
#15 0x00007fba94e80d40 in call_dispatch_alt_stack_no_free () at /home/travis/build/DynamoRIO/dynamorio/core/arch/x86/x86.asm:299
#16 0x0000000000000000 in ?? ()

This is possibly related to static linking issues here: https://bugzilla.redhat.com/show_bug.cgi?id=89698 The solution described there (using an IP address directly instead of "localhost") seems to resolve the issue.

[Carrotman42]

I am observing a crash with your reproducer, but my crash stack does not look exactly the same as yours:

(gdb) bt
#0 0x00007ffff7a89d77 in __libc_dlopen_mode () at /usr/local/google/home/chowski/dynamorio/core/lib/statsx.h:623
#1 0x00007ffff7a72020 in __nss_lookup_function () at /usr/local/google/home/chowski/dynamorio/core/lib/statsx.h:188
#2 0x00007ffff7a720fc in __nss_lookup () at /usr/local/google/home/chowski/dynamorio/core/lib/statsx.h:189
#3 0x725f32656d616e79 in ?? ()
#4 0x74736f6874656700 in ?? ()
#5 0x5f33656d616e7962 in ?? ()
#6 0x6e61637465670072 in ?? ()
#7 0x725f656d616e6e6f in ?? ()
#8 0x616e6e6f6e616300 in ?? ()
#9 0x554e203d3d20656d in ?? ()
... hundreds, goes all the way to 972 frames

I'm not quite sure what the difference is, or why my nss_* functions are claiming to be in statsx.h

[derekbruening]

Xref the original discussion here: https://groups.google.com/forum/#!topic/DynamoRIO-Users/Sk4D0w2LC7s

[ghost]

Following up on this: the issue in both backtraces seem to be during dlopen in libc. Is there a good starting point for further investigation to help determine the cause (and eventually resovle) this error?