debug a process with gdb and close the terminal

Question

I have a bug in my application which runs on a remote server. After a few hours of execution, the application gets a SIGSEGV and terminates.

I want to debug my remote application with gdb through ssh so when the program gets SIGSEGV gdb will halt and will allow me to see what happened, but I can't leave the ssh session to the server connected. I'm trying to run:

setsid gdb my_app

but when I close the ssh terminal, gdb and my app terminate immediately.

My questions:

How can I leave gdb running when closing the terminal
How can re-connect to it later?

strace on gdb on the moment of closing the terminal shows:

read(0, 0x7fff0532895f, 1)              = -1 EIO (Input/output error)
ioctl(0, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, 0x7fff05328930) = -1 EIO (Input/output error)
ioctl(0, SNDCTL_TMR_STOP or SNDRV_TIMER_IOCTL_GINFO or TCSETSW, {B38400 opost isig -icanon -echo ...}) = -1 EIO (Input/output error)
rt_sigaction(SIGINT, {0x5b3550, [INT], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGTERM, {0x5b3710, [TERM], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGHUP, {0x5b33d0, [HUP], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGQUIT, {0x5b33f0, [QUIT], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGALRM, {SIG_DFL, [], SA_RESTORER, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGTSTP, {SIG_DFL, [TSTP], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGTTOU, {SIG_DFL, [], SA_RESTORER, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGTTIN, {SIG_DFL, [], SA_RESTORER, 0x7fa0fbaa0c90}, {0x7fa0fce86ac0, [], SA_RESTORER, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGWINCH, {0x4efb70, [], SA_RESTORER, 0x7fa0fbaa0c90}, {0x7fa0fce86160, [], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGTSTP, {0x5b3410, [TSTP], SA_RESTORER|SA_RESTART, 0x7fa0fae70da0}, {SIG_DFL, [TSTP], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, 8) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, 0x7fff053288e0) = -1 EIO (Input/output error)
write(1, "quit\n", 5)                   = -1 EIO (Input/output error)
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigprocmask(SIG_BLOCK, NULL, [], 8)  = 0
rt_sigaction(SIGINT, {0x5b3550, [INT], SA_RESTORER|SA_RESTART, 0x7fa0fae70da0}, {0x5b3550, [INT], SA_RESTORER|SA_RESTART, 0x7fa0fbaa0c90}, 8) = 0
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x7fa0fbaa0c90}, {0x5b3550, [INT], SA_RESTORER|SA_RESTART, 0x7fa0fae70da0}, 8) = 0
exit_group(0)                           = ?
+++ exited with 0 +++

DirkG DirkG · Accepted Answer · 2014-12-21T12:13:25

You can use something like 'screen', it will continue to run if you detach from it (CTRL-a d), then you can reconnect later with 'screen -r'.

Or you can reattach gdb with the progam name and the PID, e.g. 'gdb my_app 1234', where the number 1234 has to be replaced by the correct PID.

debug a process with gdb and close the terminal

2 Answers