ntp segfault with latest linux-raspberry

[suthernfriend]

For solution see below!

On our pis ntpd is crashing on start with SEGFAULT which prevents the pis from syncing their time.

more info:

- core/glibc-2.24-2
- core/linux-raspberrypi-4.4.41-1
- core/linux-raspberrypi-headers-4.4.41-1
- core/linux-firmware 20161005.9c71af9-1
- extra/ntp 4.2.8.p9-1

$this->bbcode_second_pass_code('', '
[root@a36a930a ~] # strace ntpd
execve("/usr/bin/ntpd", ["ntpd"], [/* 18 vars */]) = 0
brk(NULL) = 0x1e63000
uname({sysname="Linux", nodename="a36a930a", ...}) = 0
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x76f39000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=78476, ...}) = 0
mmap2(NULL, 78476, PROT_READ, MAP_PRIVATE, 3, 0) = 0x76ef8000
close(3) = 0
open("/usr/lib/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\360\r\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=13164, ...}) = 0
mmap2(NULL, 77360, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76ee5000
mprotect(0x76ee8000, 61440, PROT_NONE) = 0
mmap2(0x76ef7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x76ef7000
close(3) = 0
open("/usr/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\320>\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=456356, ...}) = 0
mmap2(NULL, 520352, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76e65000
mprotect(0x76ed4000, 61440, PROT_NONE) = 0
mmap2(0x76ee3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x76ee3000
close(3) = 0
open("/usr/lib/libcrypto.so.1.0.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\0\337\3\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0555, st_size=1864596, ...}) = 0
mmap2(NULL, 1619736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76cd9000
mprotect(0x76e3e000, 61440, PROT_NONE) = 0
mmap2(0x76e4d000, 86016, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x164000) = 0x76e4d000
mmap2(0x76e62000, 10008, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x76e62000
close(3) = 0
open("/usr/lib/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0LI\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=138524, ...}) = 0
mmap2(NULL, 164428, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76cb0000
mprotect(0x76cc6000, 61440, PROT_NONE) = 0
mmap2(0x76cd5000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x76cd5000
mmap2(0x76cd7000, 4684, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x76cd7000
close(3) = 0
open("/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0,h\1\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1555564, ...}) = 0
mmap2(NULL, 1303872, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76b71000
mprotect(0x76c9a000, 65536, PROT_NONE) = 0
mmap2(0x76caa000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x129000) = 0x76caa000
mmap2(0x76cad000, 9536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x76cad000
close(3) = 0
open("/usr/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0`\321\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=6980840, ...}) = 0
mmap2(NULL, 180204, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76b45000
mprotect(0x76b61000, 61440, PROT_NONE) = 0
mmap2(0x76b70000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x76b70000
close(3) = 0
open("/usr/lib/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\300\n\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=18052, ...}) = 0
mmap2(NULL, 82140, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x76b30000
mprotect(0x76b34000, 61440, PROT_NONE) = 0
mmap2(0x76b43000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x76b43000
close(3) = 0
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x76f37000
set_tls(0x76f374c0, 0x76f37b98, 0x76f3c000, 0x76f374c0, 0x76f3c000) = 0
mprotect(0x76caa000, 8192, PROT_READ) = 0
mprotect(0x76b43000, 4096, PROT_READ) = 0
mprotect(0x76cd5000, 4096, PROT_READ) = 0
mprotect(0x76e4d000, 57344, PROT_READ) = 0
mprotect(0x76ee3000, 4096, PROT_READ) = 0
mprotect(0xae000, 4096, PROT_READ) = 0
mprotect(0x76f3b000, 4096, PROT_READ) = 0
munmap(0x76ef8000, 78476) = 0
set_tid_address(0x76f37068) = 584
set_robust_list(0x76f37070, 12) = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x76cb425c, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x76b9d3c0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x76cb4358, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x76b9d3c0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
ugetrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
rt_sigprocmask(SIG_SETMASK, ~[ILL TRAP BUS FPE SEGV RTMIN RT_1], [], 8) = 0
rt_sigaction(SIGILL, {sa_handler=0x76d1af54, sa_mask=~[ILL TRAP BUS FPE SEGV RTMIN RT_1], sa_flags=SA_RESTORER, sa_restorer=0x76b9d3b0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigprocmask(SIG_BLOCK, NULL, ~[ILL TRAP BUS FPE KILL SEGV STOP RTMIN RT_1], 8) = 0
rt_sigaction(SIGILL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=SA_RESTORER, sa_restorer=0x76b9d3b0}, NULL, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
mmap2(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x76f33000
brk(NULL) = 0x1e63000
brk(0x1e84000) = 0x1e84000
mprotect(0x76f33000, 4096, PROT_NONE) = 0
clone(child_stack=0x76f35fa8, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x76f364d8, tls=0x76f36930, child_tidptr=0x76f364d8) = 585
tgkill(584, 585, SIGRTMIN) = 0
+++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)

')

trying to fix it by compiling it my self with the latest kernel, but still not found a solution?

anyone else came across this problem?
thx ahead

ok got more info:

ntpd is crashing in ntpd/ntpd.c:329

$this->bbcode_second_pass_code('', '
/* pre-heat threading: create a thread and cancel it, just to exercise
* thread cancellation.
*/
static void
my_pthread_warmup(void)
{
pthread_t thread;
pthread_attr_t thr_attr;
int rc;

pthread_attr_init(&thr_attr);
#if defined(HAVE_PTHREAD_ATTR_GETSTACKSIZE) && \
defined(HAVE_PTHREAD_ATTR_SETSTACKSIZE) && \
defined(PTHREAD_STACK_MIN)
rc = pthread_attr_setstacksize(&thr_attr, PTHREAD_STACK_MIN);
if (0 != rc)
msyslog(LOG_ERR,
"my_pthread_warmup: pthread_attr_setstacksize() -> %s",
strerror(rc));
#endif
rc = pthread_create(
&thread, &thr_attr, my_pthread_warmup_worker, NULL);
pthread_attr_destroy(&thr_attr);
if (0 != rc) {
msyslog(LOG_ERR,
"my_pthread_warmup: pthread_create() -> %s",
strerror(rc));
} else {
pthread_cancel(thread);
>>> pthread_join(thread, NULL);
}
}
')

the pthread_join throws a segmentation fault
however thread seems to be valid.

im not a kernel developer but the ntp code seams fine here.
but whatever ill continue to investigate

ok more info:
it seams like pthread_join as symbol is causing a segfault:

$this->bbcode_second_pass_code('', '
if (0 != rc) {
msyslog(LOG_ERR,
"my_pthread_warmup: pthread_create() -> %s",
strerror(rc));
} else {
pthread_cancel(thread);
>>> printf("%p", pthread_join);
pthread_join(thread, NULL);
}
}
')

now its crashing at the printf ???

whatever

The solution i found after a two hours of investigation

the NEED_PTHREAD_WARMUP exercise caused everything. disabling it makes everything work again.
anyways the patch to fix it *temporary* is kinda simple:

updated PKGBUILD:

$this->bbcode_second_pass_code('', '
# Maintainer: Lukas Fleischer
# Contributor: Gaetan Bisson
# Contributor: kevin

pkgname=ntp
_pkgname=ntp #-dev
_pkgver=4.2.8p9
pkgver=${_pkgver/p/.p}
pkgrel=2
pkgdesc='Network Time Protocol reference implementation'
url='http://www.ntp.org/'
license=('custom')
arch=('i686' 'x86_64')
depends=('openssl' 'libcap' 'libedit')
backup=('etc/ntp.conf')
source=("https://www.eecis.udel.edu/~ntp/ntp_spool/ntp4/ntp-4.2/${_pkgname}-${_pkgver}.tar.gz"
'ntp.conf'
'ntpd.service'
'ntpd-segfault.patch'
'ntpdate.service')
sha1sums=('032e58e7e416ffa1cbdcbb81021785fce4ed4d4b'
'ad1d6ee2e9aca64a84a7224c88bf6008ac6c69e1'
'0cccca872385e0142888ab48d273fec0669b30a8'
'4e1506f236c46d57c6fdf19743324fbca3e514e6'
'059b382d1af0c55202e2d17f2ae065a2cbfec9ee')

options=('!emptydirs')
install=ntp.install

build() {
cd "${srcdir}/${_pkgname}-${_pkgver}"
patch ntpd/ntpd.c ../ntpd-segfault.patch
./configure --prefix=/usr --libexecdir=/usr/lib --enable-linuxcaps --enable-ntp-signd
make
}

package() {
cd "${srcdir}/${_pkgname}-${_pkgver}"

make DESTDIR="${pkgdir}" install

install -Dm644 ../ntp.conf "${pkgdir}"/etc/ntp.conf
install -Dm644 ../ntpd.service "${pkgdir}"/usr/lib/systemd/system/ntpd.service
install -Dm644 ../ntpdate.service "${pkgdir}"/usr/lib/systemd/system/ntpdate.service
install -Dm644 COPYRIGHT "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE"

install -d -o 87 "${pkgdir}"/var/lib/ntp
echo > "${pkgdir}/var/lib/ntp/.placeholder"
}
')

Patch:

$this->bbcode_second_pass_code('', '
--- a/ntpd/ntpd.c 2016-11-21 14:29:20.000000000 +0200
+++ b/ntpd/ntpd.c 2017-01-13 18:06:41.053474403 +0200
@@ -35,7 +35,7 @@
# include <pthread.h>
# endif
# if defined(linux)
-# define NEED_PTHREAD_WARMUP
+//# define NEED_PTHREAD_WARMUP
# endif
#endif

')

[crooks]

I've just experienced this exact issue after updating my Raspberry Pi. Took me a while to isolate the problem as bad time broke many other things (Unbound DNS, journalling, etc) that rely on it. Unfortunately I'm new to Arch so I need to figure out the PKGBUILD process in order to apply your patch. Yay, more learning!