***************** crashinfo --stacksummary ***************** /exports/testreports/42084/testresults/sanity2-zfs-centos7_x86_64-centos7_x86_64/oleg223-client-timeout-core (3.10.0-7.9-debug) ------- 25 stacks like that: ---------- #0 __schedule #1 schedule #2 rescuer_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=11603s(pid=729), oldest=11609s(pid=12) ........................ ata_sff 1 times bioset 3 times crypto 1 times deferwq 1 times edac-poller 1 times ext4-rsv-conver 1 times ipv6_addrconf 1 times kblockd 1 times kintegrityd 1 times kmpath_rdacd 1 times kpsmoused 1 times kthrotld 1 times kvm-irqfd-clean 1 times lru-add-drain 1 times md 1 times netns 1 times nfsiod 1 times rpciod 1 times scsi_tmf_0 1 times scsi_tmf_1 1 times watchdogd 1 times writeback 1 times xprtiod 1 times ------- 19 stacks like that: ---------- #0 __schedule #1 schedule #2 worker_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=17462), oldest=11603s(pid=4) ........................ kworker/0:0H 1 times kworker/0:1 1 times kworker/0:1H 1 times kworker/0:2 1 times kworker/1:0 1 times kworker/1:0H 1 times kworker/1:1H 1 times kworker/1:2 1 times kworker/2:0H 1 times kworker/2:1 1 times kworker/2:1H 1 times kworker/2:2 1 times kworker/3:0H 1 times kworker/3:1 1 times kworker/3:1H 1 times kworker/3:2 1 times kworker/u8:0 1 times kworker/u8:1 1 times kworker/u8:3 1 times ------- 12 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_wait_event #3 ptlrpc_main #4 kthread #5 ret_from_fork_nospec_begin youngest=3558s(pid=12832), oldest=3928s(pid=6929) ........................ ldlm_cb00_000 1 times ldlm_cb00_001 1 times ldlm_cb00_002 1 times ldlm_cb01_000 1 times ldlm_cb01_001 1 times ldlm_cb01_002 1 times ldlm_cn00_000 1 times ldlm_cn00_001 1 times ldlm_cn00_002 1 times ldlm_cn01_000 1 times ldlm_cn01_001 1 times ldlm_cn01_002 1 times ------- 12 stacks like that: ---------- #0 __schedule #1 schedule #2 smpboot_thread_fn #3 kthread #4 ret_from_fork_nospec_begin youngest=1s(pid=28), oldest=3609s(pid=29) ........................ ksoftirqd/0 1 times ksoftirqd/1 1 times ksoftirqd/2 1 times ksoftirqd/3 1 times migration/0 1 times migration/1 1 times migration/2 1 times migration/3 1 times watchdog/0 1 times watchdog/1 1 times watchdog/2 1 times watchdog/3 1 times ------- 10 stacks like that: ---------- #0 __schedule #1 schedule #2 futex_wait_queue_me #3 futex_wait #4 do_futex #5 sys_futex #6 system_call_fastpath youngest=0s(pid=31293), oldest=11601s(pid=810) ........................ JS GC Helper 1 times JS Sour~ Thread 1 times fio 2 times gssproxy 5 times rs:main Q:Reg 1 times ------- 9 stacks like that: ---------- #0 __schedule #1 schedule #2 ldlm_bl_thread_main #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=6932), oldest=8s(pid=6933) ........................ ldlm_bl_01 1 times ldlm_bl_02 1 times ldlm_bl_03 1 times ldlm_bl_04 1 times ldlm_bl_05 1 times ldlm_bl_06 1 times ldlm_bl_07 1 times ldlm_bl_08 1 times ldlm_bl_09 1 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_poll #7 system_call_fastpath youngest=0s(pid=820), oldest=11601s(pid=867) ........................ NetworkManager 1 times gdbus 2 times gmain 2 times polkitd 3 times ------- 8 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_nocb_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=27), oldest=11609s(pid=10) ........................ rcuob/0 1 times rcuob/1 1 times rcuob/2 1 times rcuob/3 1 times rcuos/0 1 times rcuos/1 1 times rcuos/2 1 times rcuos/3 1 times ------- 6 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 ep_poll #5 sys_epoll_wait #6 system_call_fastpath youngest=0s(pid=1), oldest=3920s(pid=598) ........................ dbus-daemon 1 times gssproxy 1 times systemd 1 times systemd-journal 1 times systemd-logind 1 times systemd-udevd 1 times ------- 5 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_woken #4 ptlrpcd #5 kthread #6 ret_from_fork_nospec_begin youngest=1s(pid=15342), oldest=3738s(pid=15338) ........................ ptlrpcd_00_00 1 times ptlrpcd_00_01 1 times ptlrpcd_01_00 1 times ptlrpcd_01_01 1 times ptlrpcd_rcv 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 osc_extent_wait #3 osc_extent_find #4 osc_queue_async_io #5 osc_page_cache_add #6 osc_io_commit_async #7 cl_io_commit_async #8 lov_io_commit_async #9 cl_io_commit_async #10 vvp_io_write_commit #11 vvp_io_write_start #12 cl_io_start #13 cl_io_loop #14 ll_file_io_generic #15 ll_file_aio_write #16 ll_file_write #17 vfs_write #18 sys_pwrite64 #19 system_call_fastpath youngest=2965s(pid=31294), oldest=2965s(pid=31297) ........................ fio 4 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 ksocknal_scheduler #3 kthread #4 ret_from_fork_nospec_begin youngest=1s(pid=15334), oldest=7227s(pid=15332) ........................ socknal_sd00_00 1 times socknal_sd00_01 1 times socknal_sd01_00 1 times socknal_sd01_01 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_connd #4 kthread #5 ret_from_fork_nospec_begin youngest=4942s(pid=15327), oldest=7203s(pid=15326) ........................ socknal_cd00 1 times socknal_cd01 1 times socknal_cd02 1 times socknal_cd03 1 times ------- 4 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_select #6 core_sys_select #7 sys_select #8 system_call_fastpath youngest=1s(pid=1126), oldest=11600s(pid=864) ........................ dhclient 1 times rsyslogd 1 times sshd 2 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 do_wait #3 sys_wait4 #4 system_call_fastpath youngest=3567s(pid=31078), oldest=11532s(pid=1165) ........................ auster 1 times bash 2 times ------- 3 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 n_tty_read #4 tty_read #5 vfs_read #6 sys_read #7 system_call_fastpath youngest=11597s(pid=845), oldest=11598s(pid=847) ........................ agetty 3 times ------- 3 stacks like that: ---------- #-1 native_safe_halt #0 default_idle #1 default_enter_idle #2 cpuidle_enter_state #3 cpuidle_idle_call #4 arch_cpu_idle #5 cpu_startup_entry #6 start_secondary #7 start_cpu youngest=11610s(pid=0), oldest=11610s(pid=0) ........................ swapper/1 1 times swapper/2 1 times swapper/3 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 do_nanosleep #3 hrtimer_nanosleep #4 sys_nanosleep #5 system_call_fastpath youngest=0s(pid=31289), oldest=0s(pid=31290) ........................ fio 2 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 ptlrpc_hr_main #3 kthread #4 ret_from_fork_nospec_begin youngest=7227s(pid=15324), oldest=7227s(pid=15325) ........................ ptlrpc_hr00_000 1 times ptlrpc_hr01_000 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 scsi_error_handler #3 kthread #4 ret_from_fork_nospec_begin youngest=11607s(pid=279), oldest=11607s(pid=277) ........................ scsi_eh_0 1 times scsi_eh_1 1 times ------- 2 stacks like that: ---------- #0 __schedule #1 schedule #2 rcu_gp_kthread #3 kthread #4 ret_from_fork_nospec_begin youngest=0s(pid=9), oldest=11609s(pid=8) ........................ rcu_bh 1 times rcu_sched 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 range_lock #3 ll_file_io_generic #4 ll_file_aio_write #5 ll_file_write #6 vfs_write #7 sys_pwrite64 #8 system_call_fastpath youngest=3564s(pid=31301), oldest=3564s(pid=31301) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 osc_extent_wait #3 osc_cache_wait_range #4 osc_io_fsync_end #5 cl_io_end #6 lov_io_end_wrapper #7 lov_io_fsync_end #8 cl_io_end #9 cl_io_loop #10 cl_sync_file_range #11 ll_writepages #12 do_writepages #13 __filemap_fdatawrite_range #14 filemap_write_and_wait_range #15 __generic_file_aio_write #16 __generic_file_write_iter #17 vvp_io_write_start #18 cl_io_start #19 cl_io_loop #20 ll_file_io_generic #21 ll_file_aio_write #22 ll_file_write #23 vfs_write #24 sys_pwrite64 #25 system_call_fastpath youngest=2965s(pid=31300), oldest=2965s(pid=31300) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 cl_sync_io_wait #3 osc_lock_enqueue_wait #4 osc_lock_enqueue #5 cl_lock_enqueue #6 lov_lock_enqueue #7 cl_lock_enqueue #8 cl_lock_request #9 cl_io_lock #10 cl_io_loop #11 ll_file_io_generic #12 ll_file_aio_write #13 ll_file_write #14 vfs_write #15 sys_pwrite64 #16 system_call_fastpath youngest=3565s(pid=31299), oldest=3565s(pid=31299) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 cl_sync_io_wait #3 osc_lock_enqueue_wait #4 osc_lock_enqueue #5 cl_lock_enqueue #6 lov_lock_enqueue #7 cl_lock_enqueue #8 cl_lock_request #9 cl_io_lock #10 cl_io_loop #11 ll_file_io_generic #12 ll_file_read_iter #13 ll_file_aio_read #14 ll_file_read #15 vfs_read #16 sys_pread64 #17 system_call_fastpath youngest=3565s(pid=31298), oldest=3565s(pid=31298) ........................ fio 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 pipe_wait #3 pipe_read #4 do_sync_read #5 vfs_read #6 sys_read #7 system_call_fastpath youngest=3566s(pid=31079), oldest=3566s(pid=31079) ........................ tee 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 lnet_peer_discovery #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=15337), oldest=0s(pid=15337) ........................ lnet_discovery 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 wait_for_completion_interruptible_timeout #4 lnet_monitor_thread #5 kthread #6 ret_from_fork_nospec_begin youngest=0s(pid=15336), oldest=0s(pid=15336) ........................ monitor_thread 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 lnet_acceptor #3 kthread #4 ret_from_fork_nospec_begin youngest=4942s(pid=15335), oldest=4942s(pid=15335) ........................ acceptor_001 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 ksocknal_reaper #4 kthread #5 ret_from_fork_nospec_begin youngest=0s(pid=15330), oldest=0s(pid=15330) ........................ socknal_reaper 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 mgc_requeue_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=3922s(pid=6935), oldest=3922s(pid=6935) ........................ ll_cfg_requeue 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 expired_lock_main #3 kthread #4 ret_from_fork_nospec_begin youngest=3928s(pid=6934), oldest=3928s(pid=6934) ........................ ldlm_elt 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_hrtimeout_range_clock #3 schedule_hrtimeout_range #4 poll_schedule_timeout #5 do_sys_poll #6 sys_ppoll #7 system_call_fastpath youngest=0s(pid=899), oldest=0s(pid=899) ........................ in:imjournal 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule youngest=11601s(pid=770), oldest=11601s(pid=770) ........................ nfsv4.1-svc 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kjournald2 #3 kthread #4 ret_from_fork_nospec_begin youngest=11604s(pid=480), oldest=11604s(pid=480) ........................ jbd2/nbd0-8 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 nbd_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=3595s(pid=451), oldest=3595s(pid=451) ........................ nbd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 sk_wait_data #4 tcp_recvmsg #5 inet_recvmsg #6 sock_recvmsg #7 kernel_recvmsg #8 sock_xmit #9 __nbd_ioctl #10 nbd_ioctl #11 blkdev_ioctl #12 block_ioctl #13 do_vfs_ioctl #14 sys_ioctl #15 system_call_fastpath youngest=3595s(pid=449), oldest=3595s(pid=449) ........................ nbd-client 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 khugepaged #4 kthread #5 ret_from_fork_nospec_begin youngest=1s(pid=53), oldest=1s(pid=53) ........................ khugepaged 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 ksm_scan_thread #3 kthread #4 ret_from_fork_nospec_begin youngest=11608s(pid=52), oldest=11608s(pid=52) ........................ ksmd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kswapd #3 kthread #4 ret_from_fork_nospec_begin youngest=11608s(pid=51), oldest=11608s(pid=51) ........................ kswapd0 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 schedule_timeout #3 schedule_timeout_interruptible #4 watchdog #5 kthread #6 ret_from_fork_nospec_begin youngest=89s(pid=37), oldest=89s(pid=37) ........................ khungtaskd 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 devtmpfsd #3 kthread #4 ret_from_fork_nospec_begin youngest=3927s(pid=35), oldest=3927s(pid=35) ........................ kdevtmpfs 1 times ------- 1 stacks like that: ---------- #0 __schedule #1 schedule #2 kthreadd #3 ret_from_fork_nospec_begin youngest=151s(pid=2), oldest=151s(pid=2) ........................ kthreadd 1 times ------- 1 stacks like that: ---------- #-1 native_safe_halt #0 default_idle #1 default_enter_idle #2 cpuidle_enter_state #3 cpuidle_idle_call #4 arch_cpu_idle #5 cpu_startup_entry #6 rest_init #7 start_kernel #8 x86_64_start_reservations #9 x86_64_start_kernel #10 start_cpu youngest=11610s(pid=0), oldest=11610s(pid=0) ........................ swapper/0 1 times ** Execution took 0.65s (real) 0.15s (CPU), Child processes: 0.53s