====================================== | [ 841.047731][ T12] ------------[ cut here ]------------ | [ 841.048109][ T12] conntrack cleanup blocked for 60s | [ 841.048111][ T12] WARNING: net/netfilter/nf_conntrack_core.c:2511 at 0x0, CPU#3: kworker/u16:0/12 | [ 841.048512][ T12] Modules linked in: sctp(+) xt_conntrack cls_matchall ip_gre ip6_gre gre sch_fq sch_prio xt_mark cls_basic sch_fq_codel vxlan nft_chain_nat xt_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nft_compat nf_tables cls_bpf netdevsim psample act_mirred act_tunnel_key bareudp mpls_gso mpls_iptunnel mpls_router act_gact cls_flower sch_ingress [ 841.049636][ T12] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 841.049821][ T12] Workqueue: netns cleanup_net [ 841.049973][ T12] RIP: 0010:nf_conntrack_cleanup_net_list (net/netfilter/nf_conntrack_core.c:2511 (discriminator 16)) nf_conntrack [ 841.050182][ T12] Code: 00 48 8b 54 24 18 48 8b 05 c3 a8 0a f3 48 81 c2 60 ea 00 00 48 39 c2 78 0a e8 42 2e fa f1 e9 25 fd ff ff 48 8d 3d 56 26 14 00 <67> 48 0f b9 3a eb e8 48 8b 5c 24 28 48 b8 00 00 00 00 00 fc ff df All code ======== 0: 00 48 8b add %cl,-0x75(%rax) 3: 54 push %rsp 4: 24 18 and $0x18,%al 6: 48 8b 05 c3 a8 0a f3 mov -0xcf5573d(%rip),%rax # 0xfffffffff30aa8d0 d: 48 81 c2 60 ea 00 00 add $0xea60,%rdx 14: 48 39 c2 cmp %rax,%rdx 17: 78 0a js 0x23 19: e8 42 2e fa f1 call 0xfffffffff1fa2e60 1e: e9 25 fd ff ff jmp 0xfffffffffffffd48 23: 48 8d 3d 56 26 14 00 lea 0x142656(%rip),%rdi # 0x142680 2a:* 67 48 0f b9 3a ud1 (%edx),%rdi <-- trapping instruction 2f: eb e8 jmp 0x19 31: 48 8b 5c 24 28 mov 0x28(%rsp),%rbx 36: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 3d: fc ff df Code starting with the faulting instruction =========================================== 0: 67 48 0f b9 3a ud1 (%edx),%rdi 5: eb e8 jmp 0xffffffffffffffef 7: 48 8b 5c 24 28 mov 0x28(%rsp),%rbx c: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 13: fc ff df [ 841.050652][ T12] RSP: 0018:ffa00000000c7980 EFLAGS: 00010297 [ 841.050835][ T12] RAX: 0000000100083fad RBX: ffa00000000c7af8 RCX: 0000000000000001 [ 841.051031][ T12] RDX: 0000000100083fac RSI: 0000000000000004 RDI: ffffffffc08a9030 [ 841.051223][ T12] RBP: 0000000000000001 R08: ffffffffc07667aa R09: ffe21c000175e70c [ 841.051415][ T12] R10: ffe21c000175e70d R11: 0000000000000001 R12: dffffc0000000000 [ 841.051613][ T12] R13: ff1100000baf3860 R14: fffffbfff8116c07 R15: ff1100000c0ffa00 [ 841.051840][ T12] FS: 0000000000000000(0000) GS:ff110000b758b000(0000) knlGS:0000000000000000 [ 841.052057][ T12] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 841.052566][ T12] CR2: 00007f113ace5a48 CR3: 000000000f19c001 CR4: 0000000000771ef0 [ 841.052763][ T12] PKRU: 55555554 [ 841.052859][ T12] Call Trace: [ 841.052955][ T12] [ 841.053028][ T12] ? nf_conntrack_cleanup_end (net/netfilter/nf_conntrack_core.c:2488) nf_conntrack [ 841.053185][ T12] ? kfree (mm/slub.c:6663 (discriminator 3) mm/slub.c:6871 (discriminator 3)) [ 841.053278][ T12] ? lock_acquire (./include/trace/events/lock.h:24 (discriminator 33) kernel/locking/lockdep.c:5831 (discriminator 33)) [ 841.053406][ T12] ? nf_conntrack_pernet_exit (net/netfilter/nf_conntrack_standalone.c:1129 (discriminator 4)) nf_conntrack [ 841.053562][ T12] ops_undo_list (net/core/net_namespace.c:251 (discriminator 4)) [ 841.053694][ T12] ? netns_install (net/core/net_namespace.c:223) [ 841.053828][ T12] cleanup_net (net/core/net_namespace.c:698) [ 841.053951][ T12] ? process_one_work (kernel/workqueue.c:3233 (discriminator 1)) [ 841.054076][ T12] ? net_passive_dec (net/core/net_namespace.c:660) [ 841.054199][ T12] ? rcu_is_watching (./include/linux/context_tracking.h:128 (discriminator 1) kernel/rcu/tree.c:751 (discriminator 1)) [ 841.054321][ T12] ? process_one_work (kernel/workqueue.c:3233 (discriminator 1)) [ 841.054443][ T12] ? lock_acquire (./include/trace/events/lock.h:24 (discriminator 33) kernel/locking/lockdep.c:5831 (discriminator 33)) [ 841.054562][ T12] ? rcu_is_watching (./include/linux/context_tracking.h:128 (discriminator 1) kernel/rcu/tree.c:751 (discriminator 1)) [ 841.054694][ T12] process_one_work (kernel/workqueue.c:3262) [ 841.054826][ T12] ? pwq_dec_nr_in_flight (kernel/workqueue.c:3159) [ 841.054949][ T12] ? lock_acquire.part.0 (kernel/locking/lockdep.c:470 kernel/locking/lockdep.c:5870) [ 841.055073][ T12] ? assign_work (kernel/workqueue.c:1200) [ 841.055197][ T12] worker_thread (kernel/workqueue.c:3334 (discriminator 2) kernel/workqueue.c:3421 (discriminator 2)) [ 841.055324][ T12] ? process_one_work (kernel/workqueue.c:3367) [ 841.055446][ T12] kthread (kernel/kthread.c:463) [ 841.055544][ T12] ? kthread_is_per_cpu (kernel/kthread.c:412) [ 841.055666][ T12] ? __lock_release.isra.0 (kernel/locking/lockdep.c:5536) [ 841.055799][ T12] ? rcu_is_watching (./include/linux/context_tracking.h:128 (discriminator 1) kernel/rcu/tree.c:751 (discriminator 1)) [ 841.055921][ T12] ? kthread_is_per_cpu (kernel/kthread.c:412) [ 841.056046][ T12] ret_from_fork (arch/x86/kernel/process.c:164) [ 841.056168][ T12] ? arch_exit_to_user_mode_prepare.isra.0 (arch/x86/entry/syscall_64.c:37) [ 841.056320][ T12] ? __switch_to (./include/linux/thread_info.h:140 (discriminator 2) arch/x86/kernel/process.h:16 (discriminator 2) arch/x86/kernel/process_64.c:676 (discriminator 2)) [ 841.056446][ T12] ? kthread_is_per_cpu (kernel/kthread.c:412) [ 841.056569][ T12] ret_from_fork_asm (arch/x86/entry/entry_64.S:256) | [ 841.056924][ T12] hardirqs last enabled at (8777059): __up_console_sem (./arch/x86/include/asm/irqflags.h:26 ./arch/x86/include/asm/irqflags.h:109 ./arch/x86/include/asm/irqflags.h:151 kernel/printk/printk.c:345) | [ 841.057140][ T12] hardirqs last disabled at (8777066): __up_console_sem (kernel/printk/printk.c:343 (discriminator 10)) | [ 841.057359][ T12] softirqs last enabled at (8776662): handle_softirqs (kernel/softirq.c:469 (discriminator 2) kernel/softirq.c:650 (discriminator 2)) | [ 841.057574][ T12] softirqs last disabled at (8776657): irq_exit_rcu (kernel/softirq.c:657 kernel/softirq.c:496 kernel/softirq.c:723 kernel/softirq.c:739) | [ 841.057799][ T12] ---[ end trace 0000000000000000 ]--- [ 984.491744][ T44] INFO: task modprobe:14864 blocked for more than 122 seconds. [ 984.492156][ T44] Tainted: G W 6.18.0-virtme #1 [ 984.492327][ T44] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 984.492551][ T44] task:modprobe state:D stack:25136 pid:14864 tgid:14864 ppid:37 task_flags:0x400100 flags:0x00080000 [ 984.492876][ T44] Call Trace: [ 984.492986][ T44] [ 984.493061][ T44] __schedule (kernel/sched/core.c:5259 kernel/sched/core.c:6863) [ 984.493201][ T44] ? io_schedule_timeout (kernel/sched/core.c:6719) [ 984.493325][ T44] ? __lock_release.isra.0 (kernel/locking/lockdep.c:5536) [ 984.493451][ T44] schedule (kernel/sched/core.c:6946 kernel/sched/core.c:6960) [ 984.493540][ T44] schedule_preempt_disabled (kernel/sched/core.c:7018) [ 984.493659][ T44] rwsem_down_write_slowpath (kernel/locking/rwsem.c:1187 (discriminator 7)) [ 984.493788][ T44] ? rwsem_down_read_slowpath (kernel/locking/rwsem.c:1112) [ 984.493910][ T44] ? rcu_is_watching (./include/linux/context_tracking.h:128 (discriminator 1) kernel/rcu/tree.c:751 (discriminator 1)) [ 984.494038][ T44] ? perf_trace_sched_switch (kernel/sched/core.c:8785) [ 984.494162][ T44] ? register_pernet_subsys (net/core/net_namespace.c:1430) [ 984.494280][ T44] ? lock_acquire (./include/trace/events/lock.h:24 (discriminator 33) kernel/locking/lockdep.c:5831 (discriminator 33)) [ 984.494392][ T44] down_write (kernel/locking/rwsem.c:1317 kernel/locking/rwsem.c:1326 kernel/locking/rwsem.c:1591) [ 984.494477][ T44] ? rwsem_down_write_slowpath (kernel/locking/rwsem.c:1588) [ 984.494613][ T44] ? _raw_spin_unlock (./include/linux/spinlock_api_smp.h:143 (discriminator 3) kernel/locking/spinlock.c:186 (discriminator 3)) [ 984.494732][ T44] ? __register_sysctl_table (fs/proc/proc_sysctl.c:1413) [ 984.494851][ T44] register_pernet_subsys (net/core/net_namespace.c:1430) [ 984.494970][ T44] sctp_init (net/sctp/protocol.c:1600) sctp [ 984.495104][ T44] ? gre_exit (net/ipv4/gre_demux.c:189) gre [ 984.495215][ T44] do_one_initcall (init/main.c:1283) [ 984.495334][ T44] ? trace_event_raw_event_initcall_level (init/main.c:1274) [ 984.495475][ T44] ? kasan_unpoison (mm/kasan/shadow.c:156 mm/kasan/shadow.c:182) [ 984.495594][ T44] do_init_module (kernel/module/main.c:3039) [ 984.496004][ T44] ? free_module (kernel/module/main.c:3011) [ 984.496115][ T44] ? backing_file_read_iter (fs/backing-file.c:210) [ 984.496228][ T44] init_module_from_file (kernel/module/main.c:3712) [ 984.496338][ T44] ? __do_sys_init_module (kernel/module/main.c:3677) [ 984.496454][ T44] ? __lock_release.isra.0 (kernel/locking/lockdep.c:5536) [ 984.496563][ T44] ? do_raw_spin_unlock (./arch/x86/include/asm/atomic.h:23 ./include/linux/atomic/atomic-arch-fallback.h:457 ./include/linux/atomic/atomic-instrumented.h:33 ./include/asm-generic/qspinlock.h:57 kernel/locking/spinlock_debug.c:101 kernel/locking/spinlock_debug.c:141) [ 984.496675][ T44] idempotent_init_module (kernel/module/main.c:3724) [ 984.496797][ T44] ? init_module_from_file (kernel/module/main.c:3716) [ 984.496910][ T44] ? rcu_is_watching (./include/linux/context_tracking.h:128 (discriminator 1) kernel/rcu/tree.c:751 (discriminator 1)) [ 984.497026][ T44] ? cap_capable (./include/trace/events/capability.h:26 (discriminator 33) security/commoncap.c:130 (discriminator 33)) [ 984.497138][ T44] __x64_sys_finit_module (./include/linux/file.h:62 (discriminator 1) ./include/linux/file.h:83 (discriminator 1) kernel/module/main.c:3747 (discriminator 1) kernel/module/main.c:3734 (discriminator 1) kernel/module/main.c:3734 (discriminator 1)) [ 984.497248][ T44] do_syscall_64 (arch/x86/entry/syscall_64.c:63 (discriminator 1) arch/x86/entry/syscall_64.c:94 (discriminator 1)) [ 984.497362][ T44] entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:131) [ 984.497505][ T44] RIP: 0033:0x7fba6d0de34d [ 984.497624][ T44] RSP: 002b:00007ffdf632c668 EFLAGS: 00000246 ORIG_RAX: 0000000000000139 [ 984.497799][ T44] RAX: ffffffffffffffda RBX: 0000560b81ef0b10 RCX: 00007fba6d0de34d [ 984.497965][ T44] RDX: 0000000000000000 RSI: 0000560b61e1af2e RDI: 0000000000000000 [ 984.498139][ T44] RBP: 00007ffdf632c700 R08: 0000000000000000 R09: 0000000000000000 [ 984.498306][ T44] R10: 0000000000000000 R11: 0000000000000246 R12: 0000560b61e1af2e Finger prints: nf_conntrack_cleanup_net_list:ops_undo_list:cleanup_net:process_one_work:worker_thread __up_console_sem:__up_console_sem:handle_softirqs:irq_exit_rcu:__schedule