Kernel Uninterruptible Off-CPU Flame Graph (60 secs)
Reset Zoom
Search
btrfs_write_and_wait_marked_extents.constprop.27 (7 ms, 0.76%)
entry_SYSCALL_64_fastpath (0 ms, 0.00%)
mutex_lock (115 ms, 12.42%)
mutex_lock
wait_for_completion (0 ms, 0.00%)
schedule_timeout (27 ms, 2.92%)
s..
wait_on_page_bit (13 ms, 1.40%)
sys_read (115 ms, 12.42%)
sys_read
finish_task_switch (0 ms, 0.00%)
bit_wait_io (13 ms, 1.40%)
wait_for_completion (40 ms, 4.32%)
wa..
bit_wait_io (49 ms, 5.29%)
bit..
call_usermodehelper (0 ms, 0.00%)
btrfs_write_dirty_block_groups (13 ms, 1.40%)
__mutex_lock_slowpath (50 ms, 5.40%)
__m..
btrfs_wait_cache_io (7 ms, 0.76%)
down_read (3 ms, 0.32%)
schedule (13 ms, 1.40%)
sys_munmap (0 ms, 0.00%)
schedule_preempt_disabled (50 ms, 5.40%)
sch..
vfs_fsync_range (2 ms, 0.22%)
process_one_work (0 ms, 0.00%)
ret_from_fork (90 ms, 9.72%)
ret_from..
finish_task_switch (0 ms, 0.00%)
mem_cgroup_css_offline (40 ms, 4.32%)
me..
ret_from_fork (49 ms, 5.29%)
ret..
process_one_work (90 ms, 9.72%)
process_..
finish_task_switch (0 ms, 0.00%)
call_rwsem_down_read_failed (1 ms, 0.11%)
memcg_destroy_kmem_caches (40 ms, 4.32%)
me..
vm_mmap_pgoff (1 ms, 0.11%)
schedule (0 ms, 0.00%)
memcg_offline_kmem.part.38 (107 ms, 11.55%)
memcg_offl..
synchronize_sched (95 ms, 10.26%)
synchroni..
call_rwsem_down_read_failed (1 ms, 0.11%)
flush_workqueue (0 ms, 0.00%)
finish_task_switch (48 ms, 5.18%)
fin..
wait_on_page_bit (12 ms, 1.30%)
docker (0 ms, 0.00%)
kworker/12:1 (108 ms, 11.66%)
kworker/12:1
schedule_preempt_disabled (115 ms, 12.42%)
schedule_pr..
sys_fdatasync (12 ms, 1.30%)
bit_wait_io (12 ms, 1.30%)
wait_for_completion (27 ms, 2.92%)
w..
schedule (0 ms, 0.00%)
sys_vfork (5 ms, 0.54%)
rwsem_down_write_failed_killable (0 ms, 0.00%)
rwsem_down_read_failed (1 ms, 0.11%)
finish_task_switch (1 ms, 0.11%)
wait_for_completion (0 ms, 0.00%)
schedule (0 ms, 0.00%)
finish_task_switch (50 ms, 5.40%)
fin..
rwsem_down_write_failed_killable (1 ms, 0.11%)
finish_task_switch (1 ms, 0.11%)
schedule (7 ms, 0.76%)
css_free_work_fn (0 ms, 0.00%)
rwsem_down_write_failed_killable (0 ms, 0.00%)
finish_task_switch (95 ms, 10.26%)
finish_ta..
filemap_fdatawait_range (13 ms, 1.40%)
down_write_killable (0 ms, 0.00%)
schedule (1 ms, 0.11%)
finish_task_switch (49 ms, 5.29%)
fin..
down_write_killable (1 ms, 0.11%)
call_rwsem_down_write_failed_killable (1 ms, 0.11%)
ret_from_fork (107 ms, 11.55%)
ret_from_f..
css_killed_work_fn (27 ms, 2.92%)
c..
entry_SYSCALL_64_fastpath (2 ms, 0.22%)
mem_cgroup_css_offline (95 ms, 10.26%)
mem_cgrou..
call_usermodehelper (0 ms, 0.00%)
sys_fsync (2 ms, 0.22%)
__wait_on_buffer (1 ms, 0.11%)
rcu_barrier (52 ms, 5.61%)
rcu_..
ret_from_fork (27 ms, 2.92%)
r..
wait_for_completion (0 ms, 0.00%)
proc_cgroup_show (115 ms, 12.42%)
proc_cgroup..
call_rwsem_down_write_failed_killable (1 ms, 0.11%)
mutex_lock (48 ms, 5.18%)
mut..
process_one_work (108 ms, 11.66%)
process_on..
worker_thread (27 ms, 2.92%)
w..
kworker/3:0 (102 ms, 11.01%)
kworker/3:0
schedule_preempt_disabled (16 ms, 1.73%)
io_schedule_timeout (2 ms, 0.22%)
call_rwsem_down_read_failed (0 ms, 0.00%)
release_caches (40 ms, 4.32%)
re..
schedule (0 ms, 0.00%)
bit_wait_io (1 ms, 0.11%)
schedule (2 ms, 0.22%)
worker_thread (107 ms, 11.55%)
worker_thr..
schedule (0 ms, 0.00%)
page_fault (5 ms, 0.54%)
memcg_deactivate_kmem_caches (27 ms, 2.92%)
m..
down_write_killable (0 ms, 0.00%)
schedule_timeout (49 ms, 5.29%)
sch..
_rcu_barrier (48 ms, 5.18%)
_rc..
kworker/8:2 (95 ms, 10.26%)
kworker/8:2
do_mprotect_pkey (0 ms, 0.00%)
synchronize_sched (40 ms, 4.32%)
sy..
finish_task_switch (1 ms, 0.11%)
finish_task_switch (0 ms, 0.00%)
rwsem_down_read_failed (1 ms, 0.11%)
schedule (40 ms, 4.32%)
sc..
dockerd (2 ms, 0.22%)
btrfs-transacti (29 ms, 3.13%)
b..
kthread (89 ms, 9.61%)
kthread
bit_wait_io (7 ms, 0.76%)
schedule (0 ms, 0.00%)
process_one_work (102 ms, 11.01%)
process_on..
kthread (108 ms, 11.66%)
kthread
filemap_fdatawait_range (7 ms, 0.76%)
finish_task_switch (52 ms, 5.61%)
fini..
sys_munmap (0 ms, 0.00%)
do_page_fault (1 ms, 0.11%)
jbd2/xvda-8 (49 ms, 5.29%)
jbd..
do_fsync (2 ms, 0.22%)
systemd (115 ms, 12.42%)
systemd
schedule (0 ms, 0.00%)
wait_on_page_bit (7 ms, 0.76%)
sys_select (0 ms, 0.00%)
SyS_mmap (1 ms, 0.11%)
css_killed_work_fn (107 ms, 11.55%)
css_killed..
finish_task_switch (49 ms, 5.29%)
fin..
schedule_timeout (13 ms, 1.40%)
SyS_mmap_pgoff (1 ms, 0.11%)
kthread (49 ms, 5.29%)
kth..
call_usermodehelper (0 ms, 0.00%)
vm_mmap_pgoff (0 ms, 0.00%)
mem_cgroup_css_offline (107 ms, 11.55%)
mem_cgroup..
do_page_fault (3 ms, 0.32%)
ret_from_fork (29 ms, 3.13%)
r..
process_one_work (95 ms, 10.26%)
process_o..
do_page_fault (1 ms, 0.11%)
__wait_rcu_gp (27 ms, 2.92%)
_..
process_one_work (107 ms, 11.55%)
process_on..
worker_thread (0 ms, 0.00%)
SyS_mmap_pgoff (0 ms, 0.00%)
memcg_deactivate_kmem_caches (107 ms, 11.55%)
memcg_deac..
kworker/4:0 (89 ms, 9.61%)
kworker/..
schedule_timeout (1 ms, 0.11%)
schedule (1 ms, 0.11%)
call_usermodehelper_exec (0 ms, 0.00%)
__wait_on_bit (7 ms, 0.76%)
__mutex_lock_slowpath (48 ms, 5.18%)
__m..
call_usermodehelper (0 ms, 0.00%)
schedule_timeout (7 ms, 0.76%)
wait_for_completion (52 ms, 5.61%)
wait..
__wait_rcu_gp (40 ms, 4.32%)
__..
finish_task_switch (115 ms, 12.42%)
finish_task..
finish_task_switch (40 ms, 4.32%)
fi..
schedule_timeout (12 ms, 1.30%)
down_write_killable (1 ms, 0.11%)
__wait_on_bit (49 ms, 5.29%)
__w..
call_rwsem_down_write_failed_killable (1 ms, 0.11%)
css_release_work_fn (50 ms, 5.40%)
css..
schedule_timeout (0 ms, 0.00%)
_do_fork (5 ms, 0.54%)
__mutex_lock_slowpath (16 ms, 1.73%)
rwsem_down_write_failed_killable (1 ms, 0.11%)
__wait_on_buffer (2 ms, 0.22%)
down_write_killable (0 ms, 0.00%)
schedule (40 ms, 4.32%)
sc..
memcg_destroy_kmem_caches (52 ms, 5.61%)
memc..
sys_mprotect (1 ms, 0.11%)
call_rwsem_down_write_failed_killable (1 ms, 0.11%)
worker_thread (89 ms, 9.61%)
worker_t..
css_free_work_fn (48 ms, 5.18%)
css..
ret_from_fork (95 ms, 10.26%)
ret_from_..
page_fault (0 ms, 0.00%)
finish_task_switch (1 ms, 0.11%)
schedule_timeout (0 ms, 0.00%)
rwsem_down_read_failed (5 ms, 0.54%)
schedule (5 ms, 0.54%)
__do_page_fault (3 ms, 0.32%)
css_killed_work_fn (95 ms, 10.26%)
css_kille..
finish_task_switch (50 ms, 5.40%)
fin..
commit_cowonly_roots (13 ms, 1.40%)
__wait_on_bit (12 ms, 1.30%)
finish_task_switch (5 ms, 0.54%)
css_free_work_fn (79 ms, 8.53%)
css_fre..
btrfs_sync_log (2 ms, 0.22%)
upload-systemd- (7 ms, 0.76%)
__filemap_fdatawait_range (1 ms, 0.11%)
finish_task_switch (44 ms, 4.75%)
fin..
_rcu_barrier (79 ms, 8.53%)
_rcu_ba..
__wait_rcu_gp (95 ms, 10.26%)
__wait_rc..
finish_task_switch (12 ms, 1.30%)
finish_task_switch (0 ms, 0.00%)
schedule_preempt_disabled (50 ms, 5.40%)
sch..
finish_task_switch (5 ms, 0.54%)
kthread (95 ms, 10.26%)
kthread
out_of_line_wait_on_bit (49 ms, 5.29%)
out..
__filemap_fdatawait_range (13 ms, 1.40%)
rwsem_down_write_failed_killable (1 ms, 0.11%)
worker_thread (102 ms, 11.01%)
worker_thr..
memcg_offline_kmem.part.38 (40 ms, 4.32%)
me..
systemd-watchdo (0 ms, 0.00%)
schedule_timeout (0 ms, 0.00%)
__filemap_fdatawait_range (7 ms, 0.76%)
page_fault (1 ms, 0.11%)
bit_wait_io (7 ms, 0.76%)
do_syscall_64 (6 ms, 0.65%)
schedule (1 ms, 0.11%)
memcg_deactivate_kmem_caches (95 ms, 10.26%)
memcg_dea..
kthread (0 ms, 0.00%)
schedule_preempt_disabled (48 ms, 5.18%)
sch..
rwsem_down_write_failed_killable (1 ms, 0.11%)
css_killed_work_fn (40 ms, 4.32%)
cs..
__do_page_fault (0 ms, 0.00%)
__wait_on_bit (13 ms, 1.40%)
__wait_on_bit (7 ms, 0.76%)
call_rwsem_down_write_failed_killable (0 ms, 0.00%)
finish_task_switch (1 ms, 0.11%)
process_one_work (79 ms, 8.53%)
process..
finish_task_switch (79 ms, 8.53%)
finish_..
__wait_rcu_gp (107 ms, 11.55%)
__wait_rcu..
mutex_lock (44 ms, 4.75%)
mut..
do_mprotect_pkey (1 ms, 0.11%)
wait_for_completion (79 ms, 8.53%)
wait_fo..
finish_task_switch (2 ms, 0.22%)
schedule (1 ms, 0.11%)
schedule (95 ms, 10.26%)
schedule
SyS_mmap_pgoff (1 ms, 0.11%)
call_rwsem_down_read_failed (3 ms, 0.32%)
schedule_timeout (95 ms, 10.26%)
schedule_..
_rcu_barrier (40 ms, 4.32%)
_r..
__do_page_fault (1 ms, 0.11%)
cgroup_release_agent (50 ms, 5.40%)
cgr..
__filemap_fdatawait_range (12 ms, 1.30%)
mem_cgroup_css_offline (27 ms, 2.92%)
m..
wait_for_completion (0 ms, 0.00%)
jbd2_journal_commit_transaction (49 ms, 5.29%)
jbd..
upload-mesos-me (4 ms, 0.43%)
btrfs_wait_marked_extents (7 ms, 0.76%)
vm_mmap_pgoff (1 ms, 0.11%)
schedule_timeout (79 ms, 8.53%)
schedul..
schedule (1 ms, 0.11%)
do_mprotect_pkey (0 ms, 0.00%)
mem_cgroup_css_free (40 ms, 4.32%)
me..
kthread (102 ms, 11.01%)
kthread
finish_task_switch (27 ms, 2.92%)
f..
call_usermodehelper_exec (0 ms, 0.00%)
schedule (0 ms, 0.00%)
mutex_lock (49 ms, 5.29%)
mut..
memcg_destroy_kmem_caches (79 ms, 8.53%)
memcg_d..
transaction_kthread (29 ms, 3.13%)
t..
btrfs_wait_ordered_range (13 ms, 1.40%)
rwsem_down_write_failed_killable (0 ms, 0.00%)
ret_from_fork (79 ms, 8.53%)
ret_fro..
ret_from_fork (108 ms, 11.66%)
ret_from_f..
css_free_work_fn (52 ms, 5.61%)
css_..
cgroup_release_agent (50 ms, 5.40%)
cgr..
schedule (50 ms, 5.40%)
sch..
kworker/7:1 (0 ms, 0.00%)
finish_task_switch (0 ms, 0.00%)
schedule_timeout (40 ms, 4.32%)
sc..
wait_on_page_bit (7 ms, 0.76%)
io_schedule_timeout (12 ms, 1.30%)
call_rwsem_down_write_failed_killable (0 ms, 0.00%)
finish_task_switch (40 ms, 4.32%)
fi..
kthread (27 ms, 2.92%)
k..
io_schedule_timeout (7 ms, 0.76%)
css_release_work_fn (44 ms, 4.75%)
css..
entry_SYSCALL_64_fastpath (115 ms, 12.42%)
entry_SYSCA..
worker_thread (90 ms, 9.72%)
worker_t..
schedule (5 ms, 0.54%)
kthread (79 ms, 8.53%)
kthread
call_usermodehelper_exec (0 ms, 0.00%)
wait_for_completion (0 ms, 0.00%)
filemap_fdatawait_range (12 ms, 1.30%)
seq_read (115 ms, 12.42%)
seq_read
do_page_fault (0 ms, 0.00%)
entry_SYSCALL_64_fastpath (2 ms, 0.22%)
SyS_mmap (1 ms, 0.11%)
kworker/0:1 (27 ms, 2.92%)
k..
down_read (1 ms, 0.11%)
entry_SYSCALL_64_fastpath (1 ms, 0.11%)
finish_task_switch (0 ms, 0.00%)
down_write_killable (0 ms, 0.00%)
filemap_fdatawait_range (1 ms, 0.11%)
btrfs_sync_file (2 ms, 0.22%)
out_of_line_wait_on_bit (2 ms, 0.22%)
rwsem_down_read_failed (0 ms, 0.00%)
cgroup_release_agent (0 ms, 0.00%)
call_usermodehelper (0 ms, 0.00%)
release_caches (79 ms, 8.53%)
release..
__mutex_lock_slowpath (115 ms, 12.42%)
__mutex_loc..
SyS_mmap (0 ms, 0.00%)
finish_task_switch (1 ms, 0.11%)
mem_cgroup_css_free (52 ms, 5.61%)
mem_..
schedule (50 ms, 5.40%)
sch..
sys_mprotect (0 ms, 0.00%)
mem_cgroup_css_free (48 ms, 5.18%)
mem..
filemap_write_and_wait_range (12 ms, 1.30%)
schedule (0 ms, 0.00%)
mutex_lock (50 ms, 5.40%)
mut..
schedule_preempt_disabled (49 ms, 5.29%)
sch..
finish_task_switch (13 ms, 1.40%)
schedule_timeout (5 ms, 0.54%)
finish_task_switch (1 ms, 0.11%)
schedule_timeout (52 ms, 5.61%)
sche..
cgroup_release_agent (0 ms, 0.00%)
io_schedule_timeout (13 ms, 1.40%)
finish_task_switch (107 ms, 11.55%)
finish_tas..
down_write_killable (0 ms, 0.00%)
release_caches (48 ms, 5.18%)
rel..
schedule (3 ms, 0.32%)
sys_mprotect (1 ms, 0.11%)
io_schedule_timeout (49 ms, 5.29%)
io_..
schedule_timeout (0 ms, 0.00%)
wait_for_completion (0 ms, 0.00%)
wait_for_completion (95 ms, 10.26%)
wait_for_..
kworker/1:0 (79 ms, 8.53%)
kworker..
call_rwsem_down_read_failed (5 ms, 0.54%)
finish_task_switch (0 ms, 0.00%)
vfs_fsync_range (12 ms, 1.30%)
rwsem_down_write_failed_killable (0 ms, 0.00%)
finish_task_switch (16 ms, 1.73%)
filemap_fdatawait_range (7 ms, 0.76%)
mem_cgroup_css_free (79 ms, 8.53%)
mem_cgr..
all (926 ms, 100%)
schedule_timeout (2 ms, 0.22%)
page_fault (1 ms, 0.11%)
ext4_sync_file (12 ms, 1.30%)
schedule (7 ms, 0.76%)
mutex_lock (50 ms, 5.40%)
mut..
finish_task_switch (0 ms, 0.00%)
rwsem_down_read_failed (3 ms, 0.32%)
vfs_read (115 ms, 12.42%)
vfs_read
down_read (5 ms, 0.54%)
rcu_barrier (79 ms, 8.53%)
rcu_bar..
schedule (0 ms, 0.00%)
schedule_timeout (107 ms, 11.55%)
schedule_t..
wait_for_completion (107 ms, 11.55%)
wait_for_c..
return_from_SYSCALL_64 (6 ms, 0.65%)
__mutex_lock_slowpath (44 ms, 4.75%)
__m..
wait_for_completion (40 ms, 4.32%)
wa..
schedule (49 ms, 5.29%)
sch..
finish_task_switch (7 ms, 0.76%)
schedule (1 ms, 0.11%)
__mutex_lock_slowpath (50 ms, 5.40%)
__m..
page_fault (3 ms, 0.32%)
release_caches (52 ms, 5.61%)
rele..
bit_wait_io (1 ms, 0.11%)
do_fsync (12 ms, 1.30%)
finish_task_switch (1 ms, 0.11%)
do_page_fault (5 ms, 0.54%)
synchronize_sched (27 ms, 2.92%)
s..
entry_SYSCALL_64_fastpath (0 ms, 0.00%)
cgroup_release_agent (16 ms, 1.73%)
worker_thread (95 ms, 10.26%)
worker_th..
schedule (16 ms, 1.73%)
wait_for_completion_killable (5 ms, 0.54%)
schedule (12 ms, 1.30%)
btrfs_wait_marked_extents (1 ms, 0.11%)
schedule (115 ms, 12.42%)
schedule
kworker/6:2 (107 ms, 11.55%)
kworker/6:2
schedule (27 ms, 2.92%)
s..
rcu_barrier (40 ms, 4.32%)
rc..
synchronize_sched (107 ms, 11.55%)
synchroniz..
__wait_on_buffer (49 ms, 5.29%)
__w..
__do_page_fault (5 ms, 0.54%)
__wait_on_bit (1 ms, 0.11%)
_rcu_barrier (52 ms, 5.61%)
_rcu..
schedule (52 ms, 5.61%)
sche..
call_rwsem_down_write_failed_killable (0 ms, 0.00%)
memcg_offline_kmem.part.38 (27 ms, 2.92%)
m..
finish_task_switch (0 ms, 0.00%)
process_one_work (27 ms, 2.92%)
p..
out_of_line_wait_on_bit (1 ms, 0.11%)
memcg_offline_kmem.part.38 (95 ms, 10.26%)
memcg_off..
mutex_lock (16 ms, 1.73%)
finish_task_switch (0 ms, 0.00%)
schedule_timeout (0 ms, 0.00%)
wait_on_page_bit (1 ms, 0.11%)
schedule (1 ms, 0.11%)
btrfs_commit_transaction (29 ms, 3.13%)
b..
app-reaper (1 ms, 0.11%)
schedule (0 ms, 0.00%)
schedule_timeout (1 ms, 0.11%)
java (8 ms, 0.86%)
css_free_work_fn (40 ms, 4.32%)
cs..
kthread (29 ms, 3.13%)
k..
schedule_timeout (7 ms, 0.76%)
down_read (1 ms, 0.11%)
kthread (107 ms, 11.55%)
kthread
io_schedule_timeout (7 ms, 0.76%)
schedule (49 ms, 5.29%)
sch..
process_one_work (89 ms, 9.61%)
process_..
schedule (0 ms, 0.00%)
entry_SYSCALL_64_fastpath (1 ms, 0.11%)
io_schedule_timeout (1 ms, 0.11%)
schedule_preempt_disabled (44 ms, 4.75%)
sch..
kjournald2 (49 ms, 5.29%)
kjo..
down_read (0 ms, 0.00%)
btrfs_wait_cache_io (13 ms, 1.40%)
entry_SYSCALL_64_fastpath (12 ms, 1.30%)
memcg_destroy_kmem_caches (48 ms, 5.18%)
mem..
memcg_deactivate_kmem_caches (40 ms, 4.32%)
me..
schedule (48 ms, 5.18%)
sch..
write_ctree_super (1 ms, 0.11%)
btrfs_start_dirty_block_groups (7 ms, 0.76%)
down_write_killable (1 ms, 0.11%)
sys_mprotect (0 ms, 0.00%)
schedule_timeout (0 ms, 0.00%)
finish_task_switch (7 ms, 0.76%)
__do_page_fault (1 ms, 0.11%)
bit_wait_io (2 ms, 0.22%)
__wait_on_bit (2 ms, 0.22%)
app-log-shipp (12 ms, 1.30%)
rwsem_down_write_failed_killable (0 ms, 0.00%)
worker_thread (108 ms, 11.66%)
worker_thr..
schedule_timeout (40 ms, 4.32%)
sc..
finish_task_switch (3 ms, 0.32%)
kworker/2:1 (90 ms, 9.72%)
kworker/..
schedule (44 ms, 4.75%)
sch..
call_rwsem_down_write_failed_killable (0 ms, 0.00%)
__wait_on_bit (1 ms, 0.11%)
call_rwsem_down_write_failed_killable (0 ms, 0.00%)
finish_task_switch (0 ms, 0.00%)
kthread (90 ms, 9.72%)
kthread
ret_from_fork (102 ms, 11.01%)
ret_from_f..
io_schedule_timeout (1 ms, 0.11%)
__mutex_lock_slowpath (49 ms, 5.29%)
__m..
schedule (107 ms, 11.55%)
schedule
call_usermodehelper_exec (0 ms, 0.00%)
call_usermodehelper_exec (0 ms, 0.00%)
btrfs_wait_ordered_range (7 ms, 0.76%)
write_ctree_super (2 ms, 0.22%)
rcu_barrier (48 ms, 5.18%)
rcu..
schedule (1 ms, 0.11%)
proc_single_show (115 ms, 12.42%)
proc_single..
do_mprotect_pkey (1 ms, 0.11%)
__vfs_read (115 ms, 12.42%)
__vfs_read
ret_from_fork (89 ms, 9.61%)
ret_from..
schedule (79 ms, 8.53%)
schedule
down_write_killable (1 ms, 0.11%)
worker_thread (79 ms, 8.53%)
worker_..
__filemap_fdatawait_range (7 ms, 0.76%)
ret_from_fork (0 ms, 0.00%)