zram stacktraces -> 3.16.0-0.rc3.git3.1.fc21.x86_64

[Reindl Harald]

3.16.0-0.rc3.git3.1.fc21.x86_64 seems to have a problem with zram
the trace below happens often due "dnf clean all && dnf upgrade"
is called the first time after a reboot

[root@rawhide ~]# dmesg -c
[51104.945865] =========================================================
[51104.945900] [ INFO: possible irq lock inversion dependency detected ]
[51104.945934] 3.16.0-0.rc3.git3.1.fc21.x86_64 #1 Tainted: G        W
[51104.945967] ---------------------------------------------------------
[51104.946000] kswapd0/58 just changed the state of lock:
[51104.946028]  (&zram->init_lock){++++.-}, at: [<ffffffffa0042696>]
zram_make_request+0x46/0x350 [zram]
[51104.946162] but this lock took another, RECLAIM_FS-unsafe lock in the past:
[51104.946197]  (&bdev->bd_mutex){+.+.+.}

and interrupts could create inverse lock ordering between them.

[51104.946278]
other info that might help us debug this:
[51104.946305]  Possible interrupt unsafe locking scenario:

[51104.946334]        CPU0                    CPU1
[51104.946358]        ----                    ----
[51104.946543]   lock(&bdev->bd_mutex);
[51104.946599]                                local_irq_disable();
[51104.946676]                                lock(&zram->init_lock);
[51104.946773]                                lock(&bdev->bd_mutex);
[51104.946869]   <Interrupt>
[51104.946926]     lock(&zram->init_lock);
[51104.947002]
 *** DEADLOCK ***

[51104.947109] no locks held by kswapd0/58.
[51104.947172]
the shortest dependencies between 2nd lock and 1st lock:
[51104.947281]  -> (&bdev->bd_mutex){+.+.+.} ops: 227 {
[51104.947547]     HARDIRQ-ON-W at:
[51104.947610]                       [<ffffffff811004c2>]
__lock_acquire+0x942/0x1ca0
[51104.947701]                       [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.947781]                       [<ffffffff8180ccd5>]
mutex_lock_nested+0x85/0x440
[51104.947868]                       [<ffffffff8128dbd6>] __blkdev_get+0x76/0x4e0
[51104.947953]                       [<ffffffff8128e090>] blkdev_get+0x50/0x400
[51104.948032]                       [<ffffffff813c6aff>] add_disk+0x46f/0x510
[51104.948116]                       [<ffffffff8155b005>]
sd_probe_async+0x115/0x1e0
[51104.948200]                       [<ffffffff810d1319>]
async_run_entry_fn+0x39/0x120
[51104.948287]                       [<ffffffff810c0561>]
process_one_work+0x211/0x6f0
[51104.948371]                       [<ffffffff810c0aab>] worker_thread+0x6b/0x540
[51104.948598]                       [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.948677]                       [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0
[51104.948758]     SOFTIRQ-ON-W at:
[51104.948822]                       [<ffffffff811004f2>]
__lock_acquire+0x972/0x1ca0
[51104.948905]                       [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.948985]                       [<ffffffff8180ccd5>]
mutex_lock_nested+0x85/0x440
[51104.949067]                       [<ffffffff8128dbd6>] __blkdev_get+0x76/0x4e0
[51104.949147]                       [<ffffffff8128e090>] blkdev_get+0x50/0x400
[51104.949242]                       [<ffffffff813c6aff>] add_disk+0x46f/0x510
[51104.949321]                       [<ffffffff8155b005>]
sd_probe_async+0x115/0x1e0
[51104.949548]                       [<ffffffff810d1319>]
async_run_entry_fn+0x39/0x120
[51104.949631]                       [<ffffffff810c0561>]
process_one_work+0x211/0x6f0
[51104.949713]                       [<ffffffff810c0aab>] worker_thread+0x6b/0x540
[51104.949793]                       [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.949872]                       [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0
[51104.949952]     RECLAIM_FS-ON-W at:
[51104.950016]                          [<ffffffff810ff59c>]
mark_held_locks+0x7c/0xb0
[51104.950099]                          [<ffffffff81102bcb>]
lockdep_trace_alloc+0x8b/0xf0
[51104.950182]                          [<ffffffff81224a76>]
kmem_cache_alloc_trace+0x36/0x330
[51104.950270]                          [<ffffffff81559106>]
sd_revalidate_disk+0x86/0x1e70
[51104.950354]                          [<ffffffff813c92d1>]
rescan_partitions+0x81/0x2d0
[51104.950596]                          [<ffffffff8128ded3>]
__blkdev_get+0x373/0x4e0
[51104.950675]                          [<ffffffff8128e090>] blkdev_get+0x50/0x400
[51104.950754]                          [<ffffffff813c6aff>] add_disk+0x46f/0x510
[51104.950832]                          [<ffffffff8155b005>]
sd_probe_async+0x115/0x1e0
[51104.950912]                          [<ffffffff810d1319>]
async_run_entry_fn+0x39/0x120
[51104.951015]                          [<ffffffff810c0561>]
process_one_work+0x211/0x6f0
[51104.951096]                          [<ffffffff810c0aab>]
worker_thread+0x6b/0x540
[51104.951175]                          [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.951256]                          [<ffffffff8181183c>]
ret_from_fork+0x7c/0xb0
[51104.951336]     INITIAL USE at:
[51104.951542]                      [<ffffffff810fffb7>]
__lock_acquire+0x437/0x1ca0
[51104.951621]                      [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.951720]                      [<ffffffff8180ccd5>]
mutex_lock_nested+0x85/0x440
[51104.951800]                      [<ffffffff8128dbd6>] __blkdev_get+0x76/0x4e0
[51104.951877]                      [<ffffffff8128e090>] blkdev_get+0x50/0x400
[51104.951954]                      [<ffffffff813c6aff>] add_disk+0x46f/0x510
[51104.952031]                      [<ffffffff8155b005>] sd_probe_async+0x115/0x1e0
[51104.952110]                      [<ffffffff810d1319>]
async_run_entry_fn+0x39/0x120
[51104.952212]                      [<ffffffff810c0561>]
process_one_work+0x211/0x6f0
[51104.952304]                      [<ffffffff810c0aab>] worker_thread+0x6b/0x540
[51104.952382]                      [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.952604]                      [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0
[51104.952682]   }
[51104.952725]   ... key      at: [<ffffffff832aadb8>] __key.34230+0x0/0x8
[51104.952806]   ... acquired at:
[51104.952854]    [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.952926]    [<ffffffff8180ccd5>] mutex_lock_nested+0x85/0x440
[51104.953021]    [<ffffffff8128d1eb>] revalidate_disk+0x4b/0x80
[51104.953093]    [<ffffffffa0041e6b>] disksize_store+0x11b/0x1f0 [zram]
[51104.953168]    [<ffffffff8151a598>] dev_attr_store+0x18/0x30
[51104.953242]    [<ffffffff812d8c35>] sysfs_kf_write+0x45/0x60
[51104.953317]    [<ffffffff812d7f99>] kernfs_fop_write+0xf9/0x180
[51104.953547]    [<ffffffff8124b9ea>] vfs_write+0xba/0x200
[51104.953625]    [<ffffffff8124c65c>] SyS_write+0x5c/0xd0
[51104.953699]    [<ffffffff818118e9>] system_call_fastpath+0x16/0x1b

[51104.953826] -> (&zram->init_lock){++++.-} ops: 299 {
[51104.953942]    HARDIRQ-ON-W at:
[51104.954003]                     [<ffffffff811004c2>] __lock_acquire+0x942/0x1ca0
[51104.954082]                     [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.954159]                     [<ffffffff8180f52a>] down_write+0x5a/0xc0
[51104.954246]                     [<ffffffffa0041e38>] disksize_store+0xe8/0x1f0
[zram]
[51104.954349]                     [<ffffffff8151a598>] dev_attr_store+0x18/0x30
[51104.954571]                     [<ffffffff812d8c35>] sysfs_kf_write+0x45/0x60
[51104.954649]                     [<ffffffff812d7f99>] kernfs_fop_write+0xf9/0x180
[51104.954728]                     [<ffffffff8124b9ea>] vfs_write+0xba/0x200
[51104.954804]                     [<ffffffff8124c65c>] SyS_write+0x5c/0xd0
[51104.954880]                     [<ffffffff818118e9>]
system_call_fastpath+0x16/0x1b
[51104.954960]    HARDIRQ-ON-R at:
[51104.955031]                     [<ffffffff810fff23>] __lock_acquire+0x3a3/0x1ca0
[51104.955120]                     [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.955197]                     [<ffffffff8180f481>] down_read+0x51/0xa0
[51104.955273]                     [<ffffffffa0042696>] zram_make_request+0x46/0x350
[zram]
[51104.955354]                     [<ffffffff813b4088>]
generic_make_request+0xf8/0x150
[51104.955586]                     [<ffffffff813b4158>] submit_bio+0x78/0x190
[51104.955662]                     [<ffffffff812880e0>] _submit_bh+0x140/0x230
[51104.955751]                     [<ffffffff81289daf>]
block_read_full_page+0x28f/0x350
[51104.955864]                     [<ffffffff8128c9b8>] blkdev_readpage+0x18/0x20
[51104.955942]                     [<ffffffff811d151c>]
__do_page_cache_readahead+0x32c/0x340
[51104.956029]                     [<ffffffff811d1a74>]
force_page_cache_readahead+0x34/0x60
[51104.956112]                     [<ffffffff811d1ae6>]
page_cache_sync_readahead+0x46/0x50
[51104.956194]                     [<ffffffff811c32a9>]
generic_file_read_iter+0x519/0x630
[51104.956276]                     [<ffffffff8128cf67>] blkdev_read_iter+0x37/0x40
[51104.956366]                     [<ffffffff8124b07b>] new_sync_read+0x8b/0xd0
[51104.956603]                     [<ffffffff8124b847>] vfs_read+0x97/0x180
[51104.956679]                     [<ffffffff8124c58c>] SyS_read+0x5c/0xd0
[51104.956755]                     [<ffffffff818118e9>]
system_call_fastpath+0x16/0x1b
[51104.956835]    SOFTIRQ-ON-W at:
[51104.956895]                     [<ffffffff811004f2>] __lock_acquire+0x972/0x1ca0
[51104.956974]                     [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.957061]                     [<ffffffff8180f52a>] down_write+0x5a/0xc0
[51104.957148]                     [<ffffffffa0041e38>] disksize_store+0xe8/0x1f0
[zram]
[51104.957229]                     [<ffffffff8151a598>] dev_attr_store+0x18/0x30
[51104.957306]                     [<ffffffff812d8c35>] sysfs_kf_write+0x45/0x60
[51104.957384]                     [<ffffffff812d7f99>] kernfs_fop_write+0xf9/0x180
[51104.957608]                     [<ffffffff8124b9ea>] vfs_write+0xba/0x200
[51104.957684]                     [<ffffffff8124c65c>] SyS_write+0x5c/0xd0
[51104.957760]                     [<ffffffff818118e9>]
system_call_fastpath+0x16/0x1b
[51104.957862]    SOFTIRQ-ON-R at:
[51104.957922]                     [<ffffffff811004f2>] __lock_acquire+0x972/0x1ca0
[51104.958001]                     [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.958078]                     [<ffffffff8180f481>] down_read+0x51/0xa0
[51104.958153]                     [<ffffffffa0042696>] zram_make_request+0x46/0x350
[zram]
[51104.958235]                     [<ffffffff813b4088>]
generic_make_request+0xf8/0x150
[51104.958315]                     [<ffffffff813b4158>] submit_bio+0x78/0x190
[51104.958577]                     [<ffffffff812880e0>] _submit_bh+0x140/0x230
[51104.958688]                     [<ffffffff81289daf>]
block_read_full_page+0x28f/0x350
[51104.958769]                     [<ffffffff8128c9b8>] blkdev_readpage+0x18/0x20
[51104.958847]                     [<ffffffff811d151c>]
__do_page_cache_readahead+0x32c/0x340
[51104.958930]                     [<ffffffff811d1a74>]
force_page_cache_readahead+0x34/0x60
[51104.959011]                     [<ffffffff811d1ae6>]
page_cache_sync_readahead+0x46/0x50
[51104.959127]                     [<ffffffff811c32a9>]
generic_file_read_iter+0x519/0x630
[51104.959209]                     [<ffffffff8128cf67>] blkdev_read_iter+0x37/0x40
[51104.959287]                     [<ffffffff8124b07b>] new_sync_read+0x8b/0xd0
[51104.959364]                     [<ffffffff8124b847>] vfs_read+0x97/0x180
[51104.959607]                     [<ffffffff8124c58c>] SyS_read+0x5c/0xd0
[51104.959684]                     [<ffffffff818118e9>]
system_call_fastpath+0x16/0x1b
[51104.959764]    IN-RECLAIM_FS-R at:
[51104.959837]                        [<ffffffff810fff8f>]
__lock_acquire+0x40f/0x1ca0
[51104.959927]                        [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.960005]                        [<ffffffff8180f481>] down_read+0x51/0xa0
[51104.960082]                        [<ffffffffa0042696>]
zram_make_request+0x46/0x350 [zram]
[51104.960165]                        [<ffffffff813b4088>]
generic_make_request+0xf8/0x150
[51104.960268]                        [<ffffffff813b4158>] submit_bio+0x78/0x190
[51104.960345]                        [<ffffffff812097ba>]
__swap_writepage+0x24a/0x2a0
[51104.960596]                        [<ffffffff81209849>] swap_writepage+0x39/0x80
[51104.960678]                        [<ffffffff811df1d0>]
shmem_writepage+0x1b0/0x2e0
[51104.960761]                        [<ffffffff811d8ef1>]
shrink_page_list+0x541/0xca0
[51104.960844]                        [<ffffffff811d9d2d>]
shrink_inactive_list+0x1bd/0x600
[51104.960929]                        [<ffffffff811daa63>]
shrink_lruvec+0x533/0x6e0
[51104.961015]                        [<ffffffff811dac9c>] shrink_zone+0x8c/0x1d0
[51104.961106]                        [<ffffffff811dbbcb>]
kswapd_shrink_zone+0x10b/0x1f0
[51104.961202]                        [<ffffffff811dc609>]
balance_pgdat+0x329/0x4c0
[51104.961281]                        [<ffffffff811dc95b>] kswapd+0x1bb/0x620
[51104.961357]                        [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.961576]                        [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0
[51104.961655]    INITIAL USE at:
[51104.961716]                    [<ffffffff810fffb7>] __lock_acquire+0x437/0x1ca0
[51104.961794]                    [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.961893]                    [<ffffffff8180f52a>] down_write+0x5a/0xc0
[51104.961969]                    [<ffffffffa0041e38>] disksize_store+0xe8/0x1f0
[zram]
[51104.962049]                    [<ffffffff8151a598>] dev_attr_store+0x18/0x30
[51104.962126]                    [<ffffffff812d8c35>] sysfs_kf_write+0x45/0x60
[51104.962204]                    [<ffffffff812d7f99>] kernfs_fop_write+0xf9/0x180
[51104.962295]                    [<ffffffff8124b9ea>] vfs_write+0xba/0x200
[51104.962373]                    [<ffffffff8124c65c>] SyS_write+0x5c/0xd0
[51104.962599]                    [<ffffffff818118e9>]
system_call_fastpath+0x16/0x1b
[51104.962690]  }
[51104.962733]  ... key      at: [<ffffffffa0044780>]
__key.32748+0x0/0xffffffffffffe880 [zram]
[51104.962816]  ... acquired at:
[51104.962864]    [<ffffffff810fe6a0>] check_usage_forwards+0x170/0x180
[51104.962938]    [<ffffffff810ff240>] mark_lock+0x3f0/0x6d0
[51104.963009]    [<ffffffff810fff8f>] __lock_acquire+0x40f/0x1ca0
[51104.963081]    [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.963151]    [<ffffffff8180f481>] down_read+0x51/0xa0
[51104.963243]    [<ffffffffa0042696>] zram_make_request+0x46/0x350 [zram]
[51104.963611]    [<ffffffff813b4088>] generic_make_request+0xf8/0x150
[51104.963685]    [<ffffffff813b4158>] submit_bio+0x78/0x190
[51104.963756]    [<ffffffff812097ba>] __swap_writepage+0x24a/0x2a0
[51104.963829]    [<ffffffff81209849>] swap_writepage+0x39/0x80
[51104.963912]    [<ffffffff811df1d0>] shmem_writepage+0x1b0/0x2e0
[51104.963996]    [<ffffffff811d8ef1>] shrink_page_list+0x541/0xca0
[51104.964069]    [<ffffffff811d9d2d>] shrink_inactive_list+0x1bd/0x600
[51104.964144]    [<ffffffff811daa63>] shrink_lruvec+0x533/0x6e0
[51104.964215]    [<ffffffff811dac9c>] shrink_zone+0x8c/0x1d0
[51104.964298]    [<ffffffff811dbbcb>] kswapd_shrink_zone+0x10b/0x1f0
[51104.964372]    [<ffffffff811dc609>] balance_pgdat+0x329/0x4c0
[51104.964586]    [<ffffffff811dc95b>] kswapd+0x1bb/0x620
[51104.964678]    [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.964748]    [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0

[51104.964862]
stack backtrace:
[51104.964941] CPU: 0 PID: 58 Comm: kswapd0 Tainted: G        W    
3.16.0-0.rc3.git3.1.fc21.x86_64 #1
[51104.965044] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference
Platform, BIOS 6.00
07/31/2013
[51104.965154]  0000000000000000 000000008d3cfd17 ffff88001acaf390 ffffffff81807cec
[51104.965291]  ffffffff82f7a440 ffff88001acaf3d0 ffffffff81805252 ffffffff81c67a5f
[51104.965552]  ffff88001ac43f08 ffff88001ac433a0 ffffffff81c67a5f 0000000000000000
[51104.965668] Call Trace:
[51104.965719]  [<ffffffff81807cec>] dump_stack+0x4d/0x66
[51104.965778]  [<ffffffff81805252>] print_irq_inversion_bug.part.42+0x1b8/0x1c4
[51104.965845]  [<ffffffff810fe6a0>] check_usage_forwards+0x170/0x180
[51104.965908]  [<ffffffff810ff240>] mark_lock+0x3f0/0x6d0
[51104.965989]  [<ffffffff810fe530>] ? print_shortest_lock_dependencies+0x1d0/0x1d0
[51104.966056]  [<ffffffff810fff8f>] __lock_acquire+0x40f/0x1ca0
[51104.966121]  [<ffffffff81024369>] ? sched_clock+0x9/0x10
[51104.966180]  [<ffffffff810e1e05>] ? local_clock+0x25/0x30
[51104.966240]  [<ffffffff813d828d>] ? blk_throtl_bio+0x26d/0x640
[51104.966301]  [<ffffffff813d80a0>] ? blk_throtl_bio+0x80/0x640
[51104.966362]  [<ffffffff81102104>] lock_acquire+0xa4/0x1d0
[51104.966581]  [<ffffffffa0042696>] ? zram_make_request+0x46/0x350 [zram]
[51104.966648]  [<ffffffff8180f481>] down_read+0x51/0xa0
[51104.966718]  [<ffffffffa0042696>] ? zram_make_request+0x46/0x350 [zram]
[51104.966782]  [<ffffffffa0042696>] zram_make_request+0x46/0x350 [zram]
[51104.966846]  [<ffffffff813b3d19>] ? generic_make_request_checks+0x349/0x5c0
[51104.966912]  [<ffffffff81024369>] ? sched_clock+0x9/0x10
[51104.966971]  [<ffffffff810e1e05>] ? local_clock+0x25/0x30
[51104.967030]  [<ffffffff813b4088>] generic_make_request+0xf8/0x150
[51104.967093]  [<ffffffff813b4158>] submit_bio+0x78/0x190
[51104.967152]  [<ffffffff812097ba>] __swap_writepage+0x24a/0x2a0
[51104.967213]  [<ffffffff810e1e05>] ? local_clock+0x25/0x30
[51104.967295]  [<ffffffff810fc24f>] ? lock_release_holdtime.part.28+0xf/0x200
[51104.967362]  [<ffffffff81810a67>] ? _raw_spin_unlock+0x27/0x40
[51104.967568]  [<ffffffff8120cd4c>] ? page_swapcount+0x4c/0x60
[51104.967629]  [<ffffffff81209849>] swap_writepage+0x39/0x80
[51104.967691]  [<ffffffff811df1d0>] shmem_writepage+0x1b0/0x2e0
[51104.967752]  [<ffffffff811d8ef1>] shrink_page_list+0x541/0xca0
[51104.967814]  [<ffffffff811d9d2d>] shrink_inactive_list+0x1bd/0x600
[51104.967876]  [<ffffffff811daa63>] shrink_lruvec+0x533/0x6e0
[51104.967936]  [<ffffffff811dac9c>] shrink_zone+0x8c/0x1d0
[51104.968017]  [<ffffffff811dbbcb>] kswapd_shrink_zone+0x10b/0x1f0
[51104.968079]  [<ffffffff811dc609>] balance_pgdat+0x329/0x4c0
[51104.968139]  [<ffffffff811dc95b>] kswapd+0x1bb/0x620
[51104.968196]  [<ffffffff810f54c0>] ? abort_exclusive_wait+0xb0/0xb0
[51104.968259]  [<ffffffff811dc7a0>] ? balance_pgdat+0x4c0/0x4c0
[51104.968320]  [<ffffffff810c8d48>] kthread+0x108/0x120
[51104.968378]  [<ffffffff810ff72d>] ? trace_hardirqs_on_caller+0x15d/0x200
[51104.968589]  [<ffffffff810c8c40>] ? insert_kthread_work+0x80/0x80
[51104.968651]  [<ffffffff8181183c>] ret_from_fork+0x7c/0xb0
[51104.968722]  [<ffffffff810c8c40>] ? insert_kthread_work+0x80/0x80