get an error while doing ls -R

[sandipdivekar]

1. kernel panic while doing ls -R

first I created 1000 directory and 150000 files in each directory .
kernel panic when I was tried for ls -R .

$ uname -a
Linux kqinfo 2.6.32-44.2.el6.x86_64 #1 SMP Wed Jul 21 12:48:32 EDT 2010 x86_64 x86_64 x86_64 GNU/Linux

[sandipdivekar]

error is like this -

$ dmesg

INFO: task spa_zio_write_i:3351 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
spa_zio_write D ffffffffffffffff 0 3351 2 0x00000080
ffff8800508895e0 0000000000000046 0000000000000000 0000000000000046
0000000000000010 362cc33e8101a4d9 0000000000000cc2 ffff8800362cc33e
ffff88005005a630 ffff880050889fd8 00000000000103b8 ffff88005005a630
Call Trace:
[] __mutex_lock_slowpath+0x13e/0x180
[] mutex_lock+0x2b/0x50
[] zfs_zinactive+0x4b/0x100 [zfs]
[] zfs_inactive+0x62/0x1e0 [zfs]
[] lzfs_clear_vnode+0x144/0x1b0 [lzfs]
[] clear_inode+0x8f/0x110
[] dispose_list+0x40/0x120
[] shrink_icache_memory+0x274/0x2e0
[] shrink_slab+0x125/0x190
[] do_try_to_free_pages+0x2af/0x4d0
[] try_to_free_pages+0x70/0x80
[] ? isolate_pages_global+0x0/0x250
[] __alloc_pages_nodemask+0x382/0x7e0
[] kmem_getpages+0x62/0x170
[] fallback_alloc+0x19a/0x240
[] ? cache_grow+0x2cf/0x320
[] ____cache_alloc_node+0x99/0x160
[] ? kmem_alloc_debug+0xcb/0xf0 [spl]
[] __kmalloc+0x189/0x220
[] kmem_alloc_debug+0xcb/0xf0 [spl]
[] __cv_init+0x9a/0xc0 [spl]
[] zio_create+0xa4/0x450 [zfs]
[] zio_vdev_delegated_io+0x70/0x80 [zfs]
[] ? vdev_queue_agg_io_done+0x0/0x80 [zfs]
[] vdev_queue_io_to_issue+0x42c/0x550 [zfs]
[] ? vdev_queue_agg_io_done+0x0/0x80 [zfs]
[] vdev_queue_io_done+0x82/0xd0 [zfs]
[] zio_vdev_io_done+0x88/0x190 [zfs]
[] zio_execute+0x72/0x100 [zfs]
[] taskq_thread+0x18e/0x2a0 [spl]
[] ? default_wake_function+0x0/0x20
[] ? taskq_thread+0x0/0x2a0 [spl]
[] kthread+0x96/0xa0
[] child_rip+0xa/0x20
[] ? kthread+0x0/0xa0
[] ? child_rip+0x0/0x20
INFO: task spa_zio_write_i:3354 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
spa_zio_write D ffffffffffffffff 0 3354 2 0x00000080
ffff8800510f3d00 0000000000000046 00000000cbe1ac00 ffffc90023b36800
0000000000001400 00000010a03ad190 ffff880037db8440 0000000000000200
ffff880050886670 ffff8800510f3fd8 00000000000103b8 ffff880050886670
Call Trace:
[] __mutex_lock_slowpath+0x13e/0x180
[] mutex_lock+0x2b/0x50
[] vdev_queue_io_done+0x30/0xd0 [zfs]
[] zio_vdev_io_done+0x88/0x190 [zfs]
[] zio_execute+0x72/0x100 [zfs]
[] taskq_thread+0x18e/0x2a0 [spl]
[] ? default_wake_function+0x0/0x20
[] ? taskq_thread+0x0/0x2a0 [spl]
[] kthread+0x96/0xa0
[] child_rip+0xa/0x20
[] ? kthread+0x0/0xa0
[] ? child_rip+0x0/0x20
INFO: task spa_zio_write_i:3355 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
spa_zio_write D ffffffffffffffff 0 3355 2 0x00000080
ffff88004ff89d00 0000000000000046 0000000000000000 ffff880036b44988
ffff88004ff89c80 ffffffffa036c35e ffffc900008f21f0 0000000000000202
ffff8800510dfaf0 ffff88004ff89fd8 00000000000103b8 ffff8800510dfaf0
Call Trace:
[] ? kmem_free_debug+0x2e/0x40 [spl]
[] __mutex_lock_slowpath+0x13e/0x180
[] mutex_lock+0x2b/0x50
[] vdev_queue_io_done+0x30/0xd0 [zfs]
[] zio_vdev_io_done+0x88/0x190 [zfs]
[] zio_execute+0x72/0x100 [zfs]
[] taskq_thread+0x18e/0x2a0 [spl]
[] ? default_wake_function+0x0/0x20
[] ? taskq_thread+0x0/0x2a0 [spl]
[] kthread+0x96/0xa0
[] child_rip+0xa/0x20
[] ? kthread+0x0/0xa0
[] ? child_rip+0x0/0x20
INFO: task spa_zio_write_i:3356 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
spa_zio_write D ffffffffffffffff 0 3356 2 0x00000080
ffff880051225d00 0000000000000046 0000000000000000 ffff880017fdff60
ffff880051225c80 ffffffffa036c35e ffffc90013b639b0 0000000000000202
ffff8800510df0d0 ffff880051225fd8 00000000000103b8 ffff8800510df0d0
Call Trace:
[] ? kmem_free_debug+0x2e/0x40 [spl]
[] __mutex_lock_slowpath+0x13e/0x180
[] mutex_lock+0x2b/0x50
[] vdev_queue_io_done+0x30/0xd0 [zfs]
[] zio_vdev_io_done+0x88/0x190 [zfs]
[] zio_execute+0x72/0x100 [zfs]
[] taskq_thread+0x18e/0x2a0 [spl]
[] ? default_wake_function+0x0/0x20
[] ? taskq_thread+0x0/0x2a0 [spl]
[] kthread+0x96/0xa0
[] child_rip+0xa/0x20
[] ? kthread+0x0/0xa0
[] ? child_rip+0x0/0x20
INFO: task spa_zio_write_i:3357 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
spa_zio_write D ffffffffffffffff 0 3357 2 0x00000080
ffff880050c15d00 0000000000000046 0000000000000000 ffff880018d1bb40
ffff880050c15c80 ffffffffa036c35e ffffc9000d6343a0 0000000000000202
ffff8800510de6b0 ffff880050c15fd8 00000000000103b8 ffff8800510de6b0
Call Trace:
[] ? kmem_free_debug+0x2e/0x40 [spl]
[] __mutex_lock_slowpath+0x13e/0x180
[] mutex_lock+0x2b/0x50
[] vdev_queue_io_done+0x30/0xd0 [zfs]
[] zio_vdev_io_done+0x88/0x190 [zfs]
[] zio_execute+0x72/0x100 [zfs]
[] taskq_thread+0x18e/0x2a0 [spl]
[] ? default_wake_function+0x0/0x20
[] ? taskq_thread+0x0/0x2a0 [spl]
[] kthread+0x96/0xa0
[] child_rip+0xa/0x20
[] ? kthread+0x0/0xa0
[] ? child_rip+0x0/0x20