Untitled

[155079.802542] ------------[ cut here ]------------

[155079.802575] btree trans held srcu lock (delaying memory reclaim) for 14 seconds

[155079.802634] WARNING: CPU: 4 PID: 2111010 at fs/bcachefs/btree_iter.c:3212 bch2_trans_srcu_unlock+0x14c/0x170 [bcachefs]

[155079.809922] Modules linked in: ceph libceph netfs bcachefs chacha_neon libchacha poly1305_neon lz4hc_compress lz4_compress dm_mod sd_mod sg iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi scsi_mod scsi_common binfmt_misc nls_ascii nls_cp437 vfat fat aes_ce_blk aes_ce_cipher polyval_ce polyval_generic ghash_ce gf128mul sha2_ce evdev sha256_arm64 sha1_ce arm_smccc_trng button virtio_console efi_pstore configfs nfnetlink vsock_loopback vmw_vsock_virtio_transport_common vsock efivarfs qemu_fw_cfg virtio_rng ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 btrfs blake2b_generic xor xor_neon raid6_pq xhci_pci xhci_hcd usbcore virtio_net net_failover virtio_blk usb_common failover

[155079.822033] CPU: 4 UID: 0 PID: 2111010 Comm: bch-copygc/2ae4 Tainted: G W 6.15.0-rc4+ #5 NONE

[155079.824102] Tainted: [W]=WARN

[155079.824717] Hardware name: QEMU KVM Virtual Machine, BIOS 2025.02-6 04/08/2025

[155079.826171] pstate: 60401005 (nZCv daif +PAN -UAO -TCO -DIT +SSBS BTYPE=--)

[155079.827619] pc : bch2_trans_srcu_unlock+0x14c/0x170 [bcachefs]

[155079.828981] lr : bch2_trans_srcu_unlock+0x14c/0x170 [bcachefs]

[155079.830346] sp : ffff80008290b170

[155079.831033] x29: ffff80008290b170 x28: ffff0004c8b00000 x27: 0000000000000100

[155079.832628] x26: 0000000000000002 x25: 0000000000000002 x24: ffff0004dc880000

[155079.834076] x23: ffff0004c8b039a0 x22: ffff0004dc880290 x21: 0000000000000003

[155079.835557] x20: ffff0004c8b00000 x19: ffff0004dc880000 x18: 00000000ffffffff

[155079.836999] x17: 0000000000000000 x16: ffffc502c79213c8 x15: ffff0009fa785e09

[155079.838442] x14: ffff80008290ace0 x13: ffff0009fa785e14 x12: 616c636572207972

[155079.839920] x11: 6f6d656d20676e69 x10: 79616c656428206b x9 : ffffc502c7ad7d48

[155079.841366] x8 : 0000000000000002 x7 : 0000000000000000 x6 : 000000000000000d

[155079.842820] x5 : ffff0009fa80b908 x4 : 0000000000000000 x3 : 0000000000000000

[155079.844308] x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff0000c103dc80

[155079.845756] Call trace:

[155079.846270] bch2_trans_srcu_unlock+0x14c/0x170 [bcachefs] (P)

[155079.847742] bch2_trans_begin+0x5e8/0x760 [bcachefs]

[155079.848908] bch2_btree_write_buffer_flush_locked+0xa4/0xaa0 [bcachefs]

[155079.850398] btree_write_buffer_flush_seq+0x114/0x220 [bcachefs]

[155079.852059] bch2_btree_write_buffer_flush_sync+0x78/0xf0 [bcachefs]

[155079.853429] bch2_btree_write_buffer_maybe_flush+0x11c/0x290 [bcachefs]

[155079.854846] backpointer_target_not_found+0x7c/0x408 [bcachefs]

[155079.856381] bch2_backpointer_get_node+0x14c/0x178 [bcachefs]

[155079.857586] bch2_backpointer_get_key+0xf4/0x260 [bcachefs]

[155079.858755] __bch2_move_data_phys+0x308/0x770 [bcachefs]

[155079.860095] bch2_evacuate_bucket+0x80/0xc0 [bcachefs]

[155079.861107] bch2_copygc+0x348/0x700 [bcachefs]

[155079.862006] bch2_copygc_thread+0x1d8/0x498 [bcachefs]

[155079.863007] kthread+0x120/0x220

[155079.863764] ret_from_fork+0x10/0x20

[155079.864343] ---[ end trace 0000000000000000 ]---

[163077.026688] hrtimer: interrupt took 13116939 ns

[165865.571744] Unable to handle kernel paging request at virtual address ffff0007b3100088

[165865.571784] Mem abort info:

[165865.571821] ESR = 0x0000000096000007

[165865.571831] EC = 0x25: DABT (current EL), IL = 32 bits

[165865.571839] SET = 0, FnV = 0

[165865.571845] EA = 0, S1PTW = 0

[165865.571850] FSC = 0x07: level 3 translation fault

[165865.571855] Data abort info:

[165865.571864] ISV = 0, ISS = 0x00000007, ISS2 = 0x00000000

[165865.571871] CM = 0, WnR = 0, TnD = 0, TagAccess = 0

[165865.571878] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0

[165865.571884] swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000013b138000

[165865.571890] [ffff0007b3100088] pgd=0000000000000000, p4d=1800000a3ffff403, pud=1800000a3be42403, pmd=1800000a3bca9403, pte=00680007f3100f06

[165865.571928] Internal error: Oops: 0000000096000007 [#1] SMP

[165865.616681] Modules linked in: ceph libceph netfs bcachefs chacha_neon libchacha poly1305_neon lz4hc_compress lz4_compress dm_mod sd_mod sg iscsi_tcp libiscsi_tcp libiscsi scsi_transport_iscsi scsi_mod scsi_common binfmt_misc nls_ascii nls_cp437 vfat fat aes_ce_blk aes_ce_cipher polyval_ce polyval_generic ghash_ce gf128mul sha2_ce evdev sha256_arm64 sha1_ce arm_smccc_trng button virtio_console efi_pstore configfs nfnetlink vsock_loopback vmw_vsock_virtio_transport_common vsock efivarfs qemu_fw_cfg virtio_rng ip_tables x_tables autofs4 ext4 crc16 mbcache jbd2 btrfs blake2b_generic xor xor_neon raid6_pq xhci_pci xhci_hcd usbcore virtio_net net_failover virtio_blk usb_common failover

[165865.642373] CPU: 6 UID: 0 PID: 2844499 Comm: kworker/u80:0 Tainted: G W 6.15.0-rc4+ #5 NONE

[165865.646928] Tainted: [W]=WARN

[165865.647749] Hardware name: QEMU KVM Virtual Machine, BIOS 2025.02-6 04/08/2025

[165865.650358] Workqueue: btree_update btree_interior_update_work [bcachefs]

[165865.652508] pstate: 60401005 (nZCv daif +PAN -UAO -TCO -DIT +SSBS BTYPE=--)

[165865.654526] pc : btree_node_seq_matches+0x10/0x40 [bcachefs]

[165865.655877] lr : btree_interior_update_work+0x1ac/0x9e0 [bcachefs]

[165865.657284] sp : ffff800082ddbc80

[165865.657967] x29: ffff800082ddbd20 x28: ffff0008daaa6000 x27: ffff0000c0019c28

[165865.659963] x26: 0000000000000000 x25: ffffc502c85d0968 x24: ffff0000d291be00

[165865.661851] x23: ffff0004c8b00000 x22: ffff0008daaa68c0 x21: ffff0008daaa6900

[165865.663912] x20: ffff0008daaa6058 x19: ffff00015497c000 x18: 00000000fffffff9

[165865.665547] x17: 0000000000000039 x16: ffffc502c79d49f0 x15: ffff8000a9445ab5

[165865.666957] x14: 0000000000000000 x13: 0000000000000020 x12: 0101010101010101

[165865.668428] x11: 7f7f7f7f7f7f7f7f x10: fefefeff64736063 x9 : ffffc5027613bd34

[165865.669856] x8 : ffff000827116000 x7 : 0000000000000000 x6 : 003fe323e7b7ffff

[165865.672671] x5 : 0000000000000012 x4 : 000000000000000c x3 : ffff0008daaa6558

[165865.674559] x2 : 0000000015000000 x1 : e59be2501ead6f9a x0 : ffff0007b3100000

[165865.677103] Call trace:

[165865.677626] btree_node_seq_matches+0x10/0x40 [bcachefs] (P)

[165865.678927] process_one_work+0x178/0x3d0

[165865.679785] worker_thread+0x208/0x400

[165865.680544] kthread+0x120/0x220

[165865.681218] ret_from_fork+0x10/0x20

[165865.682444] Code: aa1e03e9 d503201f f9404c00 b40000a0 (f9404400)

[165865.683692] ---[ end trace 0000000000000000 ]---

[165948.481490] Allocator stuck? Waited for 30 seconds

[165948.481564] Allocator debug:

[165948.481630] capacity87026953749

[165948.481636] reserved 7567561195

[165948.481642] hidden 113934336

[165948.481647] btree 717441024

[165948.481652] data 39694137034

[165948.481660] cached 4703156416

[165948.481662] reserved 0

[165948.481665] online_reserved 4295053

[165948.481667] nr_inodes 0

[165948.481669]

[165948.481671] freelist_wait empty

[165948.481673] open buckets allocated768

[165948.481675] open buckets total 1024

[165948.481677] open_buckets_wait waiting

[165948.481679] open_buckets_btree 701

[165948.481681] open_buckets_user 66

[165948.481683] btree reserve cache 28

 

[165948.481687] Dev 0:

[165948.481688] buckets sectors fragmented

[165948.481691] free 9916 0 0

[165948.481693] sb 3 6152 6136

[165948.481695] journal 799 3272704 0

[165948.481697] btree 52749 189519872 26540032

[165948.481699] user 42640 174231256 426140

[165948.481701] cached 21 30301 55715

[165948.481703] parity 0 0 0

[165948.481705] stripe 0 0 0

[165948.481707] need_gc_gens 0 0 0

[165948.481708] need_discard 6511 0 0

[165948.481710] unstriped 0 0 0

[165948.481712] capacity 112639

[165948.481713]

[165948.481715] reserves:

[165948.481717] stripe 3526

[165948.481718] normal 1767

[165948.481720] copygc 8

[165948.481722] btree 4

[165948.481723] btree_copygc 0

[165948.481725] reclaim 0

[165948.481727] interior_updates 0

[165948.481729]

[165948.481731] open buckets 170

[165948.481732] buckets to invalidate 0

 

[165948.481736] Dev 1:

[165948.481738] buckets sectors fragmented

[165948.481740] free 10764 0 0

[165948.481742] sb 3 6152 6136

[165948.481745] journal 799 3272704 0

[165948.481747] btree 49501 178690048 24066048

[165948.481750] user 45002 183906166 460079

[165948.481752] cached 17 17073 52559

[165948.481754] parity 0 0 0

[165948.481756] stripe 0 0 0

[165948.481758] need_gc_gens 0 0 0

[165948.481760] need_discard 6553 0 0

[165948.481762] unstriped 0 0 0

[165948.481764] capacity 112639

[165948.481766]

[165948.481767] reserves:

[165948.481769] stripe 3526

[165948.481771] normal 1767

[165948.481773] copygc 8

[165948.481774] btree 4

[165948.481776] btree_copygc 0

[165948.481778] reclaim 0

[165948.481780] interior_updates 0

[165948.481782]

[165948.481783] open buckets 178

[165948.481785] buckets to invalidate 0

 

[165948.481789] Dev 3:

[165948.481791] buckets sectors fragmented

[165948.481793] free 10298 0 0

[165948.481795] sb 3 6152 6136

[165948.481797] journal 799 3272704 0

[165948.481799] btree 46527 168088064 22486528

[165948.481802] user 48380 197764183 428369

[165948.481804] cached 24 37744 60561

[165948.481806] parity 0 0 0

[165948.481808] stripe 0 0 0

[165948.481810] need_gc_gens 0 0 0

[165948.481812] need_discard 6608 0 0

[165948.481814] unstriped 0 0 0

[165948.481816] capacity 112639

[165948.481818]

[165948.481820] reserves:

[165948.481821] stripe 3526

[165948.481823] normal 1767

[165948.481824] copygc 8

[165948.481827] btree 4

[165948.481829] btree_copygc 0

[165948.481830] reclaim 0

[165948.481832] interior_updates 0

[165948.481834]

[165948.481835] open buckets 176

[165948.481836] buckets to invalidate 0

 

[165948.481840] Dev 4:

[165948.481842] buckets sectors fragmented

[165948.481843] free 11826 0 0

[165948.481845] sb 3 6152 6136

[165948.481847] journal 799 3272704 0

[165948.481849] btree 50069 181143040 23939584

[165948.481851] user 44615 182282937 473855

[165948.481853] cached 14 9404 47940

[165948.481855] parity 0 0 0

[165948.481858] stripe 0 0 0

[165948.481860] need_gc_gens 0 0 0

[165948.481862] need_discard 5313 0 0

[165948.481864] unstriped 0 0 0

[165948.481866] capacity 112639

[165948.481868]

[165948.481870] reserves:

[165948.481872] stripe 3526

[165948.481874] normal 1767

[165948.481875] copygc 8

[165948.481877] btree 4

[165948.481878] btree_copygc 0

[165948.481880] reclaim 0

[165948.481881] interior_updates 0

[165948.481883]

[165948.481884] open buckets 177

[165948.481886] buckets to invalidate 0

 

[165948.481889] Dev 6:

[165948.481891] buckets sectors fragmented

[165948.481893] free 351617 0 0

[165948.481895] sb 1 4104 12280

[165948.481898] journal 1024 16777216 0

[165948.481900] btree 0 0 0

[165948.481902] user 518254 8469991102 21236847

[165948.481904] cached 66232 960456573 124718785

[165948.481906] parity 0 0 0

[165948.481909] stripe 0 0 0

[165948.481911] need_gc_gens 0 0 0

[165948.481912] need_discard 1244 0 0

[165948.481914] unstriped 0 0 0

[165948.481916] capacity 938372

[165948.481918]

[165948.481919] reserves:

[165948.481921] stripe 29326

[165948.481923] normal 14664

[165948.481925] copygc 2

[165948.481927] btree 1

[165948.481929] btree_copygc 0

[165948.481930] reclaim 0

[165948.481933] interior_updates 0

[165948.481934]

[165948.481935] open buckets 13

[165948.481937] buckets to invalidate 0

 

[165948.481940] Dev 7:

[165948.481942] buckets sectors fragmented

[165948.481944] free 351782 0 0

[165948.481946] sb 1 4104 12280

[165948.481948] journal 1024 16777216 0

[165948.481950] btree 0 0 0

[165948.481952] user 517943 8464596171 21537380

[165948.481954] cached 66301 962559467 123747049

[165948.481956] parity 0 0 0

[165948.481958] stripe 0 0 0

[165948.481960] need_gc_gens 0 0 0

[165948.481962] need_discard 1321 0 0

[165948.481964] unstriped 0 0 0

[165948.481966] capacity 938372

[165948.481968]

[165948.481969] reserves:

[165948.481971] stripe 29326

[165948.481973] normal 14664

[165948.481975] copygc 2

[165948.481977] btree 1

[165948.481979] btree_copygc 0

[165948.481981] reclaim 0

[165948.481982] interior_updates 0

[165948.481983]

[165948.481984] open buckets 13

[165948.481986] buckets to invalidate 0

 

[165948.481989] Dev 8:

[165948.481991] buckets sectors fragmented

[165948.481992] free 480847 0 0

[165948.481994] sb 1 4104 12280

[165948.481996] journal 1024 16777216 0

[165948.481998] btree 0 0 0

[165948.482000] user 357979 5839190781 26017483

[165948.482003] cached 96151 1384544439 190834264

[165948.482005] parity 0 0 0

[165948.482006] stripe 0 0 0

[165948.482008] need_gc_gens 0 0 0

[165948.482010] need_discard 2370 0 0

[165948.482012] unstriped 0 0 0

[165948.482015] capacity 938372

[165948.482017]

[165948.482019] reserves:

[165948.482020] stripe 29326

[165948.482021] normal 14664

[165948.482023] copygc 2

[165948.482024] btree 1

[165948.482026] btree_copygc 0

[165948.482028] reclaim 0

[165948.482029] interior_updates 0

[165948.482030]

[165948.482032] open buckets 19

[165948.482033] buckets to invalidate 0

 

[165948.482036] Dev 9:

[165948.482038] buckets sectors fragmented

[165948.482040] free 480836 0 0

[165948.482042] sb 1 4104 12280

[165948.482046] journal 1024 16777216 0

[165948.482048] btree 0 0 0

[165948.482050] user 357877 5837465968 26069076

[165948.482052] cached 95663 1384306169 183076724

[165948.482054] parity 0 0 0

[165948.482056] stripe 0 0 0

[165948.482059] need_gc_gens 0 0 0

[165948.482061] need_discard 2971 0 0

[165948.482065] unstriped 0 0 0

[165948.482068] capacity 938372

[165948.482069]

[165948.482071] reserves:

[165948.482073] stripe 29326

[165948.482074] normal 14664

[165948.482076] copygc 2

[165948.482078] btree 1

[165948.482079] btree_copygc 0

[165948.482081] reclaim 0

[165948.482082] interior_updates 0

[165948.482083]

[165948.482088] open buckets 20

[165948.482091] buckets to invalidate 0

 

[165948.482095] Dev 10:

[165948.482096] buckets sectors fragmented

[165948.482098] free 221763 0 0

[165948.482100] sb 1 4104 28664

[165948.482102] journal 512 16777216 0

[165948.482104] btree 0 0 0

[165948.482107] user 238796 6941788547 883098023

[165948.482109] cached 382 4905937 7611439

[165948.482111] parity 0 0 0

[165948.482113] stripe 0 0 0

[165948.482115] need_gc_gens 0 0 0

[165948.482117] need_discard 15412 0 0

[165948.482119] unstriped 0 0 0

[165948.482122] capacity 476866

[165948.482123]

[165948.482126] reserves:

[165948.482127] stripe 14904

[165948.482129] normal 7453

[165948.482131] copygc 2

[165948.482132] btree 1

[165948.482134] btree_copygc 0

[165948.482136] reclaim 0

[165948.482138] interior_updates 0

[165948.482139]

[165948.482141] open buckets 1

[165948.482143] buckets to invalidate 0

 

[165948.482147] Dev 11:

[165948.482150] buckets sectors fragmented

[165948.482152] free 316622 0 0

[165948.482154] sb 1 4104 28664

[165948.482157] journal 512 16777216 0

[165948.482159] btree 0 0 0

[165948.482161] user 138092 3402919923 1122079055

[165948.482163] cached 0 0 0

[165948.482165] parity 0 0 0

[165948.482166] stripe 0 0 0

[165948.482168] need_gc_gens 0 0 0

[165948.482170] need_discard 21639 0 0

[165948.482172] unstriped 0 0 0

[165948.482174] capacity 476866

[165948.482176]

[165948.482177] reserves:

[165948.482179] stripe 14904

[165948.482182] normal 7453

[165948.482184] copygc 2

[165948.482185] btree 1

[165948.482187] btree_copygc 0

[165948.482189] reclaim 0

[165948.482190] interior_updates 0

[165948.482192]

[165948.482193] open buckets 0

[165948.482194] buckets to invalidate 0

 

[165948.482198] Copygc debug:

[165948.482200] running: 1

[165948.482201] copygc_wait:0

[165948.482204] copygc_wait_at:130820788764

[165948.482206] Currently waiting for:0 B

[165948.482208] Currently waiting since:467 GiB

[165948.482210] Currently calculated wait:0 B

[165948.482212] [<0>] bch2_write+0x134/0x488 [bcachefs]

[165948.482213] [<0>] bch2_data_update_read_done+0xac/0xc8 [bcachefs]

[165948.482215] [<0>] bch2_moving_ctxt_do_pending_writes+0x120/0x3c8 [bcachefs]

[165948.482217] [<0>] bkey_nocow_lock+0x1d0/0x400 [bcachefs]

[165948.482219] [<0>] bch2_data_update_init+0x7a0/0xc10 [bcachefs]

[165948.482221] [<0>] bch2_move_extent+0x174/0x750 [bcachefs]

[165948.482222] [<0>] __bch2_move_data_phys+0x688/0x770 [bcachefs]

[165948.482224] [<0>] bch2_evacuate_bucket+0x80/0xc0 [bcachefs]

[165948.482226] [<0>] bch2_copygc+0x348/0x700 [bcachefs]

[165948.482228] [<0>] bch2_copygc_thread+0x1d8/0x498 [bcachefs]

[165948.482231] [<0>] kthread+0x120/0x220

[165948.482233] [<0>] ret_from_fork+0x10/0x20

 

[165948.482235] Journal debug:

[165948.482237] flags: replay_done,running,may_skip_flush

[165948.482239] dirty journal entries: 2038/32768

[165948.482240] seq: 3835004

[165948.482242] seq_ondisk: 3835003

[165948.482244] last_seq: 3832967

[165948.482245] last_seq_ondisk: 3832967

[165948.482247] flushed_seq_ondisk: 3835003

[165948.482249] watermark: stripe

[165948.482251] each entry reserved: 321

[165948.482253] nr flush writes: 25850

[165948.482255] nr noflush writes: 30662

[165948.482256] average write size: 1.72 MiB

[165948.482259] free buf: 2097152

[165948.482261] nr direct reclaim: 14142

[165948.482262] nr background reclaim: 17113802

[165948.482264] reclaim kicked: 0

[165948.482265] reclaim runs in: 4 ms

[165948.482267] blocked: 0

[165948.482269] current entry sectors: 4096

[165948.482271] current entry error: (No error)

[165948.482272] current entry: 62/261816

[165948.482274] unwritten entries:

[165948.482276] seq: 3835004

[165948.482277] refcount: 1

[165948.482280] io: journal_write_done+0x0/0x448 [bcachefs] r 1

[165948.482282] size: 56.0 B

[165948.482284] expires: 10 jiffies

[165948.482286] flags: need_flush_to_write_buffer

[165948.482288] last buf open

[165948.482290] space:

[165948.482291] discarded 4096:16318464

[165948.482293] clean ondisk 4096:16318464

[165948.482294] clean 4096:16318464

[165948.482296] total 4096:16777216

[165948.482298] dev 0:

[165948.482299] durability 1:

[165948.482301] nr 799

[165948.482303] bucket size 4096

[165948.482305] available 0:2

[165948.482307] discard_idx 494

[165948.482309] dirty_ondisk 494 (seq 3832967)

[165948.482310] dirty_idx 494 (seq 3832967)

[165948.482312] cur_idx 492 (seq 3834565)

[165948.482314] dev 1:

[165948.482315] durability 1:

[165948.482317] nr 799

[165948.482318] bucket size 4096

[165948.482320] available 0:3

[165948.482322] discard_idx 251

[165948.482323] dirty_ondisk 251 (seq 3832968)

[165948.482326] dirty_idx 251 (seq 3832968)

[165948.482328] cur_idx 249 (seq 3834566)

[165948.482330] dev 3:

[165948.482331] durability 1:

[165948.482332] nr 799

[165948.482334] bucket size 4096

[165948.482335] available 0:2

[165948.482337] discard_idx 515

[165948.482339] dirty_ondisk 515 (seq 3832967)

[165948.482340] dirty_idx 515 (seq 3832967)

[165948.482342] cur_idx 513 (seq 3834565)

[165948.482344] dev 4:

[165948.482345] durability 1:

[165948.482347] nr 799

[165948.482349] bucket size 4096

[165948.482351] available 0:3

[165948.482353] discard_idx 488

[165948.482355] dirty_ondisk 488 (seq 3832968)

[165948.482356] dirty_idx 488 (seq 3832968)

[165948.482358] cur_idx 486 (seq 3834566)

[165948.482359] dev 6:

[165948.482361] durability 1:

[165948.482362] nr 1024

[165948.482364] bucket size 16384

[165948.482366] available 996:16326

[165948.482367] discard_idx 934

[165948.482369] dirty_ondisk 934 (seq 3834577)

[165948.482371] dirty_idx 934 (seq 3834577)

[165948.482373] cur_idx 960 (seq 3835002)

[165948.482375] dev 7:

[165948.482376] durability 1:

[165948.482378] nr 1024

[165948.482380] bucket size 16384

[165948.482382] available 996:16027

[165948.482383] discard_idx 937

[165948.482385] dirty_ondisk 937 (seq 3834576)

[165948.482387] dirty_idx 937 (seq 3834576)

[165948.482388] cur_idx 963 (seq 3835003)

[165948.482390] dev 8:

[165948.482391] durability 1:

[165948.482393] nr 1024

[165948.482395] bucket size 16384

[165948.482396] available 996:3684

[165948.482399] discard_idx 989

[165948.482402] dirty_ondisk 989 (seq 3834575)

[165948.482403] dirty_idx 989 (seq 3834575)

[165948.482406] cur_idx 1015 (seq 3835003)

[165948.482408] dev 9:

[165948.482409] durability 1:

[165948.482410] nr 1024

[165948.482412] bucket size 16384

[165948.482414] available 996:3

[165948.482416] discard_idx 989

[165948.482418] dirty_ondisk 989 (seq 3834575)

[165948.482419] dirty_idx 989 (seq 3834575)

[165948.482421] cur_idx 1015 (seq 3834986)

[165948.482423] dev 10:

[165948.482424] durability 1:

[165948.482427] nr 512

[165948.482429] bucket size 32768

[165948.482431] available 484:32768

[165948.482433] discard_idx 175

[165948.482434] dirty_ondisk 175 (seq 3834582)

[165948.482436] dirty_idx 175 (seq 3834582)

[165948.482438] cur_idx 201 (seq 3834983)

[165948.482440] dev 11:

[165948.482441] durability 1:

[165948.482443] nr 512

[165948.482444] bucket size 32768

[165948.482446] available 484:32768

[165948.482447] discard_idx 273

[165948.482449] dirty_ondisk 273 (seq 3834581)

[165948.482451] dirty_idx 273 (seq 3834581)

[165948.482452] cur_idx 299 (seq 3834975)

[165948.482454] replicas want 2 need 2

[165950.529306] Allocator stuck? Waited for 30 seconds

[165950.529360] Allocator debug:

[165950.529395] capacity87026953749

[165950.529400] reserved 7567561195

[165950.529404] hidden 113934336

[165950.529408] btree 717441024

[165950.529412] data 39694137034

[165950.529417] cached 4703156416

[165950.529418] reserved 0

[165950.529420] online_reserved 4295053

[165950.529421] nr_inodes 0

[165950.529422]

[165950.529424] freelist_wait empty

[165950.529425] open buckets allocated768

[165950.529426] open buckets total 1024

[165950.529428] open_buckets_wait waiting

[165950.529429] open_buckets_btree 701

[165950.529430] open_buckets_user 66

[165950.529432] btree reserve cache 28

 

[165950.529434] Dev 0:

[165950.529436] buckets sectors fragmented

[165950.529437] free 9916 0 0

[165950.529439] sb 3 6152 6136

[165950.529440] journal 799 3272704 0

[165950.529442] btree 52749 189519872 26540032

[165950.529444] user 42640 174231256 426140

[165950.529445] cached 21 30301 55715

[165950.529447] parity 0 0 0

[165950.529449] stripe 0 0 0

[165950.529450] need_gc_gens 0 0 0

[165950.529452] need_discard 6511 0 0

[165950.529454] unstriped 0 0 0

[165950.529456] capacity 112639

[165950.529457]

[165950.529458] reserves:

[165950.529460] stripe 3526

[165950.529461] normal 1767

[165950.529462] copygc 8

[165950.529464] btree 4

[165950.529466] btree_copygc 0

[165950.529467] reclaim 0

[165950.529470] interior_updates 0

[165950.529472]

[165950.529473] open buckets 170

[165950.529475] buckets to invalidate 0

 

[165950.529477] Dev 1:

[165950.529479] buckets sectors fragmented

[165950.529481] free 10764 0 0

[165950.529484] sb 3 6152 6136

[165950.529486] journal 799 3272704 0

[165950.529489] btree 49501 178690048 24066048

[165950.529494] user 45002 183906166 460079

[165950.529497] cached 17 17073 52559

[165950.529501] parity 0 0 0

[165950.529504] stripe 0 0 0

[165950.529508] need_gc_gens 0 0 0

[165950.529511] need_discard 6553 0 0

[165950.529514] unstriped 0 0 0

[165950.529519] capacity 112639

[165950.529522]

[165950.529525] reserves:

[165950.529528] stripe 3526

[165950.529532] normal 1767

[165950.529536] copygc 8

[165950.529540] btree 4

[165950.529544] btree_copygc 0

[165950.529547] reclaim 0

[165950.529551] interior_updates 0

[165950.529554]

[165950.529558] open buckets 178

[165950.529560] buckets to invalidate 0

 

[165950.529568] Dev 3:

[165950.529572] buckets sectors fragmented

[165950.529577] free 10298 0 0

[165950.529581] sb 3 6152 6136

[165950.529586] journal 799 3272704 0

[165950.529590] btree 46527 168088064 22486528

[165950.529595] user 48380 197764183 428369

[165950.529600] cached 24 37744 60561

[165950.529606] parity 0 0 0

[165950.529611] stripe 0 0 0

[165950.529616] need_gc_gens 0 0 0

[165950.529620] need_discard 6608 0 0

[165950.529625] unstriped 0 0 0

[165950.529630] capacity 112639

[165950.529634]

[165950.529638] reserves:

[165950.529642] stripe 3526

[165950.529647] normal 1767

[165950.529652] copygc 8

[165950.529657] btree 4

[165950.529660] btree_copygc 0

[165950.529664] reclaim 0

[165950.529667] interior_updates 0

[165950.529670]

[165950.529675] open buckets 176

[165950.529679] buckets to invalidate 0

 

[165950.529686] Dev 4:

[165950.529690] buckets sectors fragmented

[165950.529695] free 11826 0 0

[165950.529701] sb 3 6152 6136

[165950.529705] journal 799 3272704 0

[165950.529711] btree 50069 181143040 23939584

[165950.529716] user 44615 182282937 473855

[165950.529722] cached 14 9404 47940

[165950.529725] parity 0 0 0

[165950.529727] stripe 0 0 0

[165950.529730] need_gc_gens 0 0 0

[165950.529733] need_discard 5313 0 0

[165950.529735] unstriped 0 0 0

[165950.529737] capacity 112639

[165950.529740]

[165950.529742] reserves:

[165950.529744] stripe 3526

[165950.529747] normal 1767

[165950.529748] copygc 8

[165950.529751] btree 4

[165950.529753] btree_copygc 0

[165950.529756] reclaim 0

[165950.529758] interior_updates 0

[165950.529760]

[165950.529762] open buckets 177

[165950.529764] buckets to invalidate 0

 

[165950.529767] Dev 6:

[165950.529769] buckets sectors fragmented

[165950.529771] free 351617 0 0

[165950.529772] sb 1 4104 12280

[165950.529774] journal 1024 16777216 0

[165950.529776] btree 0 0 0

[165950.529777] user 518254 8469991102 21236847

[165950.529779] cached 66232 960456573 124718785

[165950.529781] parity 0 0 0

[165950.529782] stripe 0 0 0

[165950.529784] need_gc_gens 0 0 0

[165950.529786] need_discard 1244 0 0

[165950.529787] unstriped 0 0 0

[165950.529789] capacity 938372

[165950.529790]

[165950.529791] reserves:

[165950.529792] stripe 29326

[165950.529794] normal 14664

[165950.529795] copygc 2

[165950.529797] btree 1

[165950.529798] btree_copygc 0

[165950.529799] reclaim 0

[165950.529801] interior_updates 0

[165950.529802]

[165950.529803] open buckets 13

[165950.529804] buckets to invalidate 0

 

[165950.529807] Dev 7:

[165950.529808] buckets sectors fragmented

[165950.529810] free 351782 0 0

[165950.529811] sb 1 4104 12280

[165950.529813] journal 1024 16777216 0

[165950.529814] btree 0 0 0

[165950.529816] user 517943 8464596171 21537380

[165950.529817] cached 66301 962559467 123747049

[165950.529819] parity 0 0 0

[165950.529820] stripe 0 0 0

[165950.529822] need_gc_gens 0 0 0

[165950.529823] need_discard 1321 0 0

[165950.529825] unstriped 0 0 0

[165950.529827] capacity 938372

[165950.529828]

[165950.529829] reserves:

[165950.529830] stripe 29326

[165950.529831] normal 14664

[165950.529833] copygc 2

[165950.529834] btree 1

[165950.529835] btree_copygc 0

[165950.529836] reclaim 0

[165950.529838] interior_updates 0

[165950.529839]

[165950.529840] open buckets 13

[165950.529841] buckets to invalidate 0

 

[165950.529844] Dev 8:

[165950.529845] buckets sectors fragmented

[165950.529847] free 480847 0 0

[165950.529848] sb 1 4104 12280

[165950.529850] journal 1024 16777216 0

[165950.529851] btree 0 0 0

[165950.529853] user 357979 5839190781 26017483

[165950.529854] cached 96151 1384544439 190834264

[165950.529856] parity 0 0 0

[165950.529857] stripe 0 0 0

[165950.529859] need_gc_gens 0 0 0

[165950.529860] need_discard 2370 0 0

[165950.529862] unstriped 0 0 0

[165950.529863] capacity 938372

[165950.529864]

[165950.529866] reserves:

[165950.529867] stripe 29326

[165950.529868] normal 14664

[165950.529870] copygc 2

[165950.529871] btree 1

[165950.529872] btree_copygc 0

[165950.529874] reclaim 0

[165950.529875] interior_updates 0

[165950.529876]

[165950.529877] open buckets 19

[165950.529878] buckets to invalidate 0

 

[165950.529881] Dev 9:

[165950.529882] buckets sectors fragmented

[165950.529884] free 480836 0 0

[165950.529885] sb 1 4104 12280

[165950.529887] journal 1024 16777216 0

[165950.529888] btree 0 0 0

[165950.529890] user 357877 5837465968 26069076

[165950.529891] cached 95663 1384306169 183076724

[165950.529893] parity 0 0 0

[165950.529895] stripe 0 0 0

[165950.529896] need_gc_gens 0 0 0

[165950.529898] need_discard 2971 0 0

[165950.529899] unstriped 0 0 0

[165950.529901] capacity 938372

[165950.529902]

[165950.529903] reserves:

[165950.529904] stripe 29326

[165950.529906] normal 14664

[165950.529907] copygc 2

[165950.529908] btree 1

[165950.529910] btree_copygc 0

[165950.529911] reclaim 0

[165950.529912] interior_updates 0

[165950.529913]

[165950.529915] open buckets 20

[165950.529916] buckets to invalidate 0

 

[165950.529918] Dev 10:

[165950.529919] buckets sectors fragmented

[165950.529921] free 221763 0 0

[165950.529922] sb 1 4104 28664

[165950.529924] journal 512 16777216 0

[165950.529925] btree 0 0 0

[165950.529927] user 238796 6941788547 883098023

[165950.529929] cached 382 4905937 7611439

[165950.529930] parity 0 0 0

[165950.529932] stripe 0 0 0

[165950.529933] need_gc_gens 0 0 0

[165950.529935] need_discard 15412 0 0

[165950.529936] unstriped 0 0 0

[165950.529938] capacity 476866

[165950.529939]

[165950.529940] reserves:

[165950.529941] stripe 14904

[165950.529943] normal 7453

[165950.529944] copygc 2

[165950.529945] btree 1

[165950.529947] btree_copygc 0

[165950.529948] reclaim 0

[165950.529949] interior_updates 0

[165950.529950]

[165950.529951] open buckets 1

[165950.529953] buckets to invalidate 0

 

[165950.529955] Dev 11:

[165950.529956] buckets sectors fragmented

[165950.529958] free 316622 0 0

[165950.529959] sb 1 4104 28664

[165950.529961] journal 512 16777216 0

[165950.529963] btree 0 0 0

[165950.529964] user 138092 3402919923 1122079055

[165950.529966] cached 0 0 0

[165950.529967] parity 0 0 0

[165950.529969] stripe 0 0 0

[165950.529970] need_gc_gens 0 0 0

[165950.529972] need_discard 21639 0 0

[165950.529973] unstriped 0 0 0

[165950.529975] capacity 476866

[165950.529976]

[165950.529977] reserves:

[165950.529978] stripe 14904

[165950.529980] normal 7453

[165950.529981] copygc 2

[165950.529982] btree 1

[165950.529984] btree_copygc 0

[165950.529985] reclaim 0

[165950.529986] interior_updates 0

[165950.529987]

[165950.529988] open buckets 0

[165950.529990] buckets to invalidate 0

 

[165950.529992] Copygc debug:

[165950.529993] running: 1

[165950.529994] copygc_wait:0

[165950.529996] copygc_wait_at:130820788764

[165950.529997] Currently waiting for:0 B

[165950.529999] Currently waiting since:467 GiB

[165950.530000] Currently calculated wait:0 B

[165950.530001] [<0>] bch2_copygc_wait_to_text+0x140/0x220 [bcachefs]

[165950.530003] [<0>] bch2_print_allocator_stuck+0x124/0x240 [bcachefs]

[165950.530005] [<0>] __bch2_wait_on_allocator+0x8c/0x140 [bcachefs]

[165950.530006] [<0>] __bch2_write+0x52c/0x660 [bcachefs]

[165950.530007] [<0>] bch2_write+0x134/0x488 [bcachefs]

[165950.530009] [<0>] bch2_data_update_read_done+0xac/0xc8 [bcachefs]

[165950.530010] [<0>] bch2_moving_ctxt_do_pending_writes+0x120/0x3c8 [bcachefs]

[165950.530012] [<0>] bkey_nocow_lock+0x1d0/0x400 [bcachefs]

[165950.530014] [<0>] bch2_data_update_init+0x7a0/0xc10 [bcachefs]

[165950.530015] [<0>] bch2_move_extent+0x174/0x750 [bcachefs]

[165950.530017] [<0>] __bch2_move_data_phys+0x688/0x770 [bcachefs]

[165950.530018] [<0>] bch2_evacuate_bucket+0x80/0xc0 [bcachefs]

[165950.530020] [<0>] bch2_copygc+0x348/0x700 [bcachefs]

[165950.530021] [<0>] bch2_copygc_thread+0x1d8/0x498 [bcachefs]

[165950.530023] [<0>] kthread+0x120/0x220

[165950.530024] [<0>] ret_from_fork+0x10/0x20

 

[165950.530026] Journal debug:

[165950.530028] flags: replay_done,running,may_skip_flush

[165950.530029] dirty journal entries: 2040/32768

[165950.530031] seq: 3835006

[165950.530032] seq_ondisk: 3835005

[165950.530033] last_seq: 3832967

[165950.530035] last_seq_ondisk: 3832967

[165950.530036] flushed_seq_ondisk: 3835005

[165950.530038] watermark: stripe

[165950.530039] each entry reserved: 321

[165950.530040] nr flush writes: 25852

[165950.530042] nr noflush writes: 30662

[165950.530043] average write size: 1.72 MiB

[165950.530045] free buf: 2097152

[165950.530046] nr direct reclaim: 14142

[165950.530047] nr background reclaim: 17113822

[165950.530049] reclaim kicked: 0

[165950.530050] reclaim runs in: 36 ms

[165950.530051] blocked: 0

[165950.530053] current entry sectors: 4096

[165950.530054] current entry error: (No error)

[165950.530055] current entry: 155/261816

[165950.530057] unwritten entries:

[165950.530058] seq: 3835006

[165950.530059] refcount: 1

[165950.530061] io: journal_write_done+0x0/0x448 [bcachefs] r 1

[165950.530063] size: 56.0 B

[165950.530064] expires: 9 jiffies

[165950.530066] flags: need_flush_to_write_buffer

[165950.530067] last buf open

[165950.530068] space:

[165950.530070] discarded 4096:16318464

[165950.530071] clean ondisk 4096:16318464

[165950.530072] clean 4096:16318464

[165950.530074] total 4096:16777216

[165950.530075] dev 0:

[165950.530076] durability 1:

[165950.530078] nr 799

[165950.530079] bucket size 4096

[165950.530080] available 0:2

[165950.530082] discard_idx 494

[165950.530083] dirty_ondisk 494 (seq 3832967)

[165950.530085] dirty_idx 494 (seq 3832967)

[165950.530086] cur_idx 492 (seq 3834565)

[165950.530088] dev 1:

[165950.530089] durability 1:

[165950.530090] nr 799

[165950.530091] bucket size 4096

[165950.530093] available 0:3

[165950.530094] discard_idx 251

[165950.530095] dirty_ondisk 251 (seq 3832968)

[165950.530097] dirty_idx 251 (seq 3832968)

[165950.530098] cur_idx 249 (seq 3834566)

[165950.530100] dev 3:

[165950.530101] durability 1:

[165950.530102] nr 799

[165950.530103] bucket size 4096

[165950.530105] available 0:2

[165950.530106] discard_idx 515

[165950.530108] dirty_ondisk 515 (seq 3832967)

[165950.530109] dirty_idx 515 (seq 3832967)

[165950.530110] cur_idx 513 (seq 3834565)

[165950.530112] dev 4:

[165950.530113] durability 1:

[165950.530114] nr 799

[165950.530116] bucket size 4096

[165950.530117] available 0:3

[165950.530118] discard_idx 488

[165950.530120] dirty_ondisk 488 (seq 3832968)

[165950.530121] dirty_idx 488 (seq 3832968)

[165950.530122] cur_idx 486 (seq 3834566)

[165950.530124] dev 6:

[165950.530125] durability 1:

[165950.530126] nr 1024

[165950.530128] bucket size 16384

[165950.530129] available 996:16320

[165950.530131] discard_idx 934

[165950.530132] dirty_ondisk 934 (seq 3834577)

[165950.530134] dirty_idx 934 (seq 3834577)

[165950.530135] cur_idx 960 (seq 3835004)

[165950.530136] dev 7:

[165950.530137] durability 1:

[165950.530139] nr 1024

[165950.530140] bucket size 16384

[165950.530141] available 996:16020

[165950.530143] discard_idx 937

[165950.530144] dirty_ondisk 937 (seq 3834576)

[165950.530145] dirty_idx 937 (seq 3834576)

[165950.530147] cur_idx 963 (seq 3835005)

[165950.530148] dev 8:

[165950.530149] durability 1:

[165950.530151] nr 1024

[165950.530152] bucket size 16384

[165950.530153] available 996:3671

[165950.530155] discard_idx 989

[165950.530156] dirty_ondisk 989 (seq 3834575)

[165950.530158] dirty_idx 989 (seq 3834575)

[165950.530159] cur_idx 1015 (seq 3835005)

[165950.530160] dev 9:

[165950.530162] durability 1:

[165950.530163] nr 1024

[165950.530164] bucket size 16384

[165950.530166] available 996:3

[165950.530167] discard_idx 989

[165950.530168] dirty_ondisk 989 (seq 3834575)

[165950.530170] dirty_idx 989 (seq 3834575)

[165950.530171] cur_idx 1015 (seq 3834986)

[165950.530173] dev 10:

[165950.530174] durability 1:

[165950.530175] nr 512

[165950.530177] bucket size 32768

[165950.530178] available 484:32768

[165950.530179] discard_idx 175

[165950.530181] dirty_ondisk 175 (seq 3834582)

[165950.530182] dirty_idx 175 (seq 3834582)

[165950.530183] cur_idx 201 (seq 3834983)

[165950.530185] dev 11:

[165950.530186] durability 1:

[165950.530187] nr 512

[165950.530188] bucket size 32768

[165950.530190] available 484:32768

[165950.530191] discard_idx 273

[165950.530193] dirty_ondisk 273 (seq 3834581)

[165950.530194] dirty_idx 273 (seq 3834581)

[165950.530195] cur_idx 299 (seq 3834975)

[165950.530197] replicas want 2 need 2

[165958.721124] Allocator stuck? Waited for 30 seconds

[165958.721179] Allocator debug:

[165958.721214] capacity87026953749

[165958.721219] reserved 7567561195

[165958.721223] hidden 113934336

[165958.721227] btree 717441024

[165958.721232] data 39694137034

[165958.721236] cached 4703156416

[165958.721240] reserved 0

[165958.721244] online_reserved 4296079

[165958.721249] nr_inodes 0

[165958.721253]

[165958.721257] freelist_wait empty

[165958.721261] open buckets allocated768

[165958.721267] open buckets total 1024

[165958.721268] open_buckets_wait waiting

[165958.721270] open_buckets_btree 701

[165958.721271] open_buckets_user 66

[165958.721273] btree reserve cache 28

 

[165958.721275] Dev 0:

[165958.721277] buckets sectors fragmented

[165958.721279] free 9916 0 0

[165958.721280] sb 3 6152 6136

[165958.721282] journal 799 3272704 0

[165958.721284] btree 52749 189519872 26540032

[165958.721285] user 42640 174231256 426140

[165958.721287] cached 21 30301 55715

[165958.721289] parity 0 0 0

[165958.721290] stripe 0 0 0

[165958.721292] need_gc_gens 0 0 0

[165958.721294] need_discard 6511 0 0

[165958.721295] unstriped 0 0 0

[165958.721297] capacity 112639

[165958.721298]

[165958.721299] reserves:

[165958.721301] stripe 3526

[165958.721302] normal 1767

[165958.721304] copygc 8

[165958.721306] btree 4

[165958.721308] btree_copygc 0

[165958.721312] reclaim 0

[165958.721314] interior_updates 0

[165958.721315]

[165958.721317] open buckets 170

[165958.721318] buckets to invalidate 0

 

[165958.721321] Dev 1:

[165958.721322] buckets sectors fragmented

[165958.721324] free 10764 0 0

[165958.721327] sb 3 6152 6136

[165958.721329] journal 799 3272704 0

[165958.721332] btree 49501 178690048 24066048

[165958.721334] user 45002 183906166 460079

[165958.721337] cached 17 17073 52559

[165958.721341] parity 0 0 0

[165958.721344] stripe 0 0 0

[165958.721348] need_gc_gens 0 0 0

[165958.721352] need_discard 6553 0 0

[165958.721356] unstriped 0 0 0

[165958.721359] capacity 112639

[165958.721362]

[165958.721364] reserves:

[165958.721367] stripe 3526

[165958.721370] normal 1767

[165958.721373] copygc 8

[165958.721376] btree 4

[165958.721380] btree_copygc 0

[165958.721383] reclaim 0

[165958.721386] interior_updates 0

[165958.721389]

[165958.721392] open buckets 178

[165958.721396] buckets to invalidate 0

 

[165958.721401] Dev 3:

[165958.721405] buckets sectors fragmented

[165958.721408] free 10298 0 0

[165958.721414] sb 3 6152 6136

[165958.721416] journal 799 3272704 0

[165958.721421] btree 46527 168088064 22486528

[165958.721424] user 48380 197764183 428369

[165958.721428] cached 24 37744 60561

[165958.721432] parity 0 0 0

[165958.721435] stripe 0 0 0

[165958.721439] need_gc_gens 0 0 0

[165958.721442] need_discard 6608 0 0

[165958.721446] unstriped 0 0 0

[165958.721450] capacity 112639

[165958.721453]

[165958.721456] reserves:

[165958.721460] stripe 3526

[165958.721463] normal 1767

[165958.721466] copygc 8

[165958.721471] btree 4

[165958.721474] btree_copygc 0

[165958.721477] reclaim 0

[165958.721480] interior_updates 0

[165958.721483]

[165958.721486] open buckets 176

[165958.721490] buckets to invalidate 0

 

[165958.721496] Dev 4:

[165958.721498] buckets sectors fragmented

[165958.721503] free 11826 0 0

[165958.721507] sb 3 6152 6136

[165958.721512] journal 799 3272704 0

[165958.721516] btree 50069 181143040 23939584

[165958.721520] user 44615 182282937 473855

[165958.721524] cached 14 9404 47940

[165958.721529] parity 0 0 0

[165958.721534] stripe 0 0 0

[165958.721538] need_gc_gens 0 0 0

[165958.721543] need_discard 5313 0 0

[165958.721547] unstriped 0 0 0

[165958.721552] capacity 112639

[165958.721556]

[165958.721560] reserves:

[165958.721563] stripe 3526

[165958.721567] normal 1767

[165958.721572] copygc 8

[165958.721576] btree 4

[165958.721579] btree_copygc 0

[165958.721584] reclaim 0

[165958.721588] interior_updates 0

[165958.721592]

[165958.721596] open buckets 177

[165958.721602] buckets to invalidate 0

 

[165958.721610] Dev 6:

[165958.721614] buckets sectors fragmented

[165958.721617] free 351617 0 0

[165958.721622] sb 1 4104 12280

[165958.721627] journal 1024 16777216 0

[165958.721631] btree 0 0 0

[165958.721636] user 518254 8469991102 21236847

[165958.721640] cached 66232 960456573 124718785

[165958.721645] parity 0 0 0

[165958.721651] stripe 0 0 0

[165958.721656] need_gc_gens 0 0 0

[165958.721662] need_discard 1244 0 0

[165958.721667] unstriped 0 0 0

[165958.721670] capacity 938372

[165958.721671]

[165958.721673] reserves:

[165958.721675] stripe 29326

[165958.721677] normal 14664

[165958.721680] copygc 2

[165958.721682] btree 1

[165958.721684] btree_copygc 0

[165958.721686] reclaim 0

[165958.721688] interior_updates 0

[165958.721690]

[165958.721692] open buckets 13

[165958.721694] buckets to invalidate 0

 

[165958.721698] Dev 7:

[165958.721700] buckets sectors fragmented

[165958.721702] free 351782 0 0

[165958.721705] sb 1 4104 12280

[165958.721707] journal 1024 16777216 0

[165958.721709] btree 0 0 0

[165958.721711] user 517943 8464596171 21537380

[165958.721713] cached 66301 962559467 123747049

[165958.721715] parity 0 0 0

[165958.721717] stripe 0 0 0

[165958.721719] need_gc_gens 0 0 0

[165958.721721] need_discard 1321 0 0

[165958.721723] unstriped 0 0 0

[165958.721725] capacity 938372

[165958.721727]

[165958.721728] reserves:

[165958.721729] stripe 29326

[165958.721731] normal 14664

[165958.721732] copygc 2

[165958.721734] btree 1

[165958.721735] btree_copygc 0

[165958.721736] reclaim 0

[165958.721738] interior_updates 0

[165958.721739]

[165958.721740] open buckets 13

[165958.721741] buckets to invalidate 0

 

[165958.721744] Dev 8:

[165958.721745] buckets sectors fragmented

[165958.721747] free 480847 0 0

[165958.721749] sb 1 4104 12280

[165958.721750] journal 1024 16777216 0

[165958.721752] btree 0 0 0

[165958.721753] user 357979 5839190781 26017483

[165958.721755] cached 96151 1384544439 190834264

[165958.721756] parity 0 0 0

[165958.721758] stripe 0 0 0

[165958.721760] need_gc_gens 0 0 0

[165958.721761] need_discard 2370 0 0

[165958.721763] unstriped 0 0 0

[165958.721764] capacity 938372

[165958.721765]

[165958.721767] reserves:

[165958.721768] stripe 29326

[165958.721769] normal 14664

[165958.721771] copygc 2

[165958.721772] btree 1

[165958.721773] btree_copygc 0

[165958.721775] reclaim 0

[165958.721776] interior_updates 0

[165958.721777]

[165958.721778] open buckets 19

[165958.721779] buckets to invalidate 0

 

[165958.721782] Dev 9:

[165958.721783] buckets sectors fragmented

[165958.721785] free 480836 0 0

[165958.721786] sb 1 4104 12280

[165958.721788] journal 1024 16777216 0

[165958.721790] btree 0 0 0

[165958.721791] user 357877 5837465968 26069076

[165958.721793] cached 95663 1384306169 183076724

[165958.721795] parity 0 0 0

[165958.721796] stripe 0 0 0

[165958.721798] need_gc_gens 0 0 0

[165958.721799] need_discard 2971 0 0

[165958.721801] unstriped 0 0 0

[165958.721802] capacity 938372

[165958.721804]

[165958.721805] reserves:

[165958.721806] stripe 29326

[165958.721807] normal 14664

[165958.721809] copygc 2

[165958.721810] btree 1

[165958.721812] btree_copygc 0

[165958.721813] reclaim 0

[165958.721814] interior_updates 0

[165958.721816]

[165958.721817] open buckets 20

[165958.721818] buckets to invalidate 0

 

[165958.721820] Dev 10:

[165958.721822] buckets sectors fragmented

[165958.721823] free 221763 0 0

[165958.721825] sb 1 4104 28664

[165958.721827] journal 512 16777216 0

[165958.721828] btree 0 0 0

[165958.721830] user 238796 6941788547 883098023

[165958.721831] cached 382 4905937 7611439

[165958.721833] parity 0 0 0

[165958.721834] stripe 0 0 0

[165958.721836] need_gc_gens 0 0 0

[165958.721838] need_discard 15412 0 0

[165958.721839] unstriped 0 0 0

[165958.721841] capacity 476866

[165958.721842]

[165958.721843] reserves:

[165958.721845] stripe 14904

[165958.721846] normal 7453

[165958.721847] copygc 2

[165958.721849] btree 1

[165958.721850] btree_copygc 0

[165958.721851] reclaim 0

[165958.721852] interior_updates 0

[165958.721854]

[165958.721855] open buckets 1

[165958.721856] buckets to invalidate 0

 

[165958.721859] Dev 11:

[165958.721860] buckets sectors fragmented

[165958.721862] free 316622 0 0

[165958.721863] sb 1 4104 28664

[165958.721865] journal 512 16777216 0

[165958.721866] btree 0 0 0

[165958.721868] user 138092 3402919923 1122079055

[165958.721870] cached 0 0 0

[165958.721871] parity 0 0 0

[165958.721873] stripe 0 0 0

[165958.721874] need_gc_gens 0 0 0

[165958.721876] need_discard 21639 0 0

[165958.721878] unstriped 0 0 0

[165958.721879] capacity 476866

[165958.721881]

[165958.721882] reserves:

[165958.721883] stripe 14904

[165958.721884] normal 7453

[165958.721886] copygc 2

[165958.721887] btree 1

[165958.721888] btree_copygc 0

[165958.721890] reclaim 0

[165958.721891] interior_updates 0

[165958.721892]

[165958.721893] open buckets 0

[165958.721895] buckets to invalidate 0

 

[165958.721897] Copygc debug:

[165958.721898] running: 1

[165958.721900] copygc_wait:0

[165958.721901] copygc_wait_at:130820788764

[165958.721902] Currently waiting for:0 B

[165958.721904] Currently waiting since:467 GiB

[165958.721905] Currently calculated wait:0 B

[165958.721907] [<0>] bch2_write+0x134/0x488 [bcachefs]

[165958.721908] [<0>] bch2_data_update_read_done+0xac/0xc8 [bcachefs]

[165958.721910] [<0>] bch2_moving_ctxt_do_pending_writes+0x120/0x3c8 [bcachefs]

[165958.721911] [<0>] bkey_nocow_lock+0x1d0/0x400 [bcachefs]

[165958.721913] [<0>] bch2_data_update_init+0x7a0/0xc10 [bcachefs]

[165958.721915] [<0>] bch2_move_extent+0x174/0x750 [bcachefs]

[165958.721916] [<0>] __bch2_move_data_phys+0x688/0x770 [bcachefs]

[165958.721918] [<0>] bch2_evacuate_bucket+0x80/0xc0 [bcachefs]

[165958.721919] [<0>] bch2_copygc+0x348/0x700 [bcachefs]

[165958.721921] [<0>] bch2_copygc_thread+0x1d8/0x498 [bcachefs]

[165958.721922] [<0>] kthread+0x120/0x220

[165958.721924] [<0>] ret_from_fork+0x10/0x20

 

[165958.721926] Journal debug:

[165958.721928] flags: replay_done,running,may_skip_flush

[165958.721929] dirty journal entries: 2048/32768

[165958.721931] seq: 3835014

[165958.721932] seq_ondisk: 3835013

[165958.721934] last_seq: 3832967

[165958.721935] last_seq_ondisk: 3832967

[165958.721936] flushed_seq_ondisk: 3835013

[165958.721938] watermark: stripe

[165958.721939] each entry reserved: 321

[165958.721940] nr flush writes: 25860

[165958.721942] nr noflush writes: 30662

[165958.721944] average write size: 1.72 MiB

[165958.721945] free buf: 0

[165958.721946] nr direct reclaim: 14142

[165958.721948] nr background reclaim: 17113901

[165958.721949] reclaim kicked: 0

[165958.721950] reclaim runs in: 72 ms

[165958.721952] blocked: 0

[165958.721953] current entry sectors: 4096

[165958.721955] current entry error: (No error)

[165958.721956] current entry: 62/261816

[165958.721957] unwritten entries:

[165958.721959] seq: 3835014

[165958.721960] refcount: 1

[165958.721962] io: journal_write_done+0x0/0x448 [bcachefs] r 1

[165958.721963] size: 56.0 B

[165958.721965] expires: 165 jiffies

[165958.721966] flags: need_flush_to_write_buffer

[165958.721968] last buf open

[165958.721969] space:

[165958.721971] discarded 4096:16318464

[165958.721972] clean ondisk 4096:16318464

[165958.721973] clean 4096:16318464

[165958.721975] total 4096:16777216

[165958.721976] dev 0:

[165958.721978] durability 1:

[165958.721979] nr 799

[165958.721980] bucket size 4096

[165958.721982] available 0:2

[165958.721983] discard_idx 494

[165958.721984] dirty_ondisk 494 (seq 3832967)

[165958.721986] dirty_idx 494 (seq 3832967)

[165958.721987] cur_idx 492 (seq 3834565)

[165958.721989] dev 1:

[165958.721990] durability 1:

[165958.721992] nr 799

[165958.721993] bucket size 4096

[165958.721995] available 0:3

[165958.721996] discard_idx 251

[165958.721997] dirty_ondisk 251 (seq 3832968)

[165958.721999] dirty_idx 251 (seq 3832968)

[165958.722000] cur_idx 249 (seq 3834566)

[165958.722002] dev 3:

[165958.722003] durability 1:

[165958.722004] nr 799

[165958.722005] bucket size 4096

[165958.722007] available 0:2

[165958.722008] discard_idx 515

[165958.722010] dirty_ondisk 515 (seq 3832967)

[165958.722011] dirty_idx 515 (seq 3832967)

[165958.722013] cur_idx 513 (seq 3834565)

[165958.722014] dev 4:

[165958.722016] durability 1:

[165958.722017] nr 799

[165958.722019] bucket size 4096

[165958.722020] available 0:3

[165958.722021] discard_idx 488

[165958.722023] dirty_ondisk 488 (seq 3832968)

[165958.722024] dirty_idx 488 (seq 3832968)

[165958.722026] cur_idx 486 (seq 3834566)

[165958.722027] dev 6:

[165958.722029] durability 1:

[165958.722030] nr 1024

[165958.722031] bucket size 16384

[165958.722033] available 996:16282

[165958.722034] discard_idx 934

[165958.722036] dirty_ondisk 934 (seq 3834577)

[165958.722037] dirty_idx 934 (seq 3834577)

[165958.722039] cur_idx 960 (seq 3835012)

[165958.722040] dev 7:

[165958.722041] durability 1:

[165958.722043] nr 1024

[165958.722044] bucket size 16384

[165958.722046] available 996:15957

[165958.722047] discard_idx 937

[165958.722049] dirty_ondisk 937 (seq 3834576)

[165958.722050] dirty_idx 937 (seq 3834576)

[165958.722051] cur_idx 963 (seq 3835013)

[165958.722053] dev 8:

[165958.722054] durability 1:

[165958.722055] nr 1024

[165958.722057] bucket size 16384

[165958.722058] available 996:3570

[165958.722060] discard_idx 989

[165958.722061] dirty_ondisk 989 (seq 3834575)

[165958.722063] dirty_idx 989 (seq 3834575)

[165958.722064] cur_idx 1015 (seq 3835013)

[165958.722066] dev 9:

[165958.722067] durability 1:

[165958.722068] nr 1024

[165958.722070] bucket size 16384

[165958.722071] available 996:3

[165958.722072] discard_idx 989

[165958.722074] dirty_ondisk 989 (seq 3834575)

[165958.722075] dirty_idx 989 (seq 3834575)

[165958.722077] cur_idx 1015 (seq 3834986)

[165958.722078] dev 10:

[165958.722079] durability 1:

[165958.722081] nr 512

[165958.722082] bucket size 32768

[165958.722083] available 484:32768

[165958.722085] discard_idx 175

[165958.722086] dirty_ondisk 175 (seq 3834582)

[165958.