{ "WBThrottle": { "bytes_dirtied": 184832, "bytes_wb": 88039171966, "ios_dirtied": 32, "ios_wb": 2182414, "inodes_dirtied": 27, "inodes_wb": 400856 }, "filestore": { "journal_queue_ops": 0, "journal_queue_bytes": 0, "journal_ops": 40, "journal_bytes": 389120, "journal_latency": { "avgcount": 30378365, "sum": 5629.953486229 }, "journal_wr": 30070584, "journal_wr_bytes": { "avgcount": 30070584, "sum": 699841040384 }, "journal_full": 0, "committing": 0, "commitcycle": 108796, "commitcycle_interval": { "avgcount": 108796, "sum": 551603.668761303 }, "commitcycle_latency": { "avgcount": 108796, "sum": 7605.081264868 }, "op_queue_max_ops": 50, "op_queue_ops": 0, "ops": 30378365, "op_queue_max_bytes": 104857600, "op_queue_bytes": 0, "bytes": 632653250823, "apply_latency": { "avgcount": 30378365, "sum": 39915.495949513 }, "queue_transaction_latency_avg": { "avgcount": 30378365, "sum": 10205.793829571 } }, "finisher-JournalObjectStore": { "queue_len": 0, "complete_latency": { "avgcount": 30378365, "sum": 1005.241771512 } }, "finisher-filestore-apply-0": { "queue_len": 0, "complete_latency": { "avgcount": 30362030, "sum": 47904.479831861 } }, "finisher-filestore-ondisk-0": { "queue_len": 0, "complete_latency": { "avgcount": 30361520, "sum": 47865.809512639 } }, "leveldb": { "leveldb_get": 374079, "leveldb_transaction": 30830238, "leveldb_get_latency": { "avgcount": 374079, "sum": 100.046479523 }, "leveldb_submit_latency": { "avgcount": 30721406, "sum": 810.589554699 }, "leveldb_submit_sync_latency": { "avgcount": 108832, "sum": 3491.157360690 }, "leveldb_compact": 0, "leveldb_compact_range": 0, "leveldb_compact_queue_merge": 0, "leveldb_compact_queue_len": 0 }, "mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-FileJournal::write_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.0": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.1": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.2": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.3": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:.4": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.0": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.1": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.2": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.3": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-OSD:ShardedOpWQ:order:.4": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "mutex-WBThrottle::lock": { "wait": { "avgcount": 0, "sum": 0.000000000 } }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_ack": 0, "op_commit": 0, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_tmap_up": 0, "osdop_tmap_put": 0, "osdop_tmap_get": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 66616, "map_full": 0, "map_inc": 1978, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "osd": { "op_wip": 0, "op": 20912983, "op_in_bytes": 177433006230, "op_out_bytes": 274050255881, "op_latency": { "avgcount": 20912983, "sum": 193102.443321485 }, "op_process_latency": { "avgcount": 20912983, "sum": 73559.563687090 }, "op_prepare_latency": { "avgcount": 20914566, "sum": 28167.432613242 }, "op_r": 10123557, "op_r_out_bytes": 274050255881, "op_r_latency": { "avgcount": 10123557, "sum": 18259.169071629 }, "op_r_process_latency": { "avgcount": 10123557, "sum": 6807.498932322 }, "op_r_prepare_latency": { "avgcount": 10123559, "sum": 7701.195266601 }, "op_w": 8880531, "op_w_in_bytes": 177433006080, "op_w_rlat": { "avgcount": 5571937, "sum": 98106.558598565 }, "op_w_latency": { "avgcount": 8880531, "sum": 171642.815646464 }, "op_w_process_latency": { "avgcount": 8880531, "sum": 65308.887843692 }, "op_w_prepare_latency": { "avgcount": 8882110, "sum": 18819.659563683 }, "op_rw": 1908895, "op_rw_in_bytes": 150, "op_rw_out_bytes": 0, "op_rw_rlat": { "avgcount": 4, "sum": 0.063852030 }, "op_rw_latency": { "avgcount": 1908895, "sum": 3200.458603392 }, "op_rw_process_latency": { "avgcount": 1908895, "sum": 1443.176911076 }, "op_rw_prepare_latency": { "avgcount": 1908897, "sum": 1646.577782958 }, "subop": 21450906, "subop_in_bytes": 342845956329, "subop_latency": { "avgcount": 21450906, "sum": 123894.830622819 }, "subop_w": 21450906, "subop_w_in_bytes": 342845956329, "subop_w_latency": { "avgcount": 21450906, "sum": 123894.830622819 }, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0.000000000 }, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0.000000000 }, "pull": 0, "push": 15008, "push_out_bytes": 62011019296, "push_in": 0, "push_in_bytes": 0, "recovery_ops": 9316, "loadavg": 212, "buffer_bytes": 0, "history_alloc_Mbytes": 0, "history_alloc_num": 0, "cached_crc": 0, "cached_crc_adjusted": 0, "numpg": 137, "numpg_primary": 60, "numpg_replica": 76, "numpg_stray": 1, "heartbeat_to_peers": 29, "map_messages": 21400, "map_message_epochs": 23878, "map_message_epoch_dups": 21933, "messages_delayed_for_map": 73, "stat_bytes": 1023145230336, "stat_bytes_used": 566600421376, "stat_bytes_avail": 456544808960, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 854, "tier_clean": 0, "tier_delay": 0, "tier_proxy_read": 0, "tier_proxy_write": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0, "object_ctx_cache_hit": 20557538, "object_ctx_cache_total": 20946253, "op_cache_hit": 0, "osd_tier_flush_lat": { "avgcount": 0, "sum": 0.000000000 }, "osd_tier_promote_lat": { "avgcount": 0, "sum": 0.000000000 }, "osd_tier_r_lat": { "avgcount": 0, "sum": 0.000000000 } }, "recoverystate_perf": { "initial_latency": { "avgcount": 147, "sum": 6.558216234 }, "started_latency": { "avgcount": 416, "sum": 21615847.159196867 }, "reset_latency": { "avgcount": 563, "sum": 562.987506663 }, "start_latency": { "avgcount": 563, "sum": 0.027384212 }, "primary_latency": { "avgcount": 246, "sum": 8806050.862995667 }, "peering_latency": { "avgcount": 247, "sum": 229.245101499 }, "backfilling_latency": { "avgcount": 7, "sum": 660.491642599 }, "waitremotebackfillreserved_latency": { "avgcount": 7, "sum": 1295.110434233 }, "waitlocalbackfillreserved_latency": { "avgcount": 7, "sum": 2908.407702178 }, "notbackfilling_latency": { "avgcount": 0, "sum": 0.000000000 }, "repnotrecovering_latency": { "avgcount": 130, "sum": 12798614.882800671 }, "repwaitrecoveryreserved_latency": { "avgcount": 55, "sum": 34.097589639 }, "repwaitbackfillreserved_latency": { "avgcount": 12, "sum": 8447.263042227 }, "reprecovering_latency": { "avgcount": 67, "sum": 2798.063480590 }, "activating_latency": { "avgcount": 157, "sum": 27.893482486 }, "waitlocalrecoveryreserved_latency": { "avgcount": 38, "sum": 122.487950587 }, "waitremoterecoveryreserved_latency": { "avgcount": 38, "sum": 98.164461309 }, "recovering_latency": { "avgcount": 38, "sum": 19.279819938 }, "recovered_latency": { "avgcount": 157, "sum": 0.012714835 }, "clean_latency": { "avgcount": 97, "sum": 8800837.765840068 }, "active_latency": { "avgcount": 97, "sum": 8805832.426857109 }, "replicaactive_latency": { "avgcount": 63, "sum": 12809602.838232368 }, "stray_latency": { "avgcount": 305, "sum": 283.060837993 }, "getinfo_latency": { "avgcount": 247, "sum": 52.842184866 }, "getlog_latency": { "avgcount": 216, "sum": 3.180198273 }, "waitactingchange_latency": { "avgcount": 59, "sum": 56.335245883 }, "incomplete_latency": { "avgcount": 0, "sum": 0.000000000 }, "getmissing_latency": { "avgcount": 157, "sum": 1.118182349 }, "waitupthru_latency": { "avgcount": 154, "sum": 172.084124154 } }, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 104857600, "get": 21026588, "get_sum": 182483950574, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 21026588, "put_sum": 182483950574, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 104857600, "get": 30610824, "get_sum": 431605548535, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 30610824, "put_sum": 431605548535, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 104857600, "get": 5146328, "get_sum": 241877416, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 5146328, "put_sum": 241877416, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 104857600, "get": 5146328, "get_sum": 241877416, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 5146328, "put_sum": 241877416, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hbclient": { "val": 0, "max": 104857600, "get": 8321674, "get_sum": 391118678, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 8321674, "put_sum": 391118678, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 104857600, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-osd_client_bytes": { "val": 0, "max": 524288000, "get": 20914676, "get_sum": 182408291056, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 29787194, "put_sum": 182408291056, "wait": { "avgcount": 0, "sum": 0.000000000 } }, "throttle-osd_client_messages": { "val": 0, "max": 100, "get": 23629087, "get_sum": 23629087, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 23629087, "put_sum": 23629087, "wait": { "avgcount": 91236, "sum": 243.260437234 } } }