debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 set uid:gid to 167:167 (ceph:ceph) debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable), process ceph-osd, pid 1 debug 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 pidfile_write: ignore empty --pid-file debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB debug 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close debug 2020-12-16T13:19:22.084+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close debug 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 0 starting osd.1 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal debug 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 -1 Falling back to public interface debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 0 load: jerasure load: lrc load: isa debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 debug 2020-12-16T13:19:22.328+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:0.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:1.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:2.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:3.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:4.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:5.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:6.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 0 osd.1:7.OSDShard using op scheduler ClassedOpQueueScheduler(queue=WeightedPriorityQueue, cutoff=196) debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _mount path /var/lib/ceph/osd/ceph-1 debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs mount debug 2020-12-16T13:19:22.624+0000 7fc4c3f13f40 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x18ffc00000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 1 rocksdb: do_open column families: [default] debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: RocksDB version: 6.1.2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Compile date Nov 17 2020 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: DB SUMMARY debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: CURRENT file: CURRENT debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: IDENTITY file: IDENTITY debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: MANIFEST file: MANIFEST-001369 size: 15227 Bytes debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: SST files in db dir, Total Num: 59, files: 001195.sst 001226.sst 001227.sst 001228.sst 001229.sst 001230.sst 001250.sst 001251.sst 001262.sst debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Write Ahead Log file in db.wal: 001400.log size: 255396293 ; 001402.log size: 249498342 ; debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.error_if_exists: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.create_if_missing: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_checks: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.env: 0x55a0b0301f80 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.info_log: 0x55a0b030ad80 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_file_opening_threads: 16 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.statistics: (nil) debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_fsync: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_log_file_size: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_manifest_file_size: 1073741824 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.log_file_time_to_roll: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.keep_log_file_num: 1000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.recycle_log_file_num: 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_fallocate: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_reads: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_writes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_reads: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.create_missing_column_families: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.db_log_dir: debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_dir: db.wal debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_cache_numshardbits: 6 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_subcompactions: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_flushes: -1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_ttl_seconds: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_size_limit_MB: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.manifest_preallocation_size: 4194304 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.is_fd_close_on_exec: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.advise_random_on_open: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.db_write_buffer_size: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_manager: 0x55a0b1009110 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.access_hint_on_compaction_start: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.random_access_max_buffer_size: 1048576 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.use_adaptive_mutex: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limiter: (nil) debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_recovery_mode: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_thread_tracking: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_pipelined_write: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_concurrent_memtable_write: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_max_yield_usec: 100 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_slow_yield_usec: 3 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.row_cache: None debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_filter: None debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_recovery: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.allow_ingest_behind: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.preserve_deletes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.two_write_queues: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.manual_wal_flush: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.atomic_flush: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_jobs: 2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_compactions: 2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_shutdown: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.writable_file_max_buffer_size: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.delayed_write_rate : 16777216 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_total_wal_size: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_dump_period_sec: 600 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_persist_period_sec: 600 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.stats_history_buffer_size: 1048576 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_open_files: -1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bytes_per_sync: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.wal_bytes_per_sync: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_readahead_size: 2097152 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Compression algorithms supported: debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZSTDNotFinalCompression supported: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZSTD supported: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kXpressCompression supported: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kLZ4HCCompression supported: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kLZ4Compression supported: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kBZip2Compression supported: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kZlibCompression supported: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: kSnappyCompression supported: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Fast CRC32 supported: Supported on x86 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_readonly.cc:22] Opening the db in read only mode debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-001369 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter: None debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter_factory: None debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_factory: SkipListFactory debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_factory: BlockBasedTable debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a0b0386438) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55a0b02e0b10 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_size: 268435456 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number: 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression: NoCompression debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression: Disabled debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.prefix_extractor: nullptr debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.num_levels: 7 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.level: 32767 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.enabled: false debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.window_bits: -14 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.level: 32767 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.strategy: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.enabled: false debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level0_stop_writes_trigger: 36 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_base: 67108864 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_multiplier: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_base: 268435456 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_compaction_bytes: 1677721600 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.arena_block_size: 33554432 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.disable_auto_compactions: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.table_properties_collectors: debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_support: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_num_locks: 10000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_whole_key_filtering: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_huge_page_size: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.bloom_locality: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.max_successive_merges: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.optimize_filters_for_hits: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_file_checks: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.force_consistency_checks: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.report_bg_io_stats: 0 debug 2020-12-16T13:19:22.672+0000 7fc4c3f13f40 4 rocksdb: Options.ttl: 0 debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-001369 succeeded,manifest_file_number is 1369, next_file_number is 1403, last_sequence is 144598951, log_number is 1400,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 1400 debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124762690294, "job": 1, "event": "recovery_started", "log_files": [1400, 1402]} debug 2020-12-16T13:19:22.684+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1400 mode 0 debug 2020-12-16T13:19:24.644+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1402 mode 0 debug 2020-12-16T13:19:24.648+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124764655926, "job": 1, "event": "recovery_finished"} debug 2020-12-16T13:19:24.648+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2 debug 2020-12-16T13:19:24.676+0000 7fc4c3f13f40 1 freelist init debug 2020-12-16T13:19:24.676+0000 7fc4c3f13f40 1 freelist _init_from_label debug 2020-12-16T13:19:24.680+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_alloc opening allocation metadata debug 2020-12-16T13:19:25.312+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _open_alloc loaded 48 GiB in 666948 extents available 44 GiB debug 2020-12-16T13:19:25.312+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl.cc:390] Shutdown: canceling all background work debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl.cc:563] Shutdown complete debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 1 bluefs umount debug 2020-12-16T13:19:25.316+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs mount debug 2020-12-16T13:19:25.416+0000 7fc4c3f13f40 1 bluefs _init_alloc id 1 alloc_size 0x10000 size 0x18ffc00000 debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152 debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2 debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4 debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1 debug 2020-12-16T13:19:25.464+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option compaction_readahead_size = 2097152 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option compression = kNoCompression debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option max_background_compactions = 2 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option max_write_buffer_number = 4 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option min_write_buffer_number_to_merge = 1 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option recycle_log_file_num = 4 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option writable_file_max_buffer_size = 0 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 0 set rocksdb option write_buffer_size = 268435456 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 1 rocksdb: do_open column families: [default] debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: RocksDB version: 6.1.2 debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ debug 2020-12-16T13:19:25.468+0000 7fc4c3f13f40 4 rocksdb: Compile date Nov 17 2020 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: DB SUMMARY debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: CURRENT file: CURRENT debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: IDENTITY file: IDENTITY debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: MANIFEST file: MANIFEST-001369 size: 15227 Bytes debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: SST files in db dir, Total Num: 59, files: 001195.sst 001226.sst 001227.sst 001228.sst 001229.sst 001230.sst 001250.sst 001251.sst 001262.sst debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Write Ahead Log file in db.wal: 001400.log size: 255396293 ; 001402.log size: 249498342 ; debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.error_if_exists: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.create_if_missing: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_checks: 1 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.env: 0x55a0b12b1740 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.info_log: 0x55a0b038f6a0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_file_opening_threads: 16 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.statistics: (nil) debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_fsync: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_log_file_size: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.max_manifest_file_size: 1073741824 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.log_file_time_to_roll: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.keep_log_file_num: 1000 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.recycle_log_file_num: 4 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_fallocate: 1 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_reads: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.allow_mmap_writes: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_reads: 0 debug 2020-12-16T13:19:25.472+0000 7fc4c3f13f40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.create_missing_column_families: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.db_log_dir: debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.wal_dir: db.wal debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.table_cache_numshardbits: 6 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.max_subcompactions: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_flushes: -1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_ttl_seconds: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.WAL_size_limit_MB: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.manifest_preallocation_size: 4194304 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.is_fd_close_on_exec: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.advise_random_on_open: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.db_write_buffer_size: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_manager: 0x55a0b157c8a0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.access_hint_on_compaction_start: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.random_access_max_buffer_size: 1048576 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.use_adaptive_mutex: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limiter: (nil) debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.wal_recovery_mode: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_thread_tracking: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_pipelined_write: 0 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.allow_concurrent_memtable_write: 1 debug 2020-12-16T13:19:25.476+0000 7fc4c3f13f40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_max_yield_usec: 100 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.write_thread_slow_yield_usec: 3 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.row_cache: None debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.wal_filter: None debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_recovery: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.allow_ingest_behind: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.preserve_deletes: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.two_write_queues: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.manual_wal_flush: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.atomic_flush: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_jobs: 2 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_background_compactions: 2 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.avoid_flush_during_shutdown: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.writable_file_max_buffer_size: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.delayed_write_rate : 16777216 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_total_wal_size: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_dump_period_sec: 600 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_persist_period_sec: 600 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.stats_history_buffer_size: 1048576 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.max_open_files: -1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.bytes_per_sync: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.wal_bytes_per_sync: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_readahead_size: 2097152 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Compression algorithms supported: debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZSTDNotFinalCompression supported: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZSTD supported: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kXpressCompression supported: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kLZ4HCCompression supported: 1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kLZ4Compression supported: 1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kBZip2Compression supported: 0 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kZlibCompression supported: 1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: kSnappyCompression supported: 1 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: Fast CRC32 supported: Supported on x86 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3543] Recovering from manifest file: db/MANIFEST-001369 debug 2020-12-16T13:19:25.480+0000 7fc4c3f13f40 4 rocksdb: [db/column_family.cc:477] --------------- Options for column family [default]: debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter: None debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_filter_factory: None debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_factory: SkipListFactory debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.table_factory: BlockBasedTable debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a0b0386438) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x55a0b02e1510 block_cache_name: BinnedLRUCache block_cache_options: capacity : 1288490188 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.write_buffer_size: 268435456 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number: 4 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression: NoCompression debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression: Disabled debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.prefix_extractor: nullptr debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.num_levels: 7 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.level: 32767 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.bottommost_compression_opts.enabled: false debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.window_bits: -14 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.level: 32767 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.strategy: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.compression_opts.enabled: false debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 debug 2020-12-16T13:19:25.484+0000 7fc4c3f13f40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.level0_stop_writes_trigger: 36 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_base: 67108864 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.target_file_size_multiplier: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_base: 268435456 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_compaction_bytes: 1677721600 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.arena_block_size: 33554432 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.disable_auto_compactions: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.table_properties_collectors: debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_support: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.inplace_update_num_locks: 10000 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_whole_key_filtering: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.memtable_huge_page_size: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.bloom_locality: 0 debug 2020-12-16T13:19:25.488+0000 7fc4c3f13f40 4 rocksdb: Options.max_successive_merges: 0 debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.optimize_filters_for_hits: 0 debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.paranoid_file_checks: 0 debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.force_consistency_checks: 0 debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.report_bg_io_stats: 0 debug 2020-12-16T13:19:25.492+0000 7fc4c3f13f40 4 rocksdb: Options.ttl: 0 debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3757] Recovered from manifest file:db/MANIFEST-001369 succeeded,manifest_file_number is 1369, next_file_number is 1403, last_sequence is 144598951, log_number is 1400,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/version_set.cc:3766] Column family [default] (ID 0), log number is 1400 debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1608124765507105, "job": 1, "event": "recovery_started", "log_files": [1400, 1402]} debug 2020-12-16T13:19:25.500+0000 7fc4c3f13f40 4 rocksdb: [db/db_impl_open.cc:583] Recovering log #1400 mode 0 debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate failed to allocate 0x43ce43d on bdev 1, free 0x2e50000; fallback to bdev 2 debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 1 bluefs _allocate unable to allocate 0x43ce43d on bdev 2, free 0xffffffffffffffff; fallback to slow device expander debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000 debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _allocate failed to expand slow device to fit +0x43ce43d debug 2020-12-16T13:19:27.724+0000 7fc4c3f13f40 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce43d /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4c3f13f40 time 2020-12-16T13:19:27.731533+0000 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc") ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable) 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x55a0a4e0a4fe] 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073] 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760] 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock&)+0x32) [0x55a0a54994f2] 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2] 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c] 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f] 8: (rocksdb::BuildTable(std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase*, std::vector >, std::allocator > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector >, std::allocator > > > const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::vector >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085] 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97] 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d] 11: (rocksdb::DBImpl::Recover(std::vector > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59] 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5] 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**)+0x15) [0x55a0a5955845] 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector > const*)+0x24e4) [0x55a0a58e6984] 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079] 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb] 17: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787] 18: (OSD::init()+0x380) [0x55a0a4f18dd0] 19: (main()+0x47e4) [0x55a0a4e6c1b4] 20: (__libc_start_main()+0xf3) [0x7fc4c08db6a3] 21: (_start()+0x2e) [0x55a0a4e9a39e] debug 2020-12-16T13:19:27.732+0000 7fc4c3f13f40 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4c3f13f40 time 2020-12-16T13:19:27.731533+0000 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc") ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable) 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xe5) [0x55a0a4e0a4fe] 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073] 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760] 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock&)+0x32) [0x55a0a54994f2] 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2] 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c] 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f] 8: (rocksdb::BuildTable(std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase*, std::vector >, std::allocator > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector >, std::allocator > > > const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::vector >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085] 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97] 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d] 11: (rocksdb::DBImpl::Recover(std::vector > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59] 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5] 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**)+0x15) [0x55a0a5955845] 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector > const*)+0x24e4) [0x55a0a58e6984] 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079] 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb] 17: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787] 18: (OSD::init()+0x380) [0x55a0a4f18dd0] 19: (main()+0x47e4) [0x55a0a4e6c1b4] 20: (__libc_start_main()+0xf3) [0x7fc4c08db6a3] 21: (_start()+0x2e) [0x55a0a4e9a39e] *** Caught signal (Aborted) ** in thread 7fc4c3f13f40 thread_name:ceph-osd ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable) 1: (()+0x12dd0) [0x7fc4c1c86dd0] 2: (gsignal()+0x10f) [0x7fc4c08ef70f] 3: (abort()+0x127) [0x7fc4c08d9b25] 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b6) [0x55a0a4e0a5cf] 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073] 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760] 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock&)+0x32) [0x55a0a54994f2] 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2] 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c] 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f] 11: (rocksdb::BuildTable(std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase*, std::vector >, std::allocator > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector >, std::allocator > > > const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::vector >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085] 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97] 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d] 14: (rocksdb::DBImpl::Recover(std::vector > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59] 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5] 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**)+0x15) [0x55a0a5955845] 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector > const*)+0x24e4) [0x55a0a58e6984] 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079] 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb] 20: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787] 21: (OSD::init()+0x380) [0x55a0a4f18dd0] 22: (main()+0x47e4) [0x55a0a4e6c1b4] 23: (__libc_start_main()+0xf3) [0x7fc4c08db6a3] 24: (_start()+0x2e) [0x55a0a4e9a39e] debug 2020-12-16T13:19:27.740+0000 7fc4c3f13f40 -1 *** Caught signal (Aborted) ** in thread 7fc4c3f13f40 thread_name:ceph-osd ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable) 1: (()+0x12dd0) [0x7fc4c1c86dd0] 2: (gsignal()+0x10f) [0x7fc4c08ef70f] 3: (abort()+0x127) [0x7fc4c08d9b25] 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x1b6) [0x55a0a4e0a5cf] 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a0a5488073] 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a0a5488760] 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock&)+0x32) [0x55a0a54994f2] 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a0a54b87a2] 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a0a5a7ed9c] 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a0a5a7ef1f] 11: (rocksdb::BuildTable(std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase*, std::vector >, std::allocator > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector >, std::allocator > > > const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::vector >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a0a5aa6085] 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a0a5956c97] 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector > const&, unsigned long*, bool)+0x12dd) [0x55a0a595879d] 14: (rocksdb::DBImpl::Recover(std::vector > const&, bool, bool, bool)+0xb09) [0x55a0a5959c59] 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a0a59544f5] 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string, std::allocator > const&, std::vector > const&, std::vector >*, rocksdb::DB**)+0x15) [0x55a0a5955845] 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector > const*)+0x24e4) [0x55a0a58e6984] 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a0a5386079] 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a0a539d0fb] 20: (BlueStore::_mount(bool, bool)+0x847) [0x55a0a53ef787] 21: (OSD::init()+0x380) [0x55a0a4f18dd0] 22: (main()+0x47e4) [0x55a0a4e6c1b4] 23: (__libc_start_main()+0xf3) [0x7fc4c08db6a3] 24: (_start()+0x2e) [0x55a0a4e9a39e] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- debug -696> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command assert hook 0x55a0b02985b0 debug -695> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command abort hook 0x55a0b02985b0 debug -694> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command leak_some_memory hook 0x55a0b02985b0 debug -693> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perfcounters_dump hook 0x55a0b02985b0 debug -692> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 1 hook 0x55a0b02985b0 debug -691> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf dump hook 0x55a0b02985b0 debug -690> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perfcounters_schema hook 0x55a0b02985b0 debug -689> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf histogram dump hook 0x55a0b02985b0 debug -688> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 2 hook 0x55a0b02985b0 debug -687> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf schema hook 0x55a0b02985b0 debug -686> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf histogram schema hook 0x55a0b02985b0 debug -685> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command perf reset hook 0x55a0b02985b0 debug -684> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config show hook 0x55a0b02985b0 debug -683> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config help hook 0x55a0b02985b0 debug -682> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config set hook 0x55a0b02985b0 debug -681> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config unset hook 0x55a0b02985b0 debug -680> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config get hook 0x55a0b02985b0 debug -679> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config diff hook 0x55a0b02985b0 debug -678> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command config diff get hook 0x55a0b02985b0 debug -677> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command injectargs hook 0x55a0b02985b0 debug -676> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log flush hook 0x55a0b02985b0 debug -675> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log dump hook 0x55a0b02985b0 debug -674> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command log reopen hook 0x55a0b02985b0 debug -673> 2020-12-16T13:19:21.748+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command dump_mempools hook 0x55a0b0f18068 debug -672> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: get_monmap_and_config debug -671> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: build_initial_monmap debug -670> 2020-12-16T13:19:21.764+0000 7fc4c3f13f40 10 monclient: monmap: epoch 0 fsid 39941002-eb2d-498a-bab4-04961fb33a42 last_changed 2020-12-16T13:19:21.772470+0000 created 2020-12-16T13:19:21.772470+0000 min_mon_release 0 (unknown) 0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.noname-b 1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.noname-a debug -669> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -668> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -667> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -666> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: none debug -665> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -664> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -663> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -662> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -661> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -660> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -659> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -658> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -657> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -656> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -655> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -654> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -653> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring debug -652> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: init debug -651> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx debug -650> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx debug -649> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: cephx debug -648> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding auth protocol: none debug -647> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -646> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -645> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -644> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -643> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -642> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -641> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -640> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -639> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -638> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -637> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: crc debug -636> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 5 AuthRegistry(0x7ffcedef1a30) adding con mode: secure debug -635> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring debug -634> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring debug -633> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: _reopen_session rank -1 debug -632> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient: _add_conns ranks=[1,0] debug -631> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): picked mon.noname-a con 0x55a0b0342c00 addr [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] debug -630> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): picked mon.noname-b con 0x55a0b0343000 addr [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] debug -629> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): start opening mon connection debug -628> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): start opening mon connection debug -627> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): _renew_subs debug -626> 2020-12-16T13:19:21.768+0000 7fc4c3f13f40 10 monclient(hunting): authenticate will time out at 2020-12-16T13:24:21.774889+0000 debug -625> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): get_auth_request con 0x55a0b0343000 auth_method 0 debug -624> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] debug -623> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): _init_auth method 2 debug -622> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): get_auth_request con 0x55a0b0342c00 auth_method 0 debug -621> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] debug -620> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): _init_auth method 2 debug -619> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more payload 9 debug -618> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more payload_len 9 debug -617> 2020-12-16T13:19:21.768+0000 7fc4bd495700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes debug -616> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more payload 9 debug -615> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more payload_len 9 debug -614> 2020-12-16T13:19:21.768+0000 7fc4bdc96700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes debug -613> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient(hunting): handle_auth_done global_id 14765571 payload 450 debug -612> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient: _finish_hunting 0 debug -611> 2020-12-16T13:19:21.772+0000 7fc4bd495700 1 monclient: found mon.noname-b debug -610> 2020-12-16T13:19:21.772+0000 7fc4bd495700 10 monclient: _send_mon_message to mon.noname-b at v2:172.31.36.158:3300/0 debug -609> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_monmap mon_map magic: 0 v1 debug -608> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: got monmap 24 from mon.noname-b (according to old e24) debug -607> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: dump: epoch 24 fsid 39941002-eb2d-498a-bab4-04961fb33a42 last_changed 2020-12-16T09:55:13.010421+0000 created 2019-11-21T16:05:56.597779+0000 min_mon_release 15 (octopus) 0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.f 1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.k 2: [v2:172.31.163.187:3300/0,v1:172.31.163.187:6789/0] mon.m debug -606> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _finish_auth 0 debug -605> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _check_auth_rotating renewing rotating keys (they expired before 2020-12-16T13:18:51.777665+0000) debug -604> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: _send_mon_message to mon.f at v2:172.31.36.158:3300/0 debug -603> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_config config(7 keys) v1 debug -602> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: handle_monmap mon_map magic: 0 v1 debug -601> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: got monmap 24 from mon.f (according to old e24) debug -600> 2020-12-16T13:19:21.772+0000 7fc4bcc94700 10 monclient: dump: epoch 24 fsid 39941002-eb2d-498a-bab4-04961fb33a42 last_changed 2020-12-16T09:55:13.010421+0000 created 2019-11-21T16:05:56.597779+0000 min_mon_release 15 (octopus) 0: [v2:172.31.36.158:3300/0,v1:172.31.36.158:6789/0] mon.f 1: [v2:172.31.225.8:3300/0,v1:172.31.225.8:6789/0] mon.k 2: [v2:172.31.163.187:3300/0,v1:172.31.163.187:6789/0] mon.m debug -599> 2020-12-16T13:19:21.772+0000 7fc4bb491700 4 set_mon_vals no callback set debug -598> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals bluestore_warn_on_legacy_statfs = false debug -597> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 monclient: authenticate success, global_id 14765571 debug -596> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals log_file = debug -595> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals mon_allow_pool_delete = true debug -594> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 10 monclient: get_monmap_and_config success debug -593> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals mon_cluster_log_file = debug -592> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 10 monclient: shutdown debug -591> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals osd_pool_default_pg_autoscale_mode = on debug -590> 2020-12-16T13:19:21.772+0000 7fc4bb491700 10 set_mon_vals rbd_default_features = 3 debug -589> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 set uid:gid to 167:167 (ceph:ceph) debug -588> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable), process ceph-osd, pid 1 debug -587> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 0 pidfile_write: ignore empty --pid-file debug -586> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) init /var/run/ceph/ceph-osd.1.asok debug -585> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) bind_and_listen /var/run/ceph/ceph-osd.1.asok debug -584> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command 0 hook 0x55a0b029c368 debug -583> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command version hook 0x55a0b029c368 debug -582> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command git_version hook 0x55a0b029c368 debug -581> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command help hook 0x55a0b02983c0 debug -580> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command get_command_descriptions hook 0x55a0b0298070 debug -579> 2020-12-16T13:19:21.772+0000 7fc4bc493700 5 asok(0x55a0b0344000) entry start debug -578> 2020-12-16T13:19:21.772+0000 7fc4c3f13f40 5 object store type is bluestore debug -577> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug -576> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug -575> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug -574> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluestore(/var/lib/ceph/osd/ceph-1) _set_cache_sizes cache_size 3221225472 meta 0.4 kv 0.4 data 0.2 debug -573> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command bluestore bluefs available hook 0x55a0b02980b0 debug -572> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) register_command bluefs stats hook 0x55a0b02980b0 debug -571> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev create path /var/lib/ceph/osd/ceph-1/block type kernel debug -570> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open path /var/lib/ceph/osd/ceph-1/block debug -569> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) open size 107369988096 (0x18ffc00000, 100 GiB) block_size 4096 (4 KiB) non-rotational discard not supported debug -568> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1/block size 100 GiB debug -567> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) unregister_commands bluefs stats debug -566> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 5 asok(0x55a0b0344000) unregister_commands bluestore bluefs available debug -565> 2020-12-16T13:19:21.776+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0700 /var/lib/ceph/osd/ceph-1/block) close debug -564> 2020-12-16T13:19:22.084+0000 7fc4c3f13f40 1 bdev(0x55a0b0fd0000 /var/lib/ceph/osd/ceph-1/block) close debug -563> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -562> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -561> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: cephx debug -560> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding auth protocol: none debug -559> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -558> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -557> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -556> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -555> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -554> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -553> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -552> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -551> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -550> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -549> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: crc debug -548> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032e940) adding con mode: secure debug -547> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring debug -546> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx debug -545> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx debug -544> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: cephx debug -543> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding auth protocol: none debug -542> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -541> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -540> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -539> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -538> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -537> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -536> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -535> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -534> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -533> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -532> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: crc debug -531> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b032f940) adding con mode: secure debug -530> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/osd/ceph-1/keyring debug -529> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx debug -528> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx debug -527> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: cephx debug -526> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding auth protocol: none debug -525> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure debug -524> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc debug -523> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure debug -522> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc debug -521> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure debug -520> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc debug -519> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: crc debug -518> 2020-12-16T13:19:22.316+0000 7fc4c3f13f40 5 AuthRegistry(0x55a0b0330940) adding con mode: secure