[root@rook-ceph-osd-1-7648fb96d8-ndrdm /]# ceph-bluestore-tool bluefs-bdev-expand --path /var/lib/ceph/osd/ceph-1 
inferring bluefs devices from bluestore path
1 : device size 0x18ffc00000 : own 0x[bffe10000~fffe0000] = 0xfffe0000 : using 0xfd190000(4.0 GiB) : bluestore has 0x82b360000(33 GiB) available
Expanding DB/WAL...

[root@rook-ceph-osd-1-7648fb96d8-ndrdm /]# ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-1 
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")
 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 18: (main()+0x136e) [0x55a4332f916e]
 19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 20: (_start()+0x2e) [0x55a43331b65e]
*** Caught signal (Aborted) **
 in thread 7fc4240cc240 thread_name:ceph-bluestore-
2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 18: (main()+0x136e) [0x55a4332f916e]
 19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 20: (_start()+0x2e) [0x55a43331b65e]

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (()+0x12dd0) [0x7fc4193e6dd0]
 2: (gsignal()+0x10f) [0x7fc417a2b70f]
 3: (abort()+0x127) [0x7fc417a15b25]
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 21: (main()+0x136e) [0x55a4332f916e]
 22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 23: (_start()+0x2e) [0x55a43331b65e]
2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
 in thread 7fc4240cc240 thread_name:ceph-bluestore-

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (()+0x12dd0) [0x7fc4193e6dd0]
 2: (gsignal()+0x10f) [0x7fc417a2b70f]
 3: (abort()+0x127) [0x7fc417a15b25]
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 21: (main()+0x136e) [0x55a4332f916e]
 22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 23: (_start()+0x2e) [0x55a43331b65e]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

  -447> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
  -446> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
  -445> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
  -444> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 18: (main()+0x136e) [0x55a4332f916e]
 19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 20: (_start()+0x2e) [0x55a43331b65e]

  -443> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
 in thread 7fc4240cc240 thread_name:ceph-bluestore-

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (()+0x12dd0) [0x7fc4193e6dd0]
 2: (gsignal()+0x10f) [0x7fc417a2b70f]
 3: (abort()+0x127) [0x7fc417a15b25]
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 21: (main()+0x136e) [0x55a4332f916e]
 22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 23: (_start()+0x2e) [0x55a43331b65e]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

    -4> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluestore(/var/lib/ceph/osd/ceph-1) allocate_bluefs_freespace failed to allocate on 0x3d1b0000 min_size 0x43d0000 > allocated total 0x300000 bluefs_shared_alloc_size 0x10000 allocated 0x300000 available 0x b019c8000
    -3> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _allocate failed to expand slow device to fit +0x43ce0aa
    -2> 2020-12-16T13:24:20.065+0000 7fc4240cc240 -1 bluefs _flush_range allocated: 0x0 offset: 0x0 length: 0x43ce0aa
    -1> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: In function 'int BlueFS::_flush_range(BlueFS::FileWriter*, uint64_t, uint64_t)' thread 7fc4240cc240 time 2020-12-16T13:24:20.069595+0000
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos8/DIST/centos8/MACHINE_SIZE/gigantic/release/15.2.6/rpm/el8/BUILD/ceph-15.2.6/src/os/bluestore/BlueFS.cc: 2721: ceph_abort_msg("bluefs enospc")

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0xe5) [0x7fc41a1e82cc]
 2: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 3: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 4: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 5: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 6: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 7: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 8: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 9: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 10: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 11: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 12: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 13: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 14: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 15: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 16: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 17: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 18: (main()+0x136e) [0x55a4332f916e]
 19: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 20: (_start()+0x2e) [0x55a43331b65e]

     0> 2020-12-16T13:24:20.069+0000 7fc4240cc240 -1 *** Caught signal (Aborted) **
 in thread 7fc4240cc240 thread_name:ceph-bluestore-

 ceph version 15.2.6 (cb8c61a60551b72614257d632a574d420064c17a) octopus (stable)
 1: (()+0x12dd0) [0x7fc4193e6dd0]
 2: (gsignal()+0x10f) [0x7fc417a2b70f]
 3: (abort()+0x127) [0x7fc417a15b25]
 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x1b6) [0x7fc41a1e839d]
 5: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned long)+0x1a43) [0x55a433337073]
 6: (BlueFS::_flush(BlueFS::FileWriter*, bool, bool*)+0x110) [0x55a433337760]
 7: (BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x32) [0x55a4333484f2]
 8: (BlueRocksWritableFile::Flush()+0x72) [0x55a4335204a2]
 9: (rocksdb::WritableFileWriter::Flush()+0x1fc) [0x55a43374c95c]
 10: (rocksdb::WritableFileWriter::Sync(bool)+0x1f) [0x55a43374cadf]
 11: (rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2be5) [0x55a433774085]
 12: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0xc07) [0x55a4336217c7]
 13: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool)+0x12dd) [0x55a4336232cd]
 14: (rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool)+0xb09) [0x55a433624789]
 15: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x9f5) [0x55a43361f025]
 16: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x15) [0x55a433620375]
 17: (RocksDBStore::do_open(std::ostream&, bool, bool, std::vector<KeyValueDB::ColumnFamily, std::allocator<KeyValueDB::ColumnFamily> > const*)+0x24e4) [0x55a4335af714]
 18: (BlueStore::_open_db(bool, bool, bool)+0x1239) [0x55a433395989]
 19: (BlueStore::_open_db_and_around(bool)+0x19b) [0x55a4333aca0b]
 20: (BlueStore::_fsck(BlueStore::FSCKDepth, bool)+0x1e1) [0x55a4333fe0a1]
 21: (main()+0x136e) [0x55a4332f916e]
 22: (__libc_start_main()+0xf3) [0x7fc417a176a3]
 23: (_start()+0x2e) [0x55a43331b65e]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

Aborted (core dumped)