Hi, Ceph cluster: Multisite over 2 regions Per region: 6 OSD nodes, same nodes run RGW with rgw_run_sync_thread=true 1 dedicated RGW node serving client traffic (rgw_run_sync_thread=false) Trying to add a tier-type=cloud zone that would sync to AWS (based on https://docs.ceph.com/en/latest/radosgw/cloud-sync-module/ and https://oneuptime.com/blog/post/2026-03-31-rook-cloud-sync-rgw-aws-s3/view ) Switched RGW on one OSD to sync that cloud zone, added IP address of that RGW node as zone endpoint sudo radosgw-admin zonegroup get { "id": "c1565bb4-acf9-4def-bfe1-7f189136c07b", "name": "poc-ms", "api_name": "poc-ms", "is_master": true, "endpoints": [ "http://ceph-cluster-poc1.ceph-radosgw.service.region1.consul:7480" ], "hostnames": [], "hostnames_s3website": [], "master_zone": "6e1e00de-a43d-4da9-994a-081041f6afb8", "zones": [ { "id": "3838f112-6804-4712-8af5-ab57a886e64c", "name": "poc2-region2", "endpoints": [ " http://ceph-cluster-poc2.ceph-radosgw.service.region2.consul:7480" ], "log_meta": false, "log_data": true, "bucket_index_max_shards": 11, "read_only": false, "tier_type": "", "sync_from_all": true, "sync_from": [], "redirect_zone": "", "supported_features": [ "compress-encrypted", "notification_v2", "resharding" ] }, { "id": "6e1e00de-a43d-4da9-994a-081041f6afb8", "name": "poc1-region1", "endpoints": [ " http://ceph-cluster-poc1.ceph-radosgw.service.region1.consul:7480" ], "log_meta": false, "log_data": true, "bucket_index_max_shards": 11, "read_only": false, "tier_type": "", "sync_from_all": true, "sync_from": [], "redirect_zone": "", "supported_features": [ "compress-encrypted", "notification_v2", "resharding" ] }, { "id": "8c988f68-975f-4c39-ab53-520f9fc6ee9e", "name": "poc-cloud-backup", "endpoints": [ "http://rgw_node_ip:7480" ], "log_meta": false, "log_data": true, "bucket_index_max_shards": 11, "read_only": false, "tier_type": "cloud", "sync_from_all": true, "sync_from": [], "redirect_zone": "", "supported_features": [ "compress-encrypted", "notification_v2", "resharding" ] } ], "placement_targets": [ { "name": "default-placement", "tags": [], "storage_classes": [ "STANDARD", "STANDARD_IA" ] }, { "name": "ec-only", "tags": [], "storage_classes": [ "STANDARD", "STANDARD_IA" ] } ], "default_placement": "default-placement", "realm_id": "939ac8b7-8237-4a50-bea5-fed4b1dfd74d", "sync_policy": { "groups": [ { "id": "bucket-sync", "data_flow": { "symmetrical": [ { "id": "sym-region1-region2", "zones": [ "3838f112-6804-4712-8af5-ab57a886e64c", "6e1e00de-a43d-4da9-994a-081041f6afb8" ] } ], "directional": [ { "source_zone": "6e1e00de-a43d-4da9-994a-081041f6afb8", "dest_zone": "8c988f68-975f-4c39-ab53-520f9fc6ee9e" } ] }, "pipes": [ { "id": "pipe-all", "source": { "bucket": "*", "zones": [ "*" ] }, "dest": { "bucket": "*", "zones": [ "*" ] }, "params": { "source": { "filter": { "tags": [] } }, "dest": {}, "priority": 0, "mode": "system" } } ], "status": "enabled" } ] }, "enabled_features": [ "notification_v2", "resharding" ] } Verified that: - Cloud zone has correct AWS credentials (can PUT object directly to the bucket with same credentials) sudo radosgw-admin zone get --rgw-zone=poc-cloud-backup { "id": "8c988f68-975f-4c39-ab53-520f9fc6ee9e", "name": "poc-cloud-backup", "domain_root": "poc-cloud-backup.rgw.meta:root", "control_pool": "poc-cloud-backup.rgw.control", "dedup_pool": "poc-cloud-backup.rgw.dedup", "gc_pool": "poc-cloud-backup.rgw.log:gc", "lc_pool": "poc-cloud-backup.rgw.log:lc", "log_pool": "poc-cloud-backup.rgw.log", "intent_log_pool": "poc-cloud-backup.rgw.log:intent", "usage_log_pool": "poc-cloud-backup.rgw.log:usage", "roles_pool": "poc-cloud-backup.rgw.meta:roles", "reshard_pool": "poc-cloud-backup.rgw.log:reshard", "user_keys_pool": "poc-cloud-backup.rgw.meta:users.keys", "user_email_pool": "poc-cloud-backup.rgw.meta:users.email", "user_swift_pool": "poc-cloud-backup.rgw.meta:users.swift", "user_uid_pool": "poc-cloud-backup.rgw.meta:users.uid", "otp_pool": "poc-cloud-backup.rgw.otp", "notif_pool": "poc-cloud-backup.rgw.log:notif", "topics_pool": "poc-cloud-backup.rgw.meta:topics", "account_pool": "poc-cloud-backup.rgw.meta:accounts", "group_pool": "poc-cloud-backup.rgw.meta:groups", "bucket_logging_pool": "poc-cloud-backup.rgw.log:logging", "system_key": { "access_key": "XXX", "secret_key": "XXXXX" }, "placement_pools": [ { "key": "default-placement", "val": { "index_pool": "poc-cloud-backup.rgw.buckets.index", "storage_classes": { "STANDARD": { "data_pool": "poc-cloud-backup.rgw.buckets.data-r3" }, "STANDARD_IA": { "data_pool": "poc-cloud-backup.rgw.buckets.data-ec42" } }, "data_extra_pool": "poc-cloud-backup.rgw.buckets.non-ec", "index_type": 0, "inline_data": true } }, { "key": "ec-only", "val": { "index_pool": "poc-cloud-backup.rgw.buckets.index", "storage_classes": { "STANDARD": {}, "STANDARD_IA": { "data_pool": "poc-cloud-backup.rgw.buckets.data-ec42" } }, "data_extra_pool": "poc-cloud-backup.rgw.buckets.non-ec", "index_type": 0, "inline_data": true } } ], "tier_config": { "connection": { "access_key": "XXX", "endpoint": "https://s3.eu-north-1.amazonaws.com", "host_style": "virtual", "region": "eu-north-1", "secret": "XXXXX" }, "target_path": "cloud-sync-test" }, "realm_id": "939ac8b7-8237-4a50-bea5-fed4b1dfd74d", "restore_pool": "poc-cloud-backup.rgw.log:restore" } Sync status seems OK sudo radosgw-admin bucket sync status --bucket=sync-test01 --rgw-zone=poc-cloud-backup realm 939ac8b7-8237-4a50-bea5-fed4b1dfd74d (default) zonegroup c1565bb4-acf9-4def-bfe1-7f189136c07b (poc-ms) zone 8c988f68-975f-4c39-ab53-520f9fc6ee9e (poc-cloud-backup) bucket :sync-test01[6e1e00de-a43d-4da9-994a-081041f6afb8.16963179.1 ]) current time 2026-07-24T13:48:45Z source zone 6e1e00de-a43d-4da9-994a-081041f6afb8 (poc1-region1) source bucket :sync-test01[6e1e00de-a43d-4da9-994a-081041f6afb8.16963179.1 ]) incremental sync on 11 shards bucket is caught up with source source zone 6e1e00de-a43d-4da9-994a-081041f6afb8 (poc1-region1) source bucket :sync-test01[6e1e00de-a43d-4da9-994a-081041f6afb8.16963179.1 ]) incremental sync on 11 shards bucket is caught up with source source zone 6e1e00de-a43d-4da9-994a-081041f6afb8 (poc1-region1) source bucket :sync-test01[6e1e00de-a43d-4da9-994a-081041f6afb8.16963179.1 ]) incremental sync on 11 shards bucket is caught up with source However, nothing appears on AWS And can’t find anything meaningful in RGW logs I keep adding new objects to the master zone every time I change some configs