OB 运维 | OceanBase 单机改三副本：实操手记与深思

ALTER RESOURCE POOL sys_pool ZONE_LIST=('zone1','zone2','zone3');
ALTER TENANT sys LOCALITY='F@zone1,F@zone2';
ALTER TENANT sys LOCALITY='F@zone1,F@zone2,F@zone3';

ALTER TABLE test_t1 set locality = 'F@zone3,F@zone2,F@zone1';
ALTER TABLE test_t1 locality='F,R{all_server}@ZONE1, F,R{all_server}@ZONE2, F,R{all_server}@ZONE3';

SELECT tenant_name,database_name, table_name, partition_name, index_name, tablet_id,ls_id,zone,svr_ip,role
FROM OCEANBASE.CDB_OB_TABLE_LOCATIONS a join oceanbase.DBA_OB_TENANTS b
on a.tenant_id=b.tenant_id
WHERE  DATABASE_NAME='MYDB' AND TENANT_NAME='mysql_tenant1'
order by tenant_name,database_name, table_name, partition_name, zone;

SELECT zone,svr_ip,role,count(1)
FROM OCEANBASE.CDB_OB_TABLE_LOCATIONS a join oceanbase.DBA_OB_TENANTS b
on a.tenant_id=b.tenant_id
WHERE  DATABASE_NAME='MYDB' AND TENANT_NAME='mysql_tenant1'
group by zone,svr_ip,role
order by zone;

ALTER SYSTEM SWITCH REPLICA LEADER LS = 1002 SERVER = '10.0.139.175:2882' TENANT = mysql_tenant1;

SELECT a.TENANT_ID,a.LS_ID,a.SVR_IP,a.SVR_PORT,a.ZONE,a.role,b.TENANT_NAME,b.TENANT_TYPE FROM oceanbase.CDB_OB_LS_LOCATIONS a, oceanbase.DBA_OB_TENANTS b WHERE a.TENANT_ID=b.TENANT_ID;

SELECT a.tenant_id,tenant_name,database_name,ls_id,zone,svr_ip,role,count(1)
FROM OCEANBASE.CDB_OB_TABLE_LOCATIONS a join oceanbase.DBA_OB_TENANTS b
on a.tenant_id=b.tenant_id
group by tenant_name,database_name,ls_id,zone,svr_ip,role
order by a.tenant_id,database_name,ls_id;

select TENANT_ID,LS_ID,STATUS,PRIMARY_ZONE from oceanbase.CDB_OB_LS;

[2024-10-29 00:03:56.112302] ERROR issue_dba_error (ob_Log.cpp:1875)[38115][T1060_L0_G0][T1060][YB420A008BAD-000624C0A09C1A91-0-0] [Lt-17][errcode=4388] Unexpect
ed internal error happen, please checkout the internal errcode(errcode=-4264, file="ob_log_service.cpp", Lne_no=b89, info="create_ls failed!!!")

select SVR_IP,SVR_PORT,UNIT_ID,TENANT_ID,
ceil(LOG_DISK_SIZE/1024/1024) LOG_DISK_SIZE_M,ceil
(LOG_DISK_IN_USE/1024/1024) LOG_DISK_IN_USE_M,
round(LOG_DISK_IN_USE/LOG_DISK_SIZE*100) LOG_USE_PCT,
ceil(DATA_DISK_IN_USE/1024/1024/1024) DATA_DISK_IN_USE_G,
STATUS  from oceanbase.GV$OB_UNITS 
order by SVR_IP,TENANT_ID;

select SVR_IP,ZONE,
round(LOG_DISK_CAPACITY/1024/1024/1024) LOG_DISK_CAPACITY_G,
round(LOG_DISK_ASSIGNED/1024/1024/1024) LOG_DISK_ASSIGN_G,
round(LOG_DISK_ASSIGNED/LOG_DISK_CAPACITY*100) LOG_USE_PCT
from oceanbase.GV$OB_SERVERS;

CREATE RESOURCE UNIT S4_unit_config
MEMORY_SIZE = '1536M',
MAX_CPU = 1, MIN_CPU = 1,
LOG_DISK_SIZE = '2048M',
MAX_IOPS = 10000, MIN_IOPS = 10000, IOPS_WEIGHT=1;

CREATE RESOURCE POOL mq_pool_04 
UNIT='S4_unit_config', 
UNIT_NUM=1, 
ZONE_LIST=('zone1','zone2','zone3'); 

CREATE TENANT IF NOT EXISTS mysql_tenant4 
PRIMARY_ZONE='zone1,zone2,zone3', 
RESOURCE_POOL_LIST=('mq_pool_04')
set OB_TCP_INVITED_NODES='%';

#0  oceanbase::palf::PalfEnvImpl::check_can_create_palf_handle_impl_() const (this=0x7fe7c33fc030) at ./src/logservice/palf/palf_env_impl.cpp:1197
#1  0x00007fe820571e72 in oceanbase::palf::PalfEnvImpl::create_palf_handle_impl_(long, oceanbase::palf::AccessMode const&, oceanbase::palf::PalfBaseInfo const&, oceanbase::palf::LogReplicaType, oceanbase::palf::IPalfHandleImpl*&) (this=0x7fe7c33fc030, palf_id=1001, access_mode=@0x7fe7bb14d2ec: APPEND, palf_base_info=..., replica_type=NORMAL_REPLICA, 
    ipalf_handle_impl=@0x7fe7bb14d0f0: 0x0) at ./src/logservice/palf/palf_env_impl.cpp:416
#2  0x00007fe82056aac1 in oceanbase::palf::PalfEnvImpl::create_palf_handle_impl(long, oceanbase::palf::AccessMode const&, oceanbase::palf::PalfBaseInfo const&, oceanbase::palf::IPalfHandleImpl*&) (this=0x7fe7c33fc030, palf_id=1001, access_mode=@0x7fe7bb14d2ec: APPEND, palf_base_info=..., palf_handle_impl=@0x7fe7bb14d0f0: 0x0) at ./src/logservice/palf/palf_env_impl.cpp:382
#3  0x00007fe82056a91c in oceanbase::palf::PalfEnv::create(long, oceanbase::palf::AccessMode const&, oceanbase::palf::PalfBaseInfo const&, oceanbase::palf::PalfHandle&) (
    this=0x7fe7c33fc030, id=1001, access_mode=@0x7fe7bb14d2ec: APPEND, palf_base_info=..., handle=...) at ./src/logservice/palf/palf_env.cpp:110
#4  0x00007fe81f7fd54e in oceanbase::logservice::ObLogService::create_ls_(oceanbase::share::ObLSID const&, oceanbase::common::ObReplicaType const&, oceanbase::share::ObTenantRole const&, oceanbase::palf::PalfBaseInfo const&, bool, oceanbase::logservice::ObLogHandler&, oceanbase::logservice::ObLogRestoreHandler&) (this=0x7fe7a5404030, id=..., 
    replica_type=@0x7fe7bb14d834: REPLICA_TYPE_FULL, tenant_role=..., palf_base_info=..., allow_log_sync=true, log_handler=..., restore_handler=...)
    at ./src/logservice/ob_log_service.cpp:665
#5  0x00007fe81f7fcdcc in oceanbase::logservice::ObLogService::create_ls(oceanbase::share::ObLSID const&, oceanbase::common::ObReplicaType const&, oceanbase::share::ObTenantRole const&, oceanbase::palf::PalfBaseInfo const&, bool, oceanbase::logservice::ObLogHandler&, oceanbase::logservice::ObLogRestoreHandler&) (this=0x7fe7a5404030, id=..., 
    replica_type=@0x7fe7bb14d834: REPLICA_TYPE_FULL, tenant_role=..., palf_base_info=..., allow_log_sync=true, log_handler=..., restore_handler=...)
    at ./src/logservice/ob_log_service.cpp:343
#6  0x00007fe828b07498 in oceanbase::storage::ObLS::create_ls(oceanbase::share::ObTenantRole, oceanbase::palf::PalfBaseInfo const&, oceanbase::common::ObReplicaType const&, bool) (
    this=0x7fe7996cc150, tenant_role=..., palf_base_info=..., replica_type=@0x7fe7bb14d834: REPLICA_TYPE_FULL, allow_log_sync=true) at ./src/storage/ls/ob_ls.cpp:405
#7  0x00007fe828c79ebf in oceanbase::storage::ObLSService::create_ls(oceanbase::obrpc::ObCreateLSArg const&) (this=0x7fe7909ce030, arg=...)
    at ./src/storage/tx_storage/ob_ls_service.cpp:508
#8  0x00007fe8232d8c52 in oceanbase::observer::ObRpcCreateLSP::process() (this=0x7fe7bb15a700) at ./src/observer/ob_rpc_processor_simple.cpp:1521
#9  0x00007fe81f601db0 in oceanbase::obrpc::ObRpcProcessorBase::run() (this=0x7fe7bb15a700) at ./deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp:89
#10 0x00007fe822b07e66 in oceanbase::omt::ObWorkerProcessor::process_one(oceanbase::rpc::ObRequest&) (this=0x7fe7775931a8, req=...) at ./src/observer/omt/ob_worker_processor.cpp:88
#11 0x00007fe81f2959d3 in oceanbase::omt::ObWorkerProcessor::process(oceanbase::rpc::ObRequest&) (this=0x7fe7775931a8, req=...) at ./src/observer/omt/ob_worker_processor.cpp:157
#12 0x00007fe822b06d01 in oceanbase::omt::ObThWorker::process_request(oceanbase::rpc::ObRequest&) (this=0x7fe7775930e0, req=...) at ./src/observer/omt/ob_th_worker.cpp:248
#13 0x00007fe81f294a8f in oceanbase::omt::ObThWorker::worker(long&, long&, int&) (this=0x7fe7775930e0, tenant_id=@0x7fe7bb14ec98: 1046, 
    req_recv_timestamp=@0x7fe7bb14ec90: 1729995314847463, worker_level=@0x7fe7bb14ec8c: 0) at ./src/observer/omt/ob_th_worker.cpp:387
#14 0x00007fe822b072bc in oceanbase::omt::ObThWorker::run(long) (this=0x7fe7775930e0, idx=0) at ./src/observer/omt/ob_th_worker.cpp:424
#15 0x00007fe82ea0c271 in oceanbase::lib::Thread::run() (this=0x7fe777593310) at ./deps/oblib/src/lib/thread/thread.cpp:164
#16 0x00007fe82ea0bdae in oceanbase::lib::Thread::__th_start(void*) (arg=0x7fe777593310) at ./deps/oblib/src/lib/thread/thread.cpp:322
#17 0x00007fe819435dc5 in start_thread () from /lib64/libpthread.so.0
#18 0x00007fe81916473d in clone () from /lib64/libc.so.6

bool PalfEnvImpl::check_can_create_palf_handle_impl_() const{
    bool bool_ret = true;
    // 统计Palf处理接口的个数。例如当租户的PRIMARY_ZONE三个zone时，返回3.只有一个zone时，返回1
    int64_t count = palf_handle_impl_map_.count();
    // 获取磁盘的参数，包括总大小，回收阈值，限制写入阈值等属性。其中log_disk_usage_limit_size_即为当前该租户创建指定的log_disk_size(2048M) - 租户的日志组大小(512M) = 1536M.
    const PalfDiskOptions disk_opts = disk_options_wrapper_.get_disk_opts_for_recycling_blocks();
    // MIN_DISK_SIZE_PER_PALF_INSTANCE源码定义为512M
    // 计算 (3+1)*512=2048，2048 <= 1536不成立，因此将返回bool_ret为false
    bool_ret = (count + 1) * MIN_DISK_SIZE_PER_PALF_INSTANCE <= disk_opts.log_disk_usage_limit_size_;
    return bool_ret;
}