author：sufei
说明：组内成员发现的opengauss相关bug记录并提交官方
openguass版本 2.0.0
其他版本也发现该问题
现象说明
主从配置
主库
postgres.conf
replconninfo1 = 'localhost=gauss-1-0.gauss-1-headless.gauss-test localport=5433 localservice=5434 remotehost=gauss-1-1.gauss-1-headless.gauss-test remoteport=5433 remoteservice=5434 '
listen_addresses = '*'
remote_read_mode = non_authentication
从库
postgres.conf
replconninfo1 = 'localhost=gauss-1-1.gauss-1-headless.gauss-test localport=5433 localservice=5434 remotehost=gauss-1-0.gauss-1-headless.gauss-test remoteport=5433 remoteservice=5434 '
listen_addresses = '*'
remote_read_mode = non_authentication
hba配置,主从都一样
host all all 0.0.0.0/0 sha256
问题描述
最近在实现openGuass容器化集群部署时，使用statefulset部署openGauss高可用主从集群，由于 pod的IP不是固定的，需要使用域名搭建主从。
在使用域名搭建主从时，发现从库无法同步主库信息，主要表现为
1，当创建新实例时，从库启动后连接主库进行全量备份时，提示无法连接数据库
[teledb@gs-1 data]$ gs_ctl build -D $PGDATA -b full
[2022-03-21 16:35:27.745][340][][gs_ctl]: gs_ctl full build ,datadir is /home/teledb/gsdata/gsroot/data
[2022-03-21 16:35:27.746][340][][gs_ctl]: stop failed, killing gaussdb by force ...
[2022-03-21 16:35:27.746][340][][gs_ctl]: command [ps c -eo pid,euid,cmd | grep gaussdb | grep -v grep | awk '{if($2 == curuid && $1!="-n") print "/proc/"$1"/cwd"}' curuid=`id -u`| xargs ls -l | awk '{if ($NF=="/home/teledb/gsdata/gsroot/data") print $(NF-2)}' | awk -F/ '{print $3 }' | xargs kill -9 >/dev/null 2>&1 ] path: [/home/teledb/gsdata/gsroot/data]
[2022-03-21 16:35:47.872][340][datanode1][gs_ctl]: could not connect to server.
2，使用IP搭建主从后修改为域名，从库以standy启动后，数据库状态永远为 Need repair
[2022-03-21 16:29:25.454][353][][gs_ctl]: gs_ctl query
datadir is /home/teledb/gsdata/gsroot/data
HA state:
local_role : Standy
static_connections : 0
db_state : Need repair
查看从库启动日志,数据目录下 pg_log目录，发现报错no pg_hba.conf entry for host ，
排查步骤
- 首先从库启动日志, 在数据目录下 pg_log目录，发现报错no pg_hba.conf entry for host ，但是找不出任何问题
- 然后确认使用IP搭建主从是否可行，发现使用IP能正常搭建主从，
- 然后确认是不是容器化环境问题，在物理机器上配置域名，搭建主从，并且验证PG试用域名搭建主从， 确认域名配置成功，只是gauss无法使用域名搭建主从。
- 最后根据日志分析源码，定位BUG
根因分析
根据日志定位到函数hba_getauthmethod
void hba_getauthmethod(hbaPort* port)
{
/*
* We may receive replication request from gs_basebackup, in this case, use check_hba
* to verify. But if we run gs_basebackup at the machine in ReplConnInfo, we can't
* distinguish whether this request is from a standby process or gs_basebackup...
* In this case, we still need to use check_hba_replication for compatibility.
*/
#ifdef ENABLE_MULTIPLE_NODES
if (IsDSorHaWalSender() ) {
#else
if (IsDSorHaWalSender() && is_node_internal_connection(port)) {
#endif
check_hba_replication(port);
} else {
check_hba(port);
}
}
报错日志为函数 check_hba调用报错，但是连接请求 作为复制连接，应该走check_hba_replication 函数逻辑
所以这里的判断函数 IsDSorHaWalSender() && is_node_internal_connection(port) 应该返回ture，下面看下这两个函数
bool IsDSorHaWalSender()
{
return (dummyStandbyMode == true || (AM_WAL_SENDER && AM_WAL_DB_SENDER == false));
}
/*
* Check if the current connection is a node internal connection
*/
bool is_node_internal_connection(hbaPort* port)
{
const SockAddr remote_addr = port->raddr;
const SockAddr local_addr = port->laddr;
char remote_host[NI_MAXHOST] = {0};
char local_host[NI_MAXHOST] = {0};
remote_host[0] = '\0';
local_host[0] = '\0';
(void)pg_getnameinfo_all(&remote_addr.addr,
remote_addr.salen,
remote_host,
sizeof(remote_host),
NULL,
0,
NI_NUMERICHOST | NI_NUMERICSERV);
(void)pg_getnameinfo_all(
&local_addr.addr, local_addr.salen, local_host, sizeof(local_host), NULL, 0, NI_NUMERICHOST | NI_NUMERICSERV);
if (strlen(remote_host) == 0) {
return false;
}
if (strcmp(remote_host, local_host) == 0) {
ereport(DEBUG2, (errmsg("remote host is:%s and local host is:%s", remote_host, local_host)));
return true;
}
for (int i = 1; i < MAX_REPLNODE_NUM; i++) {
if (u_sess->attr.attr_storage.ReplConnInfoArr[i] && *(u_sess->attr.attr_storage.ReplConnInfoArr[i]) != '\0' &&
strcasestr(u_sess->attr.attr_storage.ReplConnInfoArr[i], remote_host) != NULL) {
ereport(DEBUG2, (errmsg("remote host is:%s in replconninfo %s",
remote_host, u_sess->attr.attr_storage.ReplConnInfoArr[1])));
return true;
}
}
return false;
}
debug加入上述函数
Breakpoint 1, is_node_internal_connection (port=0x7fb9f0ad85b0) at hba.cpp:2010
2010 hba.cpp: No such file or directory.
(gdb) p *port
$1 = {sock = 118, noblock = false, proto = 196659, laddr = {addr = {ss_family = 2, __ss_padding = "\025\071\n\364\000\241", '\000' <repeats 111 times>, __ss_align = 0},
salen = 16}, raddr = {addr = {ss_family = 2, __ss_padding = "\353(\n\364\000\242", '\000' <repeats 111 times>, __ss_align = 0}, salen = 16},
remote_host = 0x7fb9ef7e6120 "gs-1.gs-headless.gauss-test.svc.cluster.local", remote_hostname = 0x7fb9ef7e6120 "gs-1.gs-headless.gauss-test.svc.cluster.local",
remote_hostname_resolv = 0, remote_port = 0x7fb9ef7e6188 "60200", libcomm_addrinfo = 0x0, gs_sock = {idx = 0, sid = 0, ver = 0, type = 0},
canAcceptConnections = CAC_OK, database_name = 0x7fb9ef7e6218 "", user_name = 0x7fb9ef7e61d0 "teledb", cmdline_options = 0x7fb9ef7e6260 "-c remotetype=application",
guc_options = 0x7fb9ef7e6378, hba = 0x0, md5Salt = "\235\v\032$", SessionStartTime = 700193972013293, SessionVersionNum = 92298, default_keepalives_idle = 7200,
default_keepalives_interval = 75, default_keepalives_count = 9, keepalives_idle = 7200, keepalives_interval = 75, keepalives_count = 9, gss = 0x7fb9ef67ff98, ssl = 0x0,
peer = 0x0, peer_cn = 0x0, count = 0, token = "\000\000\000\000\000\000\000\000", is_logic_conn = false, msgLog = 0x0, krbsrvname = 0x0, gss_ctx = 0x0, gss_cred = 0x0,
gss_name = 0x0, gss_outbuf = {length = 0, value = 0x0}}
(gdb) p remote_host
$11 = "10.244.0.162", '\000' <repeats 1012 times>
可以发现remote_host 返回的是IP ，
域名在port->remote_host字段， 并且函数没有进行域名校验
所以函数is_node_internal_connection走到 strcasestr(u_sess->attr.attr_storage.ReplConnInfoArr[i], remote_host) != NULL) 时，无法返回true，导致主从搭建失败。
修复
在函数is_node_internal_connection走到 strcasestr(u_sess->attr.attr_storage.ReplConnInfoArr[i], remote_host) != NULL) 时 添加域名验证，比较域名是否在字符串ReplConnInfoArr中
修复还需考虑域名长度问题，所以只比较第一个域名字段，比如gauss-1.scv.local, 只比较第一个gauss-1、修复函数如下
/*
* Check if the current connection is a node internal connection
*/
bool is_node_internal_connection(hbaPort* port)
{
const SockAddr remote_addr = port->raddr;
const SockAddr local_addr = port->laddr;
char remote_host[NI_MAXHOST] = {0};
char local_host[NI_MAXHOST] = {0};
char remote_host_prefix[NI_MAXHOST] = {0};
const char *delim = ".";
remote_host[0] = '\0';
local_host[0] = '\0';
(void)pg_getnameinfo_all(&remote_addr.addr,
remote_addr.salen,
remote_host,
sizeof(remote_host),
NULL,
0,
NI_NUMERICHOST | NI_NUMERICSERV);
(void)pg_getnameinfo_all(
&local_addr.addr, local_addr.salen, local_host, sizeof(local_host), NULL, 0, NI_NUMERICHOST | NI_NUMERICSERV);
if (strlen(remote_host) == 0) {
return false;
}
if (strcmp(remote_host, local_host) == 0) {
ereport(DEBUG2, (errmsg("remote host is:%s and local host is:%s", remote_host, local_host)));
return true;
}
strcpy(remote_host_prefix,port->remote_host);
for (int i = 1; i < MAX_REPLNODE_NUM; i++) {
if (u_sess->attr.attr_storage.ReplConnInfoArr[i] && *(u_sess->attr.attr_storage.ReplConnInfoArr[i]) != '\0' &&
(strcasestr(u_sess->attr.attr_storage.ReplConnInfoArr[i], remote_host) != NULL ||
strcasestr(u_sess->attr.attr_storage.ReplConnInfoArr[i], strtok(remote_host_prefix, delim)) != NULL)) {
ereport(DEBUG2, (errmsg("remote host is:%s in replconninfo %s",
remote_host, u_sess->attr.attr_storage.ReplConnInfoArr[1])));
return true;
}
}
return false;
}
提交到社区
提交到opengauss最新master分支
提交记录如下：https://gitee.com/opengauss/openGauss-server/pulls/1558
