44 #include <sys/socket.h>
45 #include <sys/types.h>
48 #include <netinet/in.h>
49 #include <arpa/inet.h>
50 #include <sys/param.h>
54 #include <qb/qbdefs.h>
63 #define TOKEN_RETRANSMITS_BEFORE_LOSS_CONST 4
64 #define TOKEN_TIMEOUT 1000
65 #define TOKEN_COEFFICIENT 650
66 #define JOIN_TIMEOUT 50
67 #define MERGE_TIMEOUT 200
68 #define DOWNCHECK_TIMEOUT 1000
69 #define FAIL_TO_RECV_CONST 2500
70 #define SEQNO_UNCHANGED_CONST 30
71 #define MINIMUM_TIMEOUT (int)(1000/HZ)*3
72 #define MAX_NETWORK_DELAY 50
73 #define WINDOW_SIZE 50
74 #define MAX_MESSAGES 17
75 #define MISS_COUNT_CONST 5
76 #define RRP_PROBLEM_COUNT_TIMEOUT 2000
77 #define RRP_PROBLEM_COUNT_THRESHOLD_DEFAULT 10
78 #define RRP_PROBLEM_COUNT_THRESHOLD_MIN 2
79 #define RRP_AUTORECOVERY_CHECK_TIMEOUT 1000
81 #define DEFAULT_PORT 5405
83 static char error_string_response[512];
91 if (strcmp(param_name,
"totem.token") == 0)
93 if (strcmp(param_name,
"totem.token_retransmit") == 0)
95 if (strcmp(param_name,
"totem.hold") == 0)
97 if (strcmp(param_name,
"totem.token_retransmits_before_loss_const") == 0)
99 if (strcmp(param_name,
"totem.join") == 0)
101 if (strcmp(param_name,
"totem.send_join") == 0)
103 if (strcmp(param_name,
"totem.consensus") == 0)
105 if (strcmp(param_name,
"totem.merge") == 0)
107 if (strcmp(param_name,
"totem.downcheck") == 0)
109 if (strcmp(param_name,
"totem.fail_recv_const") == 0)
111 if (strcmp(param_name,
"totem.seqno_unchanged_const") == 0)
113 if (strcmp(param_name,
"totem.rrp_token_expired_timeout") == 0)
115 if (strcmp(param_name,
"totem.rrp_problem_count_timeout") == 0)
117 if (strcmp(param_name,
"totem.rrp_problem_count_threshold") == 0)
119 if (strcmp(param_name,
"totem.rrp_problem_count_mcast_threshold") == 0)
121 if (strcmp(param_name,
"totem.rrp_autorecovery_check_timeout") == 0)
123 if (strcmp(param_name,
"totem.heartbeat_failures_allowed") == 0)
125 if (strcmp(param_name,
"totem.max_network_delay") == 0)
127 if (strcmp(param_name,
"totem.window_size") == 0)
129 if (strcmp(param_name,
"totem.max_messages") == 0)
131 if (strcmp(param_name,
"totem.miss_count_const") == 0)
141 static void totem_volatile_config_set_value (
struct totem_config *totem_config,
142 const char *key_name,
const char *deleted_key,
unsigned int default_value,
143 int allow_zero_value)
147 (deleted_key != NULL && strcmp(deleted_key, key_name) == 0) ||
148 (!allow_zero_value && *totem_get_param_by_name(totem_config, key_name) == 0)) {
149 *totem_get_param_by_name(totem_config, key_name) = default_value;
159 static void totem_volatile_config_read (
struct totem_config *totem_config,
const char *deleted_key)
163 totem_volatile_config_set_value(totem_config,
"totem.token_retransmits_before_loss_const", deleted_key,
166 totem_volatile_config_set_value(totem_config,
"totem.token", deleted_key,
TOKEN_TIMEOUT, 0);
174 totem_volatile_config_set_value(totem_config,
"totem.max_network_delay", deleted_key,
MAX_NETWORK_DELAY, 0);
176 totem_volatile_config_set_value(totem_config,
"totem.window_size", deleted_key,
WINDOW_SIZE, 0);
178 totem_volatile_config_set_value(totem_config,
"totem.max_messages", deleted_key,
MAX_MESSAGES, 0);
180 totem_volatile_config_set_value(totem_config,
"totem.miss_count_const", deleted_key,
MISS_COUNT_CONST, 0);
182 totem_volatile_config_set_value(totem_config,
"totem.token_retransmit", deleted_key,
185 totem_volatile_config_set_value(totem_config,
"totem.hold", deleted_key,
188 totem_volatile_config_set_value(totem_config,
"totem.join", deleted_key,
JOIN_TIMEOUT, 0);
190 totem_volatile_config_set_value(totem_config,
"totem.consensus", deleted_key,
193 totem_volatile_config_set_value(totem_config,
"totem.merge", deleted_key,
MERGE_TIMEOUT, 0);
195 totem_volatile_config_set_value(totem_config,
"totem.downcheck", deleted_key,
DOWNCHECK_TIMEOUT, 0);
197 totem_volatile_config_set_value(totem_config,
"totem.fail_recv_const", deleted_key,
FAIL_TO_RECV_CONST, 0);
199 totem_volatile_config_set_value(totem_config,
"totem.seqno_unchanged_const", deleted_key,
202 totem_volatile_config_set_value(totem_config,
"totem.send_join", deleted_key, 0, 1);
204 totem_volatile_config_set_value(totem_config,
"totem.rrp_problem_count_timeout", deleted_key,
207 totem_volatile_config_set_value(totem_config,
"totem.rrp_problem_count_threshold", deleted_key,
210 totem_volatile_config_set_value(totem_config,
"totem.rrp_problem_count_mcast_threshold", deleted_key,
213 totem_volatile_config_set_value(totem_config,
"totem.rrp_token_expired_timeout", deleted_key,
216 totem_volatile_config_set_value(totem_config,
"totem.rrp_autorecovery_check_timeout", deleted_key,
219 totem_volatile_config_set_value(totem_config,
"totem.heartbeat_failures_allowed", deleted_key, 0, 1);
222 static int totem_volatile_config_validate (
223 struct totem_config *totem_config,
224 const char **error_string)
226 static char local_error_reason[512];
227 const char *error_reason = local_error_reason;
230 snprintf (local_error_reason,
sizeof(local_error_reason),
231 "The max_network_delay parameter (%d ms) may not be less than (%d ms).",
237 snprintf (local_error_reason,
sizeof(local_error_reason),
238 "The token timeout parameter (%d ms) may not be less than (%d ms).",
244 snprintf (local_error_reason,
sizeof(local_error_reason),
245 "The token retransmit timeout parameter (%d ms) may not be less than (%d ms).",
251 snprintf (local_error_reason,
sizeof(local_error_reason),
252 "The token hold timeout parameter (%d ms) may not be less than (%d ms).",
258 snprintf (local_error_reason,
sizeof(local_error_reason),
259 "The join timeout parameter (%d ms) may not be less than (%d ms).",
265 snprintf (local_error_reason,
sizeof(local_error_reason),
266 "The consensus timeout parameter (%d ms) may not be less than (%d ms).",
272 snprintf (local_error_reason,
sizeof(local_error_reason),
273 "The consensus timeout parameter (%d ms) may not be less than join timeout (%d ms).",
279 snprintf (local_error_reason,
sizeof(local_error_reason),
280 "The merge timeout parameter (%d ms) may not be less than (%d ms).",
286 snprintf (local_error_reason,
sizeof(local_error_reason),
287 "The downcheck timeout parameter (%d ms) may not be less than (%d ms).",
293 snprintf (local_error_reason,
sizeof(local_error_reason),
294 "The RRP problem count timeout parameter (%d ms) may not be less than (%d ms).",
300 snprintf (local_error_reason,
sizeof(local_error_reason),
301 "The RRP problem count threshold (%d problem count) may not be less than (%d problem count).",
306 snprintf (local_error_reason,
sizeof(local_error_reason),
307 "The RRP multicast problem count threshold (%d problem count) may not be less than (%d problem count).",
313 snprintf (local_error_reason,
sizeof(local_error_reason),
314 "The RRP token expired timeout parameter (%d ms) may not be less than (%d ms).",
322 snprintf (error_string_response,
sizeof(error_string_response),
323 "parse error in config: %s\n", error_reason);
324 *error_string = error_string_response;
329 static int totem_get_crypto(
struct totem_config *totem_config)
332 const char *tmp_cipher;
333 const char *tmp_hash;
336 tmp_cipher =
"aes256";
339 if (strcmp (str,
"off") == 0) {
347 if (strcmp(str,
"none") == 0) {
350 if (strcmp(str,
"aes256") == 0) {
351 tmp_cipher =
"aes256";
353 if (strcmp(str,
"aes192") == 0) {
354 tmp_cipher =
"aes192";
356 if (strcmp(str,
"aes128") == 0) {
357 tmp_cipher =
"aes128";
359 if (strcmp(str,
"3des") == 0) {
366 if (strcmp(str,
"none") == 0) {
369 if (strcmp(str,
"md5") == 0) {
372 if (strcmp(str,
"sha1") == 0) {
375 if (strcmp(str,
"sha256") == 0) {
378 if (strcmp(str,
"sha384") == 0) {
381 if (strcmp(str,
"sha512") == 0) {
387 if ((strcmp(tmp_cipher,
"none") != 0) &&
388 (strcmp(tmp_hash,
"none") == 0)) {
401 static int totem_config_get_ip_version(
void)
408 if (strcmp(str,
"ipv4") == 0) {
411 if (strcmp(str,
"ipv6") == 0) {
420 static uint16_t generate_cluster_id (
const char *cluster_name)
425 for (i = 0; i < strlen(cluster_name); i++) {
427 value += cluster_name[i];
430 return (value & 0xFFFF);
433 static int get_cluster_mcast_addr (
434 const char *cluster_name,
436 unsigned int ringnumber,
441 char addr[INET6_ADDRSTRLEN + 1];
444 if (cluster_name == NULL) {
448 clusterid = generate_cluster_id(cluster_name) + ringnumber;
449 memset (res, 0,
sizeof(*res));
451 switch (bindnet->
family) {
453 snprintf(addr,
sizeof(addr),
"239.192.%d.%d", clusterid >> 8, clusterid % 0xFF);
456 snprintf(addr,
sizeof(addr),
"ff15::%x", clusterid);
470 static int find_local_node_in_nodelist(
struct totem_config *totem_config)
473 const char *iter_key;
475 unsigned int node_pos;
476 int local_node_pos = -1;
478 int interface_up, interface_num;
484 &bind_addr, &interface_up, &interface_num,
492 res = sscanf(iter_key,
"nodelist.node.%u.%s", &node_pos, tmp_key);
497 if (strcmp(tmp_key,
"ring0_addr") != 0) {
513 local_node_pos = node_pos;
518 return (local_node_pos);
521 static void put_nodelist_members_to_config(
struct totem_config *totem_config)
524 const char *iter_key, *iter_key2;
526 unsigned int node_pos;
531 unsigned int ringnumber = 0;
544 res = sscanf(iter_key,
"nodelist.node.%u.%s", &node_pos, tmp_key);
549 if (strcmp(tmp_key,
"ring0_addr") != 0) {
556 res = sscanf(iter_key2,
"nodelist.node.%u.ring%u%s", &node_pos, &ringnumber, tmp_key2);
557 if (res != 3 || strcmp(tmp_key2,
"_addr") != 0) {
596 const char *iter_key, *iter_key2;
598 const char *ipaddr_key;
610 ip_version = totem_config_get_ip_version();
615 res = sscanf(iter_key,
"nodelist.node.%u.%s", node_pos, tmp_key);
620 if (strcmp(tmp_key,
"ring0_addr") != 0) {
633 snprintf(tmp_key,
sizeof(tmp_key),
"nodelist.node.%u.%s", *node_pos,
634 (ipaddr_key_prefix != NULL ? ipaddr_key_prefix :
"ring0_addr"));
641 ipaddr_key = (ipaddr_key_prefix != NULL ? iter_key2 : tmp_key);
646 if (
totemip_parse(&node_addr, node_addr_str, ip_version) == -1) {
656 for (list = addrs.
next; list != &addrs; list = list->
next) {
683 static void config_convert_nodelist_to_interface(
struct totem_config *totem_config)
686 unsigned int node_pos;
690 unsigned int ringnumber = 0;
692 const char *iter_key;
701 res = sscanf(iter_key,
"nodelist.node.%u.ring%u%s", &node_pos, &ringnumber, tmp_key2);
702 if (res != 3 || strcmp(tmp_key2,
"_addr") != 0) {
720 struct totem_config *totem_config,
721 const char **error_string,
726 unsigned int ringnumber = 0;
727 int member_count = 0;
729 const char *iter_key;
730 const char *member_iter_key;
735 char *cluster_name = NULL;
742 memset (totem_config, 0,
sizeof (
struct totem_config));
745 *error_string =
"Out of memory trying to allocate ethernet interface storage area";
752 strcpy (totem_config->
rrp_mode,
"none");
756 if (totem_get_crypto(totem_config) != 0) {
757 *error_string =
"crypto_cipher requires crypto_hash with value other than none";
763 *error_string =
"totem.rrp_mode is too long";
768 strcpy (totem_config->
rrp_mode, str);
776 if (strcmp (str,
"yes") == 0) {
790 totem_config->
ip_version = totem_config_get_ip_version();
796 config_convert_nodelist_to_interface(totem_config);
803 res = sscanf(iter_key,
"totem.interface.%[^.].%s", ringnumber_key, tmp_key);
808 if (strcmp(tmp_key,
"bindnetaddr") != 0) {
814 ringnumber = atoi(ringnumber_key);
816 if (ringnumber >= INTERFACE_MAX) {
819 snprintf (error_string_response,
sizeof(error_string_response),
820 "parse error in config: interface ring number %u is bigger than allowed maximum %u\n",
821 ringnumber, INTERFACE_MAX - 1);
823 *error_string = error_string_response;
848 res = get_cluster_mcast_addr (cluster_name,
858 if (strcmp (str,
"yes") == 0) {
892 while ((member_iter_key =
icmap_iter_next(member_iter, NULL, NULL)) != NULL) {
893 if (member_count == 0) {
935 if (strcmp (str,
"udpu") == 0) {
939 if (strcmp (str,
"iba") == 0) {
955 local_node_pos = find_local_node_in_nodelist(totem_config);
956 if (local_node_pos != -1) {
961 nodeid_set = (totem_config->
node_id != 0);
976 put_nodelist_members_to_config(totem_config);
982 totem_volatile_config_read(totem_config, NULL);
986 add_totem_config_notification(totem_config);
993 struct totem_config *totem_config,
994 const char **error_string)
996 static char local_error_reason[512];
997 char parse_error[512];
998 const char *error_reason = local_error_reason;
1003 error_reason =
"No interfaces defined";
1018 error_reason =
"No multicast address specified";
1023 error_reason =
"No multicast port specified";
1028 error_reason =
"Invalid TTL (should be 0..255)";
1033 error_reason =
"Can only set ttl on multicast transport types";
1040 error_reason =
"An IPV6 network requires that a node ID be specified.";
1046 error_reason =
"Multicast address family does not match bind address family";
1051 error_reason =
"Not all bind address belong to the same IP family";
1055 error_reason =
"mcastaddr is not a correct multicast address.";
1061 if (totem_config->
version != 2) {
1062 error_reason =
"This totem parser can only parse version 2 configurations.";
1066 if (totem_volatile_config_validate(totem_config, error_string) == -1) {
1073 if (strcmp (totem_config->
rrp_mode,
"none") &&
1074 strcmp (totem_config->
rrp_mode,
"active") &&
1075 strcmp (totem_config->
rrp_mode,
"passive")) {
1076 snprintf (local_error_reason,
sizeof(local_error_reason),
1077 "The RRP mode \"%s\" specified is invalid. It must be none, active, or passive.\n", totem_config->
rrp_mode);
1081 if (strcmp (totem_config->
rrp_mode,
"none") == 0) {
1084 if (interface_max < totem_config->interface_count) {
1085 snprintf (parse_error,
sizeof(parse_error),
1086 "%d is too many configured interfaces for the rrp_mode setting %s.",
1089 error_reason = parse_error;
1093 if (totem_config->
net_mtu == 0) {
1100 snprintf (error_string_response,
sizeof(error_string_response),
1101 "parse error in config: %s\n", error_reason);
1102 *error_string = error_string_response;
1107 static int read_keyfile (
1108 const char *key_location,
1109 struct totem_config *totem_config,
1110 const char **error_string)
1114 ssize_t expected_key_len =
sizeof (totem_config->
private_key);
1116 char error_str[100];
1117 const char *error_ptr;
1119 fd = open (key_location, O_RDONLY);
1121 error_ptr = qb_strerror_r(errno, error_str,
sizeof(error_str));
1122 snprintf (error_string_response,
sizeof(error_string_response),
1123 "Could not open %s: %s\n",
1124 key_location, error_ptr);
1128 res = read (fd, totem_config->
private_key, expected_key_len);
1129 saved_errno = errno;
1133 error_ptr = qb_strerror_r (saved_errno, error_str,
sizeof(error_str));
1134 snprintf (error_string_response,
sizeof(error_string_response),
1135 "Could not read %s: %s\n",
1136 key_location, error_ptr);
1142 if (res != expected_key_len) {
1143 snprintf (error_string_response,
sizeof(error_string_response),
1144 "Could only read %d bits of 1024 bits from %s.\n",
1145 res * 8, key_location);
1152 *error_string = error_string_response;
1157 struct totem_config *totem_config,
1158 const char **error_string)
1161 char *key_location = NULL;
1175 res = read_keyfile(key_location, totem_config, error_string);
1183 if (key_len >
sizeof (totem_config->
private_key)) {
1184 sprintf(error_string_response,
"key is too long");
1191 sprintf(error_string_response,
"can't store private key");
1199 const char *filename = getenv(
"COROSYNC_TOTEM_AUTHKEY_FILE");
1202 res = read_keyfile(filename, totem_config, error_string);
1211 *error_string = error_string_response;
1216 static void debug_dump_totem_config(
const struct totem_config *totem_config)
1229 "seqno unchanged const (%d rotations) Maximum network MTU %d",
1232 "window size per rotation (%d messages) maximum messages per rotation (%d messages)",
1252 static void totem_change_notify(
1254 const char *key_name,
1259 struct totem_config *totem_config = (
struct totem_config *)user_data;
1262 const char *deleted_key = NULL;
1263 const char *error_string;
1272 param = totem_get_param_by_name((
struct totem_config *)user_data, key_name);
1278 if (!param && strcmp(key_name,
"totem.token_coefficient") != 0)
1286 deleted_key = key_name;
1296 totem_volatile_config_read (totem_config, deleted_key);
1298 debug_dump_totem_config(totem_config);
1299 if (totem_volatile_config_validate(totem_config, &error_string) == -1) {
1308 static void totem_reload_notify(
1310 const char *key_name,
1315 struct totem_config *totem_config = (
struct totem_config *)user_data;
1316 uint32_t local_node_pos;
1317 const char *error_string;
1320 if (*(uint8_t *)new_val.
data == 0) {
1321 put_nodelist_members_to_config (totem_config);
1322 totem_volatile_config_read (totem_config, NULL);
1324 debug_dump_totem_config(totem_config);
1325 if (totem_volatile_config_validate(totem_config, &error_string) == -1) {
1334 local_node_pos = find_local_node_in_nodelist(totem_config);
1335 if (local_node_pos != -1) {
1345 static void add_totem_config_notification(
struct totem_config *totem_config)
1351 totem_change_notify,
1357 totem_reload_notify,
unsigned int clear_node_high_bit
const char * icmap_iter_next(icmap_iter_t iter, size_t *value_len, icmap_value_types_t *type)
#define RRP_PROBLEM_COUNT_TIMEOUT
struct totem_interface * interfaces
unsigned int interface_count
const char * totemip_print(const struct totem_ip_address *addr)
void icmap_iter_finalize(icmap_iter_t iter)
totem_transport_t transport_number
int totemip_parse(struct totem_ip_address *totemip, const char *addr, int family)
#define DOWNCHECK_TIMEOUT
unsigned int token_hold_timeout
struct totem_ip_address member_list[PROCESSOR_COUNT_MAX]
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN]
char rrp_mode[TOTEM_RRP_MODE_BYTES]
unsigned char addr[TOTEMIP_ADDRLEN]
unsigned int rrp_problem_count_timeout
cs_error_t icmap_set_string(const char *key_name, const char *value)
int totemip_is_mcast(struct totem_ip_address *addr)
#define RRP_AUTORECOVERY_CHECK_TIMEOUT
unsigned int downcheck_timeout
unsigned int private_key_len
#define SEQNO_UNCHANGED_CONST
unsigned int max_network_delay
#define log_printf(level, format, args...)
unsigned int heartbeat_failures_allowed
unsigned int send_join_timeout
unsigned int rrp_problem_count_threshold
#define ICMAP_TRACK_DELETE
#define RRP_PROBLEM_COUNT_THRESHOLD_MIN
#define ICMAP_KEYNAME_MAXLEN
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
int totem_config_keyread(struct totem_config *totem_config, const char **error_string)
#define ICMAP_TRACK_MODIFY
unsigned int token_retransmits_before_loss_const
int totemip_iface_check(struct totem_ip_address *bindnet, struct totem_ip_address *boundto, int *interface_up, int *interface_num, int mask_high_bit)
#define FAIL_TO_RECV_CONST
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
unsigned int seqno_unchanged_const
unsigned int miss_count_const
unsigned int join_timeout
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
int totem_config_find_local_addr_in_nodelist(const char *ipaddr_key_prefix, unsigned int *node_pos)
struct totem_ip_address mcast_addr
#define LOGSYS_LEVEL_ERROR
unsigned int rrp_autorecovery_check_timeout
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
#define LOGSYS_LEVEL_DEBUG
cs_error_t icmap_set_uint16(const char *key_name, uint16_t value)
unsigned int fail_to_recv_const
#define TOKEN_COEFFICIENT
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
cs_error_t icmap_get_uint32(const char *key_name, uint32_t *u32)
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
void totemip_freeifaddrs(struct list_head *addrs)
#define MAX_NETWORK_DELAY
unsigned int token_timeout
unsigned int consensus_timeout
int totemip_getifaddrs(struct list_head *addrs)
#define PROCESSOR_COUNT_MAX
#define TOKEN_RETRANSMITS_BEFORE_LOSS_CONST
unsigned int broadcast_use
unsigned int rrp_problem_count_mcast_threshold
cs_error_t icmap_get_string(const char *key_name, char **str)
#define list_entry(ptr, type, member)
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
unsigned int max_messages
char * crypto_cipher_type
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
unsigned int merge_timeout
unsigned int token_retransmit_timeout
struct totem_ip_address bindnet
icmap_iter_t icmap_iter_init(const char *prefix)
int totemip_equal(const struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
struct totem_ip_address ip_addr
cs_error_t icmap_get_uint16(const char *key_name, uint16_t *u16)
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
unsigned int rrp_token_expired_timeout
qb_map_iter_t * icmap_iter_t
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
#define ICMAP_TRACK_PREFIX
#define RRP_PROBLEM_COUNT_THRESHOLD_DEFAULT