corosync  2.3.4
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2006 MontaVista Software, Inc.
3  * Copyright (c) 2006-2012 Red Hat, Inc.
4  *
5  * All rights reserved.
6  *
7  * Author: Steven Dake (sdake@redhat.com)
8  *
9  * This software licensed under BSD license, the text of which follows:
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are met:
13  *
14  * - Redistributions of source code must retain the above copyright notice,
15  * this list of conditions and the following disclaimer.
16  * - Redistributions in binary form must reproduce the above copyright notice,
17  * this list of conditions and the following disclaimer in the documentation
18  * and/or other materials provided with the distribution.
19  * - Neither the name of the MontaVista Software, Inc. nor the names of its
20  * contributors may be used to endorse or promote products derived from this
21  * software without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
72 #include <config.h>
73 
74 #include <pthread.h>
75 #include <assert.h>
76 #include <sys/types.h>
77 #include <sys/file.h>
78 #include <sys/poll.h>
79 #include <sys/uio.h>
80 #include <sys/mman.h>
81 #include <sys/socket.h>
82 #include <sys/un.h>
83 #include <sys/time.h>
84 #include <sys/resource.h>
85 #include <sys/stat.h>
86 #include <netinet/in.h>
87 #include <arpa/inet.h>
88 #include <unistd.h>
89 #include <fcntl.h>
90 #include <stdlib.h>
91 #include <stdio.h>
92 #include <errno.h>
93 #include <signal.h>
94 #include <sched.h>
95 #include <time.h>
96 #include <semaphore.h>
97 
98 #include <qb/qbdefs.h>
99 #include <qb/qblog.h>
100 #include <qb/qbloop.h>
101 #include <qb/qbutil.h>
102 #include <qb/qbipcs.h>
103 
104 #include <corosync/swab.h>
105 #include <corosync/corotypes.h>
106 #include <corosync/corodefs.h>
107 #include <corosync/list.h>
108 #include <corosync/totem/totempg.h>
109 #include <corosync/logsys.h>
110 #include <corosync/icmap.h>
111 
112 #include "quorum.h"
113 #include "totemsrp.h"
114 #include "logconfig.h"
115 #include "totemconfig.h"
116 #include "main.h"
117 #include "sync.h"
118 #include "timer.h"
119 #include "util.h"
120 #include "apidef.h"
121 #include "service.h"
122 #include "schedwrk.h"
123 
124 #ifdef HAVE_SMALL_MEMORY_FOOTPRINT
125 #define IPC_LOGSYS_SIZE 1024*64
126 #else
127 #define IPC_LOGSYS_SIZE 8192*128
128 #endif
129 
130 LOGSYS_DECLARE_SYSTEM ("corosync",
132  LOG_DAEMON,
133  LOG_INFO);
134 
135 LOGSYS_DECLARE_SUBSYS ("MAIN");
136 
137 #define SERVER_BACKLOG 5
138 
139 static int sched_priority = 0;
140 
141 static unsigned int service_count = 32;
142 
144 
145 static struct corosync_api_v1 *api = NULL;
146 
147 static int sync_in_process = 1;
148 
149 static qb_loop_t *corosync_poll_handle;
150 
151 struct sched_param global_sched_param;
152 
153 static corosync_timer_handle_t corosync_stats_timer_handle;
154 
155 static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid";
156 
157 static int ip_version = AF_INET;
158 
159 qb_loop_t *cs_poll_handle_get (void)
160 {
161  return (corosync_poll_handle);
162 }
163 
164 int cs_poll_dispatch_add (qb_loop_t * handle,
165  int fd,
166  int events,
167  void *data,
168 
169  int (*dispatch_fn) (int fd,
170  int revents,
171  void *data))
172 {
173  return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
174  dispatch_fn);
175 }
176 
177 int cs_poll_dispatch_delete(qb_loop_t * handle, int fd)
178 {
179  return qb_loop_poll_del(handle, fd);
180 }
181 
183 {
184  int i;
185 
186  for (i = 0; i < SERVICES_COUNT_MAX; i++) {
187  if (corosync_service[i] && corosync_service[i]->exec_dump_fn) {
189  }
190  }
191 }
192 
193 static void corosync_blackbox_write_to_file (void)
194 {
195  char fname[PATH_MAX];
196  char fdata_fname[PATH_MAX];
197  char time_str[PATH_MAX];
198  struct tm cur_time_tm;
199  time_t cur_time_t;
200  ssize_t res;
201 
202  cur_time_t = time(NULL);
203  localtime_r(&cur_time_t, &cur_time_tm);
204 
205  strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm);
206  snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld",
207  get_run_dir(),
208  time_str,
209  (long long int)getpid());
210 
211  if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
212  LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file");
213  }
214  snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_run_dir());
215  unlink(fdata_fname);
216  if (symlink(fname, fdata_fname) == -1) {
217  log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'",
218  fname, fdata_fname);
219  }
220 }
221 
222 static void unlink_all_completed (void)
223 {
224  api->timer_delete (corosync_stats_timer_handle);
225  qb_loop_stop (corosync_poll_handle);
226  icmap_fini();
227 }
228 
230 {
231  corosync_service_unlink_all (api, unlink_all_completed);
232 }
233 
234 static int32_t sig_diag_handler (int num, void *data)
235 {
237  return 0;
238 }
239 
240 static int32_t sig_exit_handler (int num, void *data)
241 {
242  log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal");
243  corosync_service_unlink_all (api, unlink_all_completed);
244  return 0;
245 }
246 
247 static void sigsegv_handler (int num)
248 {
249  (void)signal (SIGSEGV, SIG_DFL);
250  corosync_blackbox_write_to_file ();
251  qb_log_fini();
252  raise (SIGSEGV);
253 }
254 
255 /*
256  * QB wrapper for real signal handler
257  */
258 static int32_t sig_segv_handler (int num, void *data)
259 {
260 
261  sigsegv_handler(num);
262 
263  return 0;
264 }
265 
266 static void sigabrt_handler (int num)
267 {
268  (void)signal (SIGABRT, SIG_DFL);
269  corosync_blackbox_write_to_file ();
270  qb_log_fini();
271  raise (SIGABRT);
272 }
273 
274 /*
275  * QB wrapper for real signal handler
276  */
277 static int32_t sig_abrt_handler (int num, void *data)
278 {
279 
280  sigabrt_handler(num);
281 
282  return 0;
283 }
284 
285 #define LOCALHOST_IP inet_addr("127.0.0.1")
286 
287 static void *corosync_group_handle;
288 
289 static struct totempg_group corosync_group = {
290  .group = "a",
291  .group_len = 1
292 };
293 
294 static void serialize_lock (void)
295 {
296 }
297 
298 static void serialize_unlock (void)
299 {
300 }
301 
302 static void corosync_sync_completed (void)
303 {
305  "Completed service synchronization, ready to provide service.");
306  sync_in_process = 0;
307 
308  cs_ipcs_sync_state_changed(sync_in_process);
310  /*
311  * Inform totem to start using new message queue again
312  */
314 }
315 
316 static int corosync_sync_callbacks_retrieve (
317  int service_id,
318  struct sync_callbacks *callbacks)
319 {
320  if (corosync_service[service_id] == NULL) {
321  return (-1);
322  }
323 
324  if (callbacks == NULL) {
325  return (0);
326  }
327 
328  callbacks->name = corosync_service[service_id]->name;
329 
330  callbacks->sync_init = corosync_service[service_id]->sync_init;
331  callbacks->sync_process = corosync_service[service_id]->sync_process;
332  callbacks->sync_activate = corosync_service[service_id]->sync_activate;
333  callbacks->sync_abort = corosync_service[service_id]->sync_abort;
334  return (0);
335 }
336 
337 static struct memb_ring_id corosync_ring_id;
338 
339 static void member_object_joined (unsigned int nodeid)
340 {
341  char member_ip[ICMAP_KEYNAME_MAXLEN];
342  char member_join_count[ICMAP_KEYNAME_MAXLEN];
343  char member_status[ICMAP_KEYNAME_MAXLEN];
344 
345  snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
346  "runtime.totem.pg.mrp.srp.members.%u.ip", nodeid);
347  snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
348  "runtime.totem.pg.mrp.srp.members.%u.join_count", nodeid);
349  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
350  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
351 
352  if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
353  icmap_inc(member_join_count);
354  icmap_set_string(member_status, "joined");
355  } else {
356  icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid));
357  icmap_set_uint32(member_join_count, 1);
358  icmap_set_string(member_status, "joined");
359  }
360 
362  "Member joined: %s", api->totem_ifaces_print (nodeid));
363 }
364 
365 static void member_object_left (unsigned int nodeid)
366 {
367  char member_status[ICMAP_KEYNAME_MAXLEN];
368 
369  snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
370  "runtime.totem.pg.mrp.srp.members.%u.status", nodeid);
371  icmap_set_string(member_status, "left");
372 
374  "Member left: %s", api->totem_ifaces_print (nodeid));
375 }
376 
377 static void confchg_fn (
378  enum totem_configuration_type configuration_type,
379  const unsigned int *member_list, size_t member_list_entries,
380  const unsigned int *left_list, size_t left_list_entries,
381  const unsigned int *joined_list, size_t joined_list_entries,
382  const struct memb_ring_id *ring_id)
383 {
384  int i;
385  int abort_activate = 0;
386 
387  if (sync_in_process == 1) {
388  abort_activate = 1;
389  }
390  sync_in_process = 1;
391  cs_ipcs_sync_state_changed(sync_in_process);
392  memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id));
393 
394  for (i = 0; i < left_list_entries; i++) {
395  member_object_left (left_list[i]);
396  }
397  for (i = 0; i < joined_list_entries; i++) {
398  member_object_joined (joined_list[i]);
399  }
400  /*
401  * Call configuration change for all services
402  */
403  for (i = 0; i < service_count; i++) {
404  if (corosync_service[i] && corosync_service[i]->confchg_fn) {
405  corosync_service[i]->confchg_fn (configuration_type,
406  member_list, member_list_entries,
407  left_list, left_list_entries,
408  joined_list, joined_list_entries, ring_id);
409  }
410  }
411 
412  if (abort_activate) {
413  sync_abort ();
414  }
415  if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) {
416  sync_save_transitional (member_list, member_list_entries, ring_id);
417  }
418  if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
419  sync_start (member_list, member_list_entries, ring_id);
420  }
421 }
422 
423 static void priv_drop (void)
424 {
425  return; /* TODO: we are still not dropping privs */
426 }
427 
428 static void corosync_tty_detach (void)
429 {
430  int devnull;
431 
432  /*
433  * Disconnect from TTY if this is not a debug run
434  */
435 
436  switch (fork ()) {
437  case -1:
439  break;
440  case 0:
441  /*
442  * child which is disconnected, run this process
443  */
444  break;
445  default:
446  exit (0);
447  break;
448  }
449 
450  /* Create new session */
451  (void)setsid();
452 
453  /*
454  * Map stdin/out/err to /dev/null.
455  */
456  devnull = open("/dev/null", O_RDWR);
457  if (devnull == -1) {
459  }
460 
461  if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
462  || dup2(devnull, 2) < 0) {
463  close(devnull);
465  }
466  close(devnull);
467 }
468 
469 static void corosync_mlockall (void)
470 {
471  int res;
472  struct rlimit rlimit;
473 
474  rlimit.rlim_cur = RLIM_INFINITY;
475  rlimit.rlim_max = RLIM_INFINITY;
476 
477 #ifndef RLIMIT_MEMLOCK
478 #define RLIMIT_MEMLOCK RLIMIT_VMEM
479 #endif
480 
481  setrlimit (RLIMIT_MEMLOCK, &rlimit);
482 
483  res = mlockall (MCL_CURRENT | MCL_FUTURE);
484  if (res == -1) {
486  "Could not lock memory of service to avoid page faults");
487  };
488 }
489 
490 
491 static void corosync_totem_stats_updater (void *data)
492 {
493  totempg_stats_t * stats;
494  uint32_t total_mtt_rx_token;
495  uint32_t total_backlog_calc;
496  uint32_t total_token_holdtime;
497  int t, prev, i;
498  int32_t token_count;
499  char key_name[ICMAP_KEYNAME_MAXLEN];
500 
501  stats = api->totem_get_stats();
502 
503  icmap_set_uint32("runtime.totem.pg.msg_reserved", stats->msg_reserved);
504  icmap_set_uint32("runtime.totem.pg.msg_queue_avail", stats->msg_queue_avail);
505  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_tx", stats->mrp->srp->orf_token_tx);
506  icmap_set_uint64("runtime.totem.pg.mrp.srp.orf_token_rx", stats->mrp->srp->orf_token_rx);
507  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_tx", stats->mrp->srp->memb_merge_detect_tx);
508  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_merge_detect_rx", stats->mrp->srp->memb_merge_detect_rx);
509  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_tx", stats->mrp->srp->memb_join_tx);
510  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_join_rx", stats->mrp->srp->memb_join_rx);
511  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_tx", stats->mrp->srp->mcast_tx);
512  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_retx", stats->mrp->srp->mcast_retx);
513  icmap_set_uint64("runtime.totem.pg.mrp.srp.mcast_rx", stats->mrp->srp->mcast_rx);
514  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_tx", stats->mrp->srp->memb_commit_token_tx);
515  icmap_set_uint64("runtime.totem.pg.mrp.srp.memb_commit_token_rx", stats->mrp->srp->memb_commit_token_rx);
516  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_tx", stats->mrp->srp->token_hold_cancel_tx);
517  icmap_set_uint64("runtime.totem.pg.mrp.srp.token_hold_cancel_rx", stats->mrp->srp->token_hold_cancel_rx);
518  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_entered", stats->mrp->srp->operational_entered);
519  icmap_set_uint64("runtime.totem.pg.mrp.srp.operational_token_lost", stats->mrp->srp->operational_token_lost);
520  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_entered", stats->mrp->srp->gather_entered);
521  icmap_set_uint64("runtime.totem.pg.mrp.srp.gather_token_lost", stats->mrp->srp->gather_token_lost);
522  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_entered", stats->mrp->srp->commit_entered);
523  icmap_set_uint64("runtime.totem.pg.mrp.srp.commit_token_lost", stats->mrp->srp->commit_token_lost);
524  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_entered", stats->mrp->srp->recovery_entered);
525  icmap_set_uint64("runtime.totem.pg.mrp.srp.recovery_token_lost", stats->mrp->srp->recovery_token_lost);
526  icmap_set_uint64("runtime.totem.pg.mrp.srp.consensus_timeouts", stats->mrp->srp->consensus_timeouts);
527  icmap_set_uint64("runtime.totem.pg.mrp.srp.rx_msg_dropped", stats->mrp->srp->rx_msg_dropped);
528  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_gather", stats->mrp->srp->continuous_gather);
529  icmap_set_uint32("runtime.totem.pg.mrp.srp.continuous_sendmsg_failures",
531 
532  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure",
533  stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0);
534 
535  if (stats->mrp->srp->continuous_gather > MAX_NO_CONT_GATHER ||
538  "Totem is unable to form a cluster because of an "
539  "operating system or network fault. The most common "
540  "cause of this message is that the local firewall is "
541  "configured improperly.");
542  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 1);
543  } else {
544  icmap_set_uint8("runtime.totem.pg.mrp.srp.firewall_enabled_or_nic_failure", 0);
545  }
546 
547  for (i = 0; i < stats->mrp->srp->rrp->interface_count; i++) {
548  snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "runtime.totem.pg.mrp.rrp.%u.faulty", i);
549  icmap_set_uint8(key_name, stats->mrp->srp->rrp->faulty[i]);
550  }
551  total_mtt_rx_token = 0;
552  total_token_holdtime = 0;
553  total_backlog_calc = 0;
554  token_count = 0;
555  t = stats->mrp->srp->latest_token;
556  while (1) {
557  if (t == 0)
558  prev = TOTEM_TOKEN_STATS_MAX - 1;
559  else
560  prev = t - 1;
561  if (prev == stats->mrp->srp->earliest_token)
562  break;
563  /* if tx == 0, then dropped token (not ours) */
564  if (stats->mrp->srp->token[t].tx != 0 ||
565  (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx) > 0 ) {
566  total_mtt_rx_token += (stats->mrp->srp->token[t].rx - stats->mrp->srp->token[prev].rx);
567  total_token_holdtime += (stats->mrp->srp->token[t].tx - stats->mrp->srp->token[t].rx);
568  total_backlog_calc += stats->mrp->srp->token[t].backlog_calc;
569  token_count++;
570  }
571  t = prev;
572  }
573  if (token_count) {
574  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", (total_mtt_rx_token / token_count));
575  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", (total_token_holdtime / token_count));
576  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", (total_backlog_calc / token_count));
577  }
578 
580 
581  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
582  corosync_totem_stats_updater,
583  &corosync_stats_timer_handle);
584 }
585 
586 static void totem_dynamic_notify(
587  int32_t event,
588  const char *key_name,
589  struct icmap_notify_value new_val,
590  struct icmap_notify_value old_val,
591  void *user_data)
592 {
593  int res;
594  unsigned int ring_no;
595  unsigned int member_no;
596  struct totem_ip_address member;
597  int add_new_member = 0;
598  int remove_old_member = 0;
599  char tmp_str[ICMAP_KEYNAME_MAXLEN];
600 
601  res = sscanf(key_name, "nodelist.node.%u.ring%u%s", &member_no, &ring_no, tmp_str);
602  if (res != 3)
603  return ;
604 
605  if (strcmp(tmp_str, "_addr") != 0) {
606  return;
607  }
608 
609  if (event == ICMAP_TRACK_ADD && new_val.type == ICMAP_VALUETYPE_STRING) {
610  add_new_member = 1;
611  }
612 
613  if (event == ICMAP_TRACK_DELETE && old_val.type == ICMAP_VALUETYPE_STRING) {
614  remove_old_member = 1;
615  }
616 
617  if (event == ICMAP_TRACK_MODIFY && new_val.type == ICMAP_VALUETYPE_STRING &&
618  old_val.type == ICMAP_VALUETYPE_STRING) {
619  add_new_member = 1;
620  remove_old_member = 1;
621  }
622 
623  if (remove_old_member) {
625  "removing dynamic member %s for ring %u", (char *)old_val.data, ring_no);
626  if (totemip_parse(&member, (char *)old_val.data, ip_version) == 0) {
627  totempg_member_remove (&member, ring_no);
628  }
629  }
630 
631  if (add_new_member) {
633  "adding dynamic member %s for ring %u", (char *)new_val.data, ring_no);
634  if (totemip_parse(&member, (char *)new_val.data, ip_version) == 0) {
635  totempg_member_add (&member, ring_no);
636  }
637  }
638 }
639 
640 static void corosync_totem_dynamic_init (void)
641 {
642  icmap_track_t icmap_track = NULL;
643 
644  icmap_track_add("nodelist.node.",
646  totem_dynamic_notify,
647  NULL,
648  &icmap_track);
649 }
650 
651 static void corosync_totem_stats_init (void)
652 {
653  icmap_set_uint32("runtime.totem.pg.mrp.srp.mtt_rx_token", 0);
654  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_token_workload", 0);
655  icmap_set_uint32("runtime.totem.pg.mrp.srp.avg_backlog_calc", 0);
656 
657  /* start stats timer */
658  api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
659  corosync_totem_stats_updater,
660  &corosync_stats_timer_handle);
661 }
662 
663 
664 static void deliver_fn (
665  unsigned int nodeid,
666  const void *msg,
667  unsigned int msg_len,
668  int endian_conversion_required)
669 {
670  const struct qb_ipc_request_header *header;
671  int32_t service;
672  int32_t fn_id;
673  uint32_t id;
674 
675  header = msg;
676  if (endian_conversion_required) {
677  id = swab32 (header->id);
678  } else {
679  id = header->id;
680  }
681 
682  /*
683  * Call the proper executive handler
684  */
685  service = id >> 16;
686  fn_id = id & 0xffff;
687 
688  if (!corosync_service[service]) {
689  return;
690  }
691  if (fn_id >= corosync_service[service]->exec_engine_count) {
692  log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)",
693  fn_id, service, corosync_service[service]->exec_engine_count);
694  return;
695  }
696 
697  icmap_fast_inc(service_stats_rx[service][fn_id]);
698 
699  if (endian_conversion_required) {
700  assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
702  ((void *)msg);
703  }
704 
706  (msg, nodeid);
707 }
708 
710  const struct iovec *iovec,
711  unsigned int iov_len,
712  unsigned int guarantee)
713 {
714  const struct qb_ipc_request_header *req = iovec->iov_base;
715  int32_t service;
716  int32_t fn_id;
717 
718  service = req->id >> 16;
719  fn_id = req->id & 0xffff;
720 
721  if (corosync_service[service]) {
722  icmap_fast_inc(service_stats_tx[service][fn_id]);
723  }
724 
725  return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
726 }
727 
728 static void corosync_ring_id_create_or_load (
729  struct memb_ring_id *memb_ring_id,
730  const struct totem_ip_address *addr)
731 {
732  int fd;
733  int res = 0;
734  char filename[PATH_MAX];
735 
736  snprintf (filename, sizeof(filename), "%s/ringid_%s",
737  get_run_dir(), totemip_print (addr));
738  fd = open (filename, O_RDONLY, 0700);
739  /*
740  * If file can be opened and read, read the ring id
741  */
742  if (fd != -1) {
743  res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
744  close (fd);
745  }
746  /*
747  * If file could not be opened or read, create a new ring id
748  */
749  if ((fd == -1) || (res != sizeof (uint64_t))) {
750  memb_ring_id->seq = 0;
751  umask(0);
752  fd = open (filename, O_CREAT|O_RDWR, 0700);
753  if (fd != -1) {
754  res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
755  close (fd);
756  if (res == -1) {
758  "Couldn't write ringid file '%s'", filename);
759 
761  }
762  } else {
764  "Couldn't create ringid file '%s'", filename);
765 
767  }
768  }
769 
770  totemip_copy(&memb_ring_id->rep, addr);
771  assert (!totemip_zero_check(&memb_ring_id->rep));
772 }
773 
774 static void corosync_ring_id_store (
775  const struct memb_ring_id *memb_ring_id,
776  const struct totem_ip_address *addr)
777 {
778  char filename[PATH_MAX];
779  int fd;
780  int res;
781 
782  snprintf (filename, sizeof(filename), "%s/ringid_%s",
783  get_run_dir(), totemip_print (addr));
784 
785  fd = open (filename, O_WRONLY, 0777);
786  if (fd == -1) {
787  fd = open (filename, O_CREAT|O_RDWR, 0777);
788  }
789  if (fd == -1) {
791  "Couldn't store new ring id %llx to stable storage",
792  memb_ring_id->seq);
793 
795  }
797  "Storing new sequence id for ring %llx", memb_ring_id->seq);
798  res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
799  close (fd);
800  if (res != sizeof(memb_ring_id->seq)) {
802  "Couldn't store new ring id %llx to stable storage",
803  memb_ring_id->seq);
804 
806  }
807 }
808 
809 static qb_loop_timer_handle recheck_the_q_level_timer;
811 {
812  totempg_check_q_level(corosync_group_handle);
814  qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC,
815  NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer);
816  }
817 }
818 
821 };
822 
823 
825  unsigned int service,
826  unsigned int id,
827  const void *msg,
828  void *sending_allowed_private_data)
829 {
831  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
832  struct iovec reserve_iovec;
833  struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
834  int sending_allowed;
835 
836  reserve_iovec.iov_base = (char *)header;
837  reserve_iovec.iov_len = header->size;
838 
840  corosync_group_handle,
841  &reserve_iovec, 1);
842  if (pd->reserved_msgs == -1) {
843  return -EINVAL;
844  }
845 
846  sending_allowed = QB_FALSE;
847  if (corosync_quorum_is_quorate() == 1 ||
848  corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) {
849  // we are quorate
850  // now check flow control
851  if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) {
852  sending_allowed = QB_TRUE;
853  } else if (pd->reserved_msgs && sync_in_process == 0) {
854  sending_allowed = QB_TRUE;
855  } else if (pd->reserved_msgs == 0) {
856  return -ENOBUFS;
857  } else /* (sync_in_process) */ {
858  return -EINPROGRESS;
859  }
860  } else {
861  return -EHOSTUNREACH;
862  }
863 
864  return (sending_allowed);
865 }
866 
867 void corosync_sending_allowed_release (void *sending_allowed_private_data)
868 {
870  (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
871 
872  if (pd->reserved_msgs == -1) {
873  return;
874  }
876 }
877 
879 {
880  int ret = 0;
881 
882  assert (source != NULL);
883  if (source->nodeid == totempg_my_nodeid_get ()) {
884  ret = 1;
885  }
886  return ret;
887 }
888 
890  mar_message_source_t *source,
891  void *conn)
892 {
893  assert ((source != NULL) && (conn != NULL));
894  memset (source, 0, sizeof (mar_message_source_t));
895  source->nodeid = totempg_my_nodeid_get ();
896  source->conn = conn;
897 }
898 
901  qb_loop_timer_handle handle;
902  unsigned long long tv_prev;
903  unsigned long long max_tv_diff;
904 };
905 
906 static void timer_function_scheduler_timeout (void *data)
907 {
908  struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data;
909  unsigned long long tv_current;
910  unsigned long long tv_diff;
911 
912  tv_current = qb_util_nano_current_get ();
913 
914  if (timeout_data->tv_prev == 0) {
915  /*
916  * Initial call -> just pretent everything is ok
917  */
918  timeout_data->tv_prev = tv_current;
919  timeout_data->max_tv_diff = 0;
920  }
921 
922  tv_diff = tv_current - timeout_data->tv_prev;
923  timeout_data->tv_prev = tv_current;
924 
925  if (tv_diff > timeout_data->max_tv_diff) {
926  log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled for %0.4f ms "
927  "(threshold is %0.4f ms). Consider token timeout increase.",
928  (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC);
929  }
930 
931  /*
932  * Set next threshold, because token_timeout can change
933  */
934  timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8;
935  qb_loop_timer_add (corosync_poll_handle,
936  QB_LOOP_MED,
937  timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3,
938  timeout_data,
939  timer_function_scheduler_timeout,
940  &timeout_data->handle);
941 }
942 
943 
944 static void corosync_setscheduler (void)
945 {
946 #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
947  int res;
948 
949  sched_priority = sched_get_priority_max (SCHED_RR);
950  if (sched_priority != -1) {
951  global_sched_param.sched_priority = sched_priority;
952  res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
953  if (res == -1) {
955  "Could not set SCHED_RR at priority %d",
956  global_sched_param.sched_priority);
957 
958  global_sched_param.sched_priority = 0;
959 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
960  qb_log_thread_priority_set (SCHED_OTHER, 0);
961 #endif
962  } else {
963 
964  /*
965  * Turn on SCHED_RR in logsys system
966  */
967 #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
968  res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
969 #else
970  res = -1;
971 #endif
972  if (res == -1) {
974  "Could not set logsys thread priority."
975  " Can't continue because of priority inversions.");
977  }
978  }
979  } else {
981  "Could not get maximum scheduler priority");
982  sched_priority = 0;
983  }
984 #else
986  "The Platform is missing process priority setting features. Leaving at default.");
987 #endif
988 }
989 
990 static void
991 _logsys_log_printf(int level, int subsys,
992  const char *function_name,
993  const char *file_name,
994  int file_line,
995  const char *format,
996  ...) __attribute__((format(printf, 6, 7)));
997 
998 static void
999 _logsys_log_printf(int level, int subsys,
1000  const char *function_name,
1001  const char *file_name,
1002  int file_line,
1003  const char *format, ...)
1004 {
1005  va_list ap;
1006 
1007  va_start(ap, format);
1008  qb_log_from_external_source_va(function_name, file_name,
1009  format, level, file_line,
1010  subsys, ap);
1011  va_end(ap);
1012 }
1013 
1014 static void fplay_key_change_notify_fn (
1015  int32_t event,
1016  const char *key_name,
1017  struct icmap_notify_value new_val,
1018  struct icmap_notify_value old_val,
1019  void *user_data)
1020 {
1021  if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) {
1022  fprintf(stderr,"Writetofile\n");
1023  corosync_blackbox_write_to_file ();
1024  }
1025  if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) {
1026  fprintf(stderr,"statefump\n");
1028  }
1029 }
1030 
1031 static void corosync_fplay_control_init (void)
1032 {
1033  icmap_track_t track = NULL;
1034 
1035  icmap_set_string("runtime.blackbox.dump_flight_data", "no");
1036  icmap_set_string("runtime.blackbox.dump_state", "no");
1037 
1038  icmap_track_add("runtime.blackbox.dump_flight_data",
1040  fplay_key_change_notify_fn,
1041  NULL, &track);
1042  icmap_track_add("runtime.blackbox.dump_state",
1044  fplay_key_change_notify_fn,
1045  NULL, &track);
1046 }
1047 
1048 /*
1049  * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
1050  * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
1051  *
1052  * Also some RO keys cannot be determined in this stage, so they are set later in
1053  * other functions (like nodelist.local_node_pos, ...)
1054  */
1055 static void set_icmap_ro_keys_flag (void)
1056 {
1057  /*
1058  * Set RO flag for all keys of internal configuration and runtime statistics
1059  */
1060  icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
1061  icmap_set_ro_access("runtime.connections.", CS_TRUE, CS_TRUE);
1062  icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
1063  icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
1064 
1065  /*
1066  * Set RO flag for constrete keys of configuration which can't be changed
1067  * during runtime
1068  */
1069  icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
1070  icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
1071  icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
1072  icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE);
1073  icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
1074  icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);
1075  icmap_set_ro_access("qb.ipc_type", CS_FALSE, CS_TRUE);
1076  icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE);
1077  icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE);
1078 }
1079 
1080 static void main_service_ready (void)
1081 {
1082  int res;
1083 
1084  /*
1085  * This must occur after totempg is initialized because "this_ip" must be set
1086  */
1088  if (res == -1) {
1089  log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services");
1091  }
1092  cs_ipcs_init();
1093  corosync_totem_stats_init ();
1094  corosync_fplay_control_init ();
1095  corosync_totem_dynamic_init ();
1096  sync_init (
1097  corosync_sync_callbacks_retrieve,
1098  corosync_sync_completed);
1099 }
1100 
1101 static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid)
1102 {
1103  struct flock lock;
1104  enum e_corosync_done err;
1105  char pid_s[17];
1106  int fd_flag;
1107  int lf;
1108 
1109  err = COROSYNC_DONE_EXIT;
1110 
1111  lf = open (lockfile, O_WRONLY | O_CREAT, 0640);
1112  if (lf == -1) {
1113  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file.");
1114  return (COROSYNC_DONE_AQUIRE_LOCK);
1115  }
1116 
1117 retry_fcntl:
1118  lock.l_type = F_WRLCK;
1119  lock.l_start = 0;
1120  lock.l_whence = SEEK_SET;
1121  lock.l_len = 0;
1122  if (fcntl (lf, F_SETLK, &lock) == -1) {
1123  switch (errno) {
1124  case EINTR:
1125  goto retry_fcntl;
1126  break;
1127  case EAGAIN:
1128  case EACCES:
1129  log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running.");
1131  goto error_close;
1132  break;
1133  default:
1134  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't aquire lock. Error was %s",
1135  strerror(errno));
1137  goto error_close;
1138  break;
1139  }
1140  }
1141 
1142  if (ftruncate (lf, 0) == -1) {
1143  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s",
1144  strerror (errno));
1146  goto error_close_unlink;
1147  }
1148 
1149  memset (pid_s, 0, sizeof (pid_s));
1150  snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid);
1151 
1152 retry_write:
1153  if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1154  if (errno == EINTR) {
1155  goto retry_write;
1156  } else {
1157  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. "
1158  "Error was %s", strerror (errno));
1160  goto error_close_unlink;
1161  }
1162  }
1163 
1164  if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) {
1165  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. "
1166  "Error was %s", strerror (errno));
1168  goto error_close_unlink;
1169  }
1170  fd_flag |= FD_CLOEXEC;
1171  if (fcntl (lf, F_SETFD, fd_flag) == -1) {
1172  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. "
1173  "Error was %s", strerror (errno));
1175  goto error_close_unlink;
1176  }
1177 
1178  return (err);
1179 
1180 error_close_unlink:
1181  unlink (lockfile);
1182 error_close:
1183  close (lf);
1184 
1185  return (err);
1186 }
1187 
1188 int main (int argc, char **argv, char **envp)
1189 {
1190  const char *error_string;
1191  struct totem_config totem_config;
1192  int res, ch;
1193  int background, setprio, testonly;
1194  struct stat stat_out;
1195  enum e_corosync_done flock_err;
1196  uint64_t totem_config_warnings;
1197  struct scheduler_pause_timeout_data scheduler_pause_timeout_data;
1198 
1199  /* default configuration
1200  */
1201  background = 1;
1202  setprio = 0;
1203  testonly = 0;
1204 
1205  while ((ch = getopt (argc, argv, "fprtv")) != EOF) {
1206 
1207  switch (ch) {
1208  case 'f':
1209  background = 0;
1210  break;
1211  case 'p':
1212  break;
1213  case 'r':
1214  setprio = 1;
1215  break;
1216  case 't':
1217  testonly = 1;
1218  break;
1219  case 'v':
1220  printf ("Corosync Cluster Engine, version '%s'\n", VERSION);
1221  printf ("Copyright (c) 2006-2009 Red Hat, Inc.\n");
1223  return EXIT_SUCCESS;
1224 
1225  break;
1226  default:
1227  fprintf(stderr, \
1228  "usage:\n"\
1229  " -f : Start application in foreground.\n"\
1230  " -p : Does nothing. \n"\
1231  " -t : Test configuration and exit.\n"\
1232  " -r : Set round robin realtime scheduling \n"\
1233  " -v : Display version and SVN revision of Corosync and exit.\n");
1235  return EXIT_FAILURE;
1236  }
1237  }
1238 
1239  /*
1240  * Set round robin realtime scheduling with priority 99
1241  * Lock all memory to avoid page faults which may interrupt
1242  * application healthchecking
1243  */
1244  if (setprio) {
1245  corosync_setscheduler ();
1246  }
1247 
1248  corosync_mlockall ();
1249 
1250  /*
1251  * Other signals are registered later via qb_loop_signal_add
1252  */
1253  (void)signal (SIGSEGV, sigsegv_handler);
1254  (void)signal (SIGABRT, sigabrt_handler);
1255 #if MSG_NOSIGNAL != 0
1256  (void)signal (SIGPIPE, SIG_IGN);
1257 #endif
1258 
1259  if (icmap_init() != CS_OK) {
1260  log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component.");
1262  }
1263  set_icmap_ro_keys_flag();
1264 
1265  /*
1266  * Initialize the corosync_api_v1 definition
1267  */
1268  api = apidef_get ();
1269 
1270  res = coroparse_configparse(icmap_get_global_map(), &error_string);
1271  if (res == -1) {
1272  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1274  }
1275 
1276  res = corosync_log_config_read (&error_string);
1277  if (res == -1) {
1278  /*
1279  * if we are here, we _must_ flush the logsys queue
1280  * and try to inform that we couldn't read the config.
1281  * this is a desperate attempt before certain death
1282  * and there is no guarantee that we can print to stderr
1283  * nor that logsys is sending the messages where we expect.
1284  */
1285  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1286  fprintf(stderr, "%s", error_string);
1287  syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1289  }
1290 
1291  if (!testonly) {
1292  log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine ('%s'): started and ready to provide service.", VERSION);
1293  log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "");
1294  }
1295 
1296  /*
1297  * Make sure required directory is present
1298  */
1299  res = stat (get_run_dir(), &stat_out);
1300  if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) {
1301  log_printf (LOGSYS_LEVEL_ERROR, "Required directory not present %s. Please create it.", get_run_dir());
1303  }
1304 
1305  res = chdir(get_run_dir());
1306  if (res == -1) {
1307  log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to run directory %s. "
1308  "Please make sure it has correct context and rights.", get_run_dir());
1310  }
1311 
1312  res = totem_config_read (&totem_config, &error_string, &totem_config_warnings);
1313  if (res == -1) {
1314  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1316  }
1317 
1318  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) {
1319  log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored.");
1320  }
1321 
1322  if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) {
1323  log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated.");
1324  }
1325 
1326  if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED) {
1327  log_printf (LOGSYS_LEVEL_WARNING, "nodeid appears both in totem section and nodelist. Nodelist one is used.");
1328  }
1329 
1330  if (totem_config_warnings != 0) {
1331  log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
1332  }
1333 
1334  res = totem_config_keyread (&totem_config, &error_string);
1335  if (res == -1) {
1336  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1338  }
1339 
1340  res = totem_config_validate (&totem_config, &error_string);
1341  if (res == -1) {
1342  log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1344  }
1345 
1346  if (testonly) {
1348  }
1349 
1350  ip_version = totem_config.ip_version;
1351 
1352  totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
1353  totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
1354 
1356  totem_config.totem_logging_configuration.log_subsys_id = _logsys_subsys_create("TOTEM", "totem,"
1357  "totemmrp.c,totemrrp.c,totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1358  "totempg.c,totemiba.c,totemudp.c,totemudpu.c,totemnet.c");
1359 
1366  totem_config.totem_logging_configuration.log_printf = _logsys_log_printf;
1368 
1369  /*
1370  * Now we are fully initialized.
1371  */
1372  if (background) {
1373  corosync_tty_detach ();
1374  }
1375 
1376  corosync_poll_handle = qb_loop_create ();
1377 
1378  memset(&scheduler_pause_timeout_data, 0, sizeof(scheduler_pause_timeout_data));
1379  scheduler_pause_timeout_data.totem_config = &totem_config;
1380  timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1381 
1382  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1383  SIGUSR2, NULL, sig_diag_handler, NULL);
1384  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1385  SIGINT, NULL, sig_exit_handler, NULL);
1386  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1387  SIGSEGV, NULL, sig_segv_handler, NULL);
1388  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1389  SIGABRT, NULL, sig_abrt_handler, NULL);
1390  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1391  SIGQUIT, NULL, sig_exit_handler, NULL);
1392  qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1393  SIGTERM, NULL, sig_exit_handler, NULL);
1394 
1395  if (logsys_thread_start() != 0) {
1396  log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread");
1398  }
1399 
1400  if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) {
1401  corosync_exit_error (flock_err);
1402  }
1403 
1404  /*
1405  * if totempg_initialize doesn't have root priveleges, it cannot
1406  * bind to a specific interface. This only matters if
1407  * there is more then one interface in a system, so
1408  * in this case, only a warning is printed
1409  */
1410  /*
1411  * Join multicast group and setup delivery
1412  * and configuration change functions
1413  */
1415  corosync_poll_handle,
1416  &totem_config);
1417 
1419  main_service_ready);
1420 
1422  &corosync_group_handle,
1423  deliver_fn,
1424  confchg_fn);
1425 
1427  corosync_group_handle,
1428  &corosync_group,
1429  1);
1430 
1431  /*
1432  * Drop root privleges to user 'corosync'
1433  * TODO: Don't really need full root capabilities;
1434  * needed capabilities are:
1435  * CAP_NET_RAW (bindtodevice)
1436  * CAP_SYS_NICE (setscheduler)
1437  * CAP_IPC_LOCK (mlockall)
1438  */
1439  priv_drop ();
1440 
1441  schedwrk_init (
1442  serialize_lock,
1443  serialize_unlock);
1444 
1445  /*
1446  * Start main processing loop
1447  */
1448  qb_loop_run (corosync_poll_handle);
1449 
1450  /*
1451  * Exit was requested
1452  */
1453  totempg_finalize ();
1454 
1455  /*
1456  * free the loop resources
1457  */
1458  qb_loop_destroy (corosync_poll_handle);
1459 
1460  /*
1461  * free up the icmap
1462  */
1463 
1464  /*
1465  * Remove pid lock file
1466  */
1467  unlink (corosync_lock_file);
1468 
1470 
1471  return EXIT_SUCCESS;
1472 }
#define CS_TRUE
Definition: corotypes.h:51
const char * name
Definition: coroapi.h:432
int32_t cs_ipcs_q_level_get(void)
Definition: ipc_glue.c:682
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
Definition: service.c:339
void(*) in log_level_security)
Definition: totem.h:82
uint64_t gather_entered
Definition: totem.h:260
const char * get_run_dir(void)
Definition: util.c:174
int corosync_quorum_is_quorate(void)
Definition: exec/quorum.c:67
void totempg_finalize(void)
Definition: totempg.c:845
Totem Single Ring Protocol.
uint64_t memb_commit_token_rx
Definition: totem.h:255
#define LOGSYS_LEVEL_INFO
Definition: logsys.h:73
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.h:39
#define LOGSYS_LEVEL_TRACE
Definition: logsys.h:75
void totempg_trans_ack(void)
Definition: totempg.c:1542
struct totem_config * totem_config
Definition: main.c:900
#define PACKAGE_FEATURES
Definition: config.h:362
#define CS_FALSE
Definition: corotypes.h:50
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:573
uint64_t memb_join_tx
Definition: totem.h:249
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition: coroapi.h:193
qb_loop_timer_handle handle
Definition: main.c:901
void sync_abort(void)
Definition: sync.c:602
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Definition: totem.h:274
const char * totemip_print(const struct totem_ip_address *addr)
Definition: totemip.c:214
Totem Single Ring Protocol.
int(* sync_process)(void)
Definition: coroapi.h:462
void(* exec_dump_fn)(void)
Definition: coroapi.h:442
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
Definition: sync.c:175
int totemip_parse(struct totem_ip_address *totemip, const char *addr, int family)
Definition: totemip.c:263
uint64_t mcast_rx
Definition: totem.h:253
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
Definition: coroparse.c:247
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition: coroapi.h:181
qb_loop_t * cs_poll_handle_get(void)
Definition: main.c:159
void corosync_recheck_the_q_level(void *data)
Definition: main.c:810
#define corosync_exit_error(err)
Definition: exec/util.h:70
void(* exec_endian_convert_fn)(void *msg)
Definition: coroapi.h:424
int totempg_groups_joined_release(int msg_count)
Definition: totempg.c:1336
#define VERSION
Definition: config.h:425
totem_configuration_type
Definition: coroapi.h:110
#define LOCALSTATEDIR
Definition: config.h:346
struct message_header header
Definition: totemsrp.c:60
uint64_t memb_merge_detect_rx
Definition: totem.h:248
LOGSYS_DECLARE_SUBSYS("MAIN")
int guarantee
Definition: totemsrp.c:66
unsigned char addr[TOTEMIP_ADDRLEN]
Definition: coroapi.h:67
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
Definition: schedwrk.c:93
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition: icmap.c:641
void(* sync_abort)(void)
Definition: coroapi.h:464
int earliest_token
Definition: totem.h:271
uint64_t orf_token_tx
Definition: totem.h:245
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
Definition: main.c:164
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition: totempg.c:1124
uint64_t gather_token_lost
Definition: totem.h:261
#define RLIMIT_MEMLOCK
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition: totemip.c:95
icmap_map_t icmap_get_global_map(void)
Definition: icmap.c:278
unsigned long long max_tv_diff
Definition: main.c:903
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
Definition: service.c:110
int _logsys_subsys_create(const char *subsys, const char *filename)
Definition: logsys.c:436
uint64_t memb_commit_token_tx
Definition: totem.h:254
#define LOGSYS_PERROR(err_num, level, fmt, args...)
Definition: logsys.h:211
cs_error_t icmap_inc(const char *key_name)
Definition: icmap.c:1057
#define log_printf(level, format, args...)
Definition: logsys.h:217
#define MAX_NO_CONT_SENDMSG_FAILURES
Definition: totem.h:62
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition: coroapi.h:423
#define TOTEM_TOKEN_STATS_MAX
Definition: totem.h:273
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
Definition: ipc_glue.c:745
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
Definition: main.c:824
void icmap_fini(void)
Definition: icmap.c:261
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition: main.c:709
void cs_ipc_allow_connections(int32_t allow)
Definition: ipc_glue.c:155
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
Definition: service.c:394
uint64_t operational_entered
Definition: totem.h:258
#define ICMAP_TRACK_DELETE
Definition: icmap.h:77
void totempg_service_ready_register(void(*totem_service_ready)(void))
Definition: totempg.c:1511
void(* sync_activate)(void)
Definition: sync.h:46
#define ICMAP_KEYNAME_MAXLEN
Definition: icmap.h:48
int latest_token
Definition: totem.h:272
uint64_t operational_token_lost
Definition: totem.h:259
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
Definition: totem.h:58
uint64_t consensus_timeouts
Definition: totem.h:266
int totem_config_keyread(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:1156
const char * name
Definition: sync.h:48
uint64_t recovery_token_lost
Definition: totem.h:265
#define LOGSYS_LEVEL_WARNING
Definition: logsys.h:71
#define ICMAP_TRACK_MODIFY
Definition: icmap.h:78
struct corosync_exec_handler * exec_engine
Definition: coroapi.h:447
unsigned int totempg_my_nodeid_get(void)
Definition: totempg.c:1502
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition: icmap.c:611
pthread_mutex_t lock
Definition: sam.c:130
uint64_t commit_token_lost
Definition: totem.h:263
void * user_data
Definition: sam.c:126
void(* sync_abort)(void)
Definition: sync.h:47
uint64_t token_hold_cancel_rx
Definition: totem.h:257
int logsys_thread_start(void)
Definition: logsys.c:820
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
Definition: totemconfig.c:992
const void * data
Definition: icmap.h:93
#define LOGSYS_MODE_OUTPUT_SYSLOG
Definition: logsys.h:60
#define ICMAP_TRACK_ADD
Definition: icmap.h:76
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
Definition: totempg.c:1292
icmap_value_types_t type
Definition: icmap.h:91
const void * group
Definition: totempg.h:56
#define LOGSYS_LEVEL_ERROR
Definition: logsys.h:70
Linked list API.
struct totem_ip_address rep
Definition: coroapi.h:104
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: sync.c:591
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition: totempg.c:1222
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:113
uint64_t mcast_retx
Definition: totem.h:252
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
Definition: icmap.c:739
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
Definition: logconfig.c:607
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition: totem.h:75
#define LOGSYS_LEVEL_DEBUG
Definition: logsys.h:74
void cs_ipcs_init(void)
Definition: ipc_glue.c:888
totemmrp_stats_t * mrp
Definition: totem.h:288
void(* sync_activate)(void)
Definition: coroapi.h:463
int totempg_member_remove(const struct totem_ip_address *member, int ring_no)
Definition: totempg.c:1529
typedef __attribute__
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:450
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
Definition: totemconfig.h:47
uint32_t msg_queue_avail
Definition: totem.h:290
uint64_t rx_msg_dropped
Definition: totem.h:267
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
Definition: totemconfig.h:46
#define swab32(x)
Definition: swab.h:43
uint32_t continuous_gather
Definition: totem.h:268
cs_error_t icmap_set_uint64(const char *key_name, uint64_t value)
Definition: icmap.c:623
totemrrp_stats_t * rrp
Definition: totem.h:244
cs_error_t icmap_fast_inc(const char *key_name)
Definition: icmap.c:1077
#define MILLI_2_NANO_SECONDS
Definition: coroapi.h:92
void message_source_set(mar_message_source_t *source, void *conn)
Definition: main.c:889
int(* sync_process)(void)
Definition: sync.h:45
void logsys_config_apply(void)
Definition: logsys.c:770
uint64_t token_hold_cancel_tx
Definition: totem.h:256
int ip_version
Definition: totem.h:189
unsigned int token_timeout
Definition: totem.h:129
void totempg_check_q_level(void *instance)
Definition: totempg.c:1284
uint64_t mcast_tx
Definition: totem.h:251
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition: service.c:112
struct corosync_api_v1 * apidef_get(void)
Definition: apidef.c:147
void corosync_sending_allowed_release(void *sending_allowed_private_data)
Definition: main.c:867
void cs_ipcs_stats_update(void)
Definition: ipc_glue.c:751
uint64_t memb_merge_detect_tx
Definition: totem.h:247
uint32_t msg_reserved
Definition: totem.h:289
totemsrp_stats_t * srp
Definition: totem.h:283
#define SERVICES_COUNT_MAX
Definition: coroapi.h:415
unsigned long long tv_prev
Definition: main.c:902
uint64_t memb_join_rx
Definition: totem.h:250
void corosync_state_dump(void)
Definition: main.c:182
qb_loop_timer_handle corosync_timer_handle_t
Definition: coroapi.h:64
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
Definition: main.c:177
e_corosync_done
Definition: exec/util.h:47
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition: coroapi.h:456
int message_source_is_local(const mar_message_source_t *source)
Definition: main.c:878
struct totem_logging_configuration totem_logging_configuration
Definition: totem.h:163
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition: totempg.c:1172
#define LOGSYS_LEVEL_NOTICE
Definition: logsys.h:72
unsigned long long seq
Definition: coroapi.h:105
cs_error_t icmap_set_uint8(const char *key_name, uint8_t value)
Definition: icmap.c:587
uint64_t recovery_entered
Definition: totem.h:264
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
Definition: icmap.c:1233
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
Definition: totempg.c:799
void corosync_shutdown_request(void)
Definition: main.c:229
uint32_t interface_count
Definition: totem.h:232
struct sched_param global_sched_param
Definition: main.c:151
unsigned int nodeid
Definition: coroapi.h:65
uint32_t continuous_sendmsg_failures
Definition: totem.h:269
uint8_t * faulty
Definition: totem.h:231
struct memb_ring_id ring_id
Definition: totemsrp.c:64
const char *(* totem_ifaces_print)(unsigned int nodeid)
Definition: coroapi.h:243
int main(int argc, char **argv, char **envp)
Definition: main.c:1188
cs_error_t icmap_init(void)
Definition: icmap.c:221
uint64_t commit_entered
Definition: totem.h:262
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED
Definition: totemconfig.h:48
LOGSYS_DECLARE_SYSTEM("corosync", LOGSYS_MODE_OUTPUT_STDERR|LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_INFO)
#define LOGSYS_MODE_OUTPUT_STDERR
Definition: logsys.h:59
void *(* totem_get_stats)(void)
Definition: coroapi.h:388
uint64_t orf_token_rx
Definition: totem.h:246
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Definition: icmap.c:1167
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
Definition: totemconfig.c:719
void(* totem_memb_ring_id_create_or_load)(struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:191
#define ICMAP_TRACK_PREFIX
Definition: icmap.h:84
void logsys_system_fini(void)
Definition: logsys.c:285
void(* totem_memb_ring_id_store)(const struct memb_ring_id *memb_ring_id, const struct totem_ip_address *addr)
Definition: totem.h:195
int totempg_member_add(const struct totem_ip_address *member, int ring_no)
Definition: totempg.c:1522