Skip to content

Add internal metrics support: plugins #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
weyfonk wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add internal metrics support: plugins #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/amqp_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ void amqp_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
else {
Expand Down
1 change: 1 addition & 0 deletions src/imt_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,7 @@ void imt_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)
}

memcpy(pipebuf, rgptr, config.buffer_size);
status->last_plugin_off = (rgptr - (unsigned char *)((struct channels_list_entry *)ptr)->rg.base);
if (((struct ch_buf_hdr *)pipebuf)->seq != seq) {
rg_err_count++;
if (config.debug || (rg_err_count > MAX_RG_COUNT_ERR)) {
Expand Down
64 changes: 62 additions & 2 deletions src/kafka_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "plugin_cmn_json.h"
#include "plugin_cmn_avro.h"
#include "kafka_plugin.h"
#include "intstats/intstats.h"
#ifndef WITH_JANSSON
#error "--enable-kafka requires --enable-jansson"
#endif
Expand Down Expand Up @@ -267,6 +268,7 @@ void kafka_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down Expand Up @@ -338,9 +340,10 @@ void kafka_cache_purge(struct chained_cache *queue[], int index)
char *empty_pcust = NULL;
char src_mac[18], dst_mac[18], src_host[INET6_ADDRSTRLEN], dst_host[INET6_ADDRSTRLEN], ip_address[INET6_ADDRSTRLEN];
char rd_str[SRVBUFLEN], misc_str[SRVBUFLEN], dyn_kafka_topic[SRVBUFLEN], *orig_kafka_topic = NULL;
int i, j, stop, batch_idx, is_topic_dyn = FALSE, qn = 0, ret, saved_index = index;
int mv_num = 0, mv_num_save = 0;
int i, j, stop, batch_idx, is_topic_dyn = FALSE, qn = 0, err, ret, saved_index = index;
int mv_num = 0, mv_num_save = 0, duration_ms;
time_t start, duration;
struct timeval start_ms, end_ms;
pid_t writer_pid = getpid();

char *json_buf = NULL;
Expand Down Expand Up @@ -400,6 +403,7 @@ void kafka_cache_purge(struct chained_cache *queue[], int index)

Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - START (PID: %u) ***\n", config.name, config.type, writer_pid);
start = time(NULL);
gettimeofday(&start_ms, NULL);

if (config.print_markers) {
if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
Expand Down Expand Up @@ -631,6 +635,8 @@ void kafka_cache_purge(struct chained_cache *queue[], int index)
}

duration = time(NULL)-start;
gettimeofday(&end_ms, NULL);
duration_ms = (int)timeval_sub_ms(&end_ms, &start_ms);

if (config.print_markers) {
if (config.message_broker_output & PRINT_OUTPUT_JSON || config.message_broker_output & PRINT_OUTPUT_AVRO) {
Expand All @@ -657,6 +663,12 @@ void kafka_cache_purge(struct chained_cache *queue[], int index)
Log(LOG_INFO, "INFO ( %s/%s ): *** Purging cache - END (PID: %u, QN: %u/%u, ET: %u) ***\n",
config.name, config.type, writer_pid, qn, saved_index, duration);

err = saved_index - qn;
set_kafka_metric(METRICS_INT_KAFKA_FLUSH_CNT, NULL);
set_kafka_metric(METRICS_INT_KAFKA_FLUSH_MSG_SENT, &qn);
set_kafka_metric(METRICS_INT_KAFKA_FLUSH_MSG_ERR, &err);
set_kafka_metric(METRICS_INT_KAFKA_FLUSH_TIME, &duration_ms);

if (config.sql_trigger_exec) P_trigger_exec(config.sql_trigger_exec);

if (empty_pcust) free(empty_pcust);
Expand Down Expand Up @@ -692,3 +704,51 @@ void kafka_avro_schema_purge(char *avro_schema_str)
p_kafka_close(&kafka_avro_schema_host, FALSE);
}
#endif

void *kafka_generate_stats(void *cfg)
{
struct metric *met = ((struct configuration *)cfg)->met;

/*
//XXX: future metrics can be added here if they don't need event-related increments
while (met) {
if (cfg->name && strstr(met->type.label, cfg->name) == met->type.label) {
switch (met->type.id) {
case METRICS_INT_KAFKA_FLUSH_CNT:
met->int_value = kfk_metrics->flush_cnt;
break;
}
}
met = met->next;
}
*/
}

/* If val is NULL, the relevant metric is simply incremented */
void set_kafka_metric(u_int64_t id, void *val)
{
if (!config.intstats_daemon) return;

struct metric *met_tmp = config.met;

while(met_tmp) {
if (id == met_tmp->type.id
&& (config.name && strstr(met_tmp->type.label, config.name) == met_tmp->type.label)) {
switch (id) {
case METRICS_INT_KAFKA_FLUSH_CNT:
met_tmp->int_value++;
break;
case METRICS_INT_KAFKA_FLUSH_MSG_SENT:
if (val) met_tmp->int_value += *(int *)val;
break;
case METRICS_INT_KAFKA_FLUSH_MSG_ERR:
if (val) met_tmp->int_value += *(int *)val;
break;
case METRICS_INT_KAFKA_FLUSH_TIME:
if (val) met_tmp->int_value += *(u_int32_t *)val;
break;
}
}
met_tmp = met_tmp->next;
}
}
2 changes: 2 additions & 0 deletions src/kafka_plugin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ EXT void kafka_cache_purge(struct chained_cache *[], int);
#ifdef WITH_AVRO
EXT void kafka_avro_schema_purge(char *);
#endif
EXT void *kafka_generate_stats(void *);

/* global vars */
EXT void (*insert_func)(struct primitives_ptrs *, struct insert_data *); /* pointer to INSERT function */
Expand All @@ -51,6 +52,7 @@ EXT time_t refresh_deadline;

EXT struct timeval sbasetime;

EXT void set_kafka_metric(u_int64_t, void *);
#ifdef WITH_AVRO
EXT char *avro_buf;
EXT avro_schema_t avro_acct_schema;
Expand Down
1 change: 1 addition & 0 deletions src/mongodb_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ void mongodb_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
1 change: 1 addition & 0 deletions src/mysql_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ void mysql_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
18 changes: 1 addition & 17 deletions src/nfprobe_plugin/nfprobe_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -771,23 +771,6 @@ process_packet(struct FLOWTRACK *ft, struct primitives_ptrs *prim_ptrs, const st
return (PP_OK);
}

/*
* Subtract two timevals. Returns (t1 - t2) in milliseconds.
*/
u_int32_t
timeval_sub_ms(const struct timeval *t1, const struct timeval *t2)
{
struct timeval res;

res.tv_sec = t1->tv_sec - t2->tv_sec;
res.tv_usec = t1->tv_usec - t2->tv_usec;
if (res.tv_usec < 0) {
res.tv_usec += 1000000L;
res.tv_sec--;
}
return ((u_int32_t)res.tv_sec * 1000 + (u_int32_t)res.tv_usec / 1000);
}

static void
update_statistic(struct STATISTIC *s, double new, double n)
{
Expand Down Expand Up @@ -1648,6 +1631,7 @@ void nfprobe_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
1 change: 1 addition & 0 deletions src/pgsql_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ void pgsql_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
3 changes: 3 additions & 0 deletions src/plugin_hooks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ struct ch_status {
u_int8_t wakeup; /* plugin is polling */
u_int32_t backlog;
u_int64_t last_buf_off; /* offset of last committed buffer */
u_int64_t last_plugin_off; /* offset of last buffer copied by the plugin (for reporting) */
};

struct sampling {
Expand All @@ -72,6 +73,7 @@ struct plugin_type_entry {
int id;
char string[10];
void (*func)(int, struct configuration *, void *);
void * (*stats_func)(void *);
};

struct plugins_list_entry {
Expand Down Expand Up @@ -231,5 +233,6 @@ EXT void amqp_plugin(int, struct configuration *, void *);

#ifdef WITH_KAFKA
EXT void kafka_plugin(int, struct configuration *, void *);
EXT void *kafka_generate_stats(void *);
#endif
#undef EXT
28 changes: 14 additions & 14 deletions src/pmacct-data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
#define PLUGIN_ID_UNKNOWN -1

/* vars */
#if (!defined __PMACCTD_C) && (!defined __NFACCTD_C) && (!defined __SFACCTD_C) && (!defined __UACCTD_C) && (!defined __PMTELEMETRYD_C) && (!defined __PMACCT_CLIENT_C) && (!defined __PMBGPD_C) && (!defined __PMBMPD_C)
#if (!defined __PMACCTD_C) && (!defined __NFACCTD_C) && (!defined __SFACCTD_C) && (!defined __UACCTD_C) && (!defined __PMTELEMETRYD_C) && (!defined __PMACCT_CLIENT_C) && (!defined __PMBGPD_C) && (!defined __PMBMPD_C) && (!defined __INTSTATSD_C)
#define EXT extern
#else
#define EXT
Expand Down Expand Up @@ -836,31 +836,31 @@ static const struct _dictionary_line dictionary[] = {
};

static struct plugin_type_entry plugin_types_list[] = {
{PLUGIN_ID_CORE, "core", NULL},
{PLUGIN_ID_MEMORY, "memory", imt_plugin},
{PLUGIN_ID_PRINT, "print", print_plugin},
{PLUGIN_ID_NFPROBE, "nfprobe", nfprobe_plugin},
{PLUGIN_ID_SFPROBE, "sfprobe", sfprobe_plugin},
{PLUGIN_ID_CORE, "core", NULL, NULL},
{PLUGIN_ID_MEMORY, "memory", imt_plugin, NULL},
{PLUGIN_ID_PRINT, "print", print_plugin, NULL},
{PLUGIN_ID_NFPROBE, "nfprobe", nfprobe_plugin, NULL},
{PLUGIN_ID_SFPROBE, "sfprobe", sfprobe_plugin, NULL},
#ifdef WITH_MYSQL
{PLUGIN_ID_MYSQL, "mysql", mysql_plugin},
{PLUGIN_ID_MYSQL, "mysql", mysql_plugin, NULL},
#endif
#ifdef WITH_PGSQL
{PLUGIN_ID_PGSQL, "pgsql", pgsql_plugin},
{PLUGIN_ID_PGSQL, "pgsql", pgsql_plugin, NULL},
#endif
#ifdef WITH_SQLITE3
{PLUGIN_ID_SQLITE3, "sqlite3", sqlite3_plugin},
{PLUGIN_ID_SQLITE3, "sqlite3", sqlite3_plugin, NULL},
#endif
#ifdef WITH_MONGODB
{PLUGIN_ID_MONGODB, "mongodb", mongodb_plugin},
{PLUGIN_ID_MONGODB, "mongodb", mongodb_plugin, NULL},
#endif
#ifdef WITH_RABBITMQ
{PLUGIN_ID_AMQP, "amqp", amqp_plugin},
{PLUGIN_ID_AMQP, "amqp", amqp_plugin, NULL},
#endif
#ifdef WITH_KAFKA
{PLUGIN_ID_KAFKA, "kafka", kafka_plugin},
{PLUGIN_ID_KAFKA, "kafka", kafka_plugin, kafka_generate_stats},
#endif
{PLUGIN_ID_TEE, "tee", tee_plugin},
{PLUGIN_ID_UNKNOWN, "", NULL},
{PLUGIN_ID_TEE, "tee", tee_plugin, NULL},
{PLUGIN_ID_UNKNOWN, "", NULL, NULL},
};
#endif

Expand Down
1 change: 1 addition & 0 deletions src/print_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ void print_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
1 change: 1 addition & 0 deletions src/sfprobe_plugin/sfprobe_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,6 +751,7 @@ void sfprobe_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
1 change: 1 addition & 0 deletions src/sqlite3_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ void sqlite3_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down
1 change: 1 addition & 0 deletions src/tee_plugin/tee_plugin.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,7 @@ void tee_plugin(int pipe_fd, struct configuration *cfgptr, void *ptr)

pollagain = FALSE;
memcpy(pipebuf, rg->ptr, bufsz);
status->last_plugin_off = (rg->ptr - rg->base);
rg->ptr += bufsz;
}
#ifdef WITH_RABBITMQ
Expand Down