Skip to content

Commit 4fa9a47

Browse files
wazir-ahmedwazir-ahmed
committed
feat(stats): Purge query digest based on last seen
- Add handler for purge query digest command - Handle last_seen argument in purge query digest operations - Make purge logic common for both sync & async operations Signed-off-by: Wazir Ahmed <[email protected]>
1 parent 05adfe5 commit 4fa9a47

11 files changed

+311
-113
lines changed

‎include/gen_utils.h

+2
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,8 @@ bool mywildcmp(const char *p, const char *str);
339339
std::string trim(const std::string& s);
340340
char* escape_string_single_quotes_and_backslashes(char* input, bool free_it);
341341
const char* escape_string_backslash_spaces(const char* input);
342+
time_t monotonic_time_to_realtime(time_t mt);
343+
time_t realtime_to_monotonic_time(time_t rt);
342344

343345
/**
344346
* @brief Helper function that converts a MYSQL_RES into a 'SQLite3_result'.

‎include/query_processor.h

+3-3
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ class Query_Processor {
282282
std::pair<SQLite3_result*,int> get_query_digests_v2(const bool use_resultset = true);
283283
std::pair<SQLite3_result*,int> get_query_digests_reset_v2(const bool copy, const bool use_resultset = true);
284284
void get_query_digests_reset(umap_query_digest* uqd, umap_query_digest_text* uqdt);
285-
unsigned long long purge_query_digests(bool async_purge, bool parallel, char** msg);
285+
unsigned long long purge_query_digests(bool async_purge, bool parallel, time_t last_seen = 0);
286286

287287
void save_query_rules(SQLite3_result* resultset);
288288

@@ -394,8 +394,8 @@ class Query_Processor {
394394
DEFINE_HAS_METHOD_STRUCT(query_parser_first_comment_extended);
395395
DEFINE_HAS_METHOD_STRUCT(process_query_extended);
396396

397-
unsigned long long purge_query_digests_async(char** msg);
398-
unsigned long long purge_query_digests_sync(bool parallel);
397+
unsigned long long purge_query_digests_async(time_t last_seen = 0);
398+
unsigned long long purge_query_digests_sync(bool parallel, time_t last_seen = 0);
399399

400400
/**
401401
* @brief Searches for a matching rule in the supplied map, returning the destination hostgroup.

‎lib/Admin_Bootstrap.cpp

-3
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,6 @@ extern struct MHD_Daemon *Admin_HTTP_Server;
112112

113113
extern ProxySQL_Statistics *GloProxyStats;
114114

115-
template<enum SERVER_TYPE>
116-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg);
117-
118115
extern char *ssl_key_fp;
119116
extern char *ssl_cert_fp;
120117
extern char *ssl_ca_fp;

‎lib/Admin_FlushVariables.cpp

-3
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,6 @@ extern struct MHD_Daemon *Admin_HTTP_Server;
112112

113113
extern ProxySQL_Statistics *GloProxyStats;
114114

115-
template<enum SERVER_TYPE>
116-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg);
117-
118115
extern char *ssl_key_fp;
119116
extern char *ssl_cert_fp;
120117
extern char *ssl_ca_fp;

‎lib/Admin_Handler.cpp

+86-12
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,6 @@ extern struct MHD_Daemon *Admin_HTTP_Server;
114114

115115
extern ProxySQL_Statistics *GloProxyStats;
116116

117-
template<enum SERVER_TYPE>
118-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg);
119-
120117
extern char *ssl_key_fp;
121118
extern char *ssl_cert_fp;
122119
extern char *ssl_ca_fp;
@@ -276,6 +273,17 @@ const std::vector<std::string> LOAD_COREDUMP_FROM_MEMORY = {
276273
"LOAD COREDUMP TO RUNTIME" ,
277274
"LOAD COREDUMP TO RUN" };
278275

276+
const std::vector<std::string> CMD_PREFIX_PURGE_QUERY_DIGESTS = {
277+
"PURGE TABLE stats.stats_mysql_query_digest TO ",
278+
"PURGE TABLE stats_mysql_query_digest TO ",
279+
"PURGE stats.stats_mysql_query_digest TO ",
280+
"PURGE stats_mysql_query_digest TO ",
281+
"PURGE TABLE stats.stats_pgsql_query_digest TO ",
282+
"PURGE TABLE stats_pgsql_query_digest TO ",
283+
"PURGE stats.stats_pgsql_query_digest TO ",
284+
"PURGE stats_pgsql_query_digest TO ",
285+
};
286+
279287
extern unordered_map<string,std::tuple<string, vector<string>, vector<string>>> load_save_disk_commands;
280288

281289
bool is_admin_command_or_alias(const std::vector<std::string>& cmds, char *query_no_space, int query_no_space_length) {
@@ -288,6 +296,17 @@ bool is_admin_command_or_alias(const std::vector<std::string>& cmds, char *query
288296
return false;
289297
}
290298

299+
const char * match_command_prefix(const std::vector<std::string>& cmd_prefix, char *query, int query_len) {
300+
for (auto &prefix : cmd_prefix) {
301+
if ((unsigned int) query_len >= prefix.length()
302+
&& !strncasecmp(prefix.c_str(), query, prefix.length()))
303+
{
304+
return prefix.c_str();
305+
}
306+
}
307+
308+
return nullptr;
309+
}
291310

292311
template <typename S>
293312
bool FlushCommandWrapper(S* sess, const std::vector<std::string>& cmds, char *query_no_space, int query_no_space_length, const string& name, const string& direction) {
@@ -322,6 +341,35 @@ bool FlushCommandWrapper(S* sess, const string& modname, char *query_no_space, i
322341
return false;
323342
}
324343

344+
std::tuple<bool, enum SERVER_TYPE, time_t> parse_command_purge_query_digests(char *query, int query_len) {
345+
bool match = false;
346+
enum SERVER_TYPE server_type = SERVER_TYPE_MYSQL;
347+
time_t last_seen = 0;
348+
349+
const char *prefix = match_command_prefix(CMD_PREFIX_PURGE_QUERY_DIGESTS, query, query_len);
350+
if (prefix) {
351+
match = true;
352+
353+
if (strstr(prefix, "_pgsql_") != nullptr) {
354+
server_type = SERVER_TYPE_PGSQL;
355+
}
356+
357+
// parse timestamp
358+
char *ts_str = strdup(query + strlen(prefix));
359+
char *ts_end = nullptr;
360+
long long ts = strtoll(trim_spaces_in_place(ts_str), &ts_end, 10);
361+
362+
// ts_str should only contain digits and respresent a valid timestamp
363+
if ((*ts_end == 0) && (ts > 0)) {
364+
last_seen = realtime_to_monotonic_time(ts);
365+
}
366+
367+
free(ts_str);
368+
}
369+
370+
return std::make_tuple(match, server_type, last_seen);
371+
}
372+
325373
template <typename S>
326374
bool admin_handler_command_kill_connection(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa) {
327375
uint32_t id=atoi(query_no_space+16);
@@ -2498,10 +2546,8 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) {
24982546
SPA->admindb->execute("DELETE FROM stats.stats_mysql_query_digest_reset");
24992547
SPA->vacuum_stats(true);
25002548
// purge the digest map, asynchronously, in single thread
2501-
char *msg = NULL;
2502-
int r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(true, false, &msg);
2503-
SPA->send_ok_msg_to_client(sess, msg, r1, query_no_space);
2504-
free(msg);
2549+
int r1 = GloMyQPro->purge_query_digests(true, false);
2550+
SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space);
25052551
run_query=false;
25062552
goto __run_query;
25072553
}
@@ -2534,16 +2580,45 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) {
25342580
SPA->admindb->execute("DELETE FROM stats.stats_pgsql_query_digest_reset");
25352581
SPA->vacuum_stats(true);
25362582
// purge the digest map, asynchronously, in single thread
2537-
char* msg = NULL;
2538-
int r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_PGSQL>(true, false, &msg);
2539-
SPA->send_ok_msg_to_client(sess, msg, r1, query_no_space);
2540-
free(msg);
2583+
int r1 = GloPgQPro->purge_query_digests(true, false);
2584+
SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space);
25412585
run_query = false;
25422586
goto __run_query;
25432587
}
25442588
}
25452589
}
25462590
}
2591+
2592+
// handles 'PURGE stats_mysql_query_digest TO <value>'.
2593+
// any entry in stats_mysql_query_digest where last_seen is less than <value> will be deleted.
2594+
if (!strncasecmp("PURGE ", query_no_space, strlen("PURGE "))
2595+
&& sess->session_type == PROXYSQL_SESSION_ADMIN
2596+
) {
2597+
auto result = parse_command_purge_query_digests(query_no_space, query_no_space_length);
2598+
bool match = std::get<0>(result);
2599+
2600+
if (match == true) {
2601+
int ret = 0;
2602+
enum SERVER_TYPE type = std::get<1>(result);
2603+
time_t last_seen = std::get<2>(result);
2604+
2605+
if (last_seen > 0) {
2606+
if (type == SERVER_TYPE_MYSQL) {
2607+
ret = GloMyQPro->purge_query_digests(true, false, last_seen);
2608+
} else if (type == SERVER_TYPE_PGSQL) {
2609+
ret = GloPgQPro->purge_query_digests(true, false, last_seen);
2610+
}
2611+
2612+
pa->send_ok_msg_to_client(sess, NULL, ret, query_no_space);
2613+
} else {
2614+
pa->send_error_msg_to_client(sess, "Invalid timestamp");
2615+
}
2616+
2617+
run_query = false;
2618+
goto __run_query;
2619+
}
2620+
}
2621+
25472622
#ifdef DEBUG
25482623
/**
25492624
* @brief Handles the 'PROXYSQL_SIMULATOR' command. Performing the operation specified in the payload
@@ -3837,4 +3912,3 @@ void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) {
38373912
// Explicitly instantiate the required template class and member functions
38383913
template void admin_session_handler<MySQL_Session>(MySQL_Session* sess, void *_pa, PtrSize_t *pkt);
38393914
template void admin_session_handler<PgSQL_Session>(PgSQL_Session* sess, void *_pa, PtrSize_t *pkt);
3840-

‎lib/ProxySQL_Admin.cpp

-4
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,6 @@ struct MHD_Daemon *Admin_HTTP_Server;
114114

115115
extern ProxySQL_Statistics *GloProxyStats;
116116

117-
template<enum SERVER_TYPE>
118-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg);
119-
120117
extern char *ssl_key_fp;
121118
extern char *ssl_cert_fp;
122119
extern char *ssl_ca_fp;
@@ -8465,4 +8462,3 @@ void ProxySQL_Admin::enable_replicationlag_testing() {
84658462
mysql_servers_wrunlock();
84668463
}
84678464
#endif // TEST_REPLICATIONLAG
8468-

‎lib/ProxySQL_Admin_Tests.cpp

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#include <ctime>
12
#include <iostream> // std::cout
23
#include <sstream> // std::stringstream
34
#include <fstream>
@@ -56,12 +57,12 @@ bool ProxySQL_Test___Refresh_MySQL_Variables(unsigned int cnt) {
5657
}
5758

5859
template <enum SERVER_TYPE ST>
59-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg) {
60+
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, time_t last_seen) {
6061
int r = 0;
6162
if constexpr (ST == SERVER_TYPE_MYSQL) {
62-
r = GloMyQPro->purge_query_digests(async_purge, parallel, msg);
63+
r = GloMyQPro->purge_query_digests(async_purge, parallel, last_seen);
6364
} else if constexpr (ST == SERVER_TYPE_PGSQL) {
64-
r = GloPgQPro->purge_query_digests(async_purge, parallel, msg);
65+
r = GloPgQPro->purge_query_digests(async_purge, parallel, last_seen);
6566
}
6667
return r;
6768
}
@@ -147,5 +148,5 @@ int ProxySQL_Test___GenerateRandomQueryInDigestTable(int n) {
147148
return n*1000;
148149
}
149150

150-
template int ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(bool async_purge, bool parallel, char** msg);
151-
template int ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_PGSQL>(bool async_purge, bool parallel, char** msg);
151+
template int ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(bool async_purge, bool parallel, time_t last_seen);
152+
template int ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_PGSQL>(bool async_purge, bool parallel, time_t last_seen);

‎lib/ProxySQL_Admin_Tests2.cpp

+5-6
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ static void init_rand_del() {
4343
int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap);
4444
bool ProxySQL_Test___Refresh_MySQL_Variables(unsigned int cnt);
4545
template<enum SERVER_TYPE>
46-
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg);
46+
int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, time_t last_seen);
4747
int ProxySQL_Test___GenerateRandomQueryInDigestTable(int n);
4848

4949
void ProxySQL_Admin::map_test_mysql_firewall_whitelist_rules_cleanup() {
@@ -590,21 +590,20 @@ void ProxySQL_Admin::ProxySQL_Test_Handler(ProxySQL_Admin *SPA, S* sess, char *q
590590
break;
591591
case 4:
592592
// purge the digest map, synchronously, in single thread
593-
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(false, false, NULL);
593+
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(false, false, test_arg1);
594594
SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space);
595595
run_query=false;
596596
break;
597597
case 5:
598598
// purge the digest map, synchronously, in multiple threads
599-
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(false, true, NULL);
599+
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(false, true, test_arg1);
600600
SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space);
601601
run_query=false;
602602
break;
603603
case 6:
604604
// purge the digest map, asynchronously, in single thread
605-
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(true, false, &msg);
606-
SPA->send_ok_msg_to_client(sess, msg, r1, query_no_space);
607-
free(msg);
605+
r1 = ProxySQL_Test___PurgeDigestTable<SERVER_TYPE_MYSQL>(true, false, test_arg1);
606+
SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space);
608607
run_query=false;
609608
break;
610609
case 7:

‎lib/QP_query_digest_stats.cpp

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#include "gen_utils.h"
12
#include "query_processor.h"
23

34
// reverse: reverse string s in place
@@ -151,16 +152,13 @@ char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap,
151152
my_itoa(qdsp->count_star, count_star);
152153
pta[5]=qdsp->count_star;
153154

154-
time_t __now;
155-
time(&__now);
156-
unsigned long long curtime=monotonic_time();
157155
time_t seen_time;
158-
seen_time= __now - curtime/1000000 + first_seen/1000000;
156+
seen_time=monotonic_time_to_realtime(first_seen);
159157
//sprintf(qdsp->first_seen,"%ld", seen_time);
160158
my_itoa(qdsp->first_seen, seen_time);
161159
pta[6]=qdsp->first_seen;
162160

163-
seen_time= __now - curtime/1000000 + last_seen/1000000;
161+
seen_time=monotonic_time_to_realtime(last_seen);
164162
//sprintf(qdsp->last_seen,"%ld", seen_time);
165163
my_itoa(qdsp->last_seen, seen_time);
166164
pta[7]=qdsp->last_seen;
@@ -187,4 +185,3 @@ char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap,
187185
pta[13]=qdsp->rows_sent;
188186
return pta;
189187
}
190-

0 commit comments

Comments
 (0)