Skip to content

Commit bbfd07d

Browse files
alesagealesage
authored
Merge pull request #46 from farsightsec/next
Release 1.3.0
2 parents 38b70e1 + 7c914ee commit bbfd07d

20 files changed

+391
-93
lines changed

‎COPYRIGHT

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2012-2018 by Farsight Security, Inc.
1+
Copyright (c) 2012-2019 by Farsight Security, Inc.
22

33
Licensed under the Apache License, Version 2.0 (the "License");
44
you may not use this file except in compliance with the License.

‎ChangeLog

+15
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,22 @@
1+
mtbl (1.3.0)
2+
3+
* Fix mtbl_reload_now() on filesets with open iterators.
4+
* Add support for absolute pathnames in filesets
5+
* Make merge function an optional parameter for mergers and filesets to
6+
enable unmerged results.
7+
* Add dupsort function to sort unmerged results based on data.
8+
* Add filename filter option to filesets. This provides functionality similar
9+
to mtbl_fileset_partition() but resilient against fileset reloads.
10+
* Add mtbl_fileset_dup() to open an existing fileset with different options.
11+
12+
-- Farsight Security, Inc. <[email protected]> Fri, 29 Mar 2019 12:13:17 -0500
13+
114
mtbl (1.2.1)
215

316
* Fix libtool version number.
417

18+
-- Farsight Security, Inc. <[email protected]> Fri May 25 17:49:08 2018 -0500
19+
520
mtbl (1.2.0)
621

722
* Prevent fileset reloading when the fileset has iterators open.

‎Makefile.am

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ AM_LDFLAGS =
2323
##
2424
#
2525

26-
LIBMTBL_VERSION_INFO=1:2:0
26+
LIBMTBL_VERSION_INFO=2:0:1
2727

2828
include_HEADERS = mtbl/mtbl.h
2929
lib_LTLIBRARIES = mtbl/libmtbl.la

‎configure.ac

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
AC_PREREQ(2.64)
22
AC_INIT([mtbl],
3-
[1.2.1],
3+
[1.3.0],
44
[https://github.com/farsightsec/mtbl/issues],
55
[mtbl],
66
[https://github.com/farsightsec/mtbl])

‎libmy/heap.c

+9-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2012 by Farsight Security, Inc.
2+
* Copyright (c) 2012, 2019 by Farsight Security, Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -25,19 +25,21 @@ VECTOR_GENERATE(ptrvec, void *);
2525
struct heap {
2626
ptrvec *vec;
2727
heap_compare_func cmp;
28+
void *clos;
2829
};
2930

3031
static inline int
31-
cmp_wrapper(heap_compare_func cmp, const void *a, const void *b)
32+
cmp_wrapper(heap_compare_func cmp, const void *a, const void *b, void *clos)
3233
{
33-
return ((cmp(a, b) < 0) ? 1 : 0);
34+
return ((cmp(a, b, clos) < 0) ? 1 : 0);
3435
}
3536

3637
struct heap *
37-
heap_init(heap_compare_func cmp)
38+
heap_init(heap_compare_func cmp, void *clos)
3839
{
3940
struct heap *h = my_calloc(1, sizeof(*h));
4041
h->cmp = cmp;
42+
h->clos = clos;
4143
h->vec = ptrvec_init(1);
4244
return (h);
4345
}
@@ -70,7 +72,7 @@ siftdown(struct heap *h, size_t startpos, size_t pos)
7072
while (pos > startpos) {
7173
size_t parentpos = (pos - 1) >> 1;
7274
void *parent = ptrvec_value(h->vec, parentpos);
73-
int cmp = cmp_wrapper(h->cmp, newitem, parent);
75+
int cmp = cmp_wrapper(h->cmp, newitem, parent, h->clos);
7476
if (cmp == -1)
7577
return (-1);
7678
if (cmp == 0)
@@ -95,7 +97,8 @@ siftup(struct heap *h, size_t pos)
9597
if (rightpos < endpos) {
9698
int cmp = cmp_wrapper(h->cmp,
9799
ptrvec_value(h->vec, childpos),
98-
ptrvec_value(h->vec, rightpos));
100+
ptrvec_value(h->vec, rightpos),
101+
h->clos);
99102
if (cmp == -1)
100103
return (-1);
101104
if (cmp == 0)

‎libmy/heap.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33

44
struct heap;
55

6-
typedef int (*heap_compare_func)(const void *a, const void *b);
6+
typedef int (*heap_compare_func)(const void *a, const void *b, void *clos);
77

8-
struct heap *heap_init(heap_compare_func);
8+
struct heap *heap_init(heap_compare_func, void *);
99
void heap_destroy(struct heap **);
1010
void heap_reset(struct heap *);
1111
void heap_clip(struct heap *, size_t);

‎man/mtbl_dump.1

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
'\" t
22
.\" Title: mtbl_dump
33
.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author]
4-
.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
5-
.\" Date: 07/17/2015
4+
.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
5+
.\" Date: 04/01/2019
66
.\" Manual: \ \&
77
.\" Source: \ \&
88
.\" Language: English
99
.\"
10-
.TH "MTBL_DUMP" "1" "07/17/2015" "\ \&" "\ \&"
10+
.TH "MTBL_DUMP" "1" "04/01/2019" "\ \&" "\ \&"
1111
.\" -----------------------------------------------------------------
1212
.\" * Define some portability stuff
1313
.\" -----------------------------------------------------------------

‎man/mtbl_fileset.3

+47-2
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
.\" Title: mtbl_fileset
33
.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author]
44
.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
5-
.\" Date: 05/31/2018
5+
.\" Date: 03/27/2019
66
.\" Manual: \ \&
77
.\" Source: \ \&
88
.\" Language: English
99
.\"
10-
.TH "MTBL_FILESET" "3" "05/31/2018" "\ \&" "\ \&"
10+
.TH "MTBL_FILESET" "3" "03/27/2019" "\ \&" "\ \&"
1111
.\" -----------------------------------------------------------------
1212
.\" * Define some portability stuff
1313
.\" -----------------------------------------------------------------
@@ -41,6 +41,11 @@ mtbl_fileset_init(const char *\fR\fB\fIfname\fR\fR\fB, const struct mtbl_fileset
4141
.fi
4242
.sp
4343
.nf
44+
\fBstruct mtbl_fileset *
45+
mtbl_fileset_dup(struct mtbl_fileset *\fR\fB\fIorig\fR\fR\fB, const struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB);\fR
46+
.fi
47+
.sp
48+
.nf
4449
\fBvoid
4550
mtbl_fileset_destroy(struct mtbl_fileset **\fR\fB\fIf\fR\fR\fB);\fR
4651
.fi
@@ -91,6 +96,22 @@ mtbl_fileset_options_set_merge_func(
9196
.sp
9297
.nf
9398
\fBvoid
99+
mtbl_fileset_options_set_dupsort_func(
100+
struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
101+
mtbl_dupsort_func \fR\fB\fIfp\fR\fR\fB,
102+
void *\fR\fB\fIclos\fR\fR\fB);\fR
103+
.fi
104+
.sp
105+
.nf
106+
\fBvoid
107+
mtbl_fileset_options_set_filename_filter_func(
108+
struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
109+
mtbl_filename_filter_func \fR\fB\fIfp\fR\fR\fB,
110+
void *\fR\fB\fIclos\fR\fR\fB);\fR
111+
.fi
112+
.sp
113+
.nf
114+
\fBvoid
94115
mtbl_fileset_options_set_reload_interval(
95116
struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
96117
uint32_t \fR\fB\fIreload_interval\fR\fR\fB);\fR
@@ -107,6 +128,8 @@ Because the MTBL format does not allow duplicate keys, the caller must provide a
107128
.sp
108129
\fBmtbl_fileset\fR objects are created with the \fBmtbl_fileset_init\fR() function, which requires the path to a "setfile", \fIfname\fR, and a non\-NULL \fIfopt\fR argument which has been configured with a merge function \fIfp\fR\&. \fBmtbl_fileset_source\fR() should then be called in order to consume output via the \fBmtbl_source\fR(3) interface\&.
109130
.sp
131+
The \fBmtbl_fileset_dup\fR() function, which requires a \fBmtbl_fileset\fR object and a non\-NULL \fBopt\fR argument, creates a duplicate \fBmtbl_fileset\fR object that shares the underlying fileset, but with different fileset options, as specified\&.
132+
.sp
110133
Accesses via the \fBmtbl_source\fR(3) interface will implicitly check for updates to the setfile\&. However, it may be necessary to explicitly call the \fBmtbl_fileset_reload\fR() function in order to check for updates, especially if files are being removed from the setfile and the \fBmtbl_source\fR is infrequently accessed\&.
111134
.sp
112135
The \fBmtbl_fileset_reload\fR() function avoids checking for updates more frequently than every \fIreload_interval\fR seconds\&. If \fBreload_interval\fR is set to \fBMTBL_FILESET_RELOAD_INTERVAL_NEVER\fR, then \fBmtbl_fileset_reload\fR() function will only load the fileset once\&. The \fBmtbl_fileset_reload_now\fR() function can be called to bypass the \fIreload_interval\fR check\&.
@@ -130,6 +153,28 @@ See \fBmtbl_merger\fR(3)\&. An \fBmtbl_merger\fR object is used internally for t
130153
.nr an-break-flag 1
131154
.br
132155
.ps +1
156+
\fBdupsort_func\fR
157+
.RS 4
158+
.sp
159+
See \fBmtbl_merger\fR(3)\&. Used to sort the entries with duplicate keys during the merge process based on their data\&.
160+
.RE
161+
.sp
162+
.it 1 an-trap
163+
.nr an-no-space-flag 1
164+
.nr an-break-flag 1
165+
.br
166+
.ps +1
167+
\fBfname_filter_func\fR
168+
.RS 4
169+
.sp
170+
Used to filter specific files by name from a fileset\&. If the function returns \fBfalse\fR, the file\(cqs data will not be included in the results returned by any iterators on the fileset\&.
171+
.RE
172+
.sp
173+
.it 1 an-trap
174+
.nr an-no-space-flag 1
175+
.nr an-break-flag 1
176+
.br
177+
.ps +1
133178
\fBreload_interval\fR
134179
.RS 4
135180
.sp

‎man/mtbl_fileset.3.txt

+29
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ Fileset objects:
1414
^struct mtbl_fileset *
1515
mtbl_fileset_init(const char *'fname', const struct mtbl_fileset_options *'fopt');^
1616

17+
[verse]
18+
^struct mtbl_fileset *
19+
mtbl_fileset_dup(struct mtbl_fileset *'orig', const struct mtbl_fileset_options *'fopt');^
20+
1721
[verse]
1822
^void
1923
mtbl_fileset_destroy(struct mtbl_fileset **'f');^
@@ -55,6 +59,20 @@ mtbl_fileset_options_set_merge_func(
5559
mtbl_merge_func 'fp',
5660
void *'clos');^
5761

62+
[verse]
63+
^void
64+
mtbl_fileset_options_set_dupsort_func(
65+
struct mtbl_fileset_options *'fopt',
66+
mtbl_dupsort_func 'fp',
67+
void *'clos');^
68+
69+
[verse]
70+
^void
71+
mtbl_fileset_options_set_filename_filter_func(
72+
struct mtbl_fileset_options *'fopt',
73+
mtbl_filename_filter_func 'fp',
74+
void *'clos');^
75+
5876
[verse]
5977
^void
6078
mtbl_fileset_options_set_reload_interval(
@@ -96,6 +114,10 @@ which has been configured with a merge function _fp_. ^mtbl_fileset_source^()
96114
should then be called in order to consume output via the ^mtbl_source^(3)
97115
interface.
98116

117+
The ^mtbl_fileset_dup^() function, which requires a ^mtbl_fileset^ object
118+
and a non-NULL ^opt^ argument, creates a duplicate ^mtbl_fileset^ object
119+
that shares the underlying fileset, but with different fileset options, as specified.
120+
99121
Accesses via the ^mtbl_source^(3) interface will implicitly check for updates
100122
to the setfile. However, it may be necessary to explicitly call the
101123
^mtbl_fileset_reload^() function in order to check for updates, especially if
@@ -121,6 +143,13 @@ mergers in an inconsistent state.
121143
See ^mtbl_merger^(3). An ^mtbl_merger^ object is used internally for the
122144
external sort.
123145

146+
==== dupsort_func ====
147+
See ^mtbl_merger^(3). Used to sort the entries with duplicate keys during the merge process based on their data.
148+
149+
==== fname_filter_func ====
150+
Used to filter specific files by name from a fileset. If the function returns ^false^, the file's data
151+
will not be included in the results returned by any iterators on the fileset.
152+
124153
==== reload_interval ====
125154
Specifies the interval between checks for updates to the setfile, in seconds.
126155
Defaults to 60 seconds. ^MTBL_FILESET_RELOAD_INTERVAL_NEVER^ is a special value that indicates to never reload the fileset.

‎man/mtbl_iter.3

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
.\" Title: mtbl_iter
33
.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author]
44
.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
5-
.\" Date: 02/17/2017
5+
.\" Date: 03/29/2019
66
.\" Manual: \ \&
77
.\" Source: \ \&
88
.\" Language: English
99
.\"
10-
.TH "MTBL_ITER" "3" "02/17/2017" "\ \&" "\ \&"
10+
.TH "MTBL_ITER" "3" "03/29/2019" "\ \&" "\ \&"
1111
.\" -----------------------------------------------------------------
1212
.\" * Define some portability stuff
1313
.\" -----------------------------------------------------------------
@@ -48,7 +48,7 @@ mtbl_iter_destroy(struct mtbl_iter **\fR\fB\fIit\fR\fR\fB);\fR
4848
.nf
4949
\fBmtbl_res
5050
mtbl_iter_seek(struct mtbl_iter *\fR\fB\fIit\fR\fR\fB,
51-
const uint8_t **\fR\fB\fIkey\fR\fR\fB, size_t *\fR\fB\fIlen_key\fR\fR\fB);\fR
51+
const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB);\fR
5252
.fi
5353
.SH "DESCRIPTION"
5454
.sp

‎man/mtbl_iter.3.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ mtbl_iter_destroy(struct mtbl_iter **'it');^
2121
[verse]
2222
^mtbl_res
2323
mtbl_iter_seek(struct mtbl_iter *'it',
24-
const uint8_t **'key', size_t *'len_key');^
24+
const uint8_t *'key', size_t 'len_key');^
2525

2626
== DESCRIPTION ==
2727

‎man/mtbl_merger.3

+30-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
'\" t
22
.\" Title: mtbl_merger
33
.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author]
4-
.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
5-
.\" Date: 01/31/2014
4+
.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
5+
.\" Date: 03/28/2019
66
.\" Manual: \ \&
77
.\" Source: \ \&
88
.\" Language: English
99
.\"
10-
.TH "MTBL_MERGER" "3" "01/31/2014" "\ \&" "\ \&"
10+
.TH "MTBL_MERGER" "3" "03/28/2019" "\ \&" "\ \&"
1111
.\" -----------------------------------------------------------------
1212
.\" * Define some portability stuff
1313
.\" -----------------------------------------------------------------
@@ -76,13 +76,29 @@ mtbl_merger_options_set_merge_func(
7676
.fi
7777
.sp
7878
.nf
79+
\fBvoid
80+
mtbl_merger_options_set_dupsort_func(
81+
struct mtbl_merger_options *\fR\fB\fImopt\fR\fR\fB,
82+
mtbl_dupsort_func \fR\fB\fIfp\fR\fR\fB,
83+
void *\fR\fB\fIclos\fR\fR\fB);\fR
84+
.fi
85+
.sp
86+
.nf
7987
\fBtypedef void
8088
(*mtbl_merge_func)(void *\fR\fB\fIclos\fR\fR\fB,
8189
const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB,
8290
const uint8_t *\fR\fB\fIval0\fR\fR\fB, size_t \fR\fB\fIlen_val0\fR\fR\fB,
8391
const uint8_t *\fR\fB\fIval1\fR\fR\fB, size_t \fR\fB\fIlen_val1\fR\fR\fB,
8492
uint8_t **\fR\fB\fImerged_val\fR\fR\fB, size_t *\fR\fB\fIlen_merged_val\fR\fR\fB);\fR
8593
.fi
94+
.sp
95+
.nf
96+
\fBtypedef int
97+
(*mtbl_dupsort_func)(void *\fR\fB\fIclos\fR\fR\fB,
98+
const uint8_t *\fR\fB\fIkey\fR\fR\fB, size_t \fR\fB\fIlen_key\fR\fR\fB,
99+
const uint8_t *\fR\fB\fIval0\fR\fR\fB, size_t \fR\fB\fIlen_val0\fR\fR\fB,
100+
const uint8_t *\fR\fB\fIval1\fR\fR\fB, size_t \*(Aqlen_val1);\fR
101+
.fi
86102
.SH "DESCRIPTION"
87103
.sp
88104
Multiple MTBL data sources may be merged together using the \fBmtbl_merger\fR interface, which reads key\-value entries from one or more sources and provides these entries in sorted order\&. The sorted entries may be consumed via the \fBmtbl_source\fR(3) and \fBmtbl_iter\fR(3) interfaces\&.
@@ -128,6 +144,17 @@ The callee may provide an empty value as the merged value, in which case \fImerg
128144
.sp
129145
The callee may indicate an error by returning NULL in the \fImerged_val\fR argument, which will abort iteration over the \fBmtbl_merger\fR object\&.
130146
.RE
147+
.sp
148+
.it 1 an-trap
149+
.nr an-no-space-flag 1
150+
.nr an-break-flag 1
151+
.br
152+
.ps +1
153+
\fBdupsort_func\fR
154+
.RS 4
155+
.sp
156+
This option provides a comparison function for multiple data with the same key\&. It will be used to sort the data during the merge process\&. If the \fBmerge_func\fR option is set to or left NULL, the \fBdupsort_func\fR will order the results returned by the merge process for data with identical keys\&.
157+
.RE
131158
.SH "RETURN VALUE"
132159
.sp
133160
If the merge function callback is unable to provide a merged value (that is, it fails to return a non\-NULL value in its \fImerged_val\fR argument), the merge process will be aborted, and any iterators over the \fBmtbl_merger\fR object (via the \fBmtbl_source\fR(3) interface) will return \fBmtbl_res_failure\fR\&.

0 commit comments

Comments
 (0)