Merge pull request #66 from farsightsec/next

Next Release (1.5.0)

Author: reedjc

COPYRIGHT

@@ -1,3 +1,4 @@
+Copyright (c) 2022 DomainTools LLC
 Copyright (c) 2012-2021 by Farsight Security, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");

ChangeLog

@@ -1,3 +1,13 @@
+mtbl (1.5.0)
+
+ * Add reader filter function option to mtbl_fileset.
+ * Use "galloping search" instead of full binary search for mtbl_iter_seek().
+ * Make index block verification optional for more efficient reader
+   initialization.
+ * Fix underflow errors triggered by short keys and empty mtbl files.
+ * mtbl_fileset_partition() use is deprecated in favor of
+   mtbl_fileset_dup() with the fname_filter_func option set.
+
 mtbl (1.4.0)
 
  * Add -l option to specify compression level for mtbl_merge.

Makefile.am

@@ -23,7 +23,7 @@ AM_LDFLAGS =
 ##
 #
 
-LIBMTBL_VERSION_INFO=2:1:1
+LIBMTBL_VERSION_INFO=3:0:2
 
 include_HEADERS = mtbl/mtbl.h
 lib_LTLIBRARIES = mtbl/libmtbl.la
@@ -123,6 +123,17 @@ t_test_fileset_partition_SOURCES = t/test-fileset-partition.c
 t_test_fileset_partition_LDADD = mtbl/libmtbl.la
 t/test-fileset-partition.sh: t/test-fileset-partition
 
+TESTS += t/test-fileset-filter.sh
+EXTRA_DIST += t/test-fileset-filter.sh
+EXTRA_DIST += t/fileset-filter-data/animals.fileset
+EXTRA_DIST += t/fileset-filter-data/animals-1.mtbl
+EXTRA_DIST += t/fileset-filter-data/animals-2.mtbl
+EXTRA_DIST += t/fileset-filter-data/animals-3.mtbl
+check_PROGRAMS += t/test-fileset-filter
+t_test_fileset_filter_SOURCES = t/test-fileset-filter.c
+t_test_fileset_filter_LDADD = mtbl/libmtbl.la
+t/test-fileset-filter.sh: t/test-fileset-filter
+
 TESTS += t/test-fixed
 check_PROGRAMS += t/test-fixed
 t_test_fixed_SOURCES = t/test-fixed.c

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ(2.64)
 AC_INIT([mtbl],
-        [1.4.0],
+        [1.5.0],
         [https://github.com/farsightsec/mtbl/issues],
         [mtbl],
         [https://github.com/farsightsec/mtbl])

man/mtbl_fileset.3

@@ -2,12 +2,12 @@
 .\"     Title: mtbl_fileset
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
 .\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
-.\"      Date: 03/27/2019
+.\"      Date: 11/28/2022
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MTBL_FILESET" "3" "03/27/2019" "\ \&" "\ \&"
+.TH "MTBL_FILESET" "3" "11/28/2022" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -112,6 +112,14 @@ mtbl_fileset_options_set_filename_filter_func(
 .sp
 .nf
 \fBvoid
+mtbl_fileset_options_set_reader_filter_func(
+        struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
+        mtbl_reader_filter_func \fR\fB\fIfp\fR\fR\fB,
+        void *\fR\fB\fIclos\fR\fR\fB);\fR
+.fi
+.sp
+.nf
+\fBvoid
 mtbl_fileset_options_set_reload_interval(
         struct mtbl_fileset_options *\fR\fB\fIfopt\fR\fR\fB,
         uint32_t \fR\fB\fIreload_interval\fR\fR\fB);\fR
@@ -134,7 +142,7 @@ Accesses via the \fBmtbl_source\fR(3) interface will implicitly check for update
 .sp
 The \fBmtbl_fileset_reload\fR() function avoids checking for updates more frequently than every \fIreload_interval\fR seconds\&. If \fBreload_interval\fR is set to \fBMTBL_FILESET_RELOAD_INTERVAL_NEVER\fR, then \fBmtbl_fileset_reload\fR() function will only load the fileset once\&. The \fBmtbl_fileset_reload_now\fR() function can be called to bypass the \fIreload_interval\fR check\&.
 .sp
-The \fBmtbl_fileset_partition\fR() function yields two \fBstruct mtbl_merger\fR objects that are split based on the output of a callback\&. The caller is responsible for calling \fBmtbl_merger_destroy\fR() on each of these mergers\&. Calls to \fBmtbl_source_*\fR() on the fileset\(cqs source object, and calls to \fBmtbl_fileset_reload\fR() and \fBmtbl_fileset_reload_now\fR() may leave these mergers in an inconsistent state\&.
+The \fBmtbl_fileset_partition\fR() function yields two \fBstruct mtbl_merger\fR objects that are split based on the output of a callback\&. The caller is responsible for calling \fBmtbl_merger_destroy\fR() on each of these mergers\&. Calls to \fBmtbl_source_*\fR() on the fileset\(cqs source object, and calls to \fBmtbl_fileset_reload\fR() and \fBmtbl_fileset_reload_now\fR(), including those implicitly performed by operations on the fileset source, may leave these mergers in an inconsistent state\&. For this reason, \fBmtbl_fileset_partition\fR() use is deprecated in favor of \fBmtbl_fileset_dup\fR() with the \fBfname_filter_func\fR option set\&.
 .SS "Fileset options"
 .sp
 .it 1 an-trap
@@ -175,6 +183,17 @@ Used to filter specific files by name from a fileset\&. If the function returns
 .nr an-break-flag 1
 .br
 .ps +1
+\fBreader_filter_func\fR
+.RS 4
+.sp
+Used to filter specific readers from a fileset\&. If the function returns \fBfalse\fR, the reader\(cqs data will not be included in the results returned by any iterators on the fileset\&.
+.RE
+.sp
+.it 1 an-trap
+.nr an-no-space-flag 1
+.nr an-break-flag 1
+.br
+.ps +1
 \fBreload_interval\fR
 .RS 4
 .sp

man/mtbl_fileset.3.txt

@@ -73,6 +73,13 @@ mtbl_fileset_options_set_filename_filter_func(
         mtbl_filename_filter_func 'fp',
         void *'clos');^
 
+[verse]
+^void
+mtbl_fileset_options_set_reader_filter_func(
+        struct mtbl_fileset_options *'fopt',
+        mtbl_reader_filter_func 'fp',
+        void *'clos');^
+
 [verse]
 ^void
 mtbl_fileset_options_set_reload_interval(
@@ -134,8 +141,11 @@ The ^mtbl_fileset_partition^() function yields two ^struct mtbl_merger^
 objects that are split based on the output of a callback. The caller is
 responsible for calling ^mtbl_merger_destroy^() on each of these mergers.
 Calls to ^mtbl_source_*^() on the fileset's source object, and calls to
-^mtbl_fileset_reload^() and ^mtbl_fileset_reload_now^() may leave these
-mergers in an inconsistent state.
+^mtbl_fileset_reload^() and ^mtbl_fileset_reload_now^(), including those
+implicitly performed by operations on the fileset source, may leave these
+mergers in an inconsistent state. For this reason, ^mtbl_fileset_partition^()
+use is deprecated in favor of ^mtbl_fileset_dup^() with the ^fname_filter_func^
+option set.
 
 === Fileset options ===
 
@@ -150,6 +160,10 @@ See ^mtbl_merger^(3). Used to sort the entries with duplicate keys during the me
 Used to filter specific files by name from a fileset. If the function returns ^false^, the file's data
 will not be included in the results returned by any iterators on the fileset.
 
+==== reader_filter_func ====
+Used to filter specific readers from a fileset. If the function returns ^false^, the reader's data
+will not be included in the results returned by any iterators on the fileset.
+
 ==== reload_interval ====
 Specifies the interval between checks for updates to the setfile, in seconds.
 Defaults to 60 seconds.  ^MTBL_FILESET_RELOAD_INTERVAL_NEVER^ is a special value that indicates to never reload the fileset.

man/mtbl_reader.3

@@ -1,13 +1,13 @@
 '\" t
 .\"     Title: mtbl_reader
 .\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
-.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 02/03/2015
+.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>
+.\"      Date: 11/14/2022
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "MTBL_READER" "3" "02/03/2015" "\ \&" "\ \&"
+.TH "MTBL_READER" "3" "11/14/2022" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -104,7 +104,7 @@ File\-level metadata may be accessed using the \fBmtbl_metadata\fR(3) interface,
 \fBverify_checksums\fR
 .RS 4
 .sp
-Specifies whether or not the CRC32C checksum on each data block should be verified or not\&. If \fIverify_checksums\fR is enabled, a checksum mismatch will cause a runtime error\&. Note that the checksum on the index block is always verified, since the overhead of doing this once when the reader object is instantiated is minimal\&. The default is to not verify data block checksums\&.
+Specifies whether or not the CRC32C checksum on the index block and each data block should be verified or not\&. If \fIverify_checksums\fR is enabled, a checksum mismatch will cause a fatal runtime error\&. The default is to not verify index or data block checksums\&.
 .RE
 .sp
 .it 1 an-trap

man/mtbl_reader.3.txt

@@ -76,11 +76,10 @@ metadata object is valid only as long as the reader object exists.
 
 ==== verify_checksums ====
 
-Specifies whether or not the CRC32C checksum on each data block should be
-verified or not. If _verify_checksums_ is enabled, a checksum mismatch will
-cause a runtime error. Note that the checksum on the index block is always
-verified, since the overhead of doing this once when the reader object is
-instantiated is minimal. The default is to not verify data block checksums.
+Specifies whether or not the CRC32C checksum on the index block and each
+data block should be verified or not. If _verify_checksums_ is enabled, a
+checksum mismatch will cause a fatal runtime error. The default is to not verify
+index or data block checksums.
 
 ==== madvise_random ====
 

mtbl.spec

@@ -1,5 +1,5 @@
 Name:           mtbl
-Version:        1.3.0
+Version:        1.5.0
 Release:        1%{?dist}
 Summary:	immutable sorted string table utilities
 
@@ -17,7 +17,7 @@ mtbl is a C library implementation of the Sorted String Table (SSTable)
 data structure. mtbl exposes primitives for creating, searching and
 merging SSTable files.
 
-This package contains the shared library for libmbtl and the mtbl
+This package contains the shared library for libmtbl and the mtbl
 command-line tools.
 
 %package devel

mtbl/block.c

@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2012, 2014 by Farsight Security, Inc.
+ * Copyright (c) 2022 DomainTools LLC
+ * Copyright (c) 2012, 2014, 2017 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -249,33 +250,58 @@ block_iter_seek_to_last(struct block_iter *bi)
 	}
 }
 
-void 
+static int
+compare_restart_point(struct block_iter *bi, const uint32_t i, const uint8_t *target, size_t target_len) {
+	uint32_t shared, non_shared, value_length;
+	uint64_t region_offset = get_restart_point(bi, i);
+	const uint8_t *key_ptr = decode_entry(bi->data + region_offset,
+						      bi->data + bi->restarts,
+						      &shared, &non_shared, &value_length);
+	/* check for corruption */
+	assert(key_ptr != NULL && shared == 0);
+	return bytes_compare(key_ptr, non_shared, target, target_len);
+}
+
+void
 block_iter_seek(struct block_iter *bi, const uint8_t *target, size_t target_len)
 {
-	/* binary search in restart array to find the first restart point
-	 * with a key >= target
+	/* 
+	 * If the restart_index is not zero and not equal to the number of
+	 * restarts, then begin with galloping search in the restart array to find
+	 * the first restart point with a key >= target, otherwise just do binary
+	 * search from the start of the restart array
 	 */
 	uint32_t left = 0;
 	uint32_t right = bi->num_restarts - 1;
+	if (bi->num_restarts != bi->restart_index && bi->restart_index != 0) {
+		/* Start galloping from the current restart index */
+		uint32_t i = bi->restart_index;
+		right = i;
+		uint32_t incr = 1;
+		while (compare_restart_point(bi, i, target, target_len) < 0) {
+			left = i;
+			i += incr;
+			/* Stop galloping if i is past the end of the restart array */
+			if (i > bi->num_restarts - 1) {
+				right = bi->num_restarts - 1;
+				break;
+			}
+			right = i;
+			incr *= 2;
+		}
+	}
+
+	/* binary search */
 	while (left < right) {
 		uint32_t mid = (left + right + 1) / 2;
-		uint64_t region_offset = get_restart_point(bi, mid);
-		uint32_t shared, non_shared, value_length;
-		const uint8_t *key_ptr = decode_entry(bi->data + region_offset,
-						      bi->data + bi->restarts,
-						      &shared, &non_shared, &value_length);
-		if (key_ptr == NULL || (shared != 0)) {
-			/* corruption */
-			return;
-		}
-		if (bytes_compare(key_ptr, non_shared, target, target_len) < 0) {
+		if (compare_restart_point(bi, mid, target, target_len) < 0) {
 			/* key at "mid" is smaller than "target", therefore all
 			 * keys before "mid" are uninteresting
 			 */
 			left = mid;
 		} else {
 			/* key at "mid" is larger than "target", therefore all
-			 * keys at or before "mid" are uninteresting
+			 * keys at or after "mid" are uninteresting
 			 */
 			right = mid - 1;
 		}

mtbl/bytes.h

@@ -22,7 +22,7 @@ bytes_shortest_separator(ubuf *start, const uint8_t *limit, size_t len_limit)
 	if (diff_byte < 0xFF && diff_byte + 1 < limit[diff_index]) {
 		ubuf_data(start)[diff_index]++;
 		ubuf_clip(start, diff_index + 1);
-	} else if (diff_index < min_length - sizeof(uint16_t)) {
+	} else if (diff_index + sizeof(uint16_t) < min_length) {
 		/* awww yeah, big endian arithmetic on strings */
 		uint16_t u_start, u_limit, u_between;
 		memcpy(&u_start, &ubuf_data(start)[diff_index], sizeof(u_start));

mtbl/fileset.c

@@ -1,4 +1,5 @@
 /*
+ * Copyright (c) 2022 DomainTools LLC
  * Copyright (c) 2012-2019 by Farsight Security, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -29,6 +30,8 @@ struct mtbl_fileset_options {
 	void				*dupsort_clos;
 	mtbl_filename_filter_func	fname_filter;
 	void				*fname_filter_clos;
+	mtbl_reader_filter_func		reader_filter;
+	void				*reader_filter_clos;
 };
 
 struct shared_fileset {
@@ -48,6 +51,8 @@ struct mtbl_fileset {
 	struct mtbl_source		*source;
 	mtbl_filename_filter_func	fname_filter;
 	void				*fname_filter_clos;
+	mtbl_reader_filter_func		reader_filter;
+	void				*reader_filter_clos;
 };
 
 struct fileset_iter {
@@ -180,6 +185,14 @@ mtbl_fileset_options_set_filename_filter_func(struct mtbl_fileset_options *opt,
 	opt->fname_filter_clos = clos;
 }
 
+void
+mtbl_fileset_options_set_reader_filter_func(struct mtbl_fileset_options *opt,
+					mtbl_reader_filter_func reader_filter, void *clos)
+{
+	opt->reader_filter = reader_filter;
+	opt->reader_filter_clos = clos;
+}
+
 void
 mtbl_fileset_options_set_reload_interval(struct mtbl_fileset_options *opt,
 					 uint32_t reload_interval)
@@ -215,6 +228,8 @@ mtbl_fileset_set_options(struct mtbl_fileset *f, const struct mtbl_fileset_optio
 	mtbl_merger_options_set_dupsort_func(f->mopt, opt->dupsort, opt->dupsort_clos);
 	f->fname_filter = opt->fname_filter;
 	f->fname_filter_clos = opt->fname_filter_clos;
+	f->reader_filter = opt->reader_filter;
+	f->reader_filter_clos = opt->reader_filter_clos;
 	f->merger = mtbl_merger_init(f->mopt);
 	f->source = mtbl_source_init(fileset_source_iter,
 				     fileset_source_get,
@@ -290,11 +305,25 @@ fs_reinit_merger(struct mtbl_fileset *f)
 		f->merger = mtbl_merger_init(f->mopt);
 	}
 	assert(f->merger != NULL);
-	while (my_fileset_get(f->shared_fs->my_fs, i++, &fname, (void **) &reader))
-		if ((reader != NULL) && ((f->fname_filter == NULL) ||
-					 f->fname_filter(fname, f->fname_filter_clos))) {
+	while (my_fileset_get(f->shared_fs->my_fs, i++, &fname, (void **) &reader)) {
+		if (reader == NULL) {
+			continue;
+		}
+
+		/*
+		* Add the reader's source to the merger unless filtered out by the
+		* fname_filter or reader_filter.
+		*/
+		if (
+			(f->fname_filter == NULL
+				|| f->fname_filter(fname, f->fname_filter_clos))
+			&&
+			(f->reader_filter == NULL
+				|| f->reader_filter(reader, f->reader_filter_clos))
+		   ) {
 			mtbl_merger_add_source(f->merger, mtbl_reader_source(reader));
 		}
+	}
 }
 
 void

mtbl/libmtbl.sym

@@ -98,3 +98,8 @@ global:
         mtbl_fileset_options_set_dupsort_func;
         mtbl_fileset_dup;
 } LIBMTBL_1.1.0;
+
+LIBMTBL_1.4.0 {
+global:
+        mtbl_fileset_options_set_reader_filter_func;
+} LIBMTBL_1.2.0;