From c3e5c1c5781629ffaf477d9a4fe2a31172e081b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julien=20=C3=89LIE?= Date: Sun, 17 Sep 2023 11:48:48 +0200 Subject: [PATCH] Add new ovflushcount parameter in inn.conf The icdsynccount parameter was used for the flushing of both the active and history files, and buffindexed (though undocumented for that use with buffindexed). Add a new parameter for that use. close #280 --- doc/pod/buffindexed.conf.pod | 3 +++ doc/pod/inn.conf.pod | 12 ++++++++++++ doc/pod/news.pod | 7 +++++++ include/inn/innconf.h | 1 + lib/innconf.c | 1 + samples/inn.conf.in | 1 + storage/buffindexed/buffindexed.c | 3 ++- tests/overview/api-t.c | 5 ++--- tests/overview/overview-t.c | 2 +- 9 files changed, 30 insertions(+), 5 deletions(-) diff --git a/doc/pod/buffindexed.conf.pod b/doc/pod/buffindexed.conf.pod index bc9fd191b..7ef80be9a 100644 --- a/doc/pod/buffindexed.conf.pod +++ b/doc/pod/buffindexed.conf.pod @@ -35,6 +35,9 @@ method. You can see the current usage of the buffers with the B<-o> option to B. +The overview data of received articles are flushed to disk every +I articles, as set in F. + In the F file, blank lines and lines beginning with a number sign (C<#>) are ignored. All other lines must be of the format: diff --git a/doc/pod/inn.conf.pod b/doc/pod/inn.conf.pod index 1220876b5..1cd12d5b3 100644 --- a/doc/pod/inn.conf.pod +++ b/doc/pod/inn.conf.pod @@ -670,6 +670,18 @@ set to true and the storage method for these newsgroups does not have self-expire functionality, storing overview data will fail. The default is unset. +=item I + +How many articles received between flushing their overview data to disk. +This parameter is only used for the buffindexed overview storage method, and +defaults to C<50>. (Flushing to disk is parameterized differently for other +methods: I in F, I and I +in F, and always after each article arrival for tradindexed.) + +See I (related to flushes of the I and I files) +for more information about this trade-off between faster speed and more data +loss if B crashes. + =item I Which overview storage method to use. Currently supported values are diff --git a/doc/pod/news.pod b/doc/pod/news.pod index 691c94e1b..5edea60a6 100644 --- a/doc/pod/news.pod +++ b/doc/pod/news.pod @@ -35,6 +35,13 @@ NoCeM issuers, and make sure the right PGP keys are present on your system. =item * +Added a new I parameter in F, defaulting to C<50>, to +parameterize the number of articles received between flushing their overview +data to disk, when using the buffindexed overview storage method. (In +previous releases, that number was ten times the value of I.) + +=item * + B now supports high-precision timestamps like C<2023-07-29T04:15:01.889064+02:00> that syslog can be parameterized to use; daily Usenet reports otherwise indicated all these logs as unknown entries, diff --git a/include/inn/innconf.h b/include/inn/innconf.h index 4fcc5823d..9a9d9b01e 100644 --- a/include/inn/innconf.h +++ b/include/inn/innconf.h @@ -80,6 +80,7 @@ struct innconf { bool mergetogroups; /* Refile articles from to.* into to */ bool nfswriter; /* Use NFS writer functionality */ unsigned long overcachesize; /* fd size cache for tradindexed */ + unsigned long ovflushcount; /* Articles between buffindexed flushes */ char *ovgrouppat; /* Newsgroups to store overview for */ char *ovmethod; /* Which overview method to use */ bool storeonxref; /* SMstore use Xref to detemine class? */ diff --git a/lib/innconf.c b/lib/innconf.c index 2929bbec8..bbb1826b5 100644 --- a/lib/innconf.c +++ b/lib/innconf.c @@ -210,6 +210,7 @@ static const struct config config_table[] = { {K(nfswriter), BOOL(false) }, {K(nnrpdcheckart), BOOL(true) }, {K(overcachesize), UNUMBER(128) }, + {K(ovflushcount), UNUMBER(50) }, {K(ovgrouppat), STRING(NULL) }, {K(storeonxref), BOOL(true) }, {K(tradindexedmmap), BOOL(true) }, diff --git a/samples/inn.conf.in b/samples/inn.conf.in index 404ba2214..2e761ff67 100644 --- a/samples/inn.conf.in +++ b/samples/inn.conf.in @@ -70,6 +70,7 @@ groupbaseexpiry: true mergetogroups: false nfswriter: false overcachesize: 128 +ovflushcount: 50 #ovgrouppat: storeonxref: true useoverchan: false diff --git a/storage/buffindexed/buffindexed.c b/storage/buffindexed/buffindexed.c index 0a272130a..962526f0a 100644 --- a/storage/buffindexed/buffindexed.c +++ b/storage/buffindexed/buffindexed.c @@ -54,7 +54,7 @@ ** do file write for each update. Instead we'll do it at every ** OVBUFF_SYNC_COUNT updates. */ -#define OVBUFF_SYNC_COUNT (innconf->icdsynccount * 10 + 1) +#define OVBUFF_SYNC_COUNT (innconf->ovflushcount) /* ovbuff header */ #define OVBUFFMASIZ 8 @@ -763,6 +763,7 @@ ovbuffinit_disks(void) ovbuff->usedblk = 0; ovbuff->freeblk = 0; ovbuff->updated = 0; + /* Force a flush. */ ovbuff->dirty = OVBUFF_SYNC_COUNT + 1; notice("buffindexed: no magic cookie found for ovbuff %d, " "initializing", diff --git a/tests/overview/api-t.c b/tests/overview/api-t.c index d0a156486..6f458dd05 100644 --- a/tests/overview/api-t.c +++ b/tests/overview/api-t.c @@ -59,8 +59,7 @@ group_del(void *entry) /* Build a stripped-down innconf struct that contains only those settings that the overview backend cares about. (May still be missing additional bits - for ovdb.) - innconf->icdsynccount defines OVBUFF_SYNC_COUNT. */ + for ovdb.) */ static void fake_innconf(void) { @@ -76,10 +75,10 @@ fake_innconf(void) memset(innconf, 0, sizeof(*innconf)); innconf->enableoverview = true; innconf->groupbaseexpiry = true; - innconf->icdsynccount = 10; innconf->keepmmappedthreshold = 1024; innconf->nfsreader = false; innconf->overcachesize = 20; + innconf->ovflushcount = 50; innconf->ovgrouppat = NULL; innconf->pathdb = xstrdup("ov-tmp"); innconf->pathetc = xstrdup("etc"); diff --git a/tests/overview/overview-t.c b/tests/overview/overview-t.c index 51d2f5c16..2d279cd2a 100644 --- a/tests/overview/overview-t.c +++ b/tests/overview/overview-t.c @@ -87,10 +87,10 @@ fake_innconf(void) memset(innconf, 0, sizeof(*innconf)); innconf->enableoverview = true; innconf->groupbaseexpiry = true; - innconf->icdsynccount = 10; innconf->keepmmappedthreshold = 1024; innconf->nfsreader = false; innconf->overcachesize = 20; + innconf->ovflushcount = 50; innconf->ovgrouppat = NULL; innconf->ovmethod = xstrdup(STRING(OVTYPE)); innconf->pathdb = xstrdup("ov-tmp");