From 3704498c7a4fe6f437e0fff7cc34fc4e069db338 Mon Sep 17 00:00:00 2001
From: Colin Dellow <cldellow@gmail.com>
Date: Thu, 7 Dec 2023 00:44:20 -0500
Subject: [PATCH] buffer sqlite writes if needed

Fixes #596, I think.

Time to process Antarctica drops from 9m5s to 3m51s
---
 include/mbtiles.h | 14 ++++++++++++++
 src/mbtiles.cpp   | 45 ++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/include/mbtiles.h b/include/mbtiles.h
index 8ed4c670..ad12126d 100644
--- a/include/mbtiles.h
+++ b/include/mbtiles.h
@@ -7,6 +7,14 @@
 #include <vector>
 #include "sqlite_modern_cpp.h"
 
+struct PendingStatement {
+	int zoom;
+	int x;
+	int y;
+	std::string data;
+	bool isMerge;
+};
+
 /** \brief Write to MBTiles (sqlite) database
 *
 * (note that sqlite_modern_cpp.h is very slightly changed from the original, for blob support and an .init method)
@@ -17,6 +25,12 @@ class MBTiles {
 	std::mutex m;
 	bool inTransaction;
 
+	std::shared_ptr<std::vector<PendingStatement>> pendingStatements1, pendingStatements2;
+	std::mutex pendingStatementsMutex;
+
+	void insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge);
+	void flushPendingStatements();
+
 public:
 	MBTiles();
 	virtual ~MBTiles();
diff --git a/src/mbtiles.cpp b/src/mbtiles.cpp
index 4517a0d2..88caf9b1 100644
--- a/src/mbtiles.cpp
+++ b/src/mbtiles.cpp
@@ -13,7 +13,10 @@ using namespace sqlite;
 using namespace std;
 namespace bio = boost::iostreams;
 
-MBTiles::MBTiles() {}
+MBTiles::MBTiles():
+  pendingStatements1(std::make_shared<std::vector<PendingStatement>>()),
+  pendingStatements2(std::make_shared<std::vector<PendingStatement>>())
+{}
 
 MBTiles::~MBTiles() {
 	if (db && inTransaction) db << "COMMIT;"; // commit all the changes if open
@@ -52,18 +55,46 @@ void MBTiles::writeMetadata(string key, string value) {
 	db << "REPLACE INTO metadata (name,value) VALUES (?,?);" << key << value;
 	m.unlock();
 }
-	
-void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) {
-	int tmsY = pow(2,zoom) - 1 - y;
+
+void MBTiles::insertOrReplace(int zoom, int x, int y, const std::string& data, bool isMerge) {
+	// NB: assumes we have the `m` mutex
+	int tmsY = pow(2, zoom) - 1 - y;
 	int s = isMerge ? 1 : 0;
-	m.lock();
 	preparedStatements[s].reset();
-	preparedStatements[s] << zoom << x << tmsY && *data;
+	preparedStatements[s] << zoom << x << tmsY && data;
 	preparedStatements[s].execute();
-	m.unlock();
+}
+
+void MBTiles::flushPendingStatements() {
+	// NB: assumes we have the `m` mutex
+
+	for (int i = 0; i < 2; i++) {
+		while(!pendingStatements2->empty()) {
+			const PendingStatement& stmt = pendingStatements2->back();
+			insertOrReplace(stmt.zoom, stmt.x, stmt.y, stmt.data, stmt.isMerge);
+			pendingStatements2->pop_back();
+		}
+
+		std::lock_guard<std::mutex> lock(pendingStatementsMutex);
+		pendingStatements1.swap(pendingStatements2);
+	}
+}
+	
+void MBTiles::saveTile(int zoom, int x, int y, string *data, bool isMerge) {
+	// If the lock is available, write directly to SQLite.
+	if (m.try_lock()) {
+		insertOrReplace(zoom, x, y, *data, isMerge);
+		flushPendingStatements();
+		m.unlock();
+	} else {
+		// Else buffer the write for later, copying its binary blob.
+		const std::lock_guard<std::mutex> lock(pendingStatementsMutex);
+		pendingStatements1->push_back({zoom, x, y, *data, isMerge});
+	}
 }
 
 void MBTiles::closeForWriting() {
+	flushPendingStatements();
 	db << "CREATE UNIQUE INDEX IF NOT EXISTS tile_index on tiles (zoom_level, tile_column, tile_row);";
 	preparedStatements[0].used(true);
 	preparedStatements[1].used(true);