From 50a539940b4e7db268b45a40a5dec7cdbefac49c Mon Sep 17 00:00:00 2001 From: "buaa.byl" Date: Thu, 29 Dec 2016 09:59:37 +0800 Subject: [PATCH] debug issuse #43 --- bugs/Makefile | 23 ++++++ bugs/main.cpp | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++ unqlite.c | 35 +++++++++ 3 files changed, 256 insertions(+) create mode 100644 bugs/Makefile create mode 100644 bugs/main.cpp diff --git a/bugs/Makefile b/bugs/Makefile new file mode 100644 index 0000000..c7c9f1a --- /dev/null +++ b/bugs/Makefile @@ -0,0 +1,23 @@ +CFLAGS = -I .. + +all:b a404 a200 + +a404:main.cpp unqlite-modify.o + g++ $(CFLAGS) -D TEST_COMMIT $^ -o $@ + +a200:main.cpp unqlite-modify.o + g++ $(CFLAGS) $^ -o $@ + +b:main.cpp unqlite-org.o + g++ $(CFLAGS) -D TEST_COMMIT $^ -o $@ + +test: + a404 > 404 + a200 > 200 + +unqlite-modify.o:../unqlite-modify.c + gcc $(CFLAGS) -c $< -o $@ + +unqlite-org.o:../unqlite.c + gcc $(CFLAGS) -c $< -o $@ + diff --git a/bugs/main.cpp b/bugs/main.cpp new file mode 100644 index 0000000..2c6e7bf --- /dev/null +++ b/bugs/main.cpp @@ -0,0 +1,198 @@ +//@brief debug step +//@description +// +//## 1th: analysis +// I call unqlite_close and unqlite_open when commit. +// this will force flush everything, and reload +// as I found, all data have wrote to database file. +// +// so the bug maybe inside unqlite_commit or before. +// +// I have read SQLite3 source codes some years ago. +// The SQLite3 pages cache is complex, so I guess maybe cache mismatch? +// the debug step will be: +// 1. when flush (commit) to database, is all dirty pages delete? +// 1.1. If not, how unqlite do? +// 1.2. If yes, when it reload from disk? +// +// 2. is there any dirty pages not mark? +// 2.1. If not, why? +// 2.2. If yes, dump all pages for every change or read, and have a look. +// +//## 2nd: debuging +// I analysis unqlite_commit, it merge dirty pages, +// then wrote to database file, unref that pages when sync. +// +// and I try to log pages number, +// such like `unqlite_kv_store` and `unqlite_kv_fetch`. +// +// when update key `162`, unqlite_kv_store trigger pages overflow, +// and reassign new pages, so maybe unqlite using old page after commit. +// +// I add this codes to `pager_commit_phase1(Pager *pPager)` +// /* release all pages */ +// { +// Page *p; +// +// while (1) { +// p = pPager->pAll; +// if (p == NULL) { +// break; +// } +// pager_unlink_page(pPager, p); +// } +// } +// /* release all pages */ +// +// and the bug gone. +// +// I dump all pages number, have some try: +// 1. release page 7, the bug gone. +// 2. release page 1, buggy. +// +// And I found `unqlite_kv_fetch` access page 7, +// and trigger page 4 overflow to page 4 and page 5. +// +// +#include +#include +#include + +extern "C" { +#include "unqlite.h" +} + +struct TestStruct +{ + double d1; + double d2; +}; + +#define WR_KEY 162 +#define N 165 + +int main(int argc, char **argv) +{ +#ifdef TEST_COMMIT + std::string filename("TEST404." + std::to_string(time(nullptr))); +#else + std::string filename("TEST200." + std::to_string(time(nullptr))); +#endif + TestStruct test = { 1.0, 1.1 }; + TestStruct test_modify = { 2.0, 1.1 }; + + // Fill database + { + unqlite *db = nullptr; + + int res = unqlite_open(&db, filename.c_str(), UNQLITE_OPEN_CREATE); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_open %d", res); + return res; + } + + for (int64_t i = 0; i < N; ++i) + //for (int64_t i = N - 1; i >= 0; --i) + { + int res = unqlite_kv_store(db, &i, sizeof(int64_t), &test, sizeof(TestStruct)); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_kv_store %d", res); + return res; + } + } + + res = unqlite_close(db); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_close %d", res); + return res; + } + } + + printf("\n\n\n"); + + // Reopen + { + unqlite *db = nullptr; + + int res = unqlite_open(&db, filename.c_str(), UNQLITE_OPEN_CREATE); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_open %d", res); + return res; + } + printf("\n"); + + res = unqlite_begin(db); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_begin %d", res); + return res; + } + printf("\n"); + + // Write pair with key 162 again + int64_t i_bug = 0; + i_bug = WR_KEY; + res = unqlite_kv_store(db, &i_bug, sizeof(int64_t), &test_modify, sizeof(TestStruct)); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_kv_store %d", res); + return res; + } + printf("\n"); + + //// Force commit +#ifdef TEST_COMMIT + res = unqlite_commit(db); +#endif + //res = unqlite_rollback(db); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_commit %d", res); + return res; + } + printf("\n"); + + + //res = unqlite_close(db); + //if (res != UNQLITE_OK) + //{ + // printf("Error: unqlite_close %d", res); + // return res; + //} + //res = unqlite_open(&db, filename.c_str(), UNQLITE_OPEN_CREATE); + //if (res != UNQLITE_OK) + //{ + // printf("Error: unqlite_open %d", res); + // return res; + //} + + + TestStruct test1= {}; + unqlite_int64 buf_size = sizeof(TestStruct); + + //for (int64_t i = WR_KEY; i < N; ++i) + for (int64_t i = 0; i < N; ++i) + //for (int64_t i = 100; i < N; ++i) + //for (int64_t i = N - 1; i >= 0; --i) + { + res = unqlite_kv_fetch(db, &i, sizeof(int64_t), &test1, &buf_size); + if (res != UNQLITE_OK) { + printf("!!! Error: key %d not found !!!\n", (int)i); + break; + } + } + + res = unqlite_close(db); + if (res != UNQLITE_OK) + { + printf("Error: unqlite_close %d", res); + return res; + } + } + + return 0; +} diff --git a/unqlite.c b/unqlite.c index d42fbdd..877afae 100644 --- a/unqlite.c +++ b/unqlite.c @@ -11,6 +11,7 @@ * or visit: * http://unqlite.org/licensing.html */ +#include /* * Copyright (C) 2012, 2016 Symisc Systems, S.U.A.R.L [M.I.A.G Mrad Chems Eddine ]. * All rights reserved. @@ -57188,6 +57189,40 @@ static int pager_commit_phase1(Pager *pPager) unqliteGenError(pPager->pDb,"IO error while writing dirty pages, rollback your database"); return rc; } + + + { + Page *p; + + //release one page + p = pPager->pAll; + while (p) { + //if (p->pgno == 1) { // bug exists + if (p->pgno == 7) { // bug fixed + pager_unlink_page(pPager, p); + break; + } + p = p->pNext; + } + + ////release all pages, bug fixed + //while (1) { + // p = pPager->pAll; + // if (p == NULL) { + // break; + // } + // pager_unlink_page(pPager, p); + //} + + //dump pages exists + p = pPager->pAll; + while (p) { + printf(" -- pgno %d exists\n", p->pgno); + p = p->pNext; + } + } + + /* If the file on disk is not the same size as the database image, * then use unqliteOsTruncate to grow or shrink the file here. */