Skip to content

Refactor Index #113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 16, 2024
Merged

Refactor Index #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e89427b
Move requireIndexing method out of Index
Mar 26, 2024
9052457
Inject database dependency into constructor of Index
Mar 28, 2024
18deb2d
Fix failing tests by using an in memory database
Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 41 additions & 154 deletions phpdotnet/phd/Index.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,14 @@ class Index extends Format
'phpdoc' => 'PI_PHPDOCHandler',
);

private $db;
private IndexRepository $indexRepository;
private $currentchunk;
private $ids = array();
private $currentid;
private $chunks = array();
private $isChunk = array();
protected $nfo = array();
private $isSectionChunk = array();
private $log = '';
private $previousId = "";
private $inChangelog = false;
private $currentChangelog = array();
Expand All @@ -101,6 +100,11 @@ class Index extends Format
private $POST_REPLACEMENT_INDEXES = array();
private $POST_REPLACEMENT_VALUES = array();

public function __construct(IndexRepository $indexRepository) {
$this->indexRepository = $indexRepository;
parent::__construct();
}

public function transformFromMap($open, $tag, $name, $attrs, $props) {
}
public function TEXT($value) {
Expand All @@ -112,50 +116,6 @@ public function createLink($for, &$desc = null, $type = Format::SDESC) {
public function appendData($data) {
}

/**
* Checks if indexing is needed.
*
* This is determined the following way:
* 0. If no index file exists, indexing is required.
* 1. If the config option --no-index is supplied, nothing is indexed
* 2. If the config option --force-index is supplied, indexing is required
* 3. If no option is given, the file modification time of the index and
* the manual docbook file are compared. If the index is older than
* the docbook file, indexing will be done.
*
* @return boolean True if indexing is required.
*/
final static public function requireIndexing()
{
$indexfile = Config::output_dir() . 'index.sqlite';
if (!file_exists($indexfile)) {
return true;
}

if (Config::no_index()) {
return false;
}

if (Config::force_index()) {
return true;
}

$db = new \SQLite3($indexfile);
$indexingCount = $db->query('SELECT COUNT(time) FROM indexing')
->fetchArray(SQLITE3_NUM);
if ($indexingCount[0] == 0) {
return true;
}

$indexing = $db->query('SELECT time FROM indexing')
->fetchArray(SQLITE3_ASSOC);
$xmlLastModification = filemtime(Config::xml_file());
if ($indexing['time'] > $xmlLastModification) {
return false;
}
return true;
}

public function update($event, $value = null)
{
switch($event) {
Expand All @@ -172,58 +132,24 @@ public function update($event, $value = null)
break;
case Render::INIT:
if ($value) {
if (Config::memoryindex()) {
$db = new \SQLite3(":memory:");
} else {
$db = new \SQLite3(Config::output_dir() . 'index.sqlite');
$db->exec('DROP TABLE IF EXISTS ids');
$db->exec('DROP TABLE IF EXISTS indexing');
$db->exec('DROP TABLE IF EXISTS changelogs');
}

$create = <<<SQL
CREATE TABLE ids (
docbook_id TEXT,
filename TEXT,
parent_id TEXT,
sdesc TEXT,
ldesc TEXT,
element TEXT,
previous TEXT,
next TEXT,
chunk INTEGER
);
CREATE TABLE changelogs (
membership TEXT, -- How the extension in distributed (pecl, core, bundled with/out external dependencies..)
docbook_id TEXT,
parent_id TEXT,
version TEXT,
description TEXT
);
CREATE TABLE indexing (
time INTEGER PRIMARY KEY
);
SQL;
$db->exec('PRAGMA default_synchronous=OFF');
$db->exec('PRAGMA count_changes=OFF');
$db->exec('PRAGMA cache_size=100000');
$db->exec($create);

if (Config::memoryindex()) {
Config::set_indexcache($db);
}

$this->db = $db;
$this->indexRepository->init();
$this->chunks = array();
} else {
print_r($this->chunks);
}
break;
case Render::FINALIZE:
$retval = $this->db->exec("BEGIN TRANSACTION; INSERT INTO indexing (time) VALUES ('" . time() . "'); COMMIT");
$this->commit();
if ($this->db->lastErrorCode()) {
trigger_error($this->db->lastErrorMsg(), E_USER_WARNING);
$this->indexRepository->saveIndexingTime(time());
if ($this->indexRepository->commit(
$this->commit,
$this->POST_REPLACEMENT_INDEXES,
$this->POST_REPLACEMENT_VALUES,
$this->changelog,
)) {
$this->commit = [];
}
if ($this->indexRepository->lastErrorCode()) {
trigger_error($this->indexRepository->lastErrorMsg(), E_USER_WARNING);
}
break;
}
Expand Down Expand Up @@ -266,7 +192,7 @@ protected function storeInfo($elm, $id, $filename, $isChunk = true) {
);
// Append "next" to the previously inserted row
if ($isChunk) {
$this->POST_REPLACEMENT_VALUES[$this->previousId] = $this->db->escapeString($id);
$this->POST_REPLACEMENT_VALUES[$this->previousId] = $id;
$this->previousId = $id;
}
}
Expand All @@ -285,34 +211,33 @@ public function appendID() {
$sdesc = $lastChunk["sdesc"];
}

$this->commit[++$idx] = sprintf(
"INSERT INTO ids (docbook_id, filename, parent_id, sdesc, ldesc, element, previous, next, chunk) VALUES('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d);\n",
$this->db->escapeString($lastChunkId),
$this->db->escapeString($lastChunk["filename"]),
$this->db->escapeString($this->currentchunk),
$this->db->escapeString($sdesc),
$this->db->escapeString($lastChunk["ldesc"]),
$this->db->escapeString($lastChunk["element"]),
$this->db->escapeString($lastChunk["previous"]),
$this->db->escapeString($lastChunk["chunk"] ? "POST-REPLACEMENT" : ""),
$this->db->escapeString($lastChunk["chunk"])
);
$this->commit[++$idx] = [
"docbook_id" => $lastChunkId,
"filename" => $lastChunk["filename"],
"parent_id" => $this->currentchunk,
"sdesc" => $sdesc,
"ldesc" => $lastChunk["ldesc"],
"element" => $lastChunk["element"],
"previous" => $lastChunk["previous"],
"next" => ($lastChunk["chunk"] ? "POST-REPLACEMENT" : ""),
"chunk" => $lastChunk["chunk"],
];
if ($lastChunk["chunk"]) {
$this->POST_REPLACEMENT_INDEXES[] = array("docbook_id" => $lastChunkId, "idx" => $idx);
}
if ($array === true) {
foreach($lastChunk["sdesc"] as $sdesc) {
$this->commit[++$idx] = sprintf(
"INSERT INTO ids (docbook_id, filename, parent_id, sdesc, ldesc, element, previous, next, chunk) VALUES('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', 0);\n",
$this->db->escapeString($lastChunkId),
$this->db->escapeString($lastChunk["filename"]),
$this->db->escapeString($this->currentchunk),
$this->db->escapeString($sdesc),
$this->db->escapeString($lastChunk["ldesc"]),
$this->db->escapeString($lastChunk["element"]),
$this->db->escapeString($lastChunk["previous"]),
$this->db->escapeString($lastChunk["chunk"] ? "POST-REPLACEMENT" : "")
);
$this->commit[++$idx] = [
"docbook_id" => $lastChunkId,
"filename" => $lastChunk["filename"],
"parent_id" => $this->currentchunk,
"sdesc" => $sdesc,
"ldesc" => $lastChunk["ldesc"],
"element" => $lastChunk["element"],
"previous" => $lastChunk["previous"],
"next" => ($lastChunk["chunk"] ? "POST-REPLACEMENT" : ""),
"chunk" => 0,
];
$this->POST_REPLACEMENT_INDEXES[] = array("docbook_id" => $lastChunkId, "idx" => $idx);
}
}
Expand Down Expand Up @@ -475,44 +400,6 @@ public function format_row($open, $name, $attrs, $props) {
}
}



public function commit() {
if (isset($this->commit) && $this->commit) {
$search = $this->db->escapeString("POST-REPLACEMENT");
$none = $this->db->escapeString("");

foreach($this->POST_REPLACEMENT_INDEXES as $a) {
if (isset($this->POST_REPLACEMENT_VALUES[$a["docbook_id"]])) {
$replacement = $this->POST_REPLACEMENT_VALUES[$a["docbook_id"]];
$this->commit[$a["idx"]] = str_replace($search, $replacement, $this->commit[$a["idx"]]);
} else {
// If there are still post replacement, then they don't have
// any 'next' page
$this->commit[$a["idx"]] = str_replace($search, $none, $this->commit[$a["idx"]]);
}
}

$this->db->exec('BEGIN TRANSACTION; '.implode("", $this->commit).' COMMIT');
$log = "";
foreach($this->changelog as $id => $arr) {
foreach($arr as $entry) {
$log .= sprintf(
"INSERT INTO changelogs (membership, docbook_id, parent_id, version, description) VALUES('%s', '%s', '%s', '%s', '%s');\n",
$this->db->escapeString($entry[0] ?? ''),
$this->db->escapeString($id),
$this->db->escapeString($entry[1]),
$this->db->escapeString($entry[2]),
$this->db->escapeString($entry[3])
);
}
}
$this->db->exec('BEGIN TRANSACTION; ' . $log. ' COMMIT');
$this->log = "";
$this->commit = array();
}
}

public function processFilename() {
static $dbhtml = null;
if ($dbhtml == null) {
Expand Down
119 changes: 119 additions & 0 deletions phpdotnet/phd/IndexRepository.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
<?php
namespace phpdotnet\phd;

class IndexRepository
{
public function __construct(
private \SQLite3 $db
) {}

public function init(): void {
$create = <<<SQL
CREATE TABLE ids (
docbook_id TEXT,
filename TEXT,
parent_id TEXT,
sdesc TEXT,
ldesc TEXT,
element TEXT,
previous TEXT,
next TEXT,
chunk INTEGER
);
CREATE TABLE changelogs (
membership TEXT, -- How the extension in distributed (pecl, core, bundled with/out external dependencies..)
docbook_id TEXT,
parent_id TEXT,
version TEXT,
description TEXT
);
CREATE TABLE indexing (
time INTEGER PRIMARY KEY
);
SQL;
$this->db->exec('DROP TABLE IF EXISTS ids');
$this->db->exec('DROP TABLE IF EXISTS indexing');
$this->db->exec('DROP TABLE IF EXISTS changelogs');
$this->db->exec('PRAGMA default_synchronous=OFF');
$this->db->exec('PRAGMA count_changes=OFF');
$this->db->exec('PRAGMA cache_size=100000');
$this->db->exec($create);
}

public function saveIndexingTime(int $time): void {
$this->db->exec("BEGIN TRANSACTION; INSERT INTO indexing (time) VALUES ('" . $time . "'); COMMIT");
}

public function commit(
array $commitList,
array $postReplacementIndexes,
array $postReplacementValues,
array $changelog,
): bool {
if (!$commitList) {
return false;
}

foreach ($postReplacementValues as $key => $postReplacementValue) {
$postReplacementValues[$key] = $this->db->escapeString($postReplacementValue);
}

foreach ($commitList as $key => $commit) {
$commitList[$key] = sprintf(
"INSERT INTO ids (docbook_id, filename, parent_id, sdesc, ldesc, element, previous, next, chunk) VALUES('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d);\n",
$this->db->escapeString($commit["docbook_id"]),
$this->db->escapeString($commit["filename"]),
$this->db->escapeString($commit["parent_id"]),
$this->db->escapeString($commit["sdesc"]),
$this->db->escapeString($commit["ldesc"]),
$this->db->escapeString($commit["element"]),
$this->db->escapeString($commit["previous"]),
$this->db->escapeString($commit["next"]),
$this->db->escapeString($commit["chunk"])
);
}

$search = $this->db->escapeString("POST-REPLACEMENT");
$none = $this->db->escapeString("");

foreach($postReplacementIndexes as $a) {
if (isset($postReplacementValues[$a["docbook_id"]])) {
$replacement = $postReplacementValues[$a["docbook_id"]];
$commitList[$a["idx"]] = str_replace($search, $replacement, $commitList[$a["idx"]]);
} else {
// If there are still post replacement, then they don't have
// any 'next' page
$commitList[$a["idx"]] = str_replace($search, $none, $commitList[$a["idx"]]);
}
}

$this->db->exec('BEGIN TRANSACTION; '.implode("", $commitList).' COMMIT');
$this->saveChangelogs($changelog);
return true;
}

private function saveChangelogs(array $changelog): void {
$log = "";
foreach($changelog as $id => $arr) {
foreach($arr as $entry) {
$log .= sprintf(
"INSERT INTO changelogs (membership, docbook_id, parent_id, version, description) VALUES('%s', '%s', '%s', '%s', '%s');\n",
$this->db->escapeString($entry[0] ?? ''),
$this->db->escapeString($id),
$this->db->escapeString($entry[1]),
$this->db->escapeString($entry[2]),
$this->db->escapeString($entry[3])
);
}
}
$this->db->exec('BEGIN TRANSACTION; ' . $log. ' COMMIT');
}

public function lastErrorCode(): int {
return $this->db->lastErrorCode();
}

public function lastErrorMsg(): string {
return $this->db->lastErrorMsg();
}
}
2 changes: 1 addition & 1 deletion phpdotnet/phd/TestRender.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public function __construct(
) {}

public function run() {
if ($this->index && $this->index::requireIndexing()) {
if ($this->index && requireIndexing($this->config)) {
if (!file_exists($this->config->output_dir())) {
mkdir($this->config->output_dir(), 0755);
}
Expand Down
Loading
Loading