-
Notifications
You must be signed in to change notification settings - Fork 56
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #9 from hydradatabase/memory_and_parallel_scan
PostgreSQL memory allocator and multi thread scan
- Loading branch information
Showing
22 changed files
with
646 additions
and
254 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| CREATE EXTENSION quack; | ||
| SET client_min_messages to 'DEBUG3'; | ||
| CREATE TABLE t(a INT); | ||
| INSERT INTO t SELECT g % 10 from generate_series(1,1000000) g; | ||
| SELECT COUNT(*) FROM t; | ||
| DEBUG: -- (DuckDB/PostgresHeapBind) Column name: a, Type: INTEGER -- | ||
| DEBUG: -- (DuckDB/PostgresHeapScanGlobalState) Running 1 threads -- | ||
| count | ||
| --------- | ||
| 1000000 | ||
| (1 row) | ||
|
|
||
| SELECT a, COUNT(*) FROM t WHERE a > 5 GROUP BY a ORDER BY a; | ||
| DEBUG: -- (DuckDB/PostgresHeapBind) Column name: a, Type: INTEGER -- | ||
| DEBUG: -- (DuckDB/PostgresHeapScanGlobalState) Running 1 threads -- | ||
| a | count | ||
| ---+-------- | ||
| 6 | 100000 | ||
| 7 | 100000 | ||
| 8 | 100000 | ||
| 9 | 100000 | ||
| (4 rows) | ||
|
|
||
| SET quack.max_threads_per_query to 4; | ||
| SELECT COUNT(*) FROM t; | ||
| DEBUG: -- (DuckDB/PostgresHeapBind) Column name: a, Type: INTEGER -- | ||
| DEBUG: -- (DuckDB/PostgresHeapScanGlobalState) Running 4 threads -- | ||
| count | ||
| --------- | ||
| 1000000 | ||
| (1 row) | ||
|
|
||
| SELECT a, COUNT(*) FROM t WHERE a > 5 GROUP BY a ORDER BY a; | ||
| DEBUG: -- (DuckDB/PostgresHeapBind) Column name: a, Type: INTEGER -- | ||
| DEBUG: -- (DuckDB/PostgresHeapScanGlobalState) Running 4 threads -- | ||
| a | count | ||
| ---+-------- | ||
| 6 | 100000 | ||
| 7 | 100000 | ||
| 8 | 100000 | ||
| 9 | 100000 | ||
| (4 rows) | ||
|
|
||
| SET quack.max_threadS_per_query TO default; | ||
| SET client_min_messages TO default; | ||
| DROP TABLE t; |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| #pragma once | ||
|
|
||
| #include "duckdb.hpp" | ||
|
|
||
| extern "C" { | ||
| #include "postgres.h" | ||
| #include "access/tableam.h" | ||
| #include "access/heapam.h" | ||
| } | ||
|
|
||
| #include <mutex> | ||
|
|
||
| namespace quack { | ||
|
|
||
| class PostgresHeapSeqScanThreadInfo { | ||
| public: | ||
| PostgresHeapSeqScanThreadInfo(); | ||
| ~PostgresHeapSeqScanThreadInfo(); | ||
| void EndScan(); | ||
|
|
||
| public: | ||
| TupleDesc m_tuple_desc; | ||
| bool m_inited; | ||
| bool m_read_next_page; | ||
| bool m_page_tuples_all_visible; | ||
| int m_output_vector_size; | ||
| BlockNumber m_block_number; | ||
| Buffer m_buffer; | ||
| OffsetNumber m_current_tuple_index; | ||
| int m_page_tuples_left; | ||
| HeapTupleData m_tuple; | ||
| }; | ||
|
|
||
| class PostgresHeapSeqScan { | ||
| private: | ||
| class ParallelScanState { | ||
| public: | ||
| ParallelScanState() : m_nblocks(InvalidBlockNumber), m_last_assigned_block_number(InvalidBlockNumber) { | ||
| } | ||
| BlockNumber AssignNextBlockNumber(); | ||
| std::mutex m_lock; | ||
| BlockNumber m_nblocks; | ||
| BlockNumber m_last_assigned_block_number; | ||
| }; | ||
|
|
||
| public: | ||
| PostgresHeapSeqScan(RangeTblEntry *table); | ||
| ~PostgresHeapSeqScan(); | ||
| PostgresHeapSeqScan(const PostgresHeapSeqScan &other) = delete; | ||
| PostgresHeapSeqScan &operator=(const PostgresHeapSeqScan &other) = delete; | ||
| PostgresHeapSeqScan &operator=(PostgresHeapSeqScan &&other) = delete; | ||
| PostgresHeapSeqScan(PostgresHeapSeqScan &&other); | ||
|
|
||
| public: | ||
| void InitParallelScanState(); | ||
| void | ||
| SetSnapshot(Snapshot snapshot) { | ||
| m_snapshot = snapshot; | ||
| } | ||
|
|
||
| public: | ||
| Relation GetRelation(); | ||
| TupleDesc GetTupleDesc(); | ||
| bool ReadPageTuples(duckdb::DataChunk &output, PostgresHeapSeqScanThreadInfo &threadScanInfo); | ||
| bool IsValid() const; | ||
|
|
||
| private: | ||
| Page PreparePageRead(PostgresHeapSeqScanThreadInfo &threadScanInfo); | ||
|
|
||
| private: | ||
| Relation m_rel = nullptr; | ||
| Snapshot m_snapshot = nullptr; | ||
| ParallelScanState m_parallel_scan_state; | ||
| }; | ||
|
|
||
| } // namespace quack |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| #pragma once | ||
|
|
||
| #include "duckdb/common/allocator.hpp" | ||
|
|
||
| namespace quack { | ||
|
|
||
| struct QuackAllocatorData : public duckdb::PrivateAllocatorData { | ||
| explicit QuackAllocatorData() { | ||
| } | ||
| }; | ||
|
|
||
| duckdb::data_ptr_t QuackAllocate(duckdb::PrivateAllocatorData *private_data, duckdb::idx_t size); | ||
| void QuackFree(duckdb::PrivateAllocatorData *private_data, duckdb::data_ptr_t ptr, duckdb::idx_t idx); | ||
| duckdb::data_ptr_t QuackReallocate(duckdb::PrivateAllocatorData *private_data, duckdb::data_ptr_t pointer, | ||
| duckdb::idx_t old_size, duckdb::idx_t size); | ||
|
|
||
| } // namespace quack |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.