Merge branch 'main' of github.com:Leo-XM-Zeng/pg_duckdb into pg_duckd…

…b_15

.github/workflows/build_and_test.yaml

@@ -3,28 +3,7 @@ on:
   push:
     branches: ["main"]
     tags: ["v*"]
-    paths:
-      - 'src/**'
-      - 'test/**'
-      - 'include/**'
-      - 'sql/**'
-      - Makefile
-      - Makefile.global
-      - duckdb.control
-      - third_party/duckdb
-      - '.github/workflows/**'
   pull_request:
-    paths:
-      - 'src/**'
-      - 'test/**'
-      - 'include/**'
-      - 'sql/**'
-      - Makefile
-      - Makefile.global
-      - duckdb.control
-      - third_party/duckdb
-      - '.github/workflows/**'
-      - .clang-format
 
 jobs:
   format:
@@ -48,6 +27,7 @@ jobs:
         run: ruff check --output-format=github .
       - name: Run ruff format
         run: ruff format --diff
+
   build-and-test:
     name: 'Build and test'
     strategy:
@@ -73,6 +53,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/[email protected]
         with:
+          key: ${{ matrix.os }}
           create-symlink: true
 
       - name: Checkout pg_duckdb extension code
@@ -81,29 +62,37 @@ jobs:
           submodules: 'recursive'
           path: duckdb
 
-      - name: Compute DuckDB SHA
+      - name: Checkout PostgreSQL code
+        run: |
+          rm -rf postgres
+          git clone --branch ${{ matrix.version }} --single-branch --depth 1 https://github.com/postgres/postgres.git
+
+      - name: Compute Version SHAs
         id: versions
         run: |
-          pushd duckdb
-          DUCKDB_SHA=`git ls-tree HEAD third_party/duckdb --format='%(objectname)'`
+          pushd duckdb/third_party/duckdb
+          DUCKDB_SHA=`git rev-parse HEAD`
           echo "duckdb_sha=${DUCKDB_SHA}" >> "$GITHUB_OUTPUT"
           echo "Got DUCKDB_SHA='${DUCKDB_SHA}'"
+          popd
+          pushd postgres
+          POSTGRES_SHA=`git rev-parse HEAD`
+          echo "postgres_sha=${POSTGRES_SHA}" >> "$GITHUB_OUTPUT"
+          echo "Got POSTGRES_SHA='${POSTGRES_SHA}'"
 
       - name: Setup PG build cache
         id: cache-pg-build
         uses: actions/cache@v4
         with:
           path: postgres/inst
-          key: pg-build-${{ matrix.version }}
+          key: pg-build-${{ matrix.os }}-${{ steps.versions.outputs.postgres_sha }}
 
-      - name: Checkout and build PostgreSQL code
+      - name: Build PostgreSQL code
         if: steps.cache-pg-build.outputs.cache-hit != 'true'
         run: |
-          rm -rf postgres
-          git clone --branch ${{ matrix.version }} --single-branch --depth 1 https://github.com/postgres/postgres.git
           pushd postgres
           git branch
-          ./configure --prefix=$PWD/inst/ --enable-cassert --enable-debug --with-openssl --with-icu --with-libxml
+          ./configure --prefix=$PWD/inst/ --enable-cassert --enable-debug --with-openssl --with-icu --with-libxml --with-lz4
           make -j8 install
 
       - uses: actions/setup-python@v5
@@ -115,7 +104,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: duckdb/third_party/duckdb/build/
-          key: duckdb-build-${{ steps.versions.outputs.duckdb_sha }}
+          key: duckdb-build-${{ matrix.os }}-${{ steps.versions.outputs.duckdb_sha }}
 
       - name: Install pytest and other test requirements
         run: python3 -m pip install -r duckdb/requirements.txt
@@ -124,8 +113,7 @@ jobs:
         id: build
         run: |
           pushd duckdb
-          make
-          make install
+          make -j8 install
 
       - name: Run make installcheck
         id: installcheck

CONTRIBUTING.md

@@ -86,8 +86,8 @@ coding styles.
 ### Postgres C Guidelines
 
 * Use memory contexts to allocate memory, e.g. `palloc` or `palloc0`.
-* Casing is very inconsistent in Postgres, we use `CamelCase` for function names and `snake_case` for variables.
-* If you copied code from Postgres, try to keep the Postgres style and prefer to keep it in a separate function so that it is easy to update when the Postgres code changes.
+* Casing is very inconsistent in Postgres, we use `CamelCase` for function names and `snake_case` for variables. For the C implementation of a SQL function (e.g. `duckdb.install_extension(...)`, use snake_case.
+* If you copied code from Postgres, try to keep the Postgres style and prefer to keep it in a separate function/file so that it is easy to update when the Postgres code changes.
 * TODO: Add more guidelines here.
 
 ### DuckDB C++ Guidelines

Makefile

@@ -4,41 +4,12 @@ MODULE_big = pg_duckdb
 EXTENSION = pg_duckdb
 DATA = pg_duckdb.control $(wildcard sql/pg_duckdb--*.sql)
 
-SRCS = src/scan/heap_reader.cpp \
-	   src/scan/index_scan_utils.cpp \
-	   src/scan/postgres_index_scan.cpp \
-	   src/scan/postgres_scan.cpp \
-	   src/scan/postgres_seq_scan.cpp \
-	   src/utility/copy.cpp \
-	   src/vendor/pg_explain.cpp \
-	   src/pgduckdb_metadata_cache.cpp \
-	   src/pgduckdb_detoast.cpp \
-	   src/pgduckdb_duckdb.cpp \
-	   src/pgduckdb_filter.cpp \
-	   src/pgduckdb_hooks.cpp \
-	   src/pgduckdb_memory_allocator.cpp \
-	   src/pgduckdb_node.cpp \
-	   src/pgduckdb_options.cpp \
-	   src/pgduckdb_planner.cpp \
-	   src/pgduckdb_ruleutils.cpp \
-	   src/pgduckdb_types.cpp \
-	   src/pgduckdb.cpp \
-	   src/catalog/pgduckdb_storage.cpp \
-	   src/catalog/pgduckdb_schema.cpp \
-	   src/catalog/pgduckdb_table.cpp \
-	   src/catalog/pgduckdb_transaction.cpp \
-	   src/catalog/pgduckdb_transaction_manager.cpp \
-	   src/catalog/pgduckdb_catalog.cpp
-
+SRCS = $(wildcard src/*.cpp src/*/*.cpp)
 OBJS = $(subst .cpp,.o, $(SRCS))
 
-
-C_SRCS = src/vendor/pg_ruleutils_15.c \
-		 src/vendor/pg_ruleutils_16.c \
-		 src/vendor/pg_ruleutils_17.c
+C_SRCS = $(wildcard src/*.c src/*/*.c)
 OBJS += $(subst .c,.o, $(C_SRCS))
 
-
 DUCKDB_BUILD_CXX_FLAGS=
 DUCKDB_BUILD_TYPE=
 
@@ -50,6 +21,9 @@ else
 	DUCKDB_BUILD_TYPE = release
 endif
 
+DUCKDB_LIB = libduckdb$(DLSUFFIX)
+FULL_DUCKDB_LIB = third_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/src/$(DUCKDB_LIB)
+
 override PG_CPPFLAGS += -Iinclude -Ithird_party/duckdb/src/include -Ithird_party/duckdb/third_party/re2
 override PG_CXXFLAGS += -std=c++17 -Wno-sign-compare ${DUCKDB_BUILD_CXX_FLAGS}
 
@@ -60,19 +34,11 @@ COMPILE.cc.bc = $(CXX) -Wno-ignored-attributes -Wno-register $(BITCODE_CXXFLAGS)
 %.bc : %.cpp
 	$(COMPILE.cc.bc) $(SHLIB_LINK) -I$(INCLUDE_SERVER) -o $@ $<
 
-# determine the name of the duckdb library that is built
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-	DUCKDB_LIB = libduckdb.dylib
-endif
-ifeq ($(UNAME_S),Linux)
-	DUCKDB_LIB = libduckdb.so
-endif
-
-all: duckdb $(OBJS)
-
 include Makefile.global
 
+# shlib is the final output product - make duckdb and all .o dependencies
+$(shlib): $(FULL_DUCKDB_LIB) $(OBJS)
+
 NO_INSTALLCHECK = 1
 
 PYTEST_CONCURRENCY = auto
@@ -91,22 +57,20 @@ pycheck: all install
 
 check: installcheck pycheck
 
-FULL_DUCKDB_LIB = third_party/duckdb/build/$(DUCKDB_BUILD_TYPE)/src/$(DUCKDB_LIB)
-duckdb: third_party/duckdb/Makefile $(FULL_DUCKDB_LIB)
-
+duckdb: $(FULL_DUCKDB_LIB)
 
 third_party/duckdb/Makefile:
 	git submodule update --init --recursive
 
-$(FULL_DUCKDB_LIB):
+$(FULL_DUCKDB_LIB): third_party/duckdb/Makefile
 	$(MAKE) -C third_party/duckdb \
 	$(DUCKDB_BUILD_TYPE) \
 	DISABLE_SANITIZER=1 \
 	ENABLE_UBSAN=0 \
 	BUILD_UNITTESTS=OFF \
 	EXTENSION_CONFIGS="../pg_duckdb_extensions.cmake"
 
-install-duckdb: $(FULL_DUCKDB_LIB)
+install-duckdb: $(FULL_DUCKDB_LIB) $(shlib)
 	$(install_bin) -m 755 $(FULL_DUCKDB_LIB) $(DESTDIR)$(PG_LIB)
 
 clean-duckdb:

test/regression/Makefile

@@ -9,7 +9,7 @@ check-regression-duckdb:
 	--temp-instance=./tmp_check \
 	--temp-config regression.conf \
 	--load-extension=pg_duckdb \
-	--schedule schedule
+	$(if $(TEST),$(TEST),--schedule=schedule)
 
 clean-regression:
 	rm -fr $(CURDIR)/tmp_check