WIP: Add new custom event loop for I/O layer

Introduce ev.h and ev.c, establishing the foundation for the new custom event loop, `pgagroal_ev`. Replace previous dependencies on libev with the custom event loop. For Linux, implement support for io_uring with a fallback to epoll if io_uring is unavailable. For BSD, implement support for kqueue. Changelog ========= 2025-01-17: - Fix regression added to the Debug build. `waitpid()` inside `#if DEBUG` would block waiting for the running child processes. Add WNOHANG to the function options so the main process does not block. - Remove '\n' from `pagroal_log_*()` - Run uncrustify.sh - Remove `printf()` debugging - Modify CD/CI pipeline to improve function verify_running and enable the CD/CI for kqueue, which was disabled. - Change `pgagroal_log_error()` that informs errors in `kill()` to `pgagroal_log_debug()`. I suppose these errors should happen if pgagroal is not closely following up the death of its children. Therefore, there is no need to flood the log with error messages. Keep as debug because it is informative of something that could be improved. 2025-01-16: - Remove `pgagroal_ev_*_stop()` functions from main.c and vault.c as these functions are not intended to be called from a child process inheriting the ev loop. This does not make sense in io_uring and for epoll and kqueue, just closing the fds is enough as a precaution to prevent their use. - Enhance debugging steps in `sigchld_handler()` to help debug child processes termination. - Add error handling to `kill()` to help debug child processes termination. - Allow return error on `pgagroal_ev_io_stop()` instead of exit so the caller function can handle errors appropriately. - Add a call to `setpgid()` when worker is created to ensure child processes do not exit immediately if a SIGINT is issued in the controlling terminal. - Add a call to `pgagroal_ev_fork()` when a worker is created. 2025-01-13: - Fix ASan report in `pgagroal_ev_io_stop()`
decarv · Jan 17, 2025 · 2e31f0b · 2e31f0b
1 parent 92ac986
commit 2e31f0b
Show file tree

Hide file tree

Showing 40 changed files with 5,033 additions and 601 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -23,9 +23,9 @@ What is the version of pgagroal ?
 
 What is the version of PostgreSQL ?
 
-**libev**
+**liburing**
 
-What is the version of libev ?
+What is the version of liburing ?
 
 **OpenSSL**
 

diff --git a/.github/config/pg_hba.conf b/.github/config/pg_hba.conf
@@ -0,0 +1,4 @@
+local   all             all trust
+host    all             all all   trust
+local   replication     all peer
+host    replication     all all   trust
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,7 +11,7 @@ on:
 jobs:
   build-linux:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
     - uses: actions/checkout@v3
@@ -21,8 +21,8 @@ jobs:
         sudo wget --quiet --output-document /etc/apt/trusted.gpg.d/apt.postgresql.org.asc https://www.postgresql.org/media/keys/ACCC4CF8.asc
     - name: Update system
       run: sudo apt update
-    - name: Install libev
-      run: sudo apt install -y libev4 libev-dev
+    - name: Install liburing
+      run: sudo apt install -y liburing-dev
     - name: Install systemd
       run: sudo apt install -y libsystemd-dev
     - name: Install rst2man
@@ -37,14 +37,43 @@ jobs:
       run: sudo apt install graphviz
     - name: Install doxygen
       run: sudo apt install doxygen
+    - name: Install crudini
+      run: sudo apt install -y crudini
     - name: Install clang
       run: sudo apt install -y clang
     - name: Install PostgreSQL
-      run: sudo apt install -y postgresql
-    - name: Start postgres
+      run: sudo apt install -y postgresql postgresql-contrib
+    - name: Start postgres & setup test table
       run: |
         version=$(pg_config --version | grep -Eo "[0-9]{1,2}" | head -1)
-        sudo -u postgres /usr/lib/postgresql/${version}/bin/pg_ctl start -D /etc/postgresql/${version}/main/
+        sudo cp pg_hba.conf /etc/postgresql/${version}/main
+        sudo -u postgres /usr/lib/postgresql/${version}/bin/pg_ctl start -D /etc/postgresql/${version}/main/ -o "-p 5432"
+        netstat -tuln | grep '127.0.0.1:5432' || (echo "Nothing is listening on 127.0.0.1:5432"; exit 1)
+        netstat -tuln | grep '::1:5432' || (echo "Nothing is listening on ::1:5432"; exit 1)
+        PGPASSWORD="postgres" pgbench -i -s 1 -h localhost -p 5432 -U postgres -d postgres
+      working-directory: /home/runner/work/pgagroal/pgagroal/.github/config/
+    - name: Define functions `verify_running` and `verify_shutdown`
+      run: |
+        echo 'verify_running() {
+          echo "Confirming pgagroal is listening on port 2345"
+          netstat -tuln | grep "127.0.0.1:2345" || (echo "Nothing is listening on 127.0.0.1:2345"; exit 1)
+          netstat -tuln | grep "::1:2345" || (echo "Nothing is listening on ::1:2345"; exit 1)
+          echo "[*] Running pgagroal-cli ping"
+          ./pgagroal-cli ping
+          echo "[*] Running queries with psql"
+          PGPASSWORD="postgres" psql -h 127.0.0.1 -p 2345 -U postgres -d postgres -c "SELECT * FROM pgbench_accounts LIMIT 50;" > /dev/null
+          PGPASSWORD="postgres" psql -h ::1 -p 2345 -U postgres -d postgres -c "SELECT * FROM pgbench_accounts LIMIT 50;" > /dev/null
+        }
+
+        verify_shutdown() {
+          echo "[*] Running pgagroal-cli shutdown immediate"
+          ./pgagroal-cli shutdown immediate
+          sleep 5
+          echo "[*] Confirming there are no dangling pgagroal processes"
+          pgrep pgagroal > /dev/null && echo "[E] Dangling pgagroal child processes: $(wc -l < <(pgrep pgagroal))" && exit 1
+          echo "rm -f /tmp/pgagroal.2345.pid"
+          rm -f /tmp/pgagroal.2345.pid
+        }' > /tmp/functions.sh
     - name: GCC/mkdir
       run: mkdir build
       working-directory: /home/runner/work/pgagroal/pgagroal/
@@ -54,28 +83,52 @@ jobs:
     - name: GCC/make
       run: make
       working-directory: /home/runner/work/pgagroal/pgagroal/build/
-    - name: GCC/Run pgagroal & confirm pgagroal is running
+    - name: GCC/Run pgagroal as daemon with 'io_uring' backend
       run: |
         sudo mkdir -p /etc/pgagroal
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_type file
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_path /dev/null
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal ev_backend io_uring
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
         sudo cp ../../doc/etc/*.conf /etc/pgagroal
-        ./pgagroal >> /dev/null 2>&1 &
-        pid=$!
+        sudo sysctl kernel.io_uring_disabled=0
+        ./pgagroal -d || exit 1
         sleep 5
-        ./pgagroal-cli ping
       working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
-    - name: GCC/Stop pgagroal & postgres
+    - name: GCC/Run verify_running for 'io_uring' backend
       run: |
-        ./pgagroal-cli shutdown
-        version=$(pg_config --version | grep -Eo "[0-9]{1,2}" | head -1)
-        sudo -u postgres /usr/lib/postgresql/${version}/bin/pg_ctl stop -D /etc/postgresql/${version}/main/
+        source /tmp/functions.sh
+        verify_running
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: GCC/Run verify_shutdown for 'io_uring' backend
+      run: |
+        source /tmp/functions.sh
+        verify_shutdown
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: GCC/Run pgagroal as daemon with 'epoll' backend
+      run: |
+        sudo mkdir -p /etc/pgagroal
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_type file
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_path /dev/null
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal ev_backend epoll
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
+        sudo cp ../../doc/etc/*.conf /etc/pgagroal
+        ./pgagroal -d || exit
+        sleep 5
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: GCC/Run verify_running for 'epoll' backend
+      run: |
+        source /tmp/functions.sh
+        verify_running
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: GCC/Run verify_shutdown for 'epoll' backend
+      run: |
+        source /tmp/functions.sh
+        verify_shutdown
       working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
     - name: rm -Rf
       run: rm -Rf build/
       working-directory: /home/runner/work/pgagroal/pgagroal/
-    - name: Start postgres
-      run: |
-        version=$(pg_config --version | grep -Eo "[0-9]{1,2}" | head -1)
-        sudo -u postgres /usr/lib/postgresql/${version}/bin/pg_ctl start -D /etc/postgresql/${version}/main/
     - name: CLANG/mkdir
       run: mkdir build
       working-directory: /home/runner/work/pgagroal/pgagroal/
@@ -85,38 +138,60 @@ jobs:
     - name: CLANG/make
       run: make
       working-directory: /home/runner/work/pgagroal/pgagroal/build/
-    - name: CLANG/Run pgagroal & confirm pgagroal is running
+    - name: CLANG/Run pgagroal as daemon with 'io_uring' backend
       run: |
         sudo mkdir -p /etc/pgagroal
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_type file
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_path /dev/null
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal ev_backend io_uring
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
         sudo cp ../../doc/etc/*.conf /etc/pgagroal
-        ./pgagroal >> /dev/null 2>&1 &
-        pid=$!
+        sudo sysctl kernel.io_uring_disabled=0
+        ./pgagroal -d || exit 1
         sleep 5
-        ./pgagroal-cli ping
       working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
-    - name: CLANG/Stop pgagroal & postgres
+    - name: CLANG/Run verify_running for 'io_uring' backend
       run: |
-        ./pgagroal-cli shutdown
-        version=$(pg_config --version | grep -Eo "[0-9]{1,2}" | head -1)
-        sudo -u postgres /usr/lib/postgresql/${version}/bin/pg_ctl stop -D /etc/postgresql/${version}/main/
+        source /tmp/functions.sh
+        verify_running
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: CLANG/Run verify_shutdown for 'io_uring' backend
+      run: |
+        source /tmp/functions.sh
+        verify_shutdown
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: CLANG/Run pgagroal as daemon with 'epoll' backend
+      run: |
+        sudo mkdir -p /etc/pgagroal
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_type file
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal log_path /dev/null
+        crudini --set ../../doc/etc/pgagroal.conf pgagroal ev_backend epoll
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
+        sudo cp ../../doc/etc/*.conf /etc/pgagroal
+        ./pgagroal -d || exit 1
+        sleep 5
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: CLANG/Run verify_running for 'epoll' backend
+      run: |
+        source /tmp/functions.sh
+        verify_running
+      working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
+    - name: CLANG/Run verify_shutdown for 'epoll' backend
+      run: |
+        source /tmp/functions.sh
+        verify_shutdown
       working-directory: /home/runner/work/pgagroal/pgagroal/build/src/
-
-
 
   build-macos:
-
+  
     runs-on: macos-latest
-
+  
     steps:
     - uses: actions/checkout@v3
-    - name: Install Homebrew
-      run: /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
     - name: Update system
       run: brew update
     - name: Install openssl
       run: brew install openssl
-    - name: Install libev
-      run: brew install libev
     - name: Install zstd
       run: brew install zstd
     - name: Install lz4
@@ -135,10 +210,42 @@ jobs:
       run: |
         latest_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
         brew install ${latest_pg} || true  # `|| true` prevents install errors from breaking the run
-    - name: Start postgres
+    - name: Start PostgreSQL & setup test table
       run: |
         installed_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
         brew services start ${installed_pg}
+        sleep 5
+        /opt/homebrew/opt/${installed_pg}/bin/pgbench -i -s 1 -h localhost -p 5432 -U $(whoami) -d postgres
+        exit 0
+    - name: Define functions `verify_running` and `verify_shutdown`
+      run: |
+        echo 'verify_running() {
+          echo "[*] Confirming pgagroal is listening on port 2345"
+          netstat -an | grep "\.2345 .*LISTEN" || (echo "Nothing is listening on port 2345"; exit 1)
+          echo "[*] Running pgagroal-cli ping"
+          ./pgagroal-cli ping
+          echo "[*] Running queries with psql"
+          installed_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
+          PGPASSWORD="postgres" /opt/homebrew/opt/${installed_pg}/bin/psql -h 127.0.0.1 -p 2345 -U $(whoami) -d postgres -c "SELECT * FROM pgbench_accounts LIMIT 50;" > /dev/null
+          PGPASSWORD="postgres" /opt/homebrew/opt/${installed_pg}/bin/psql -h ::1 -p 2345 -U $(whoami) -d postgres -c "SELECT * FROM pgbench_accounts LIMIT 50;" > /dev/null
+        }
+  
+        verify_shutdown() {
+          echo "[*] Getting pgid"
+          pgid=$(ps -o pgid= -p $(cat /tmp/pgagroal.2345.pid) | grep -o "[0-9]*")
+          echo "[*] Running pgagroal-cli shutdown immediate"
+          ./pgagroal-cli shutdown immediate
+          sleep 5
+          echo "[*] Confirming there are no dangling pgagroal processes"
+          if pgrep -g $pgid > /tmp/dangling; then
+            echo "[E] Dangling pgagroal child processes:
+                  $(cat /tmp/dangling)"
+            exit 1
+          else
+            echo "Removing PID file"
+            rm -f /tmp/pgagroal.2345.pid
+          fi
+        }' > /tmp/functions.sh
     - name: GCC/mkdir
       run: mkdir build
       working-directory: /Users/runner/work/pgagroal/pgagroal/
@@ -148,49 +255,63 @@ jobs:
     - name: GCC/make
       run: make
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/
-    - name: GCC/Run pgagroal & confirm pgagroal is running
-      run: |
-         sudo mkdir -p /etc/pgagroal
-         sudo cp ../../doc/etc/*.conf /etc/pgagroal
-         ./pgagroal >> /dev/null 2>&1 &
-         pid=$!
-         sleep 5
-         ./pgagroal-cli ping
+    - name: GCC/Run pgagroal as daemon with 'kqueue' backend
+      run: |
+        sudo mkdir -p /etc/pgagroal
+        sed -i '' 's/^log_type =.*$/log_type = file/' ../../doc/etc/pgagroal.conf
+        sed -i '' 's|^log_path =.*$|log_path = /dev/null|' ../../doc/etc/pgagroal.conf
+        sed -i '' 's/^ev_backend =.*$/ev_backend = kqueue/' ../../doc/etc/pgagroal.conf
+        cat ../../doc/etc/pgagroal.conf
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
+        sudo cp ../../doc/etc/*.conf /etc/pgagroal
+        ./pgagroal -d || exit 1
+        sleep 5
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
-    - name: GCC/Stop pgagroal & postgres
+    - name: GCC/Run verify_running for 'kqueue' backend
       run: |
-        ./pgagroal-cli shutdown
-        installed_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
-        brew services stop ${installed_pg}
+        source /tmp/functions.sh
+        verify_running
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
-    - name: rm -Rf
-      run: rm -Rf build/
-      working-directory: /Users/runner/work/pgagroal/pgagroal/
-    - name: Start postgres
+    - name: GCC/Run verify_shutdown for 'kqueue' backend
       run: |
-        installed_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
-        brew services start ${installed_pg}
+        source /tmp/functions.sh
+        verify_shutdown
+      working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
+    - name: GCC/Clean up build
+      run: rm -Rf build
+      working-directory: /Users/runner/work/pgagroal/pgagroal/
     - name: CLANG/mkdir
       run: mkdir build
       working-directory: /Users/runner/work/pgagroal/pgagroal/
     - name: CLANG/cmake
-      run: export CC=/usr/bin/clang && export OPENSSL_ROOT_DIR=`brew --prefix openssl` && cmake -DCMAKE_BUILD_TYPE=Debug ..
+      run: |
+        export CC=/usr/bin/clang
+        export OPENSSL_ROOT_DIR=$(brew --prefix openssl)
+        cmake -DCMAKE_BUILD_TYPE=Debug ..
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/
     - name: CLANG/make
       run: make
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/
-    - name: CLANG/Run pgagroal & confirm pgagroal is running
+    - name: CLANG/Run pgagroal as daemon with 'kqueue' backend
       run: |
         sudo mkdir -p /etc/pgagroal
+        sed -i '' 's/^log_type =.*$/log_type = file/' ../../doc/etc/pgagroal.conf
+        sed -i '' 's|^log_path =.*$|log_path = /dev/null|' ../../doc/etc/pgagroal.conf
+        sed -i '' 's/^ev_backend =.*$/ev_backend = kqueue/' ../../doc/etc/pgagroal.conf
+        cat ../../doc/etc/pgagroal.conf
+        echo "host all all all trust" > ../../doc/etc/pgagroal_hba.conf
         sudo cp ../../doc/etc/*.conf /etc/pgagroal
-        ./pgagroal >> /dev/null 2>&1 &
-        pid=$!
+        ./pgagroal -d || exit 1
         sleep 5
-        ./pgagroal-cli ping
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
-    - name: CLANG/Stop pgagroal & postgres
+    - name: CLANG/Run verify_running for 'kqueue' backend
       run: |
-        ./pgagroal-cli shutdown
-        installed_pg=$(brew search postgresql | grep postgresql@ | tail -n 1)
-        brew services stop ${installed_pg}
+        source /tmp/functions.sh
+        verify_running
       working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
+    - name: CLANG/Run verify_shutdown for 'kqueue' backend
+      run: |
+        source /tmp/functions.sh
+        verify_shutdown
+      working-directory: /Users/runner/work/pgagroal/pgagroal/build/src/
+