Skip to content

Commit

Permalink
Add tlist function pushdown support and Bug fixes
Browse files Browse the repository at this point in the history 
Support function pushdown in the target list (for PGSpider)
Bug fixes
- Fix FETCH ... WITH TIES issue
- Fix sqlite_fdw does not bind the correct numeric value when it is sub-query
  • Loading branch information
@khieuvm
khieuvm authored Jan 14, 2021
1 parent 0f673fc commit 1745a69
Show file tree
Hide file tree
Showing 24 changed files with 1,552 additions and 295 deletions.
232 changes: 164 additions & 68 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,16 @@
def NODE_NAME = 'AWS_Instance_CentOS'
def MAIL_TO = '$DEFAULT_RECIPIENTS'
def MAIL_SUBJECT = '[CI PGSpider] SQLite FDW Test FAILED ' + BRANCH_NAME
def BUILD_INFO = 'Jenkins job: ' + env.BUILD_URL
def BRANCH_NAME = 'Branch [' + env.BRANCH_NAME + ']'
def BUILD_INFO = 'Jenkins job: ' + env.BUILD_URL + '\n'

def POSTGRES_DOCKER_PATH = '/home/jenkins/Docker/Server/Postgres'
def ENHANCE_TEST_DOCKER_PATH = '/home/jenkins/Docker'
def TEST_TYPE = 'SQLITE'

START_EXISTED_TEST = ''
START_ENHANCE_TEST = ''
INIT_ENHANCE_TEST = ''

def retrySh(String shCmd) {
def MAX_RETRY = 10
script {
int status = 1;
for (int i = 0; i < MAX_RETRY; i++) {
status = sh(returnStatus: true, script: shCmd)
if (status == 0) {
echo "SUCCESS: " + shCmd
break
} else {
echo "RETRY: " + shCmd
sleep 5
}
}
if (status != 0) {
sh(shCmd)
}
}
}

pipeline {
agent {
Expand All @@ -33,7 +21,7 @@ pipeline {
options {
gitLabConnection('GitLabConnection')
}
triggers {
triggers {
gitlab(
triggerOnPush: true,
triggerOnMergeRequest: false,
Expand All @@ -46,76 +34,184 @@ pipeline {
)
}
stages {
stage('Build') {
stage('Start_containers_Existed_Test') {
steps {
sh '''
rm -rf postgresql-13beta2 || true
tar -zxvf /home/jenkins/Postgres/postgresql-13beta2.tar.gz > /dev/null
cd postgresql-13beta2
./configure --prefix=$(pwd)/install > /dev/null
make clean && make > /dev/null
'''
dir("postgresql-13beta2/contrib") {
sh 'rm -rf sqlite_fdw || true'
retrySh('git clone -b ' + env.GIT_BRANCH + ' ' + env.GIT_URL)
script {
if (env.GIT_URL != null) {
BUILD_INFO = BUILD_INFO + "Git commit: " + env.GIT_URL.replace(".git", "/commit/") + env.GIT_COMMIT + "\n"
}
}
catchError() {
sh """
cd ${POSTGRES_DOCKER_PATH}
docker build -t postgresserver .
docker run -d --name postgresserver_for_sqlite_existed_test postgresserver
"""
}
}
post {
failure {
echo '** BUILD FAILED !!! NEXT STAGE WILL BE SKIPPED **'
emailext subject: "${MAIL_SUBJECT}", body: BUILD_INFO + "\nGit commit: " + env.GIT_URL.replace(".git", "/commit/") + env.GIT_COMMIT + "\n" + '${BUILD_LOG, maxLines=200, escapeHtml=false}', to: "${MAIL_TO}", attachLog: false
script {
START_EXISTED_TEST = 'FAILED'
}
updateGitlabCommitStatus name: 'Build', state: 'failed'
}
success {
script {
START_EXISTED_TEST = 'SUCCESS'
}
updateGitlabCommitStatus name: 'Build', state: 'success'
}
}
}
stage('sqlite_fdw_test') {
stage('make_check_Existed_Test') {
steps {
dir("postgresql-13beta2/contrib/sqlite_fdw") {
catchError() {
sh '''
chmod +x *.sh
./test.sh
'''
catchError() {
sh """
rm -rf make_check_existed_test.out || true
docker exec postgresserver_for_sqlite_existed_test /bin/bash -c 'su -c "/tmp/sqlite_existed_test.sh ${env.GIT_BRANCH}" postgres'
docker cp postgresserver_for_sqlite_existed_test:/home/postgres/postgresql-13beta2/contrib/sqlite_fdw/make_check.out make_check_existed_test.out
"""
}
script {
status = sh(returnStatus: true, script: "grep -q 'All [0-9]* tests passed' 'make_check_existed_test.out'")
if (status != 0) {
unstable(message: "Set UNSTABLE result")
sh 'docker cp postgresserver_for_sqlite_existed_test:/home/postgres/postgresql-13beta2/contrib/sqlite_fdw/regression.diffs regression.diffs'
sh 'cat regression.diffs || true'
updateGitlabCommitStatus name: 'make_check', state: 'failed'
} else {
updateGitlabCommitStatus name: 'make_check', state: 'success'
}
}
}
}
stage('Start_containers_Enhance_Test') {
steps {
catchError() {
sh """
cd ${ENHANCE_TEST_DOCKER_PATH}
docker-compose up --build -d
"""
}
}
post {
failure {
script {
status = sh(returnStatus: true, script: "grep -q 'All [0-9]* tests passed' 'make_check.out'")
if (status != 0) {
unstable(message: "Set UNSTABLE result")
emailext subject: "${MAIL_SUBJECT}", body: BUILD_INFO + "\nGit commit: " + env.GIT_URL.replace(".git", "/commit/") + env.GIT_COMMIT + "\n" + '${FILE,path="make_check.out"}', to: "${MAIL_TO}", attachLog: false
sh 'cat regression.diffs || true'
updateGitlabCommitStatus name: 'sqlite_fdw_test', state: 'failed'
} else {
updateGitlabCommitStatus name: 'sqlite_fdw_test', state: 'success'
}
START_ENHANCE_TEST = 'FAILED'
}
updateGitlabCommitStatus name: 'Build', state: 'failed'
}
success {
script {
START_ENHANCE_TEST = 'SUCCESS'
}
updateGitlabCommitStatus name: 'Build', state: 'success'
}
}
}
stage('sqlite_fdw_test_extra') {
stage('Initialize_for_Enhance_Test') {
steps {
dir("postgresql-13beta2/contrib/sqlite_fdw") {
catchError() {
sh '''
chmod +x *.sh
./test_extra.sh
'''
catchError() {
sh """
docker exec mysqlserver1_enhance_test /bin/bash -c '/tmp/start_enhance_test.sh'
docker exec mysqlserver2_enhance_test /bin/bash -c '/tmp/start_enhance_test.sh'
docker exec postgresserver1_enhance_test /bin/bash -c '/tmp/start_enhance_test_1.sh'
docker exec postgresserver2_enhance_test /bin/bash -c '/tmp/start_enhance_test_2.sh'
docker exec tinybraceserver1_enhance_test /bin/bash -c '/tmp/start_enhance_test_1.sh'
docker exec -d -w /usr/local/tinybrace tinybraceserver1_enhance_test /bin/bash -c 'bin/tbserver &'
docker exec tinybraceserver2_enhance_test /bin/bash -c '/tmp/start_enhance_test_2.sh'
docker exec -d -w /usr/local/tinybrace tinybraceserver2_enhance_test /bin/bash -c 'bin/tbserver &'
docker exec influxserver1_enhance_test /bin/bash -c '/tmp/start_enhance_test.sh'
docker exec influxserver2_enhance_test /bin/bash -c '/tmp/start_enhance_test.sh'
docker exec -d gridserver1_enhance_test /bin/bash -c '/tmp/start_enhance_test_1.sh'
sleep 10
docker exec -d gridserver2_enhance_test /bin/bash -c '/tmp/start_enhance_test_2.sh'
sleep 10
docker exec pgspiderserver1_enhance_test /bin/bash -c 'su -c "/tmp/start_enhance_test.sh ${env.GIT_BRANCH} ${TEST_TYPE}" pgspider'
"""
}
}
post {
failure {
script {
INIT_ENHANCE_TEST = 'FAILED'
}
updateGitlabCommitStatus name: 'Init_Data', state: 'failed'
}
success {
script {
status = sh(returnStatus: true, script: "grep -q 'All [0-9]* tests passed' 'make_check.out'")
if (status != 0) {
unstable(message: "Set UNSTABLE result")
emailext subject: "${MAIL_SUBJECT}", body: BUILD_INFO + "\nGit commit: " + env.GIT_URL.replace(".git", "/commit/") + env.GIT_COMMIT + "\n" + '${FILE,path="make_check.out"}', to: "${MAIL_TO}", attachLog: false
sh 'cat regression.diffs || true'
updateGitlabCommitStatus name: 'sqlite_fdw_test_extra', state: 'failed'
} else {
updateGitlabCommitStatus name: 'sqlite_fdw_test_extra', state: 'success'
}
INIT_ENHANCE_TEST = 'SUCCESS'
}
updateGitlabCommitStatus name: 'Init_Data', state: 'success'
}
}
}
stage('make_check_Enhance_Test') {
steps {
catchError() {
sh """
rm -rf make_check_enhance_test.out regression.diffs || true
docker exec -w /home/pgspider/GIT/PGSpider/contrib/pgspider_core_fdw pgspiderserver1_enhance_test /bin/bash -c 'su -c "chmod a+x *.sh" pgspider'
docker exec -w /home/pgspider/GIT/PGSpider/contrib/pgspider_core_fdw pgspiderserver1_enhance_test /bin/bash -c "sed -i 's/enhance\\\\\\\\\\/BasicFeature1_File_4ARG enhance\\\\\\\\\\/BasicFeature1_File_AllARG enhance\\\\\\\\\\/BasicFeature1_GridDB_4ARG enhance\\\\\\\\\\/BasicFeature1_GridDB_AllARG enhance\\\\\\\\\\/BasicFeature1_InfluxDB_4ARG enhance\\\\\\\\\\/BasicFeature1_InfluxDB_AllARG enhance\\\\\\\\\\/BasicFeature1_MySQL_4ARG enhance\\\\\\\\\\/BasicFeature1_MySQL_AllARG enhance\\\\\\\\\\/BasicFeature1_PostgreSQL_4ARG enhance\\\\\\\\\\/BasicFeature1_PostgreSQL_AllARG enhance\\\\\\\\\\/BasicFeature1_SQLite_4ARG enhance\\\\\\\\\\/BasicFeature1_SQLite_AllARG enhance\\\\\\\\\\/BasicFeature1_TinyBrace_4ARG enhance\\\\\\\\\\/BasicFeature1_TinyBrace_AllARG enhance\\\\\\\\\\/BasicFeature1_t_max_range enhance\\\\\\\\\\/BasicFeature1_tmp_t15_4ARG enhance\\\\\\\\\\/BasicFeature1_tmp_t15_AllARG enhance\\\\\\\\\\/BasicFeature2_JOIN_Multi_Tbl enhance\\\\\\\\\\/BasicFeature2_SELECT_Muli_Tbl enhance\\\\\\\\\\/BasicFeature2_UNION_Multi_Tbl enhance\\\\\\\\\\/BasicFeature_Additional_Test enhance\\\\\\\\\\/BasicFeature_ComplexCommand enhance\\\\\\\\\\/BasicFeature_For_Bug_54 enhance\\\\\\\\\\/BasicFeature_For_Bug_60/enhance\\\\\\\\\\/BasicFeature1_SQLite_4ARG enhance\\\\\\\\\\/BasicFeature1_SQLite_AllARG/g' test_enhance.sh"
docker exec -w /home/pgspider/GIT/PGSpider/contrib/pgspider_core_fdw pgspiderserver1_enhance_test /bin/bash -c 'su -c "./test_enhance.sh" pgspider'
docker cp pgspiderserver1_enhance_test:/home/pgspider/GIT/PGSpider/contrib/pgspider_core_fdw/make_check.out make_check_enhance_test.out
"""
}
script {
status = sh(returnStatus: true, script: "grep -q 'All [0-9]* tests passed' 'make_check_enhance_test.out'")
if (status != 0) {
unstable(message: "Set UNSTABLE result")
sh 'docker cp pgspiderserver1_enhance_test:/home/pgspider/GIT/PGSpider/contrib/pgspider_core_fdw/regression.diffs regression.diffs'
sh 'cat regression.diffs || true'
updateGitlabCommitStatus name: 'make_check', state: 'failed'
} else {
updateGitlabCommitStatus name: 'make_check', state: 'success'
}
}
}
}
}
}
post {
success {
script {
prevResult = 'SUCCESS'
if (currentBuild.previousBuild != null) {
prevResult = currentBuild.previousBuild.result.toString()
}
if (prevResult != 'SUCCESS') {
emailext subject: '[CI SQLITE_FDW] InfluxDB_Test BACK TO NORMAL on ' + BRANCH_NAME, body: BUILD_INFO + '\n---------EXISTED_TEST---------\n' + '${FILE,path="make_check_existed_test.out"}' + '\n---------ENHANCE_TEST---------\n' + '${FILE,path="make_check_enhance_test.out"}', to: "${MAIL_TO}", attachLog: false
}
}
}
unsuccessful {
script {
if (START_EXISTED_TEST == 'FAILED') {
if (START_ENHANCE_TEST == 'FAILED') {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Start Containers FAILED | ENHANCE_TEST: Start Containers FAILED ' + BRANCH_NAME, body: BUILD_INFO + '${BUILD_LOG, maxLines=200, escapeHtml=false}', to: "${MAIL_TO}", attachLog: false
} else if (INIT_ENHANCE_TEST == 'FAILED') {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Start Containers FAILED | ENHANCE_TEST: Initialize FAILED ' + BRANCH_NAME, body: BUILD_INFO + '${BUILD_LOG, maxLines=200, escapeHtml=false}', to: "${MAIL_TO}", attachLog: false
} else {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Start Containers FAILED | ENHANCE_TEST: Result make check ' + BRANCH_NAME, body: BUILD_INFO + '${FILE,path="make_check_enhance_test.out"}', to: "${MAIL_TO}", attachLog: false
}
} else {
if (START_ENHANCE_TEST == 'FAILED') {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Result make check | ENHANCE_TEST: Start Containers FAILED ' + BRANCH_NAME, body: BUILD_INFO + '${FILE,path="make_check_existed_test.out"}', to: "${MAIL_TO}", attachLog: false
} else if (INIT_ENHANCE_TEST == 'FAILED') {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Result make check | ENHANCE_TEST: Initialize FAILED ' + BRANCH_NAME, body: BUILD_INFO + '${FILE,path="make_check_existed_test.out"}', to: "${MAIL_TO}", attachLog: false
} else {
emailext subject: '[CI SQLITE_FDW] EXISTED_TEST: Result make check | ENHANCE_TEST: Result make check ' + BRANCH_NAME, body: BUILD_INFO + '\n---------EXISTED_TEST---------\n' + '${FILE,path="make_check_existed_test.out"}' + '\n---------ENHANCE_TEST---------\n' + '${FILE,path="make_check_enhance_test.out"}', to: "${MAIL_TO}", attachLog: false
}
}
}
}
always {
sh """
docker stop postgresserver_for_sqlite_existed_test
docker rm postgresserver_for_sqlite_existed_test
cd ${ENHANCE_TEST_DOCKER_PATH}
docker-compose down
"""
}
}
}
4 changes: 2 additions & 2 deletions META.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
"name": "sqlite_fdw",
"abstract": "Foreign Data Wrapper for SQLite databases",
"description": "PostgreSQL extension which implements a Foreign Data Wrapper (FDW) for SQLite databases.",
"version": "1.2.0",
"version": "1.3.1",
"maintainer": "mochizk",
"license": "postgresql",
"provides": {
"sqlite_fdw": {
"abstract": "Foreign Data Wrapper for SQLite databases",
"file": "sqlite_fdw.c",
"docfile": "README.md",
"version": "1.2.0"
"version": "1.3.1"
}
},
"prereqs": {
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ OBJS = connection.o option.o deparse.o sqlite_query.o sqlite_fdw.o
EXTENSION = sqlite_fdw
DATA = sqlite_fdw--1.0.sql

REGRESS = extra/sqlite_fdw_post extra/float4 extra/float8 extra/int4 extra/int8 extra/numeric extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/timestamp sqlite_fdw type aggregate
REGRESS = extra/sqlite_fdw_post extra/float4 extra/float8 extra/int4 extra/int8 extra/numeric extra/join extra/limit extra/aggregates extra/prepare extra/select_having extra/select extra/insert extra/update extra/timestamp sqlite_fdw type aggregate selectfunc

SQLITE_LIB = sqlite3

Expand Down
Loading

0 comments on commit 1745a69

Please sign in to comment.