diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9565f8b..b37b7be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,6 @@ jobs: run: | docker run -d -p 5433:5433 -p 5444:5444 \ -e ODBCSYSINI=/var/odbc-loader/tests/config \ - -v ${{ github.workspace }}:/var/odbc-loader \ --add-host=host.docker.internal:host-gateway \ --name vertica_docker \ vertica/vertica-ce:12.0.4-0 @@ -48,12 +47,14 @@ jobs: docker exec -u root vertica_docker yum -y install devtoolset-7 docker exec -u root vertica_docker yum -y install unixODBC-devel docker exec -u root vertica_docker yum -y install pcre-devel + docker cp ${{ github.workspace }} vertica_docker:/var/odbc-loader + docker exec -u dbadmin vertica_docker /bin/bash -c "sudo chown -R dbadmin:verticadba /var/odbc-loader" docker exec -w /var/odbc-loader -u dbadmin vertica_docker /bin/bash -c "source /opt/rh/devtoolset-7/enable; \ make; \ make install" - name: Install ODBC clients run: | - wget https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.31-1.el7.x86_64.rpm + docker exec -w /var/odbc-loader -u root vertica_docker wget https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.31-1.el7.x86_64.rpm docker exec -w /var/odbc-loader -u root vertica_docker rpm -i mysql-connector-odbc-8.0.31-1.el7.x86_64.rpm - name: Run Tests run: | diff --git a/Makefile b/Makefile index df46dd5..ca6ddf9 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ SDK_HOME ?= /opt/vertica/sdk SHELL = /bin/bash VSQL ?= /opt/vertica/bin/vsql LOADER_DEBUG = 0 -TARGET ?= /opt/vertica/packages/odbc-loader/lib +TARGET ?= ./lib -ALL_CXXFLAGS := $(CXXFLAGS) -I $(SDK_HOME)/include -I $(SDK_HOME)/examples/HelperLibraries -fPIC -shared -Wall -g -std=c++11 -lpcrecpp -lpcre -D_GLIBCXX_USE_CXX11_ABI=0 +ALL_CXXFLAGS := $(CXXFLAGS) -I $(SDK_HOME)/include -I $(SDK_HOME)/examples/HelperLibraries -fPIC -shared -Wall -g -std=c++11 -D_GLIBCXX_USE_CXX11_ABI=0 ALL_CXXFLAGS += -DLOADER_DEBUG=$(LOADER_DEBUG) build: $(TARGET)/ODBCLoader.so @@ -56,4 +56,4 @@ test: ## Actual build target $(TARGET)/ODBCLoader.so: ODBCLoader.cpp $(SDK_HOME)/include/Vertica.cpp $(SDK_HOME)/include/BuildInfo.h mkdir -p $(TARGET) - $(CXX) $(ALL_CXXFLAGS) -o $@ $(SDK_HOME)/include/Vertica.cpp ODBCLoader.cpp -lodbc + $(CXX) $(ALL_CXXFLAGS) -o $@ $(SDK_HOME)/include/Vertica.cpp ODBCLoader.cpp -lodbc -lpcrecpp -lpcre diff --git a/README.md b/README.md index b4bc5b8..90ac124 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ COPY myschema.myverticatable ; ``` where ``rowset`` is an optional parameter to define the number of rows fetched from the remote database in each SQLFetch() call (default = 100). Increasing this parameter can improve the performance but will also increase memory usage. - + This will cause Vertica to connect to the remote database identified by the given "connect" string and execute the given query. It will then fetch the results of the query and load them into the table ``myschema.myverticatable``. ``myverticatable`` must have the same number of columns as ``remote_table``. The column types must also match up, or the ODBC driver for the remote database must be able to cast the column types to the Vertica types. If necessary, you can always explicitly cast on the remote side by modifying the query, or on the local side with a Vertica COPY expression. @@ -184,10 +184,11 @@ The following error has been reported, during the deloyment phase, on a few Linu undefined symbol: _ZNK7pcrecpp2RE13GlobalReplaceERKNS_11StringPieceEPSs ``` -To fix this issue you might want to... +#### To fix this issue you might want to... **STEP 1: get rid of the standard pcre packages**: Remove ``pcre-devel`` and ``pcre-cpp`` packages (if installed) using the appropriate package management commands. For example: + ``` # yum remove pcre-devel pcre-cpp ``` @@ -205,6 +206,30 @@ Remove ``pcre-devel`` and ``pcre-cpp`` packages (if installed) using the appropr # echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf && rm /etc/ld.so.cache && ldconfig ``` +#### But if existing version PCRE must be kept, you could... + +**STEP 1: install PCRE from sources to a dedicated location**: +``` +# tar xzvf pcre-8.45.tar.gz +# cd pcre-8.45 +# ./configure CXXFLAGS='-std=c++11 -D_GLIBCXX_USE_CXX11_ABI=0' --prefix=/opt/pcre +# make && make install +``` + +**STEP 2: set PATHs for PCRE header files and libraries**: +``` +echo 'export LD_LIBRARY_PATH=/opt/pcre/lib:${LD_LIBRARY_PATH}' >> /home/dbadmin/.bashrc + +export CPLUS_INCLUDE_PATH=/opt/pcre/include:${CPLUS_INCLUDE_PATH} +export LIBRARY_PATH=/opt/pcre/lib:${LIBRARY_PATH} +export LD_LIBRARY_PATH=/opt/pcre/lib:${LD_LIBRARY_PATH} + +# restart vertica database to effect settings +admintools -t stop_db -d testdb; admintools -t start_db -d testdb + +# Building and installing the library as mentioned before +``` + ## Sample ODBC Configurations The following two configuration files ```odbc.ini``` and ```odbcinst.ini``` have been used to define two data sources: **pmf** to connect to PostgreSQL and **mmf** to connect to MySQL: ``` diff --git a/ddl/install.sql b/ddl/install.sql index 582f368..4d0fcfa 100644 --- a/ddl/install.sql +++ b/ddl/install.sql @@ -1,4 +1,5 @@ -CREATE OR REPLACE LIBRARY ODBCLoaderLib AS '/opt/vertica/packages/odbc-loader/lib/ODBCLoader.so'; +\set libfile '\''`pwd`'/lib/ODBCLoader.so\''; +CREATE OR REPLACE LIBRARY ODBCLoaderLib AS :libfile; CREATE OR REPLACE PARSER ODBCLoader AS LANGUAGE 'C++' NAME 'ODBCLoaderFactory' LIBRARY ODBCLoaderLib FENCED; CREATE OR REPLACE SOURCE ODBCSource AS LANGUAGE 'C++' NAME 'ODBCSourceFactory' LIBRARY ODBCLoaderLib FENCED; --CREATE OR REPLACE PARSER ODBCLoader AS LANGUAGE 'C++' NAME 'ODBCLoaderFactory' LIBRARY ODBCLoaderLib NOT FENCED;