From 7d3042b80d3ef8b877615c9956ea0c584f5a278f Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Mon, 6 Nov 2017 10:56:11 -0600 Subject: [PATCH 1/6] Implement Solr 5.x-7.x testing via Docker Adds a docker-compose.yml, Dockerfile, and Solr configuration skeleton for testing Pysolr against Solr 5.x, 6.x, and 7.x. Testing against Solr 4.x is still implemented using the start-solr-test-server.sh script and a local download of Solr 4. Due to time constraints, this does not include tests for Pysolr's SolrCloud support for Solr 5.x+. These tests have been marked as "failure allowed" in the Travis configuration, so that they don't break build status in the short-term. --- .travis.yml | 57 + docker-compose.yml | 33 + docker/Dockerfile | 15 + docker/cores/core0/conf/currency.xml | 67 + docker/cores/core0/conf/elevate.xml | 38 + .../core0/conf/mapping-ISOLatin1Accent.txt | 246 +++ docker/cores/core0/conf/protwords.txt | 21 + docker/cores/core0/conf/schema.xml | 762 +++++++ docker/cores/core0/conf/solrconfig.xml | 1909 +++++++++++++++++ docker/cores/core0/conf/spellings.txt | 2 + docker/cores/core0/conf/stopwords.txt | 14 + docker/cores/core0/conf/stopwords_en.txt | 54 + docker/cores/core0/conf/synonyms.txt | 29 + docker/cores/core0/core.properties | 1 + docker/cores/solr.xml | 45 + start-solr-test-server.sh | 87 +- tests/test_admin.py | 6 + tests/utils.py | 54 + 18 files changed, 3412 insertions(+), 28 deletions(-) create mode 100644 .travis.yml create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile create mode 100644 docker/cores/core0/conf/currency.xml create mode 100644 docker/cores/core0/conf/elevate.xml create mode 100644 docker/cores/core0/conf/mapping-ISOLatin1Accent.txt create mode 100644 docker/cores/core0/conf/protwords.txt create mode 100644 docker/cores/core0/conf/schema.xml create mode 100644 docker/cores/core0/conf/solrconfig.xml create mode 100644 docker/cores/core0/conf/spellings.txt create mode 100644 docker/cores/core0/conf/stopwords.txt create mode 100644 docker/cores/core0/conf/stopwords_en.txt create mode 100644 docker/cores/core0/conf/synonyms.txt create mode 100644 docker/cores/core0/core.properties create mode 100644 docker/cores/solr.xml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..a63cd18a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,57 @@ +sudo: required +language: python +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "pypy" + +cache: + apt: true + pip: true + directories: + - $HOME/download-cache + +env: + - SOLR_VERSION="4.10.4" SOLRCLOUD=false + - SOLR_VERSION="4.10.4" SOLRCLOUD=true + - SOLR_VERSION="5.5" SOLRCLOUD=false + - SOLR_VERSION="5.5" SOLRCLOUD=true + - SOLR_VERSION="6.6" SOLRCLOUD=false + - SOLR_VERSION="6.6" SOLRCLOUD=true + - SOLR_VERSION="7.1" SOLRCLOUD=false + - SOLR_VERSION="7.1" SOLRCLOUD=true + +matrix: + allow_failures: + - env: SOLR_VERSION="5.5" SOLRCLOUD=true + - env: SOLR_VERSION="6.6" SOLRCLOUD=true + - env: SOLR_VERSION="7.1" SOLRCLOUD=false + - env: SOLR_VERSION="7.1" SOLRCLOUD=true + - python: 'pypy' + +services: + - docker + +addons: + apt_packages: + - default-jdk + +before_install: + - sudo apt-get update + - sudo apt-get install docker-ce + +install: + - "pip install 'requests>2'" + - "pip install ." + - 'if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then travis_retry pip install faulthandler; fi' + - 'if [[ "${SOLRCLOUD:-false}" == "true" ]]; then pip install -e .[solrcloud]; fi' + +script: + - python run-tests.py + +notifications: + # irc: "irc.freenode.org#pysolr" + email: false diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..54471f17 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +version: '2' + +services: + solr-base: + restart: "no" + build: + context: ./docker + ports: + - "8983:8983" + + solr-5.5: + extends: solr-base + build: + args: + PY_SOLR_VERSION: 5.5 + + solr-6.6: + extends: solr-base + build: + args: + PY_SOLR_VERSION: 6.6 + + solr-7.0: + extends: solr-base + build: + args: + PY_SOLR_VERSION: 7.0 + + solr-7.1: + extends: solr-base + build: + args: + PY_SOLR_VERSION: 7.1 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..563295e3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,15 @@ +ARG PY_SOLR_VERSION +FROM solr:${PY_SOLR_VERSION} + +ARG PY_SOLR_VERSION + +ENV CORES_DIR /opt/solr/server/solr + +USER root +COPY ./cores $CORES_DIR + +# rewrite solrconfig.xml luceneMatchVersion to match current version and re-chown +RUN perl -p -i -e "s|.*|${SOLR_VERSION}|"g ${CORES_DIR}/core0/conf/solrconfig.xml && \ + chown -R solr:solr $CORES_DIR + +USER solr diff --git a/docker/cores/core0/conf/currency.xml b/docker/cores/core0/conf/currency.xml new file mode 100644 index 00000000..3a9c58af --- /dev/null +++ b/docker/cores/core0/conf/currency.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/cores/core0/conf/elevate.xml b/docker/cores/core0/conf/elevate.xml new file mode 100644 index 00000000..25d5cebe --- /dev/null +++ b/docker/cores/core0/conf/elevate.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + diff --git a/docker/cores/core0/conf/mapping-ISOLatin1Accent.txt b/docker/cores/core0/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 00000000..ede77425 --- /dev/null +++ b/docker/cores/core0/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "À" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "ß" => "ss" +# "\t" => " " +# "\n" => "" + +# À => A +"\u00C0" => "A" + +# Á => A +"\u00C1" => "A" + +#  => A +"\u00C2" => "A" + +# à => A +"\u00C3" => "A" + +# Ä => A +"\u00C4" => "A" + +# Å => A +"\u00C5" => "A" + +# Æ => AE +"\u00C6" => "AE" + +# Ç => C +"\u00C7" => "C" + +# È => E +"\u00C8" => "E" + +# É => E +"\u00C9" => "E" + +# Ê => E +"\u00CA" => "E" + +# Ë => E +"\u00CB" => "E" + +# Ì => I +"\u00CC" => "I" + +# Í => I +"\u00CD" => "I" + +# Î => I +"\u00CE" => "I" + +# Ï => I +"\u00CF" => "I" + +# IJ => IJ +"\u0132" => "IJ" + +# Ð => D +"\u00D0" => "D" + +# Ñ => N +"\u00D1" => "N" + +# Ò => O +"\u00D2" => "O" + +# Ó => O +"\u00D3" => "O" + +# Ô => O +"\u00D4" => "O" + +# Õ => O +"\u00D5" => "O" + +# Ö => O +"\u00D6" => "O" + +# Ø => O +"\u00D8" => "O" + +# Œ => OE +"\u0152" => "OE" + +# Þ +"\u00DE" => "TH" + +# Ù => U +"\u00D9" => "U" + +# Ú => U +"\u00DA" => "U" + +# Û => U +"\u00DB" => "U" + +# Ü => U +"\u00DC" => "U" + +# Ý => Y +"\u00DD" => "Y" + +# Ÿ => Y +"\u0178" => "Y" + +# à => a +"\u00E0" => "a" + +# á => a +"\u00E1" => "a" + +# â => a +"\u00E2" => "a" + +# ã => a +"\u00E3" => "a" + +# ä => a +"\u00E4" => "a" + +# å => a +"\u00E5" => "a" + +# æ => ae +"\u00E6" => "ae" + +# ç => c +"\u00E7" => "c" + +# è => e +"\u00E8" => "e" + +# é => e +"\u00E9" => "e" + +# ê => e +"\u00EA" => "e" + +# ë => e +"\u00EB" => "e" + +# ì => i +"\u00EC" => "i" + +# í => i +"\u00ED" => "i" + +# î => i +"\u00EE" => "i" + +# ï => i +"\u00EF" => "i" + +# ij => ij +"\u0133" => "ij" + +# ð => d +"\u00F0" => "d" + +# ñ => n +"\u00F1" => "n" + +# ò => o +"\u00F2" => "o" + +# ó => o +"\u00F3" => "o" + +# ô => o +"\u00F4" => "o" + +# õ => o +"\u00F5" => "o" + +# ö => o +"\u00F6" => "o" + +# ø => o +"\u00F8" => "o" + +# œ => oe +"\u0153" => "oe" + +# ß => ss +"\u00DF" => "ss" + +# þ => th +"\u00FE" => "th" + +# ù => u +"\u00F9" => "u" + +# ú => u +"\u00FA" => "u" + +# û => u +"\u00FB" => "u" + +# ü => u +"\u00FC" => "u" + +# ý => y +"\u00FD" => "y" + +# ÿ => y +"\u00FF" => "y" + +# ff => ff +"\uFB00" => "ff" + +# fi => fi +"\uFB01" => "fi" + +# fl => fl +"\uFB02" => "fl" + +# ffi => ffi +"\uFB03" => "ffi" + +# ffl => ffl +"\uFB04" => "ffl" + +# ſt => ft +"\uFB05" => "ft" + +# st => st +"\uFB06" => "st" diff --git a/docker/cores/core0/conf/protwords.txt b/docker/cores/core0/conf/protwords.txt new file mode 100644 index 00000000..1dfc0abe --- /dev/null +++ b/docker/cores/core0/conf/protwords.txt @@ -0,0 +1,21 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +# Some non-words that normally won't be encountered, +# just to test that they won't be stemmed. +dontstems +zwhacky + diff --git a/docker/cores/core0/conf/schema.xml b/docker/cores/core0/conf/schema.xml new file mode 100644 index 00000000..58a714cb --- /dev/null +++ b/docker/cores/core0/conf/schema.xml @@ -0,0 +1,762 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/cores/core0/conf/solrconfig.xml b/docker/cores/core0/conf/solrconfig.xml new file mode 100644 index 00000000..36f931de --- /dev/null +++ b/docker/cores/core0/conf/solrconfig.xml @@ -0,0 +1,1909 @@ + + + + + + 4.10.4 + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + ${solr.hdfs.home:} + + ${solr.hdfs.confdir:} + + ${solr.hdfs.blockcache.enabled:true} + + ${solr.hdfs.blockcache.global:true} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + false + + + 2 + + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + text + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + true + json + true + + + + + + + + {!xport} + xsort + false + + + + query + + + + + + + + + + + + + + + + explicit + + + velocity + browse + layout + Solritas + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 + + text + 100% + *:* + 10 + *,score + + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0 + + text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename + 3 + + + on + true + cat + manu_exact + content_type + author_s + ipod + GB + 1 + cat,inStock + after + price + 0 + 600 + 50 + popularity + 0 + 10 + 3 + manufacturedate_dt + NOW/YEAR-10YEARS + NOW + +1YEAR + before + after + + + on + content features title name + true + html + <b> + </b> + 0 + title + 0 + name + 3 + 200 + content + 750 + + + on + false + 5 + 2 + 5 + true + true + 5 + 3 + + + + + spellcheck + + + + + + + + + + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + + + explicit + true + + + + + + + + + + + + + + + + text_general + + + + + + default + text + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + + + + + + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + + + text + true + + + tvComponent + + + + + + + lingo + + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + clustering/carrot2 + + + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + kmeans + org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm + + + + + + + true + true + + name + + id + + features + + true + + + + false + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + *:* + 10 + *,score + + + clustering + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + + + + + 5 + + + + + + + + + + + + + + + + + + *:* + + + diff --git a/docker/cores/core0/conf/spellings.txt b/docker/cores/core0/conf/spellings.txt new file mode 100644 index 00000000..d7ede6f5 --- /dev/null +++ b/docker/cores/core0/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history \ No newline at end of file diff --git a/docker/cores/core0/conf/stopwords.txt b/docker/cores/core0/conf/stopwords.txt new file mode 100644 index 00000000..ae1e83ee --- /dev/null +++ b/docker/cores/core0/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/docker/cores/core0/conf/stopwords_en.txt b/docker/cores/core0/conf/stopwords_en.txt new file mode 100644 index 00000000..2c164c0b --- /dev/null +++ b/docker/cores/core0/conf/stopwords_en.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +# Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +such +that +the +their +then +there +these +they +this +to +was +will +with diff --git a/docker/cores/core0/conf/synonyms.txt b/docker/cores/core0/conf/synonyms.txt new file mode 100644 index 00000000..7f721283 --- /dev/null +++ b/docker/cores/core0/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + diff --git a/docker/cores/core0/core.properties b/docker/cores/core0/core.properties new file mode 100644 index 00000000..59f7c530 --- /dev/null +++ b/docker/cores/core0/core.properties @@ -0,0 +1 @@ +name=core0 diff --git a/docker/cores/solr.xml b/docker/cores/solr.xml new file mode 100644 index 00000000..94d60b6a --- /dev/null +++ b/docker/cores/solr.xml @@ -0,0 +1,45 @@ + + + + + + + + + ${host:} + ${jetty.port:8983} + ${hostContext:solr} + ${zkClientTimeout:30000} + ${genericCoreNodeNames:true} + + + + ${socketTimeout:0} + ${connTimeout:0} + + + diff --git a/start-solr-test-server.sh b/start-solr-test-server.sh index 3c2e08ce..4b96b566 100755 --- a/start-solr-test-server.sh +++ b/start-solr-test-server.sh @@ -7,7 +7,9 @@ if [ ! -t 0 ]; then exec 1>test-solr.stdout.log 2>test-solr.stderr.log fi -SOLR_VERSION=4.10.4 +# SOLR_VERSION=4.10.4 +DEFAULT_SOLR_VERSION=4.10.4 +SOLR_VERSION=${SOLR_VERSION:-4.10.4} ROOT=$(cd `dirname $0`; pwd) APP=$ROOT/solr-app @@ -167,40 +169,69 @@ function prepare() { prepare_core $ROOT/solr/cloud-configs cloud } +function prepare_docker() { + docker-compose build solr-${SOLR_VERSION} +} + +function stop_docker_solr() { + docker-compose stop +} + +function start_docker_solr() { + docker-compose up -d solr-${SOLR_VERSION} +} + if [ $# -eq 0 ]; then echo "$0 [prepare] [start] [stop]" exit fi while [ $# -gt 0 ]; do - if [ "$1" = "prepare" ]; then - prepare - elif [ "$1" = "stop" ]; then - stop_solr - elif [ "$1" = "start" ]; then - echo 'Starting Solr' - confirm_down non-cloud 8983 - confirm_down cloud-zk 8992 - confirm_down cloud-node0 8993 - confirm_down cloud-node1 8994 - - start_solr $ROOT/solr/cloud-zk-node 8992 zk -DzkRun - wait_for ZooKeeper 8992 - upload_configs localhost:9992 $ROOT/solr/cloud-configs/cloud/conf - - start_solr $ROOT/solr/non-cloud 8983 non-cloud - start_solr $ROOT/solr/cloud-node0 8993 cloud-node0 -DzkHost=localhost:9992 - start_solr $ROOT/solr/cloud-node1 8994 cloud-node1 -DzkHost=localhost:9992 - wait_for simple-solr 8983 - wait_for cloud-node0 8993 - wait_for cloud-node1 8994 - create_collection 8993 core0 localhost:8993_solr,localhost:8994_solr - create_collection 8993 core1 localhost:8993_solr,localhost:8994_solr - echo 'Solr started' + if [ "$SOLR_VERSION" = "$DEFAULT_SOLR_VERSION" ]; then + if [ "$1" = "prepare" ]; then + echo "Starting Solr ${SOLR_VERSION} locally" + prepare + elif [ "$1" = "stop" ]; then + echo 'Stopping Solr' + stop_solr + elif [ "$1" = "start" ]; then + echo 'Starting Solr' + confirm_down non-cloud 8983 + confirm_down cloud-zk 8992 + confirm_down cloud-node0 8993 + confirm_down cloud-node1 8994 + + start_solr $ROOT/solr/cloud-zk-node 8992 zk -DzkRun + wait_for ZooKeeper 8992 + upload_configs localhost:9992 $ROOT/solr/cloud-configs/cloud/conf + + start_solr $ROOT/solr/non-cloud 8983 non-cloud + start_solr $ROOT/solr/cloud-node0 8993 cloud-node0 -DzkHost=localhost:9992 + start_solr $ROOT/solr/cloud-node1 8994 cloud-node1 -DzkHost=localhost:9992 + wait_for simple-solr 8983 + wait_for cloud-node0 8993 + wait_for cloud-node1 8994 + create_collection 8993 core0 localhost:8993_solr,localhost:8994_solr + create_collection 8993 core1 localhost:8993_solr,localhost:8994_solr + echo 'Solr started' + else + echo "Unknown command: $1" + exit 1 + fi else - echo "Unknown command: $1" - exit 1 + if [ "$1" = "prepare" ]; then + echo "Starting Solr ${SOLR_VERSION} via Docker" + prepare_docker + elif [ "$1" = "stop" ]; then + echo 'Stopping Solr' + stop_docker_solr + elif [ "$1" = "start" ]; then + echo 'Starting Solr' + start_docker_solr + wait_for "Solr ${SOLR_VERSION}" 8983 10 + # Solr doesn't seem to be completely ready until ~1s after it lights up its port + sleep 2 + fi fi - shift done diff --git a/tests/test_admin.py b/tests/test_admin.py index 734a85e8..4541e19e 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -6,6 +6,8 @@ from pysolr import SolrCoreAdmin +from .utils import SolrVersion + class SolrCoreAdminTestCase(unittest.TestCase): def setUp(self): @@ -16,6 +18,10 @@ def test_status(self): self.assertIn('name="defaultCoreName"', self.solr_admin.status()) self.assertIn('', self.solr_admin.status(core="core0")) + @unittest.skipIf(SolrVersion() >= 5, 'Solr 5+ does not use defaultCoreName') + def test_status_has_defaultCoreName(self): + self.assertTrue('name="defaultCoreName"' in self.solr_admin.status()) + def test_create(self): self.assertIn('0', self.solr_admin.create("wheatley")) diff --git a/tests/utils.py b/tests/utils.py index 0682ce90..25fdf32b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -2,7 +2,11 @@ from __future__ import absolute_import, unicode_literals +import os import subprocess # NOQA: B404 +from functools import total_ordering + +SOLR_VERSION = os.environ.get('SOLR_VERSION', '4.10.4') def _process(action): @@ -19,3 +23,53 @@ def start_solr(): def stop_solr(): _process("stop") + + +def solr_version_to_tuple(version): + """Given a string, integer, or tuple return a version tuple (x, y, z)""" + if isinstance(version, str): + version = version.split('.') + elif isinstance(version, int): + version = version, + + if len(version) < 2: + version += 0, + + if len(version) < 3: + version += 0, + + return tuple((int(v) for v in version)) + + +@total_ordering +class SolrVersion(object): + def __init__(self, version=SOLR_VERSION): + self.version = solr_version_to_tuple(version) + + def __eq__(self, other): + return self.version == solr_version_to_tuple(other) + + def __lt__(self, other): + return self.version < solr_version_to_tuple(other) + + +if __name__ == "__main__": + solr = SolrVersion('5.5') + assert(solr < 6) + assert(solr < '6') + assert(solr < '6.0') + assert(solr < '6.0.0') + assert(solr < (6,)) + assert(solr < (6, 0)) + assert(solr < (6, 0, 0)) + assert(solr > 5) + assert(solr > '5') + assert(solr > '5.4') + assert(solr > '5.4.2') + assert(solr > (5, 4)) + assert(solr > '4.10.4') + assert(solr > (4, 10, 4)) + assert(solr > ()) + assert(solr > (0,)) + assert(solr > (0, 0)) + assert(solr > (0, 0, 1)) From 8cc54baff58fc3fbd2da9ead43317398dfd078f9 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Thu, 9 Nov 2017 17:00:53 -0600 Subject: [PATCH 2/6] Remove comments from schema.xml/solrconfig.xml --- docker/cores/core0/conf/schema.xml | 981 ++++----------- docker/cores/core0/conf/solrconfig.xml | 1524 +----------------------- 2 files changed, 294 insertions(+), 2211 deletions(-) diff --git a/docker/cores/core0/conf/schema.xml b/docker/cores/core0/conf/schema.xml index 58a714cb..923ea7c4 100644 --- a/docker/cores/core0/conf/schema.xml +++ b/docker/cores/core0/conf/schema.xml @@ -22,741 +22,260 @@ (i.e. ./solr/conf/schema.xml by default) or located where the classloader for the Solr webapp can find it. - This example schema is the recommended starting point for users. - It should be kept correct and concise, usable out-of-the-box. + NOTE: This schema file has been stripped down and customized for use in + testing the pysolr library. All comments and unnecessary definitions have been + removed. For more information, on how to customize this file, please see http://wiki.apache.org/solr/SchemaXml - - PERFORMANCE NOTE: this schema includes many optional features and should not - be used for benchmarking. To improve performance one could - - set stored="false" for all fields possible (esp large fields) when you - only need to search on the field but don't need to return the original - value. - - set indexed="false" if you don't need to search on the field, but only - return the field as a result of searching on other indexed fields. - - remove all unneeded copyField statements - - for best index size and searching performance, set "index" to false - for all general text fields, use copyField to copy them to the - catchall "text" field, and use that for searching. - - For maximum indexing performance, use the ConcurrentUpdateSolrServer - java client. - - Remember to run the JVM in server mode, and use a higher logging level - that avoids logging every request --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/cores/core0/conf/solrconfig.xml b/docker/cores/core0/conf/solrconfig.xml index 36f931de..da0bbccf 100644 --- a/docker/cores/core0/conf/solrconfig.xml +++ b/docker/cores/core0/conf/solrconfig.xml @@ -18,59 +18,14 @@ 4.10.4 - @@ -83,584 +38,59 @@ - - - - ${solr.data.dir:} - - - - - ${solr.hdfs.home:} - ${solr.hdfs.confdir:} - ${solr.hdfs.blockcache.enabled:true} - ${solr.hdfs.blockcache.global:true} - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - true - - - + true - - - - - - - - - - + ${solr.ulog.dir:} - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - + + ${solr.autoCommit.maxTime:15000} + false + + + ${solr.autoSoftCommit.maxTime:-1} + - - - - - - - 1024 + + + + - - - - - - - - - - - - - - - - - - - - - - true - - - - - 20 - - - 200 - - - + 20 + 200 + - - - + @@ -670,231 +100,26 @@ - false - 2 - - - - - - - - - - - - - - explicit 10 text - - - - - - - - - - + + explicit @@ -904,18 +129,6 @@ - - true @@ -924,11 +137,6 @@ - - {!xport} @@ -941,35 +149,17 @@ - - - - - - - - explicit - velocity browse layout Solritas - edismax text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 @@ -988,7 +178,6 @@ text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename 3 - on true cat @@ -1015,7 +204,6 @@ before after - on content features title name true @@ -1031,7 +219,6 @@ content 750 - on false 5 @@ -1043,168 +230,27 @@ 3 - spellcheck + - - - - - - - - - - + true ignored_ - - true links ignored_ + - - - - - - - - - - - - - - - solrpingquery @@ -1212,15 +258,8 @@ all - - - explicit @@ -1228,124 +267,24 @@ - - - - - - - - - - - text_general - - - default text solr.DirectSolrSpellChecker - internal - 0.5 - 2 - 1 - 5 - 4 - 0.01 - - wordbreak solr.WordBreakSolrSpellChecker @@ -1354,66 +293,11 @@ true 10 - - - - - - - - - - text - default wordbreak on @@ -1431,42 +315,8 @@ - - - - text @@ -1477,92 +327,32 @@ - - + lingo - - org.carrot2.clustering.lingo.LingoClusteringAlgorithm - - clustering/carrot2 - - stc org.carrot2.clustering.stc.STCClusteringAlgorithm - - kmeans org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm - - + true true - name - id - features - true - - - false - - edismax text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 @@ -1576,16 +366,8 @@ - - true @@ -1596,22 +378,11 @@ - - - string elevate.xml - explicit @@ -1622,76 +393,37 @@ - - - - + 100 - - + - 70 - 0.5 - [-\w ,/\n\"']{20,200} - - + ]]> ]]> - - - - - + - - + + + - - - - - - - - - + @@ -1705,8 +437,7 @@ - 10 @@ -1717,10 +448,7 @@ - WORD - - en US @@ -1728,180 +456,16 @@ - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - + - 5 - - - - - - - - - - - - - - *:* From 4dd6cccab30b28b2a332e1aa6a1f59942a8078be Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:41:03 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docker/cores/core0/conf/currency.xml | 8 ++--- docker/cores/core0/conf/elevate.xml | 4 +-- docker/cores/core0/conf/protwords.txt | 1 - docker/cores/core0/conf/spellings.txt | 2 +- docker/cores/core0/conf/synonyms.txt | 1 - docker/cores/solr.xml | 6 ++-- tests/test_admin.py | 2 +- tests/utils.py | 48 +++++++++++++-------------- 8 files changed, 35 insertions(+), 37 deletions(-) diff --git a/docker/cores/core0/conf/currency.xml b/docker/cores/core0/conf/currency.xml index 3a9c58af..654de41f 100644 --- a/docker/cores/core0/conf/currency.xml +++ b/docker/cores/core0/conf/currency.xml @@ -58,10 +58,10 @@ - + - - - + + + diff --git a/docker/cores/core0/conf/elevate.xml b/docker/cores/core0/conf/elevate.xml index 25d5cebe..8f3aa80f 100644 --- a/docker/cores/core0/conf/elevate.xml +++ b/docker/cores/core0/conf/elevate.xml @@ -29,10 +29,10 @@ - + - + diff --git a/docker/cores/core0/conf/protwords.txt b/docker/cores/core0/conf/protwords.txt index 1dfc0abe..4341c051 100644 --- a/docker/cores/core0/conf/protwords.txt +++ b/docker/cores/core0/conf/protwords.txt @@ -18,4 +18,3 @@ # just to test that they won't be stemmed. dontstems zwhacky - diff --git a/docker/cores/core0/conf/spellings.txt b/docker/cores/core0/conf/spellings.txt index d7ede6f5..162a044d 100644 --- a/docker/cores/core0/conf/spellings.txt +++ b/docker/cores/core0/conf/spellings.txt @@ -1,2 +1,2 @@ pizza -history \ No newline at end of file +history diff --git a/docker/cores/core0/conf/synonyms.txt b/docker/cores/core0/conf/synonyms.txt index 7f721283..0ef0e8da 100644 --- a/docker/cores/core0/conf/synonyms.txt +++ b/docker/cores/core0/conf/synonyms.txt @@ -26,4 +26,3 @@ Television, Televisions, TV, TVs # Synonym mappings can be used for spelling correction too pixima => pixma - diff --git a/docker/cores/solr.xml b/docker/cores/solr.xml index 94d60b6a..d68cd6a9 100644 --- a/docker/cores/solr.xml +++ b/docker/cores/solr.xml @@ -17,11 +17,11 @@ --> diff --git a/tests/test_admin.py b/tests/test_admin.py index 4541e19e..939aef2b 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -18,7 +18,7 @@ def test_status(self): self.assertIn('name="defaultCoreName"', self.solr_admin.status()) self.assertIn('', self.solr_admin.status(core="core0")) - @unittest.skipIf(SolrVersion() >= 5, 'Solr 5+ does not use defaultCoreName') + @unittest.skipIf(SolrVersion() >= 5, "Solr 5+ does not use defaultCoreName") def test_status_has_defaultCoreName(self): self.assertTrue('name="defaultCoreName"' in self.solr_admin.status()) diff --git a/tests/utils.py b/tests/utils.py index 25fdf32b..5503fd69 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -6,7 +6,7 @@ import subprocess # NOQA: B404 from functools import total_ordering -SOLR_VERSION = os.environ.get('SOLR_VERSION', '4.10.4') +SOLR_VERSION = os.environ.get("SOLR_VERSION", "4.10.4") def _process(action): @@ -28,15 +28,15 @@ def stop_solr(): def solr_version_to_tuple(version): """Given a string, integer, or tuple return a version tuple (x, y, z)""" if isinstance(version, str): - version = version.split('.') + version = version.split(".") elif isinstance(version, int): - version = version, + version = (version,) if len(version) < 2: - version += 0, + version += (0,) if len(version) < 3: - version += 0, + version += (0,) return tuple((int(v) for v in version)) @@ -54,22 +54,22 @@ def __lt__(self, other): if __name__ == "__main__": - solr = SolrVersion('5.5') - assert(solr < 6) - assert(solr < '6') - assert(solr < '6.0') - assert(solr < '6.0.0') - assert(solr < (6,)) - assert(solr < (6, 0)) - assert(solr < (6, 0, 0)) - assert(solr > 5) - assert(solr > '5') - assert(solr > '5.4') - assert(solr > '5.4.2') - assert(solr > (5, 4)) - assert(solr > '4.10.4') - assert(solr > (4, 10, 4)) - assert(solr > ()) - assert(solr > (0,)) - assert(solr > (0, 0)) - assert(solr > (0, 0, 1)) + solr = SolrVersion("5.5") + assert solr < 6 + assert solr < "6" + assert solr < "6.0" + assert solr < "6.0.0" + assert solr < (6,) + assert solr < (6, 0) + assert solr < (6, 0, 0) + assert solr > 5 + assert solr > "5" + assert solr > "5.4" + assert solr > "5.4.2" + assert solr > (5, 4) + assert solr > "4.10.4" + assert solr > (4, 10, 4) + assert solr > () + assert solr > (0,) + assert solr > (0, 0) + assert solr > (0, 0, 1) From 31c9ebcbc66ababe4fd64a67c86a2cf37522a3d4 Mon Sep 17 00:00:00 2001 From: Jason Novinger Date: Tue, 27 Jun 2023 12:55:35 -0500 Subject: [PATCH 4/6] Disable spellcheck in XML files --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9cfc94fe..53ce67a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,4 +85,4 @@ max-statements = 54 [tool.codespell] ignore-words-list = "dekstop,nwe,yello" -skip="./.*" +skip="./.*,*.xml" From 9a7fa17cf7e55bf8dc1abfd8244ac3418179186c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 9 Dec 2023 03:16:05 +0100 Subject: [PATCH 5/6] Update pyproject.toml: codespell --skip="./.*,*.xml" --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7976829..b8bcb31f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,4 +86,4 @@ max-statements = 54 [tool.codespell] ignore-words-list = "dekstop,nwe,wahtever,yello" -skip="./.*" +skip="./.*,*.xml" From 98c77c02506c976e4367cf7f38d2a16ee4f2113d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 May 2024 18:10:12 +0200 Subject: [PATCH 6/6] Delete .travis.yml --- .travis.yml | 57 ----------------------------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a63cd18a..00000000 --- a/.travis.yml +++ /dev/null @@ -1,57 +0,0 @@ -sudo: required -language: python -python: - - "2.7" - - "3.3" - - "3.4" - - "3.5" - - "3.6" - - "pypy" - -cache: - apt: true - pip: true - directories: - - $HOME/download-cache - -env: - - SOLR_VERSION="4.10.4" SOLRCLOUD=false - - SOLR_VERSION="4.10.4" SOLRCLOUD=true - - SOLR_VERSION="5.5" SOLRCLOUD=false - - SOLR_VERSION="5.5" SOLRCLOUD=true - - SOLR_VERSION="6.6" SOLRCLOUD=false - - SOLR_VERSION="6.6" SOLRCLOUD=true - - SOLR_VERSION="7.1" SOLRCLOUD=false - - SOLR_VERSION="7.1" SOLRCLOUD=true - -matrix: - allow_failures: - - env: SOLR_VERSION="5.5" SOLRCLOUD=true - - env: SOLR_VERSION="6.6" SOLRCLOUD=true - - env: SOLR_VERSION="7.1" SOLRCLOUD=false - - env: SOLR_VERSION="7.1" SOLRCLOUD=true - - python: 'pypy' - -services: - - docker - -addons: - apt_packages: - - default-jdk - -before_install: - - sudo apt-get update - - sudo apt-get install docker-ce - -install: - - "pip install 'requests>2'" - - "pip install ." - - 'if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then travis_retry pip install faulthandler; fi' - - 'if [[ "${SOLRCLOUD:-false}" == "true" ]]; then pip install -e .[solrcloud]; fi' - -script: - - python run-tests.py - -notifications: - # irc: "irc.freenode.org#pysolr" - email: false