Merge branch 'master' of github.com:netarchivesuite/solrwayback

CHANGES.md

@@ -2,10 +2,17 @@
 
 
 
+
 5.2.0
 -----
 Geo search was not working for Solr 9.4 in cloud mode. The fix still works against Solr7 but also with Solr9.4 cloud mode.
 
+5.1.2
+-----
+Bug fix. Chunking was not removed in all cases. This was only relevant for WARC-files that are created with chunking. (not Heritrix) 
+
+Dockerfile has been updated to build SolrWayback bundle 5.1.0. See: https://github.com/netarchivesuite/solrwayback/pull/456 implemented by @c-vandendyck-kbr
+
 5.1.1
 -----
 Little cleanup in log messages due to shard-splitting to avoid repeated stack traces.

Dockerfile

@@ -7,11 +7,11 @@
 # where <path/to/WARCs> is a file path that only contains WARC files and directories.
 
 # When the container is running, run the following commands to start Solr and Tomcat:
-# export SOLRWAYBACK_VERSION=4.4.2
-# export APACHE_TOMCAT_VERSION=8.5.60
-# export SOLR_VERSION=7.7.3
-# ./unpacked-bundle/solrwayback_package_$SOLRWAYBACK_VERSION/solr-$SOLR_VERSION/bin/solr start
-# ./unpacked-bundle/solrwayback_package_$SOLRWAYBACK_VERSION/apache-tomcat-$APACHE_TOMCAT_VERSION/bin/startup.sh
+# export SOLRWAYBACK_VERSION=5.1.0
+# export APACHE_TOMCAT_VERSION=9
+# export SOLR_VERSION=9
+# ./unpacked-bundle/solrwayback_package_$SOLRWAYBACK_VERSION/solr-$SOLR_VERSION/bin/solr start -c
+# ./unpacked-bundle/solrwayback_package_$SOLRWAYBACK_VERSION/tomcat-$APACHE_TOMCAT_VERSION/bin/startup.sh
 
 # You should now verify that the following links works with a browser:
 # http://localhost:8080/solrwayback/
@@ -23,9 +23,9 @@
 
 FROM ubuntu:22.04
 
-ENV SOLRWAYBACK_VERSION 4.4.2
-ENV APACHE_TOMCAT_VERSION 8.5.60
-ENV SOLR_VERSION 7.7.3
+ENV SOLRWAYBACK_VERSION 5.1.0
+ENV APACHE_TOMCAT_VERSION 9
+ENV SOLR_VERSION 9
 
 RUN apt-get update --assume-yes --quiet
 RUN apt-get install wget unzip --assume-yes --quiet
@@ -53,9 +53,9 @@ RUN cp unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/properties/sol
 RUN cp unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/properties/solrwaybackweb.properties .
 
 # Verify that apache-tomcat works
-RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/apache-tomcat-${APACHE_TOMCAT_VERSION}/bin/startup.sh
-RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/apache-tomcat-${APACHE_TOMCAT_VERSION}/bin/shutdown.sh
+RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/tomcat-${APACHE_TOMCAT_VERSION}/bin/startup.sh
+RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/tomcat-${APACHE_TOMCAT_VERSION}/bin/shutdown.sh
 
 # Verify that solr works
-RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/solr-${SOLR_VERSION}/bin/solr start
+RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/solr-${SOLR_VERSION}/bin/solr start -c
 RUN unpacked-bundle/solrwayback_package_${SOLRWAYBACK_VERSION}/solr-${SOLR_VERSION}/bin/solr stop -all

README.md

@@ -136,6 +136,7 @@ Documents in SolrWayback are indexed through the [warc-indexer](https://github.c
  * (Optional) chrome/(chromium) installed for page previews to work. (headless chrome) 
 
 ## Build and usage for developers.
+ * Make sure to use a supported Java version, if necessary set the `JAVA_HOME` accordingly (e.g. `export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-11.0.23.0.9-1.fc39.x86_64`)
  * Build the application with: `mvn package`
  * Deploy the `target/solrwayback-*.war` file in a web-container
  * Copy `properties/solrwayback.properties` and `properties/solrwaybackweb.properties` to the `user/home/` folder.  
@@ -144,7 +145,7 @@ Documents in SolrWayback are indexed through the [warc-indexer](https://github.c
 
 
 ## Build, Index and test with Docker
-The docker container will download the SolrWayback bundle 4.4.2.
+The docker container will download the SolrWayback bundle 5.1.0.
 You can index WARC files from a folder outside the docker container and index them. 
 A containerized sample can be found [here](./Dockerfile)
 Read the docker file for documentation.
@@ -186,13 +187,13 @@ SolrWayback requires both Solr and Tomcat to be running. These processes are sta
 
 * Start tomcat: `tomcat-9/bin/startup.sh`  
 * Stop tomcat:  `tomcat-9/bin/shutdown.sh`  
-* (For windows navigate to `tomcat-9/bin/` and type `startup.bat` or `shutdown.bat`)  
+* (For windows navigate to `tomcat-9\bin\` and type `startup.bat` or `shutdown.bat`)  
 * To see Tomcat is running open: http://localhost:8080/solrwayback/  
 
 #### Solr:  
 * Start solr: `solr-9/bin/solr start -c -m 4g`  (start with 8g or 16g if you have an index with over 100M records.)
 * Stop solr: `solr-9/bin/solr stop -all`  
-* (For windows navigate to `solr-9/bin/` and type `solr.cmd start -c -m 4g` or `solr.cmd stop -all`)    
+* (For windows navigate to `solr-9\bin\` and type `solr.cmd start -c -m 4g` or `solr.cmd stop -all`)    
 * To see Solr is running open: http://localhost:8983/solr/#/netarchivebuilder  
 * For Solrwayback bundle version before v.5 the '-c' parameter must be omitted.
 
@@ -233,8 +234,8 @@ You can then enable faceting on these fields in the property file: `solrwaybackw
 
 Indexing works a little different on Windows. This also works on Linux and Mac, however we recommend using the `warc-indexer.sh` as above. 
 
-* **Step 1:** Copy ARC/WARC files into the folder: `indexing/warcs1`
-* **Step 2:** To index the files call `indexing/batch_warcs1_folder.bat` (batch_warcs1_folder.sh for Linux and Mac)
+* **Step 1:** Copy ARC/WARC files into the folder: `indexing\warcs1`
+* **Step 2:** To index the files call `indexing\batch_warcs1_folder.bat` (batch_warcs1_folder.sh for Linux and Mac)
 * **Note:** There is a batch_warcs2_folder.sh similar script to show how to easily add new WARC files to the collection without indexing the old ones again.
 
 

pom.xml

@@ -4,7 +4,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>dk.kb.netarchivesuite.solrwayback</groupId>
   <artifactId>solrwayback</artifactId>
-  <version>5.1.1</version>
+  <version>5.1.2-SNAPSHOT</version>
   <packaging>war</packaging>
   <name>solrwayback</name>
   <url>https://maven.apache.org</url>

src/bundle/properties/solrwaybackweb.properties

@@ -83,7 +83,7 @@ leaflet.attribution=&copy; <a href="https://www.openstreetmap.org/copyright">Ope
 # Wordcloud stop words. 
 
 # Danish
-#wordcloud.stopwords=ad,af,aldrig,alle,alt,anden,andet,andre,at,bare,begge,blev,blive,bliver,da,de,dem,den,denne,der,deres,det,dette,dig,din,dine,disse,dit,dog,du,efter,ej,eller,en,end,ene,eneste,enhver,er,et,fem,fik,fire,flere,fleste,for,fordi,forrige,fra,f�,f�r,f�r,god,godt,ham,han,hans,har,havde,have,hej,helt,hende,hendes,her,hos,hun,hvad,hvem,hver,hvilken,hvis,hvor,hvordan,hvorfor,hvorn�r,i,ikke,ind,ingen,intet,ja,jeg,jer,jeres,jo,kan,kom,komme,kommer,kun,kunne,lad,lav,lidt,lige,lille,man,mange,med,meget,men,mens,mere,mig,min,mine,mit,mod,m�,ned,nej,ni,nogen,noget,nogle,nu,ny,nyt,n�r,n�r,n�ste,n�sten,og,ogs�,okay,om,op,os,otte,over,p�,se,seks,selv,ser,ses,sig,sige,sin,sine,sit,skal,skulle,som,stor,store,syv,s�,s�dan,tag,tage,thi,ti,til,to,tre,ud,under,var,ved,vi,vil,ville,vor,vores,v�re,v�ret,alene,allerede,alligevel,altid,bag,blandt,burde,b�r,dens,derefter,derfor,derfra,deri,dermed,derp�,derved,egen,ellers,endnu,ens,enten,flest,foran,f�rst,gennem,gjorde,gjort,g�r,g�re,g�rende,hel,heller,hen,henover,herefter,heri,hermed,herp�,hvilke,hvilkes,hvorefter,hvorfra,hvorhen,hvori,hvorimod,hvorved,igen,igennem,imellem,imens,imod,indtil,langs,lave,lavet,ligesom,l�ngere,mellem,mest,mindre,mindst,m�ske,nemlig,nogensinde,nok,omkring,overalt,samme,sammen,selvom,senere,siden,stadig,synes,syntes,s�ledes,temmelig,tidligere,tilbage,tit,uden,udover,undtagen,via,vore,v�r,�vrigt
+#wordcloud.stopwords=ad,af,aldrig,alle,alt,anden,andet,andre,at,bare,begge,blev,blive,bliver,da,de,dem,den,denne,der,deres,det,dette,dig,din,dine,disse,dit,dog,du,efter,ej,eller,en,end,ene,eneste,enhver,er,et,fem,fik,fire,flere,fleste,for,fordi,forrige,fra,få,før,god,godt,ham,han,hans,har,havde,have,hej,helt,hende,hendes,her,hos,hun,hvad,hvem,hver,hvilken,hvis,hvor,hvordan,hvorfor,hvornår,i,ikke,ind,ingen,intet,ja,jeg,jer,jeres,jo,kan,kom,komme,kommer,kun,kunne,lad,lav,lidt,lige,lille,man,mange,med,meget,men,mens,mere,mig,min,mine,mit,mod,må,ned,nej,ni,nogen,noget,nogle,nu,ny,nyt,når,nær,n�ste,næsten,og,også,okay,om,op,os,otte,over,på,se,seks,selv,ser,ses,sig,sige,sin,sine,sit,skal,skulle,som,stor,store,syv,s�,s�dan,tag,tage,thi,ti,til,to,tre,ud,under,var,ved,vi,vil,ville,vor,vores,være,været,alene,allerede,alligevel,altid,bag,blandt,burde,bør,dens,derefter,derfor,derfra,deri,dermed,derpå,derved,egen,ellers,endnu,ens,enten,flest,foran,først,gennem,gjorde,gjort,gør,gøre,gørende,hel,heller,hen,henover,herefter,heri,hermed,herpå,hvilke,hvilkes,hvorefter,hvorfra,hvorhen,hvori,hvorimod,hvorved,igen,igennem,imellem,imens,imod,indtil,langs,lave,lavet,ligesom,længere,mellem,mest,mindre,mindst,måske,nemlig,nogensinde,nok,omkring,overalt,samme,sammen,selvom,senere,siden,stadig,synes,syntes,således,temmelig,tidligere,tilbage,tit,uden,udover,undtagen,via,vore,vær,øvrigt
 
 # English
 wordcloud.stopwords=i,me,my,myself,we,our,ours,ourselves,you,your,yours,yourself,yourselves,he,him,his,himself,she,her,hers,herself,it,its,itself,they,them,their,theirs,themselves,what,which,who,whom,this,that,these,those,am,is,are,was,were,be,been,being,have,has,had,having,do,does,did,doing,a,an,the,and,but,if,or,because,as,until,while,of,at,by,for,with,about,against,between,into,through,during,before,after,above,below,to,from,up,down,in,out,on,off,over,under,again,further,then,once,here,there,when,where,why,how,all,any,both,each,few,more,most,other,some,such,no,nor,not,only,own,same,so,than,too,very,s,t,can,will,just,don,should,now

src/main/java/dk/kb/netarchivesuite/solrwayback/service/SolrWaybackResource.java

@@ -961,7 +961,7 @@ private Response viewImpl(String source_file_path, long offset,Boolean showToolb
    //log.debug("setting contentype:"+contentType);
 //          
 
-   ResponseBuilder response = Response.ok(arcEntry.getBinaryRaw()).type(contentType );
+   ResponseBuilder response = Response.ok(arcEntry.getBinaryNoChunking()).type(contentType );
 
     if (arcEntry.isHasBeenDecompressed()){ //Will have if playback (HTML, Twitter, etc.) has replaced the content
     	response.header("Content-Encoding", "identity"); //Not required, but will make it easier to see it has been applied.

src/test/java/dk/kb/netarchivesuite/solrwayback/javascript/JavascriptTests.java → src/test/java/dk/kb/netarchivesuite/solrwayback/javascript/JavascriptTests.java_DISABLED

@@ -1,3 +1,12 @@
+
+/**
+ *  This unit class has been disabled until a solution is found for both java11, 15 and 17   
+ *
+ */
+
+
+
+
 package dk.kb.netarchivesuite.solrwayback.javascript;
 
 

src/test/resources/properties/solrwaybackweb_unittest.properties

@@ -83,7 +83,7 @@ leaflet.attribution=&copy; <a href="https://www.openstreetmap.org/copyright">Ope
 # Wordcloud stop words. 
 
 # Danish
-#wordcloud.stopwords=ad,af,aldrig,alle,alt,anden,andet,andre,at,bare,begge,blev,blive,bliver,da,de,dem,den,denne,der,deres,det,dette,dig,din,dine,disse,dit,dog,du,efter,ej,eller,en,end,ene,eneste,enhver,er,et,fem,fik,fire,flere,fleste,for,fordi,forrige,fra,f�,f�r,f�r,god,godt,ham,han,hans,har,havde,have,hej,helt,hende,hendes,her,hos,hun,hvad,hvem,hver,hvilken,hvis,hvor,hvordan,hvorfor,hvorn�r,i,ikke,ind,ingen,intet,ja,jeg,jer,jeres,jo,kan,kom,komme,kommer,kun,kunne,lad,lav,lidt,lige,lille,man,mange,med,meget,men,mens,mere,mig,min,mine,mit,mod,m�,ned,nej,ni,nogen,noget,nogle,nu,ny,nyt,n�r,n�r,n�ste,n�sten,og,ogs�,okay,om,op,os,otte,over,p�,se,seks,selv,ser,ses,sig,sige,sin,sine,sit,skal,skulle,som,stor,store,syv,s�,s�dan,tag,tage,thi,ti,til,to,tre,ud,under,var,ved,vi,vil,ville,vor,vores,v�re,v�ret,alene,allerede,alligevel,altid,bag,blandt,burde,b�r,dens,derefter,derfor,derfra,deri,dermed,derp�,derved,egen,ellers,endnu,ens,enten,flest,foran,f�rst,gennem,gjorde,gjort,g�r,g�re,g�rende,hel,heller,hen,henover,herefter,heri,hermed,herp�,hvilke,hvilkes,hvorefter,hvorfra,hvorhen,hvori,hvorimod,hvorved,igen,igennem,imellem,imens,imod,indtil,langs,lave,lavet,ligesom,l�ngere,mellem,mest,mindre,mindst,m�ske,nemlig,nogensinde,nok,omkring,overalt,samme,sammen,selvom,senere,siden,stadig,synes,syntes,s�ledes,temmelig,tidligere,tilbage,tit,uden,udover,undtagen,via,vore,v�r,�vrigt
+#wordcloud.stopwords=ad,af,aldrig,alle,alt,anden,andet,andre,at,bare,begge,blev,blive,bliver,da,de,dem,den,denne,der,deres,det,dette,dig,din,dine,disse,dit,dog,du,efter,ej,eller,en,end,ene,eneste,enhver,er,et,fem,fik,fire,flere,fleste,for,fordi,forrige,fra,få,før,god,godt,ham,han,hans,har,havde,have,hej,helt,hende,hendes,her,hos,hun,hvad,hvem,hver,hvilken,hvis,hvor,hvordan,hvorfor,hvornår,i,ikke,ind,ingen,intet,ja,jeg,jer,jeres,jo,kan,kom,komme,kommer,kun,kunne,lad,lav,lidt,lige,lille,man,mange,med,meget,men,mens,mere,mig,min,mine,mit,mod,må,ned,nej,ni,nogen,noget,nogle,nu,ny,nyt,når,nær,n\uFFFDste,næsten,og,også,okay,om,op,os,otte,over,på,se,seks,selv,ser,ses,sig,sige,sin,sine,sit,skal,skulle,som,stor,store,syv,s\uFFFD,s\uFFFDdan,tag,tage,thi,ti,til,to,tre,ud,under,var,ved,vi,vil,ville,vor,vores,være,været,alene,allerede,alligevel,altid,bag,blandt,burde,bør,dens,derefter,derfor,derfra,deri,dermed,derpå,derved,egen,ellers,endnu,ens,enten,flest,foran,først,gennem,gjorde,gjort,gør,gøre,gørende,hel,heller,hen,henover,herefter,heri,hermed,herpå,hvilke,hvilkes,hvorefter,hvorfra,hvorhen,hvori,hvorimod,hvorved,igen,igennem,imellem,imens,imod,indtil,langs,lave,lavet,ligesom,længere,mellem,mest,mindre,mindst,måske,nemlig,nogensinde,nok,omkring,overalt,samme,sammen,selvom,senere,siden,stadig,synes,syntes,således,temmelig,tidligere,tilbage,tit,uden,udover,undtagen,via,vore,vær,øvrigt
 
 # English
 wordcloud.stopwords=i,me,my,myself,we,our,ours,ourselves,you,your,yours,yourself,yourselves,he,him,his,himself,she,her,hers,herself,it,its,itself,they,them,their,theirs,themselves,what,which,who,whom,this,that,these,those,am,is,are,was,were,be,been,being,have,has,had,having,do,does,did,doing,a,an,the,and,but,if,or,because,as,until,while,of,at,by,for,with,about,against,between,into,through,during,before,after,above,below,to,from,up,down,in,out,on,off,over,under,again,further,then,once,here,there,when,where,why,how,all,any,both,each,few,more,most,other,some,such,no,nor,not,only,own,same,so,than,too,very,s,t,can,will,just,don,should,now