Merge branch 'master' into 1670-stockNumber

hbz · Jul 25, 2023 · 35135d5 · 35135d5
2 parents 6761244 + fb1551f
commit 35135d5
Show file tree

Hide file tree

Showing 79 changed files with 284 additions and 108 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -26,7 +26,7 @@ jobs:
       run: |
         git clone https://github.com/metafacture/metafacture-fix.git
         cd metafacture-fix
-        git reset --hard 0.4.0
+        git reset --hard 0.5.1
         ./gradlew publishToMavenLocal
         cd ..
     - name: Build with Maven

diff --git a/pom.xml b/pom.xml
@@ -105,7 +105,7 @@
 		<dependency>
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafix</artifactId>
-			<version>0.4.0</version>
+			<version>0.5.1</version>
 		</dependency>
 		<dependency>
 			<groupId>com.github.jsonld-java</groupId>
@@ -118,11 +118,6 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
-		<dependency>
-			<groupId>xml-apis</groupId>
-			<artifactId>xml-apis</artifactId>
-			<version>1.4.01</version>
-		</dependency>
 		<dependency>
 			<groupId>com.google.gdata</groupId>
 			<artifactId>core</artifactId>
@@ -132,27 +127,18 @@
 			<groupId>org.elasticsearch</groupId>
 			<artifactId>elasticsearch</artifactId>
 			<version>5.6.3</version>
+			<exclusions>
+				<exclusion>
+					<groupId>log4j</groupId>
+					<artifactId>log4j-api</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.elasticsearch.client</groupId>
 			<artifactId>transport</artifactId>
 			<version>5.6.3</version>
 		</dependency>
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-api</artifactId>
-			<version>2.17.1</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-core</artifactId>
-			<version>2.17.1</version>
-		</dependency>
-		<dependency>
-			<groupId>commons-validator</groupId>
-			<artifactId>commons-validator</artifactId>
-			<version>1.5.1</version>
-		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
@@ -178,11 +164,6 @@
 			<artifactId>jaxb-api</artifactId>
 			<version>2.3.0</version>
 		</dependency>
-		<dependency>
-			<groupId>xerces</groupId>
-			<artifactId>xercesImpl</artifactId>
-			<version>2.12.2</version>
-		</dependency>
 	</dependencies>
 	<build>
 		<resources>

diff --git a/src/main/java/de/hbz/lobid/helper/CompareJsonMaps.java b/src/main/java/de/hbz/lobid/helper/CompareJsonMaps.java
@@ -13,8 +13,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -33,7 +33,7 @@
  */
 @SuppressWarnings("javadoc")
 public final class CompareJsonMaps {
-	final static Logger logger = LogManager.getLogger(CompareJsonMaps.class);
+	final static Logger logger = LoggerFactory.getLogger(CompareJsonMaps.class);
 	Stack<String> stack = new Stack<>();
 	static final String JSON_LD_CONTEXT = "[@context";
 	private static boolean IGNORE_CONTEXT = true;

diff --git a/src/main/java/de/hbz/lobid/helper/Etikett.java b/src/main/java/de/hbz/lobid/helper/Etikett.java
@@ -8,8 +8,8 @@
 
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -24,7 +24,7 @@
  */
 @XmlRootElement
 public class Etikett {
-	final static Logger logger = LogManager.getLogger(Etikett.class);
+	final static Logger logger = LoggerFactory.getLogger(Etikett.class);
 
 	public enum EtikettType {
 		CACHE, CONTEXT, STORE

diff --git a/src/main/java/de/hbz/lobid/helper/EtikettMaker.java b/src/main/java/de/hbz/lobid/helper/EtikettMaker.java
@@ -12,8 +12,8 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -30,7 +30,7 @@ public class EtikettMaker implements EtikettMakerInterface {
 	private static final String ID = "id";
 	private String contextLocation = "web/conf/context.jsonld";
 
-	final static Logger logger = LogManager.getLogger(EtikettMaker.class);
+	final static Logger logger = LoggerFactory.getLogger(EtikettMaker.class);
 
 	/**
 	 * A map with URIs as key

diff --git a/src/main/java/de/hbz/lobid/helper/HttpPoster.java b/src/main/java/de/hbz/lobid/helper/HttpPoster.java
@@ -1,10 +1,14 @@
 package de.hbz.lobid.helper;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.io.*;
-import java.net.*;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.MalformedURLException;
 
 /**
  * Uploads data using {@link URLConnection} with POST method and returns the response.
@@ -17,7 +21,7 @@ public final class HttpPoster {
     public final String POST = "POST";
     private String contentType = "application/json";
     private URL url;
-    private static final Logger LOG = LogManager.getLogger(HttpPoster.class);
+    private static final Logger LOG = LoggerFactory.getLogger(HttpPoster.class);
 
     /**
      * Creates an instance of {@link HttpPoster}.

diff --git a/src/main/java/org/lobid/resources/ElasticsearchIndexer.java b/src/main/java/org/lobid/resources/ElasticsearchIndexer.java
@@ -15,24 +15,20 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.StreamSupport;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
+
 import com.google.common.collect.SortedSetMultimap;
 import com.google.common.collect.TreeMultimap;
-import com.google.gdata.util.common.base.Pair;
 import com.google.gdata.util.common.io.CharStreams;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
@@ -46,15 +42,12 @@
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.MultiMatchQueryBuilder;
-import org.elasticsearch.index.query.Operator;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.rest.action.admin.indices.AliasesNotFoundException;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
+
 import org.metafacture.framework.ObjectReceiver;
 import org.metafacture.framework.annotations.In;
 import org.metafacture.framework.annotations.Out;
@@ -71,8 +64,7 @@
 public class ElasticsearchIndexer
 		extends DefaultObjectPipe<HashMap<String, String>, ObjectReceiver<Void>> {
 
-	private static final Logger LOG =
-			LogManager.getLogger(ElasticsearchIndexer.class);
+	private static final Logger LOG = LoggerFactory.getLogger(ElasticsearchIndexer.class);
 	private String hostname;
 	private String clustername;
 	private BulkRequestBuilder bulkRequest;
@@ -227,7 +219,7 @@ public void process(final HashMap<String, String> json) {
 				LOG.warn("Retry indexing record" + json.get(Properties.ID.getName())
 						+ ":" + e.getMessage() + " (" + retries + " more retries)");
 			} catch (final Exception ex) {
-				LOG.warn(ex);
+				LOG.warn(ex.getMessage());
 			}
 		}
 	}

diff --git a/src/main/java/org/lobid/resources/EtikettJson.java b/src/main/java/org/lobid/resources/EtikettJson.java
@@ -13,8 +13,9 @@
 
 import com.github.jsonldjava.utils.JsonUtils;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.metafacture.framework.ObjectReceiver;
 import org.metafacture.framework.annotations.In;
 import org.metafacture.framework.annotations.Out;
@@ -32,7 +33,7 @@
 @Out(String.class)
 public final class EtikettJson
     extends DefaultObjectPipe<String, ObjectReceiver<String>> {
-  private static final Logger LOG = LogManager.getLogger(EtikettJson.class);
+  private static final Logger LOG = LoggerFactory.getLogger(EtikettJson.class);
   private String labelsDirectoryName = "labels";
   private String contextFilenameLocation;
   private boolean generateContext = false;

diff --git a/src/main/java/org/lobid/resources/JsonLdEtikett.java b/src/main/java/org/lobid/resources/JsonLdEtikett.java
@@ -9,8 +9,9 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.metafacture.framework.ObjectReceiver;
 import org.metafacture.framework.annotations.In;
 import org.metafacture.framework.annotations.Out;
@@ -27,7 +28,7 @@
 @Out(HashMap.class)
 public final class JsonLdEtikett extends
 		DefaultObjectPipe<Map<String, Object>, ObjectReceiver<Map<String, Object>>> {
-	private static final Logger LOG = LogManager.getLogger(JsonLdEtikett.class);
+	private static final Logger LOG = LoggerFactory.getLogger(JsonLdEtikett.class);
 	private static String labelsDirectoryName = "labels";
 	private static String contextFilenameLocation = "web/conf/context.jsonld";
 

diff --git a/src/main/java/org/lobid/resources/run/AlmaMarcXmlFix2lobidJsonEs.java b/src/main/java/org/lobid/resources/run/AlmaMarcXmlFix2lobidJsonEs.java
@@ -11,8 +11,6 @@
 import java.util.stream.Stream;
 
 import de.hbz.lobid.helper.HttpPoster;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.lobid.resources.ElasticsearchIndexer;
 import org.lobid.resources.EtikettJson;
 import org.lobid.resources.JsonToElasticsearchBulkMap;
@@ -28,6 +26,9 @@
 import org.metafacture.xml.XmlElementSplitter;
 import org.metafacture.metafix.Metafix;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import de.hbz.lobid.helper.Email;
 
 /**
@@ -57,8 +58,7 @@ public class AlmaMarcXmlFix2lobidJsonEs {
     private static String triggerWebhookData;
     private static String kind = "";
     private static boolean switchAutomatically = false;
-    private static final Logger LOG =
-        LogManager.getLogger(AlmaMarcXmlFix2lobidJsonEs.class);
+    private static final Logger LOG = LoggerFactory.getLogger(AlmaMarcXmlFix2lobidJsonEs.class);
     public static boolean threadAlreadyStarted = false;
     private static String switchAlias1;
     private static String switchAlias2;
@@ -132,6 +132,7 @@ public void run() {
                 fixVariables.put("rpb2.ttl", "./maps/rpb2.ttl");
                 fixVariables.put("rpb-spatial.ttl", "./maps/rpb-spatial.ttl");
                 fixVariables.put("rpb.ttl", "./maps/rpb.ttl");
+                fixVariables.put("hbzId2zdbId.tsv", "./maps/hbzId2zdbId.tsv.gz");
 
                 XmlElementSplitter xmlElementSplitter = new XmlElementSplitter();
                 xmlElementSplitter.setElementName("record");

diff --git a/src/main/java/org/lobid/resources/run/SwitchEsAlmaAlias.java b/src/main/java/org/lobid/resources/run/SwitchEsAlmaAlias.java
@@ -10,8 +10,9 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
@@ -34,8 +35,7 @@ public final class SwitchEsAlmaAlias {
   private static final int DOCS_MINIMUM = 83000000;
   private static final int BYTES_MINIMUM = 49 * 1024 * 1024 * 1024;
   private static ObjectMapper objectMapper = new ObjectMapper();
-  private static final Logger LOG =
-      LogManager.getLogger(SwitchEsAlmaAlias.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SwitchEsAlmaAlias.class);
   private static StringBuilder logMessages = new StringBuilder(512);
 
   private SwitchEsAlmaAlias() {

diff --git a/src/main/java/org/lobid/resources/run/UpdateAliases.java b/src/main/java/org/lobid/resources/run/UpdateAliases.java
@@ -14,8 +14,9 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
@@ -40,7 +41,7 @@
  */
 public class UpdateAliases {
   private static ObjectMapper objectMapper = new ObjectMapper();
-  private static final Logger LOG = LogManager.getLogger(UpdateAliases.class);
+  private static final Logger LOG = LoggerFactory.getLogger(UpdateAliases.class);
   private static boolean success = false;
   private static String logMessage;
   private static StringBuilder logMessages = new StringBuilder(512);

diff --git a/src/main/resources/alma/alma.fix b/src/main/resources/alma/alma.fix
@@ -57,6 +57,7 @@ retain(
   "ismn[]",
   "isPartOf[]",
   "language[]",
+  "langNote[]",
   "license[]",
   "medium[]",
   "natureOfContent[]",

diff --git a/src/main/resources/alma/fix/identifiers.fix b/src/main/resources/alma/fix/identifiers.fix
@@ -43,6 +43,7 @@ if exists("hbzId")
   paste("deprecatedUri", "~http://lobid.org/resources/", "hbzId", "~#!", join_char: "")
 end
 
+
 # 020 - International Standard Book Number (R) - $a (NR)
 # source data sometimes provides repeated subfield $a even if this is not valid marc
 
@@ -153,3 +154,10 @@ set_array("stockNumber[]")
 do list(path:"028??", "var":"$i")
   copy_field("$i.a", "stockNumber[].$append")
 end
+
+copy_field("zdbId","@hbzId")
+lookup("@hbzId","zdbId2oldHbzId",delete:"true")
+if exists("@hbzId")
+  paste("deprecatedUri", "~http://lobid.org/resources/", "@hbzId", "~#!", join_char: "")
+end
+copy_field("@hbzId","hbzId")
diff --git a/src/main/resources/alma/fix/maps.fix b/src/main/resources/alma/fix/maps.fix
@@ -35,6 +35,8 @@ put_filemap("$[almaMmsId2rpbId]","almaMmsId2rpbId", sep_char:"\t",key_column:"0"
 # lobid Organisations id -> label
 put_filemap("$[lobidOrgLabels]","lobidOrgLabels", sep_char:"\t",key_column:"0",value_column:"1",expected_columns:"-1")
 
+# map zdbId to old hbzId(HT Nummer) based on the last aleph transformation
+put_filemap("$[hbzId2zdbId.tsv]","zdbId2oldHbzId", sep_char:"\t",key_column:"1",value_column:"0",expected_columns:"-1")
 
 put_map("rswk-indicator",
   "p": "Person",