Skip to content

Commit

Permalink
Added basic Dockerfile that runs a stubbed-out IndexWorker class.
Browse files Browse the repository at this point in the history
  • Loading branch information
mbjones committed Jun 8, 2022
1 parent 839eb96 commit b06e0ff
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 14 deletions.
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,15 @@ registered workers in parallel, and workers in turn process the associated objec

See LICENSE.txt for the details of distributing this software.

## Building Docker image

```bash
nerdctl build -t dataone-index-worker:2.4.0 -f docker/Dockerfile --build-arg TAG=2.4.0 .
```

## History

This is a refactored version of the original DataONE [d1_cn_index_processor](https://github.com/DataONEorg/d1_cn_index_processor) that runs completely independently of other
DataONE Coordinating Node services. It is intended to be deployed in a Kubernetes cluster environment, but is written such
that it can be deployed in other environments as well as needed.

25 changes: 25 additions & 0 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Use an OpenJDK runtime as a parent image
FROM openjdk:8-jre-alpine

ARG TAG=2.0.0
ENV TAG=${TAG}

# Set the working directory
WORKDIR /var/lib/dataone-indexer

RUN apk update
# bash is needed by the openssl install
RUN apk add bash
#RUN apk add g++ libc-dev openssl-dev libxml2 libxml2-dev

# The most recently built jar file is copied from the maven build directory to this dir by maven, so that
# it can be copyied to the image.
COPY ../target/dataone-index-worker-$TAG.jar .

# Run the Worker process
# Note: docker --build-arg only allows one argument (one token only, multiple tokens inside quotes doesn't work, so have
# to specify java options directly on command line.
# Set classpath to include /opt/local/metadig/log4j.properties, if it exists, so that logging can be changed without
# having to rebuild the container. Note that on k8s, this dir is mapped to the persistent volume, so will be /data/metadig/log4j.properties
CMD java -Dlog4j2.formatMsgNoLookups=true -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:+UseSerialGC -cp ./dataone-index-worker-$TAG.jar org.dataone.cn.index.IndexWorker
#CMD sh -c 'trap "exit" TERM; while true; do sleep 1; done'
60 changes: 46 additions & 14 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,24 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
<version>${solr.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${solr.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -100,18 +112,26 @@
<version>1.3.163</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
</dependency>

<dependency>
<groupId>org.dataone</groupId>
<artifactId>d1_cn_common</artifactId>
<version>${d1_cn_common.version}</version>
<type>jar</type>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.dataone</groupId>
Expand All @@ -130,11 +150,23 @@
<artifactId>d1_cn_index_common</artifactId>
<version>${d1_cn_index_common.version}</version>
<type>jar</type>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.dataone</groupId>
<artifactId>d1_cn_index_generator</artifactId>
<version>${d1_cn_index_generator.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.hazelcast</groupId>
Expand Down Expand Up @@ -244,11 +276,6 @@
<artifactId>commons-daemon</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand All @@ -259,20 +286,15 @@
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
<version>1.7.36</version>
</dependency>
<dependency>
<groupId>net.minidev</groupId>
Expand All @@ -283,6 +305,16 @@
<groupId>org.apache.jena</groupId>
<artifactId>jena-tdb</artifactId>
<version>1.1.2</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
Expand Down
54 changes: 54 additions & 0 deletions src/main/java/org/dataone/cn/index/IndexWorker.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/**
* This work was created by participants in the DataONE project, and is
* jointly copyrighted by participating institutions in DataONE. For
* more information on DataONE, see our web site at http://dataone.org.
*
* Copyright 2022
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.dataone.cn.index;

/**
* Worker class to process index tasks and submit results to store.
*/
public class IndexWorker {

/**
* Commandline main for the IndexWorker to be started.
* @param args
*/
public static void main(String[] args) {

System.out.println("Starting index worker...");
IndexWorker worker = new IndexWorker();

// TODO: read RabbitMQ and SOLR config info from environment

// TODO: register this worker with RabbitMQ

worker.handleIndexTask("123");
System.out.println("Done.");
}

/**
* Callback for processing a specific indexing task
* @param message the message describing the task to be processed
*/
public void handleIndexTask(String message) {
System.out.println("Handling task: " + message);
// TODO: Handle the index task
}
}

0 comments on commit b06e0ff

Please sign in to comment.