Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: track changes for oersi-etl #329

Draft
wants to merge 10 commits into
base: master
Choose a base branch
from
18 changes: 18 additions & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
name: Publish package to GitHub Packages
on:
push:
branches:
- '*-rc*'
jobs:
publish:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Publish package
run: ./gradlew publish
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

43 changes: 42 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ editorconfig {
'**/*.bzip2',
'**/*.gzip',
'**/*.xz',
'gradlew*'
'gradlew*',
'**/bin'
]
}

Expand All @@ -56,6 +57,7 @@ subprojects {
apply plugin: 'maven'
apply plugin: 'jacoco'
apply plugin: 'checkstyle'
apply plugin: 'maven-publish'

check.dependsOn(editorconfigCheck)
check.dependsOn(javadoc)
Expand Down Expand Up @@ -94,6 +96,7 @@ subprojects {
options {
addBooleanOption 'Xwerror', true
}
exclude '**/harvester2/**'
}

artifacts {
Expand Down Expand Up @@ -241,9 +244,43 @@ gradle.projectsEvaluated {
}
}

//'upload' vs. 'publish, see
//https://medium.com/dot-debug/deploying-artifacts-to-maven-using-gradle-b669acc1b6f8'
/*
task publish {
dependsOn tasks.uploadArchives
}
*/

publishing {
publications {
mavenArtifacts(MavenPublication) {
from components.java
afterEvaluate {
groupId = project.group
artifactId = project.name
}
}
}

repositories {
maven {
name = 'localEmbedded'
// run generated gradle task `./gradlew
// publishMavenArtifactsPublicationToLocalEmbeddedRepository`
// to publish all subprojects into the same local embedded repo:
url = "file://${rootDir}/build/repo"
}
maven {
name = "GitHubPackages"
url = "https://maven.pkg.github.com/metafacture/metafacture-core"
credentials {
username = System.getenv("GITHUB_ACTOR")
password = System.getenv("GITHUB_TOKEN")
}
}
}
}
}
}

Expand Down Expand Up @@ -315,6 +352,10 @@ def getSnapshotVersion() {
logger.lifecycle('Release branch found')
return "${extractVersionFromBranch(grgit.branch.current().name)}-SNAPSHOT"
}
if (grgit.branch.current().name.contains('-rc')) {
logger.lifecycle('Release candidate branch found')
return "${grgit.branch.current().name}"
}
logger.lifecycle('Feature branch found')
return "feature-${grgit.branch.current().name}-SNAPSHOT"
}
Expand Down
2 changes: 1 addition & 1 deletion config/checkstyle/checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<module name="SuppressWarningsFilter" />
<module name="SuppressionSingleFilter">
<property name="checks" value=".*"/>
<property name="files" value="generated-src"/>
<property name="files" value="generated-src|harvester2"/>
</module>
<module name="TreeWalker">
<module name="AbstractClassName"/>
Expand Down
2 changes: 1 addition & 1 deletion metafacture-biblio/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ dependencies {
api project(':metafacture-framework')
implementation project(':metafacture-commons')
implementation project(':metafacture-flowcontrol')
implementation 'org.dspace:oclc-harvester2:0.1.12'
implementation ('xalan:xalan:2.7.0') {
exclude group: 'xalan', module: 'serializer'
exclude group: 'xercesImpl', module: 'xercesImpl'
exclude group: 'xml-apis', module: 'xml-apis'
}
implementation 'log4j:log4j:1.2.17'
implementation 'org.slf4j:slf4j-api:1.7.7'
testImplementation 'junit:junit:4.12'
testImplementation 'org.mockito:mockito-core:2.5.5'
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2013 Pascal Christoph.
/* Copyright 2013, 2022 Pascal Christoph and others.
* Licensed under the Eclipse Public License 1.0 */

package org.metafacture.biblio;
Expand All @@ -10,7 +10,7 @@
import org.metafacture.framework.annotations.Out;
import org.metafacture.framework.helpers.DefaultObjectPipe;

import ORG.oclc.oai.harvester2.app.RawWrite;
import org.oclc.oai.harvester2.app.RawWrite;
import org.xml.sax.SAXException;

import java.io.ByteArrayInputStream;
Expand All @@ -19,7 +19,7 @@
import java.io.InputStreamReader;
import java.io.Reader;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPathException;

/**
* Opens an OAI-PMH stream and passes a reader to the receiver.
Expand Down Expand Up @@ -111,10 +111,10 @@ public void process(final String baseUrl) {
catch (final SAXException e) {
e.printStackTrace();
}
catch (final TransformerException e) {
catch (final NoSuchFieldException e) {
e.printStackTrace();
}
catch (final NoSuchFieldException e) {
catch (final XPathException e) {
e.printStackTrace();
}
try {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
/**
* Copyright 2006 OCLC, Online Computer Library Center Licensed under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a copy of the
* License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is
* distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and limitations under the License.
*/
package org.oclc.oai.harvester2.app;

import java.io.*;
import java.lang.NoSuchFieldException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.HashMap;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathException;
import javax.xml.xpath.XPathExpressionException;
import org.oclc.oai.harvester2.verb.*;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

public class RawWrite {

public static void main(String[] args) {
try {
System.out.println(new Date());

HashMap<String, Object> options = getOptions(args);
List<?> rootArgs = (List<?>) options.get("rootArgs");
String baseURL = null;
if (rootArgs.size() > 0) {
baseURL = (String) rootArgs.get(0);
} else {
throw new IllegalArgumentException();
}

OutputStream out = System.out;
String outFileName = (String) options.get("-out");
String from = (String) options.get("-from");
String until = (String) options.get("-until");
String metadataPrefix = (String) options.get("-metadataPrefix");
if (metadataPrefix == null) metadataPrefix = "oai_dc";
String resumptionToken = (String) options.get("-resumptionToken");
String setSpec = (String) options.get("-setSpec");

if (resumptionToken != null) {
if (outFileName != null)
out = new FileOutputStream(outFileName, true);
run(baseURL, resumptionToken, out);
} else {
if (outFileName != null)
out = new FileOutputStream(outFileName);
run(baseURL, from, until, metadataPrefix, setSpec, out);
}

if (out != System.out) out.close();
System.out.println(new Date());
} catch (IllegalArgumentException e) {
System.err.println("RawWrite <-from date> <-until date> <-metadataPrefix prefix> <-setSpec setName> <-resumptionToken token> <-out fileName> baseURL");
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}

public static void run(String baseURL, String resumptionToken,
OutputStream out)
throws IOException, ParserConfigurationException, SAXException, XPathExpressionException,
NoSuchFieldException {
ListRecords listRecords = new ListRecords(baseURL, resumptionToken);
while (listRecords != null) {
NodeList errors = listRecords.getErrors();
if (errors != null && errors.getLength() > 0) {
System.out.println("Found errors");
int length = errors.getLength();
for (int i = 0; i < length; ++i) {
Node item = errors.item(i);
System.out.println(item);
}
System.out.println("Error record: " + listRecords.toString());
break;
}
// System.out.println(listRecords);
out.write(listRecords.toString().getBytes("UTF-8"));
out.write("\n".getBytes("UTF-8"));
resumptionToken = listRecords.getResumptionToken();
System.out.println("resumptionToken: " + resumptionToken);
if (resumptionToken == null || resumptionToken.length() == 0) {
listRecords = null;
} else {
listRecords = new ListRecords(baseURL, resumptionToken);
}
}
out.write("</harvest>\n".getBytes("UTF-8"));
}

public static void run(String baseURL, String from, String until,
String metadataPrefix, String setSpec,
OutputStream out)
throws IOException, ParserConfigurationException, SAXException, XPathException,
NoSuchFieldException {
out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n".getBytes("UTF-8"));
out.write("<harvest>\n".getBytes("UTF-8"));
out.write(new Identify(baseURL).toString().getBytes("UTF-8"));
out.write("\n".getBytes("UTF-8"));
out.write(new ListMetadataFormats(baseURL).toString().getBytes("UTF-8"));
out.write("\n".getBytes("UTF-8"));
ListSets listSets = new ListSets(baseURL);
while (listSets != null) {
out.write(listSets.toString().getBytes("UTF-8"));
out.write("\n".getBytes("UTF-8"));
String resumptionToken = listSets.getResumptionToken();
System.out.println("resumptionToken: " + resumptionToken);
if (resumptionToken == null || resumptionToken.length() == 0) {
listSets = null;
} else {
listSets = new ListSets(baseURL, resumptionToken);
}
}
ListRecords listRecords = new ListRecords(baseURL, from, until, setSpec,
metadataPrefix);
while (listRecords != null) {
NodeList errors = listRecords.getErrors();
if (errors != null && errors.getLength() > 0) {
System.out.println("Found errors");
int length = errors.getLength();
for (int i = 0; i < length; ++i) {
Node item = errors.item(i);
System.out.println(item);
}
System.out.println("Error record: " + listRecords.toString());
break;
}
// System.out.println(listRecords);
out.write(listRecords.toString().getBytes("UTF-8"));
out.write("\n".getBytes("UTF-8"));
String resumptionToken = listRecords.getResumptionToken();
System.out.println("resumptionToken: " + resumptionToken);
if (resumptionToken == null || resumptionToken.length() == 0) {
listRecords = null;
} else {
listRecords = new ListRecords(baseURL, resumptionToken);
}
}
out.write("</harvest>\n".getBytes("UTF-8"));
}

private static HashMap<String, Object> getOptions(String[] args) {
HashMap<String, Object> options = new HashMap<>();
ArrayList<String> rootArgs = new ArrayList<>();
options.put("rootArgs", rootArgs);

for (int i = 0; i < args.length; ++i) {
if (args[i].charAt(0) != '-') {
rootArgs.add(args[i]);
} else if (i + 1 < args.length) {
options.put(args[i], args[++i]);
} else {
throw new IllegalArgumentException();
}
}
return options;
}
}
Loading