Skip to content

Commit c4679de

Browse files
author
Vladimir Kotal
committed
introduce HeadHandler, use it in GitRepository
1 parent c827872 commit c4679de

File tree

3 files changed

+300
-18
lines changed

3 files changed

+300
-18
lines changed

opengrok-indexer/src/main/java/org/opengrok/indexer/history/GitRepository.java

+13-18
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.opengrok.indexer.logger.LoggerFactory;
4848
import org.opengrok.indexer.util.BufferSink;
4949
import org.opengrok.indexer.util.Executor;
50+
import org.opengrok.indexer.util.HeadHandler;
5051
import org.opengrok.indexer.util.StringUtils;
5152
import org.opengrok.indexer.util.Version;
5253

@@ -396,23 +397,17 @@ private String getFirstRevision(String fullpath) throws IOException {
396397

397398
Executor executor = new Executor(Arrays.asList(argv), new File(getDirectoryName()),
398399
RuntimeEnvironment.getInstance().getInteractiveCommandTimeout());
399-
int status = executor.exec();
400-
401-
try (BufferedReader in = new BufferedReader(
402-
new InputStreamReader(executor.getOutputStream()))) {
403-
String line;
400+
HeadHandler headHandler = new HeadHandler(1);
401+
int status = executor.exec(false, headHandler);
404402

405-
if ((line = in.readLine()) != null) {
406-
return line.trim();
407-
}
403+
String line;
404+
if (headHandler.count() > 0 && (line = headHandler.get(0)) != null) {
405+
return line.trim();
408406
}
409407

410-
if (status != 0) {
411-
LOGGER.log(Level.WARNING,
412-
"Failed to get first revision for: \"{0}\" Exit code: {1}",
413-
new Object[]{fullpath, String.valueOf(status)});
414-
return null;
415-
}
408+
LOGGER.log(Level.WARNING,
409+
"Failed to get first revision for: \"{0}\" Exit code: {1}",
410+
new Object[]{fullpath, String.valueOf(status)});
416411

417412
return null;
418413
}
@@ -449,10 +444,10 @@ public Annotation annotate(File file, String revision) throws IOException {
449444
cmd.add("--");
450445
cmd.add(getPathRelativeToCanonicalRepositoryRoot(file.getCanonicalPath()));
451446

452-
Executor exec = new Executor(cmd, new File(getDirectoryName()),
447+
Executor executor = new Executor(cmd, new File(getDirectoryName()),
453448
RuntimeEnvironment.getInstance().getInteractiveCommandTimeout());
454449
GitAnnotationParser parser = new GitAnnotationParser(file.getName());
455-
int status = exec.exec(true, parser);
450+
int status = executor.exec(true, parser);
456451

457452
// File might have changed its location if it was renamed.
458453
// Try to lookup its original name and get the annotation again.
@@ -468,10 +463,10 @@ public Annotation annotate(File file, String revision) throws IOException {
468463
}
469464
cmd.add("--");
470465
cmd.add(findOriginalName(file.getCanonicalPath(), revision));
471-
exec = new Executor(cmd, new File(getDirectoryName()),
466+
executor = new Executor(cmd, new File(getDirectoryName()),
472467
RuntimeEnvironment.getInstance().getInteractiveCommandTimeout());
473468
parser = new GitAnnotationParser(file.getName());
474-
status = exec.exec(true, parser);
469+
status = executor.exec(true, parser);
475470
}
476471

477472
if (status != 0) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
24+
package org.opengrok.indexer.util;
25+
26+
import java.io.BufferedInputStream;
27+
import java.io.BufferedReader;
28+
import java.io.IOException;
29+
import java.io.InputStream;
30+
import java.io.InputStreamReader;
31+
import java.nio.charset.Charset;
32+
import java.nio.charset.StandardCharsets;
33+
import java.util.ArrayList;
34+
import java.util.List;
35+
36+
/**
37+
* The purpose of this class is to provide {@code StreamHandler} that limits the output
38+
* to specified number of lines. Compared to {@code SpoolHandler} it consumes
39+
* limited amount of heap.
40+
*/
41+
public class HeadHandler implements Executor.StreamHandler {
42+
private final int maxLines;
43+
44+
private final List<String> lines = new ArrayList<>();
45+
private final Charset charset;
46+
47+
private static final int bufferedReaderSize = 200;
48+
49+
/**
50+
* Charset of the underlying reader is set to UTF-8.
51+
* @param maxLines maximum number of lines to store
52+
*/
53+
public HeadHandler(int maxLines) {
54+
this.maxLines = maxLines;
55+
this.charset = StandardCharsets.UTF_8;
56+
}
57+
58+
/**
59+
* @param maxLines maximum number of lines to store
60+
* @param charset character set
61+
*/
62+
public HeadHandler(int maxLines, Charset charset) {
63+
this.maxLines = maxLines;
64+
this.charset = charset;
65+
}
66+
67+
/**
68+
* @return number of lines read
69+
*/
70+
public int count() {
71+
return lines.size();
72+
}
73+
74+
/**
75+
* @param index index
76+
* @return line at given index. Will be non {@code null} for valid index.
77+
*/
78+
public String get(int index) {
79+
return lines.get(index);
80+
}
81+
82+
// for testing
83+
static int getBufferedReaderSize() {
84+
return bufferedReaderSize;
85+
}
86+
87+
@Override
88+
public void processStream(InputStream input) throws IOException {
89+
try (BufferedInputStream bufStream = new BufferedInputStream(input);
90+
BufferedReader reader = new BufferedReader(new InputStreamReader(bufStream, this.charset),
91+
bufferedReaderSize)) {
92+
int lineNum = 0;
93+
while (lineNum < maxLines) {
94+
String line = reader.readLine();
95+
if (line == null) { // EOF
96+
return;
97+
}
98+
lines.add(line);
99+
lineNum++;
100+
}
101+
102+
// Read and forget the rest.
103+
byte[] buf = new byte[1024];
104+
while ((bufStream.read(buf)) != -1) {
105+
;
106+
}
107+
}
108+
}
109+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
22+
*/
23+
24+
package org.opengrok.indexer.util;
25+
26+
import org.junit.Test;
27+
import org.junit.runner.RunWith;
28+
import org.junit.runners.Parameterized;
29+
import org.opengrok.indexer.logger.LoggerFactory;
30+
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.util.ArrayList;
34+
import java.util.Arrays;
35+
import java.util.Collection;
36+
import java.util.List;
37+
import java.util.concurrent.ThreadLocalRandom;
38+
import java.util.logging.Level;
39+
import java.util.logging.Logger;
40+
41+
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
42+
import static org.junit.jupiter.api.Assertions.assertEquals;
43+
import static org.junit.jupiter.api.Assertions.assertTrue;
44+
45+
/**
46+
* Test the behavior of the {@code HeadHandler} class. The main aspect to check is that the
47+
* input stream is read whole.
48+
*/
49+
@RunWith(Parameterized.class)
50+
public class HeadHandlerTest {
51+
@Parameterized.Parameters
52+
public static Collection<Object[]> data() {
53+
List<Object[]> tests = new ArrayList<>();
54+
for (int i = 0; i < ThreadLocalRandom.current().nextInt(4, 8); i++) {
55+
tests.add(new Object[]{ThreadLocalRandom.current().nextInt(2, 10),
56+
ThreadLocalRandom.current().nextInt(1, 10),
57+
HeadHandler.getBufferedReaderSize() * ThreadLocalRandom.current().nextInt(1, 40)});
58+
}
59+
return tests;
60+
}
61+
62+
private int lineCnt;
63+
private int headLineCnt;
64+
private int totalCharCount;
65+
66+
private static final Logger LOGGER = LoggerFactory.getLogger(HeadHandlerTest.class);
67+
68+
/**
69+
* @param lineCnt number of lines to generate
70+
* @param headLineCnt maximum number of lines to get
71+
*/
72+
public HeadHandlerTest(int lineCnt, int headLineCnt, int totalCharCount) {
73+
this.lineCnt = lineCnt;
74+
this.headLineCnt = headLineCnt;
75+
this.totalCharCount = totalCharCount;
76+
}
77+
78+
private static class RandomInputStream extends InputStream {
79+
private final int maxCharCount;
80+
private int charCount;
81+
private final int maxLines;
82+
private int lines;
83+
84+
private final String letters;
85+
86+
private static final String alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
87+
private static final String numbers = "0123456789";
88+
89+
private String result = "";
90+
91+
int[] lineBreaks;
92+
93+
/**
94+
* Generate alphanumeric characters.
95+
* @param maxCharCount number of characters to generate
96+
* @param maxLines number of lines to generate
97+
*/
98+
public RandomInputStream(int maxCharCount, int maxLines) {
99+
if (maxLines > maxCharCount) {
100+
throw new IllegalArgumentException("maxLines must be smaller than or equal to maxCharCount");
101+
}
102+
103+
if (maxCharCount <= 0) {
104+
throw new IllegalArgumentException("maxCharCount must be positive number");
105+
}
106+
107+
if (maxLines <= 1) {
108+
throw new IllegalArgumentException("maxLines must be strictly bigger than 1");
109+
}
110+
111+
this.maxCharCount = maxCharCount;
112+
this.charCount = 0;
113+
this.maxLines = maxLines;
114+
this.lines = 0;
115+
116+
letters = alpha + alpha.toLowerCase() + numbers;
117+
118+
// Want the newlines generally to appear within the first half of the generated data
119+
// so that the handler has significant amount of data to read after it is done reading the lines.
120+
lineBreaks = new int[maxLines - 1];
121+
for (int i = 0; i < lineBreaks.length; i++) {
122+
lineBreaks[i] = ThreadLocalRandom.current().nextInt(1, maxCharCount / 2);
123+
}
124+
}
125+
126+
int getCharCount() {
127+
return charCount;
128+
}
129+
130+
String getResult() {
131+
return result;
132+
}
133+
134+
@Override
135+
public int read() throws IOException {
136+
int ret;
137+
if (charCount < maxCharCount) {
138+
if (charCount > 0 && lines < maxLines - 1 && charCount == lineBreaks[lines]) {
139+
ret = '\n';
140+
lines++;
141+
} else {
142+
ret = letters.charAt(ThreadLocalRandom.current().nextInt(0, letters.length()));
143+
}
144+
result += String.format("%c", ret);
145+
charCount++;
146+
return ret;
147+
}
148+
149+
return -1;
150+
}
151+
}
152+
153+
@Test
154+
public void testHeadHandler() throws IOException {
155+
LOGGER.log(Level.INFO, "testing HeadHandler with: {0}/{1}/{2}",
156+
new Object[]{lineCnt, headLineCnt, totalCharCount});
157+
158+
RandomInputStream rndStream = new RandomInputStream(totalCharCount, lineCnt);
159+
HeadHandler handler = new HeadHandler(headLineCnt);
160+
assertTrue(totalCharCount >= HeadHandler.getBufferedReaderSize(),
161+
"number of characters to generate must be bigger than " +
162+
"HeadHandler internal buffer size");
163+
handler.processStream(rndStream);
164+
assertTrue(handler.count() <= headLineCnt,
165+
"HeadHandler should not get more lines than was asked to");
166+
assertEquals(totalCharCount, rndStream.getCharCount(),
167+
"HeadHandler should read all the characters from input stream");
168+
String[] headLines = new String[handler.count()];
169+
for (int i = 0; i < handler.count(); i++) {
170+
String line = handler.get(i);
171+
LOGGER.log(Level.INFO, "line [{0}]: {1}", new Object[]{i, line});
172+
headLines[i] = line;
173+
}
174+
assertArrayEquals(headLines,
175+
Arrays.copyOfRange(rndStream.getResult().split("\n"), 0, handler.count()),
176+
"the lines retrieved by HeadHandler needs to match the input");
177+
}
178+
}

0 commit comments

Comments
 (0)