refactor of systemtest base in preparation for #8

src/test/java/io/amient/kafka/hadoop/DefaultSystemTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2014 Michal Harish, [email protected]
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.amient.kafka.hadoop;
+
+import io.amient.kafka.hadoop.testutils.SystemTestBase;
+import kafka.producer.KeyedMessage;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DefaultSystemTest extends SystemTestBase {
+    @Test
+    public void canFollowKafkaPartitionsIncrementally()
+            throws IOException, ClassNotFoundException, InterruptedException {
+
+        //produce text data
+        simpleProducer.send(new KeyedMessage<>("topic01", "key1", "payloadA"));
+        simpleProducer.send(new KeyedMessage<>("topic01", "key2", "payloadB"));
+        simpleProducer.send(new KeyedMessage<>("topic01", "key1", "payloadC"));
+
+        //run the first job
+        runSimpleJob("topic01", "canFollowKafkaPartitions");
+
+        //produce more data
+        simpleProducer.send(new KeyedMessage<>("topic01", "key1", "payloadD"));
+        simpleProducer.send(new KeyedMessage<>("topic01", "key2", "payloadE"));
+
+        //run the second job
+        Path result = runSimpleJob("topic01", "canFollowKafkaPartitions");
+
+        //check results
+        Path part0offset0 = new Path(result, "topic01/0/topic01-0-0000000000000000000");
+        assertTrue(localFileSystem.exists(part0offset0));
+        assertEquals(String.format("payloadA%npayloadC%n"), readFullyAsString(part0offset0, 20));
+
+        Path part0offset2 = new Path(result, "topic01/0/topic01-0-0000000000000000002");
+        assertTrue(localFileSystem.exists(part0offset2));
+        assertEquals(String.format("payloadD%n"), readFullyAsString(part0offset2, 20));
+
+        Path part1offset0 = new Path(result, "topic01/1/topic01-1-0000000000000000000");
+        assertTrue(localFileSystem.exists(part1offset0));
+        assertEquals(String.format("payloadB%n"), readFullyAsString(part1offset0, 20));
+
+        Path part1offset1 = new Path(result, "topic01/1/topic01-1-0000000000000000001");
+        assertTrue(localFileSystem.exists(part1offset1));
+        assertEquals(String.format("payloadE%n"), readFullyAsString(part1offset1, 20));
+
+    }
+
+
+}

src/test/java/io/amient/kafka/hadoop/InvalidInputSystemTest.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2014 Michal Harish, [email protected]
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.amient.kafka.hadoop;
+
+import io.amient.kafka.hadoop.io.KafkaInputFormat;
+import io.amient.kafka.hadoop.io.MultiOutputFormat;
+import io.amient.kafka.hadoop.testutils.MyJsonTimestampExtractor;
+import io.amient.kafka.hadoop.testutils.SystemTestBase;
+import kafka.producer.KeyedMessage;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class InvalidInputSystemTest extends SystemTestBase {
+
+    @Test(expected = java.lang.Error.class)
+    public void failsNormallyWithInvalidInput() throws IOException, ClassNotFoundException, InterruptedException {
+        //configure inputs, timestamp extractor and the output path format
+        KafkaInputFormat.configureKafkaTopics(conf, "topic02");
+        KafkaInputFormat.configureZkConnection(conf, zkConnect);
+        HadoopJobMapper.configureTimestampExtractor(conf, MyJsonTimestampExtractor.class.getName());
+        MultiOutputFormat.configurePathFormat(conf, "'t={T}/d='yyyy-MM-dd'/h='HH");
+
+        //produce and run
+        simpleProducer.send(new KeyedMessage<>("topic02", "1", "{invalid-json-should-fail-in-extractor"));
+        runSimpleJob("topic02", "failsNormallyWithInvalidInput");
+    }
+}

src/test/java/io/amient/kafka/hadoop/TimestampExtractorSystemTest.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2014 Michal Harish, [email protected]
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.amient.kafka.hadoop;
+
+import io.amient.kafka.hadoop.io.KafkaInputFormat;
+import io.amient.kafka.hadoop.io.MultiOutputFormat;
+import io.amient.kafka.hadoop.testutils.MyJsonTimestampExtractor;
+import io.amient.kafka.hadoop.testutils.SystemTestBase;
+import kafka.producer.KeyedMessage;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TimestampExtractorSystemTest extends SystemTestBase {
+
+
+    @Test
+    public void canUseTimestampInPartitions() throws IOException, ClassNotFoundException, InterruptedException {
+
+        //produce some json data
+        String message5 = "{\"version\":5,\"timestamp\":1402944501425,\"id\": 1}";
+        simpleProducer.send(new KeyedMessage<>("topic02", "1", message5));
+        String message1 = "{\"version\":1,\"timestamp\":1402945801425,\"id\": 2}";
+        simpleProducer.send(new KeyedMessage<>("topic02", "2", message1));
+        String message6 = "{\"version\":6,\"timestamp\":1402948801425,\"id\": 1}";
+        simpleProducer.send(new KeyedMessage<>("topic02", "1", message6));
+        //testing a null message - with timestamp extractor this means skip message
+        simpleProducer.send(new KeyedMessage<>("topic02", "1", (String)null));
+
+        //configure inputs, timestamp extractor and the output path format
+        KafkaInputFormat.configureKafkaTopics(conf, "topic02");
+        KafkaInputFormat.configureZkConnection(conf, zkConnect);
+        HadoopJobMapper.configureTimestampExtractor(conf, MyJsonTimestampExtractor.class.getName());
+        MultiOutputFormat.configurePathFormat(conf, "'t={T}/d='yyyy-MM-dd'/h='HH");
+
+        Path outDir = runSimpleJob("topic02", "canUseTimestampInPartitions");
+
+        Path h18 = new Path(outDir, "t=topic02/d=2014-06-16/h=18/topic02-1-0000000000000000000");
+        assertTrue(localFileSystem.exists(h18));
+        assertEquals(String.format("%s%n", message5), readFullyAsString(h18, 100));
+
+        Path h19 = new Path(outDir, "t=topic02/d=2014-06-16/h=19/topic02-0-0000000000000000000");
+        assertTrue(localFileSystem.exists(h19));
+        assertEquals(String.format("%s%n", message1), readFullyAsString(h19, 100));
+
+        Path h20 = new Path(outDir, "t=topic02/d=2014-06-16/h=20/topic02-1-0000000000000000000");
+        assertTrue(localFileSystem.exists(h20));
+        assertEquals(String.format("%s%n", message6), readFullyAsString(h20, 100));
+    }
+
+
+}

src/test/java/io/amient/kafka/hadoop/TimestampExtractorTest.java → src/test/java/io/amient/kafka/hadoop/TimestampExtractorUnitTest.java

@@ -16,7 +16,7 @@
 
 import static org.junit.Assert.assertEquals;
 
-public class TimestampExtractorTest {
+public class TimestampExtractorUnitTest {
 
     private MapDriver<MsgMetadataWritable, BytesWritable, MsgMetadataWritable, BytesWritable> mapDriver;