upgrading qio

q3panel/src/au/edu/qimr/panel/Q3ClinVar.java

@@ -68,16 +68,13 @@
 import org.qcmg.common.util.LoadReferencedClasses;
 import org.qcmg.common.util.Pair;
 import org.qcmg.common.util.TabTokenizer;
-
 import org.qcmg.common.vcf.VcfRecord;
 import org.qcmg.common.vcf.VcfUtils;
 import org.qcmg.common.vcf.header.VcfHeader;
 import org.qcmg.common.vcf.header.VcfHeaderRecord;
 import org.qcmg.common.vcf.header.VcfHeaderUtils;
-import org.qcmg.tab.TabbedFileReader;
-import org.qcmg.tab.TabbedHeader;
-import org.qcmg.tab.TabbedRecord;
-import org.qcmg.vcf.VCFFileWriter;
+import org.qcmg.qio.record.StringFileReader;
+import org.qcmg.qio.record.RecordWriter;
 
 public class Q3ClinVar {
 
@@ -483,22 +480,18 @@ private void loadTiledAlignerData() throws Exception {
 
 		logger.info("loading genome tiles alignment data");
 
-		try (TabbedFileReader reader = new TabbedFileReader(new File(refTiledAlignmentFile))) {
+		try (StringFileReader reader = new StringFileReader(new File(refTiledAlignmentFile))) {		 
+			List<String> headerList = reader.getHeader();
 
-			TabbedHeader header = reader.getHeader();
-			List<String> headerList = new ArrayList<>();
-			for (String head : header) {
-				headerList.add(head);
-			}
 			positionToActualLocation.loadMap(headerList);
 			int i = 0;
-			for (TabbedRecord rec : reader) {
+			for (String rec : reader) {
 				if (++i % 1000000 == 0) {
 					logger.info("hit " + (i / 1000000) + "M records");
 				}
-				String tile = rec.getData().substring(0, TILE_SIZE);
+				String tile = rec.substring(0, TILE_SIZE);
 				if (ampliconTiles.contains(tile)) {
-					String countOrPosition = rec.getData().substring(rec.getData().indexOf('\t') + 1);
+					String countOrPosition = rec.substring(rec.indexOf('\t') + 1);
 					if (countOrPosition.charAt(0) == 'C') {
 						frequentlyOccurringRefTiles.add(tile);
 					} else {
@@ -1055,7 +1048,7 @@ private void writeHaplotypesCsv(boolean filter) throws IOException {
 
 		//make a new header		
 		String outputFileName = filter ? outputFileNameBase + "vcf" : outputFileNameBase + "diag.unfiltered.vcf";
-		try (VCFFileWriter writer = new VCFFileWriter(new File(outputFileName))) {
+		try (RecordWriter<VcfRecord> writer = new RecordWriter<>(new File(outputFileName))) {
 
 			/*
 			 * Setup the VcfHeader
@@ -1084,7 +1077,7 @@ private void writeHaplotypesCsv(boolean filter) throws IOException {
 		}
 
 		outputFileName = filter ?  outputFileNameBase + "diag.detailed.vcf" : outputFileNameBase + "diag.unfiltered_detailed.vcf";
-		try (VCFFileWriter writer = new VCFFileWriter(new File(outputFileName))) {
+		try (RecordWriter<VcfRecord> writer = new RecordWriter<>(new File(outputFileName))) {
 
 			/*
 			 * Setup the VcfHeader

q3panel/src/au/edu/qimr/panel/Q3Panel.java

@@ -89,11 +89,9 @@
 import org.qcmg.common.vcf.header.VcfHeaderRecord;
 import org.qcmg.common.vcf.header.VcfHeaderUtils;
 import org.qcmg.common.vcf.header.VcfHeaderUtils.VcfInfoType;
-import org.qcmg.tab.TabbedFileReader;
-import org.qcmg.tab.TabbedHeader;
-import org.qcmg.tab.TabbedRecord;
+import org.qcmg.qio.record.StringFileReader;
 import org.qcmg.vcf.VCFFileReader;
-import org.qcmg.vcf.VCFFileWriter;
+import org.qcmg.qio.record.RecordWriter;
 
 public class Q3Panel {
 
@@ -390,12 +388,12 @@ private void loadTranscripts() throws IOException {
 
 			logger.info("Number of unique chromosomes in bed file: " + uniqueChrs.size());
 
-			try (TabbedFileReader reader = new TabbedFileReader(new File(geneTranscriptsFile))) {
+			try (StringFileReader reader = new StringFileReader(new File(geneTranscriptsFile))) {
 				String currentTranscriptId = null;
-				for (TabbedRecord rec : reader) {
-					String contig = rec.getData().substring(0, rec.getData().indexOf(Constants.TAB));
+				for (String rec : reader) {
+					String contig = rec.substring(0, rec.indexOf(Constants.TAB));
 					if (uniqueChrs.containsKey(contig)) {
-						String [] params = TabTokenizer.tokenize(rec.getData());
+						String [] params = TabTokenizer.tokenize(rec);
 						String [] column8 = params[8].split(Constants.SEMI_COLON_STRING); 
 						Optional<String> optionalId = Arrays.stream(column8).filter(s -> s.trim().startsWith("transcript_id")).findAny();
 						Optional<String> optionalExonNumber = Arrays.stream(column8).filter(s -> s.trim().startsWith("exon_number")).findAny();
@@ -671,9 +669,9 @@ private void mapBedToAmplicons() throws IOException, Exception {
 		 */
 		if (bedFile != null && new File(bedFile).exists()) {
 			int bedId = 0;
-			try (TabbedFileReader reader = new TabbedFileReader(new File(bedFile));) {
-				for (TabbedRecord rec : reader) {
-					String [] params = TabTokenizer.tokenize(rec.getData());
+			try (StringFileReader reader = new StringFileReader(new File(bedFile));) {
+				for (String rec : reader) {
+					String [] params = TabTokenizer.tokenize(rec);
 					ChrPosition cp = new ChrRangePosition(params[0], Integer.parseInt(params[1]), Integer.parseInt(params[2]));
 					bedToAmpliconMap.put(new Contig(++bedId, cp), new ArrayList<Contig>(1));
 				}
@@ -1037,7 +1035,7 @@ private void addFormatFieldValues() {
 	}
 
 	private void writeMutationsToFile() throws IOException {
-		try (VCFFileWriter writer = new VCFFileWriter(new File(outputFileNameBase + ".vcf"))) {
+		try (RecordWriter<VcfRecord> writer = new RecordWriter<>(new File(outputFileNameBase + ".vcf"))) {
 
 			/*
 			 * Setup the VcfHeader
@@ -1082,7 +1080,7 @@ private void writeMutationsToFile() throws IOException {
 
 	private void writeMutationsToFileNew(List<VcfRecord> vcfs) throws IOException {
 		String filename = outputFileNameBase + "_2.vcf";
-		try (VCFFileWriter writer = new VCFFileWriter(new File(filename))) {
+		try (RecordWriter<VcfRecord> writer = new RecordWriter<>(new File(filename))) {
 
 			/*
 			 * Setup the VcfHeader
@@ -1393,20 +1391,18 @@ private void loadTiledAlignerData() {
 
 		logger.info("loading genome tiles alignment data");
 
-		try (TabbedFileReader reader = new TabbedFileReader(new File(refTiledAlignmentFile))) {
+		try (StringFileReader reader = new StringFileReader(new File(refTiledAlignmentFile))) {
 
-			TabbedHeader header = reader.getHeader();
-			List<String> headerList = new ArrayList<>();
-			for (String head : header) {
-				headerList.add(head);
-			}
+
+			List<String> headerList = reader.getHeader();
+
 			positionToActualLocation.loadMap(headerList);
 			int i = 0;
-			for (TabbedRecord rec : reader) {
+			for (String rec : reader) {
 				if (++i % 1000000 == 0) {
 					logger.info("hit " + (i / 1000000) + "M records");
 				}
-				queue.add(rec.getData());
+				queue.add(rec);
 			}
 			/*
 			 * kill off threads waiting on queue

q3tiledaligner/src/au/edu/qimr/tiledaligner/ReadTiledAligerFile.java

@@ -23,7 +23,7 @@
 
 import org.qcmg.common.util.Constants;
 import org.qcmg.common.util.NumberUtils;
-import org.qcmg.string.StringFileReader;
+import org.qcmg.qio.record.StringFileReader;
 
 public class ReadTiledAligerFile {
 

q3tiledaligner/src/au/edu/qimr/tiledaligner/TiledAligner.java

@@ -25,7 +25,7 @@
 import org.qcmg.common.log.QLoggerFactory;
 import org.qcmg.common.model.BLATRecord;
 import org.qcmg.common.util.NumberUtils;
-import org.qcmg.string.StringFileReader;
+import org.qcmg.qio.record.StringFileReader;
 
 public class TiledAligner {
 

q3tiledaligner/src/au/edu/qimr/tiledaligner/util/TiledAlignerUtil.java

@@ -37,10 +37,8 @@
 import org.qcmg.common.sw.SmithWatermanGotoh;
 import org.qcmg.common.util.Constants;
 import org.qcmg.common.util.NumberUtils;
-import org.qcmg.tab.TabbedFileReader;
-import org.qcmg.tab.TabbedHeader;
-import org.qcmg.tab.TabbedRecord;
 
+import org.qcmg.qio.record.StringFileReader;
 import gnu.trove.list.TIntList;
 import gnu.trove.list.TLongList;
 import gnu.trove.list.array.TIntArrayList;
@@ -530,20 +528,20 @@ public static int nonContinuousCount(long [][] array, long l, int arrayStartPosi
 	 * @param tiledInput
 	 */
 	public static void getTiledDataInMap(String tiledAlignerFile,  Map<String, String> tiledInput) {
-		try (TabbedFileReader reader = new TabbedFileReader(new File(tiledAlignerFile))) {
+		try (StringFileReader reader = new StringFileReader(new File(tiledAlignerFile))) {
 
 			int i = 0;
 			int matches = 0;
 			int mapSize = tiledInput.size();
-			for (TabbedRecord rec : reader) {
+			for (String rec : reader) {
 				if (++i % 1000000 == 0) {
 					logger.info("hit " + (i / 1000000) + "M records, matches: " + matches);
 				}
-				String data = rec.getData();
-				int tabindex = data.indexOf(Constants.TAB);
-				if (tiledInput.containsKey(data.substring(0, tabindex))) {
+
+				int tabindex = rec.indexOf(Constants.TAB);
+				if (tiledInput.containsKey(rec.substring(0, tabindex))) {
 					matches++;
-					tiledInput.put(data.substring(0, tabindex), data.substring(tabindex + 1));
+					tiledInput.put(rec.substring(0, tabindex), rec.substring(tabindex + 1));
 
 					/*
 					 * If we have found all elements in the map - might as well stop looking
@@ -561,8 +559,8 @@ public static void getTiledDataInMap(String tiledAlignerFile,  Map<String, Strin
 		}
 	}
 
-	public static TabbedHeader getTiledAlignerHeader(String file) throws IOException {
-		try (TabbedFileReader reader = new TabbedFileReader(new File(file))) {
+	public static List<String> getTiledAlignerHeader(String file) throws IOException {
+		try (StringFileReader reader = new StringFileReader(new File(file))) {
 			return reader.getHeader();
 		}
 	}
@@ -859,14 +857,12 @@ public static String[] getIntelligentSwDiffs(String ref, String sequence, float
 	 * @return
 	 */
 	public static int getSWScore(String diffs) {
-		int matchCount = 0, mismatchCount = 0, gapCount = 0;
+		int matchCount = 0, mismatchCount = 0; 
 		for (char c : diffs.toCharArray()) {
 			if ('|' == c) {
 				matchCount++;
 			} else if ('.' == c) {
 				mismatchCount++;
-			} else if (' ' == c) {
-//				gapCount++;
 			} else {
 				logger.warn("found [" + c + "] in sw diffs string!!!");
 			}

qbamannotate/src/org/qcmg/qbamannotate/Annotator.java

@@ -15,10 +15,10 @@
 import htsjdk.samtools.SAMFileWriterFactory;
 import htsjdk.samtools.SAMRecord;
 
-import org.qcmg.ma.MAFileReader;
-import org.qcmg.ma.MARecord;
 import org.qcmg.picard.HeaderUtils;
 import org.qcmg.picard.SAMFileReaderFactory;
+import org.qcmg.qio.ma.MaFileReader;
+import org.qcmg.qio.ma.MaRecord;
 
 public final class Annotator {
 	private final boolean modifyProgramLine;
@@ -31,10 +31,10 @@ public final class Annotator {
 	private SAMFileWriter fileWriter;
 	private SAMFileHeader header;
 	private SamReader samFileReader;
-	private MAFileReader maFileReader;
+	private MaFileReader maFileReader;
 	private Iterator<SAMRecord> samIterator;
-	private Iterator<MARecord> maIterator;
-	private MARecord nextMARecord;
+	private Iterator<MaRecord> maIterator;
+	private MaRecord nextMARecord;
 	private SAMRecord nextSAMRecord;
 	private AnnotatorType type = new Frag();
 
@@ -140,7 +140,7 @@ private void nextMARecord() {
 	private void marchAndAnnotate() throws Exception {
 		try {
 			samFileReader = SAMFileReaderFactory.createSAMFileReader( inputBAMFile) ; //new SAMFileReader(inputBAMFile);
-			maFileReader = new MAFileReader(inputMAFile);
+			maFileReader = new MaFileReader(inputMAFile);
 			header = samFileReader.getFileHeader();
 			header.setSortOrder(SAMFileHeader.SortOrder.unsorted);
 			if (modifyProgramLine) {

qbamannotate/src/org/qcmg/qbamannotate/AnnotatorType.java

@@ -8,7 +8,7 @@
 
 import java.util.regex.Pattern;
 
-import org.qcmg.ma.MARecord;
+import org.qcmg.qio.ma.MaRecord;
 
 import htsjdk.samtools.SAMRecord;
 
@@ -19,7 +19,7 @@ public abstract class AnnotatorType {
 	public abstract boolean annotate(final SAMRecord record) throws Exception;
 
 	public abstract boolean annotate(final SAMRecord record,
-			final MARecord maRecord) throws Exception;
+			final MaRecord maRecord) throws Exception;
 
 	public void markRecordUnmatched(final SAMRecord record) {
 		Object zmObj = record.getAttribute("ZM");
@@ -47,7 +47,7 @@ static void setZMAttribute(final SAMRecord record, int n) {
 		record.setAttribute("ZM", Integer.toString(n));
 	}
 
-	static int compareTriplet(final SAMRecord record, final MARecord maRecord) {
+	static int compareTriplet(final SAMRecord record, final MaRecord maRecord) {
 		return compareTriplet(record.getReadName(), maRecord.getDefLine()
 				.getReadName());
 	}

qbamannotate/src/org/qcmg/qbamannotate/Frag.java

@@ -6,8 +6,8 @@
  */
 package org.qcmg.qbamannotate;
 
-import org.qcmg.ma.MADirection;
-import org.qcmg.ma.MARecord;
+import org.qcmg.qio.ma.MaDirection;
+import org.qcmg.qio.ma.MaRecord;
 
 import htsjdk.samtools.SAMRecord;
 
@@ -19,7 +19,7 @@ public boolean annotate(final SAMRecord record) throws Exception {
 	}
 
 	@Override
-	public boolean annotate(SAMRecord record, MARecord maRecord) throws Exception {
+	public boolean annotate(SAMRecord record, MaRecord maRecord) throws Exception {
 		int n = maRecord.getDefLine().getNumberMappings();
 		setZMAttribute(record, n);
 		return annotate(record);

qbamannotate/src/org/qcmg/qbamannotate/LongMatePair.java

@@ -18,20 +18,21 @@
 import javax.xml.transform.Transformer;
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.stream.StreamSource;
+
+import org.qcmg.qio.ma.MaDirection;
+import org.qcmg.qio.ma.MaRecord;
+
 import javax.xml.transform.stream.StreamResult;
 import java.io.*;
 
 import htsjdk.samtools.SAMRecord;
 
-import org.qcmg.ma.MADirection;
-import org.qcmg.ma.MARecord;
-
 public class LongMatePair extends AnnotatorType {
 	final int isizeLowerLimit;
 	final int isizeUpperLimit;
 	private String zpAnnotation;
 	private SAMRecord record;
-	private MARecord maRecord;
+	private MaRecord maRecord;
 	private int annotatedCount = 0;
 	private final Map<String, Integer> zpToCount = new HashMap<String, Integer>();
 	private String xmlReport;
@@ -65,7 +66,7 @@ public boolean annotate(final SAMRecord record) throws Exception {
 	}
 
 	@Override
-	public boolean annotate(SAMRecord record, MARecord maRecord) throws Exception {
+	public boolean annotate(SAMRecord record, MaRecord maRecord) throws Exception {
 		this.record = record;
 		this.maRecord = maRecord;
 		performZMAnnotation();
@@ -96,10 +97,10 @@ public String generateReport() throws Exception {
 
 	private void performZMAnnotation() {
 		assert 0 == compareTriplet(record, maRecord);
-		if (maRecord.getDefLine().getDirection() == MADirection.F3
+		if (maRecord.getDefLine().getDirection() == MaDirection.F3
 				&& record.getReadPairedFlag() && record.getFirstOfPairFlag()) {
 			setZMAttribute();
-		} else if (maRecord.getDefLine().getDirection() == MADirection.R3
+		} else if (maRecord.getDefLine().getDirection() == MaDirection.R3
 				&& record.getReadPairedFlag() && record.getSecondOfPairFlag()) {
 			setZMAttribute();
 		} else {