VCFHeader, VCFCodec and VCFHeaderLine refactoring to enable support for VCF4.3/BCF2.2 and bug fixes.

Author: cmnbroad

src/main/java/htsjdk/variant/bcf2/BCF2Codec.java

@@ -62,7 +62,7 @@ public class BCF2Codec extends BinaryFeatureCodec<VariantContext> {
 
     /** sizeof a BCF header (+ min/max version). Used when trying to detect when a streams starts with a bcf header */
     public static final int SIZEOF_BCF_HEADER =  BCFVersion.MAGIC_HEADER_START.length + 2*Byte.BYTES;
-    
+
     private BCFVersion bcfVersion = null;
 
     private VCFHeader header = null;
@@ -501,10 +501,10 @@ protected BCF2GenotypeFieldDecoders.Decoder getGenotypeFieldDecoder(final String
     protected void error(final String message) throws RuntimeException {
         throw new TribbleException(String.format("%s, at record %d with position %d:", message, recordNo, pos));
     }
-    
+
     /** try to read a BCFVersion from an uncompressed BufferedInputStream.
      * The buffer must be large enough to contain {@link #SIZEOF_BCF_HEADER}
-     * 
+     *
      * @param uncompressedBufferedInput the uncompressed input stream
      * @return the BCFVersion if it can be decoded, or null if not found.
      * @throws IOException
@@ -515,5 +515,5 @@ public static BCFVersion tryReadBCFVersion(final BufferedInputStream uncompresse
         uncompressedBufferedInput.reset();
         return bcfVersion;
     }
-    
+
 }

src/main/java/htsjdk/variant/bcf2/BCF2Utils.java

@@ -27,7 +27,11 @@
 
 import htsjdk.samtools.util.FileExtensions;
 import htsjdk.tribble.TribbleException;
-import htsjdk.variant.vcf.*;
+import htsjdk.variant.vcf.VCFConstants;
+import htsjdk.variant.vcf.VCFHeader;
+import htsjdk.variant.vcf.VCFHeaderLine;
+import htsjdk.variant.vcf.VCFIDHeaderLine;
+import htsjdk.variant.vcf.VCFSimpleHeaderLine;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -93,10 +97,15 @@ public static ArrayList<String> makeDictionary(final VCFHeader header) {
         // set up the strings dictionary
         for ( VCFHeaderLine line : header.getMetaDataInInputOrder() ) {
             if ( line.shouldBeAddedToDictionary() ) {
-                final VCFIDHeaderLine idLine = (VCFIDHeaderLine)line;
-                if ( ! seen.contains(idLine.getID())) {
-                    dict.add(idLine.getID());
-                    seen.add(idLine.getID());
+                if (!line.isIDHeaderLine()) {
+                    //TODO: how do we ensure that shouldBeAddedToDictionary==true only when isIDHeaderLine==true
+                    throw new TribbleException(String.format(
+                            "The header line %s cannot be added to the BCF dictionary if its not an ID header line",
+                            line));
+                }
+                if ( ! seen.contains(line.getID())) {
+                    dict.add(line.getID());
+                    seen.add(line.getID());
                 }
             }
         }
@@ -291,7 +300,7 @@ else if ( o.getClass().isArray() ) {
      * Are the elements and their order in the output and input headers consistent so that
      * we can write out the raw genotypes block without decoding and recoding it?
      *
-     * If the order of INFO, FILTER, or contrig elements in the output header is different than
+     * If the order of INFO, FILTER, or contig elements in the output header is different than
      * in the input header we must decode the blocks using the input header and then recode them
      * based on the new output order.
      *
@@ -308,15 +317,15 @@ public static boolean headerLinesAreOrderedConsistently(final VCFHeader outputHe
         if ( ! nullAsEmpty(outputHeader.getSampleNamesInOrder()).equals(nullAsEmpty(genotypesBlockHeader.getSampleNamesInOrder())) )
             return false;
 
-        final Iterator<? extends VCFIDHeaderLine> outputLinesIt = outputHeader.getIDHeaderLines().iterator();
-        final Iterator<? extends VCFIDHeaderLine> inputLinesIt = genotypesBlockHeader.getIDHeaderLines().iterator();
+        final Iterator<VCFSimpleHeaderLine> outputLinesIt = outputHeader.getIDHeaderLines().iterator();
+        final Iterator<VCFSimpleHeaderLine> inputLinesIt = genotypesBlockHeader.getIDHeaderLines().iterator();
 
         while ( inputLinesIt.hasNext() ) {
             if ( ! outputLinesIt.hasNext() ) // missing lines in output
                 return false;
 
-            final VCFIDHeaderLine outputLine = outputLinesIt.next();
-            final VCFIDHeaderLine inputLine = inputLinesIt.next();
+            final VCFSimpleHeaderLine outputLine = outputLinesIt.next();
+            final VCFSimpleHeaderLine inputLine = inputLinesIt.next();
 
             if ( ! inputLine.getClass().equals(outputLine.getClass()) || ! inputLine.getID().equals(outputLine.getID()) )
                 return false;

src/main/java/htsjdk/variant/variantcontext/writer/VCFWriter.java

@@ -27,15 +27,19 @@
 
 import htsjdk.samtools.SAMSequenceDictionary;
 import htsjdk.samtools.util.IOUtil;
+import htsjdk.samtools.util.Log;
 import htsjdk.samtools.util.RuntimeIOException;
+import htsjdk.tribble.TribbleException;
 import htsjdk.tribble.index.IndexCreator;
+import htsjdk.utils.ValidationUtils;
 import htsjdk.variant.variantcontext.VariantContext;
 import htsjdk.variant.variantcontext.VariantContextBuilder;
 import htsjdk.variant.vcf.VCFConstants;
 import htsjdk.variant.vcf.VCFEncoder;
 import htsjdk.variant.vcf.VCFHeader;
 import htsjdk.variant.vcf.VCFHeaderLine;
 import htsjdk.variant.vcf.VCFHeaderVersion;
+import htsjdk.variant.vcf.VCFUtils;
 
 import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
@@ -45,14 +49,15 @@
 import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.nio.file.Path;
+import java.util.stream.Collectors;
 
 /**
  * this class writes VCF files
  */
 class VCFWriter extends IndexingVariantContextWriter {
+    protected final static Log logger = Log.getInstance(VCFWriter.class);
 
-    private static final String VERSION_LINE =
-            VCFHeader.METADATA_INDICATOR + VCFHeaderVersion.VCF4_2.getFormatString() + "=" + VCFHeaderVersion.VCF4_2.getVersionString();
+    private static final String DEFAULT_VERSION_LINE = VCFHeader.DEFAULT_VCF_VERSION.toHeaderVersionLine();
 
 	// Initialized when the header is written to the output stream
 	private VCFEncoder vcfEncoder = null;
@@ -164,7 +169,7 @@ public void writeHeader(final VCFHeader header) {
     }
 
     public static String getVersionLine() {
-        return VERSION_LINE;
+        return DEFAULT_VERSION_LINE;
     }
 
     public static VCFHeader writeHeader(VCFHeader header,
@@ -175,12 +180,18 @@ public static VCFHeader writeHeader(VCFHeader header,
         try {
             rejectVCFV43Headers(header);
 
-            // the file format field needs to be written first
+            // Validate that the file version we're writing is version-compatible this header's version.
+            validateHeaderVersion(header, versionLine);
+
+            // The file format field needs to be written first; below any file format lines
+            // embedded in the header will be removed
             writer.write(versionLine + "\n");
 
             for (final VCFHeaderLine line : header.getMetaDataInSortedOrder() ) {
-                if ( VCFHeaderVersion.isFormatString(line.getKey()) )
+                // Remove the fileformat header lines
+                if ( VCFHeaderVersion.isFormatString(line.getKey()) ) {
                     continue;
+                }
 
                 writer.write(VCFHeader.METADATA_INDICATOR);
                 writer.write(line.toString());
@@ -189,14 +200,9 @@ public static VCFHeader writeHeader(VCFHeader header,
 
             // write out the column line
             writer.write(VCFHeader.HEADER_INDICATOR);
-            boolean isFirst = true;
-            for (final VCFHeader.HEADER_FIELDS field : header.getHeaderFields() ) {
-                if ( isFirst )
-                    isFirst = false; // don't write out a field separator
-                else
-                    writer.write(VCFConstants.FIELD_SEPARATOR);
-                writer.write(field.toString());
-            }
+            writer.write(header.getHeaderFields().stream()
+                    .map(f -> f.name())
+                    .collect(Collectors.joining(VCFConstants.FIELD_SEPARATOR)).toString());
 
             if ( header.hasGenotypingData() ) {
                 writer.write(VCFConstants.FIELD_SEPARATOR);
@@ -217,6 +223,33 @@ public static VCFHeader writeHeader(VCFHeader header,
         return header;
     }
 
+    /**
+     * Given a header and a requested target output version, see if the header's version is compatible with the
+     * requested version.
+     * @param header
+     * @param requestedVersionLine
+     */
+    private static void validateHeaderVersion(final VCFHeader header, final String requestedVersionLine) {
+        ValidationUtils.nonNull(header);
+        ValidationUtils.nonNull(requestedVersionLine);
+
+        final VCFHeaderVersion vcfCurrentVersion = header.getVCFHeaderVersion();
+        final VCFHeaderVersion vcfRequestedVersion = VCFHeaderVersion.getHeaderVersion(requestedVersionLine);
+        if (!vcfCurrentVersion.equals(vcfRequestedVersion)) {
+            final String message = String.format("Attempting to write a %s VCF header to a %s VCFWriter",
+                    vcfRequestedVersion,
+                    vcfCurrentVersion.getVersionString());
+            if (!VCFHeaderVersion.versionsAreCompatible(VCFHeaderVersion.getHeaderVersion(requestedVersionLine), vcfCurrentVersion)) {
+                if (VCFUtils.getStrictVCFVersionValidation()) {
+                    throw new TribbleException(message);
+                }
+            }
+            if (VCFUtils.getVerboseVCFLogging()) {
+                logger.warn(message);
+            }
+        }
+    }
+
     /**
      * attempt to close the VCF file
      */