Add VCF 4.3 writing

Author: Anders Leung

src/main/java/htsjdk/samtools/Defaults.java

@@ -1,6 +1,7 @@
 package htsjdk.samtools;
 
 import htsjdk.samtools.util.Log;
+import htsjdk.variant.variantcontext.writer.VCF42To43VersionTransitionPolicy;
 
 import java.io.File;
 import java.util.Collections;
@@ -110,6 +111,8 @@ public class Defaults {
      */
     public static final boolean DISABLE_SNAPPY_COMPRESSOR;
 
+    public static final VCF42To43VersionTransitionPolicy VCF_VERSION_TRANSITION_POLICY;
+
 
     public static final String SAMJDK_PREFIX = "samjdk.";
     static {
@@ -134,6 +137,10 @@ public class Defaults {
         SAM_FLAG_FIELD_FORMAT = SamFlagField.valueOf(getStringProperty("sam_flag_field_format", SamFlagField.DECIMAL.name()));
         SRA_LIBRARIES_DOWNLOAD = getBooleanProperty("sra_libraries_download", false);
         DISABLE_SNAPPY_COMPRESSOR = getBooleanProperty(DISABLE_SNAPPY_PROPERTY_NAME, false);
+        VCF_VERSION_TRANSITION_POLICY = VCF42To43VersionTransitionPolicy.valueOf(getStringProperty(
+            "vcf_version_transition_policy",
+            VCF42To43VersionTransitionPolicy.DO_NOT_TRANSITION.name()
+        ));
     }
 
     /**
@@ -157,6 +164,7 @@ public static SortedMap<String, Object> allDefaults(){
         result.put("CUSTOM_READER_FACTORY", CUSTOM_READER_FACTORY);
         result.put("SAM_FLAG_FIELD_FORMAT", SAM_FLAG_FIELD_FORMAT);
         result.put("DISABLE_SNAPPY_COMPRESSOR", DISABLE_SNAPPY_COMPRESSOR);
+        result.put("VCF_VERSION_TRANSITION_POLICY", VCF_VERSION_TRANSITION_POLICY);
         return Collections.unmodifiableSortedMap(result);
     }
 

src/main/java/htsjdk/samtools/cram/structure/CRAMCompressionRecord.java

@@ -364,7 +364,7 @@ public void assignReadName() {
      * that the record's read bases, quality scores, and mate graph are not stored directly as part of the
      * record. Normalization is the process of resolving these values, and is performed at Slice granularity,
      * across all records in a Slice.
-     * (see {@link Slice#normalizeCRAMRecords(List, CRAMReferenceRegion, SubstitutionMatrix)}).
+     * (see {@link Slice#normalizeCRAMRecords(List, CRAMReferenceRegion)}).
      */
     void setIsNormalized() { isNormalized = true; }
 
@@ -373,7 +373,7 @@ public void assignReadName() {
      * scores, and mate graph are not stored directly as part of the record. These values must be resolved
      * through the separate process of normalization, which is performed at Slice granularity (all records in a
      * Slice are normalized at the same time).
-     * (see {@link Slice#normalizeCRAMRecords(List, CRAMReferenceRegion, SubstitutionMatrix)}).
+     * (see {@link Slice#normalizeCRAMRecords(List, CRAMReferenceRegion)}).
      * @return true if this record is normalized
      */
     public boolean isNormalized() { return isNormalized; }

src/main/java/htsjdk/variant/variantcontext/VariantContext.java

@@ -399,6 +399,8 @@ private static void validateFilters(final VariantContext variantContext) {
                 if (!VALID_FILTER.matcher(filter).matches()) {
                     throw new IllegalStateException("Filter '" + filter +
                             "' contains an illegal character. It must conform to the regex ;'" + VALID_FILTER);
+                } else if (filter.equals("0")) {
+                    throw new IllegalStateException("Filter cannot use reserved string '0'");
                 }
             }
         }
@@ -451,7 +453,7 @@ protected VariantContext(final String source,
                              final Map<String, Object> attributes,
                              final boolean fullyDecoded,
                              final EnumSet<Validation> validationToPerform ) {
-        if ( contig == null ) { throw new IllegalArgumentException("Contig cannot be null"); }
+        if ( contig == null || contig.isEmpty() ) { throw new IllegalArgumentException("Contig cannot be null or the empty string"); }
         this.contig = contig;
         this.start = start;
         this.stop = stop;

src/main/java/htsjdk/variant/variantcontext/writer/BCF2Writer.java

@@ -43,6 +43,7 @@
 import htsjdk.variant.vcf.VCFConstants;
 import htsjdk.variant.vcf.VCFContigHeaderLine;
 import htsjdk.variant.vcf.VCFHeader;
+import htsjdk.variant.vcf.VCFHeaderVersion;
 import htsjdk.variant.vcf.VCFUtils;
 
 import java.io.ByteArrayOutputStream;
@@ -169,7 +170,8 @@ public void writeHeader(VCFHeader header) {
             // write out the header into a byte stream, get its length, and write everything to the file
             final ByteArrayOutputStream capture = new ByteArrayOutputStream();
             final OutputStreamWriter writer = new OutputStreamWriter(capture);
-            this.header = VCFWriter.writeHeader(this.header, writer, VCFWriter.getVersionLine(), "BCF2 stream");
+            // TODO implement logic to select VCF version based on BCF version
+            this.header = VCFWriter.writeHeader(this.header, writer, VCFHeaderVersion.VCF4_2, "BCF2 stream");
             writer.append('\0'); // the header is null terminated by a byte
             writer.close();
 

src/main/java/htsjdk/variant/variantcontext/writer/VCF42To43VersionTransitionPolicy.java

@@ -0,0 +1,30 @@
+package htsjdk.variant.variantcontext.writer;
+
+/**
+ * Defines the policy VCFWriter will use when writing a VCF file from a version before 4.3.
+ *
+ * This is necessary as VCF 4.3 is stricter than 4.2, meaning that some valid 4.2 files are invalid 4.3
+ * and automatically transitioning from pre 4.3 to 4.3+ is not always possible.
+ * This class is a temporary workaround to allow opt-in 4.3 writing support in a way that does not break
+ * workflows that may process 4.2 files that are invalid 4.3, but should be removed once proper versioning
+ * support for VCF is incorporated into htsjdk
+ */
+public enum VCF42To43VersionTransitionPolicy {
+    /**
+     * Write pre 4.3 files as 4.2, to which automatic transitioning should always be possible, and
+     * write 4.3+ files as 4.3.
+     */
+    DO_NOT_TRANSITION,
+
+    /**
+     * Inspect the headers of pre 4.3 files to determine if they can be automatically transitioned to 4.3,
+     * and if automatic transition is possible write them as 4.3, or else write them as 4.2.
+     */
+    TRANSITION_IF_POSSIBLE,
+
+    /**
+     * Inspect the headers of pre 4.3 files to determine if they can be automatically transitioned to 4.3,
+     * and abort with an error if automatic transition is not possible
+     */
+    FAIL_IF_CANNOT_TRANSITION,
+}

src/main/java/htsjdk/variant/variantcontext/writer/VCFWriter.java

@@ -25,6 +25,7 @@
 
 package htsjdk.variant.variantcontext.writer;
 
+import htsjdk.samtools.Defaults;
 import htsjdk.samtools.SAMSequenceDictionary;
 import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.Log;
@@ -57,8 +58,6 @@
 class VCFWriter extends IndexingVariantContextWriter {
     protected final static Log logger = Log.getInstance(VCFWriter.class);
 
-    private static final String DEFAULT_VERSION_LINE = VCFHeader.DEFAULT_VCF_VERSION.toHeaderVersionLine();
-
 	// Initialized when the header is written to the output stream
 	private VCFEncoder vcfEncoder = null;
 
@@ -76,6 +75,8 @@ class VCFWriter extends IndexingVariantContextWriter {
     // is the header or body written to the output stream?
     private boolean outputHasBeenWritten;
 
+    private VCF42To43VersionTransitionPolicy transitionPolicy = Defaults.VCF_VERSION_TRANSITION_POLICY;
+
     /*
      * The VCF writer uses an internal Writer, based by the ByteArrayOutputStream lineBuffer,
      * to temp. buffer the header and per-site output before flushing the per line output
@@ -155,37 +156,55 @@ private void writeAndResetBuffer() throws IOException {
 
     @Override
     public void writeHeader(final VCFHeader header) {
-
         // note we need to update the mHeader object after this call because they header
         // may have genotypes trimmed out of it, if doNotWriteGenotypes is true
         setHeader(header);
         try {
-            writeHeader(this.mHeader, writer, getVersionLine(), getStreamName());
+            writeHeader(this.mHeader, writer, getOutputVersion(header), getStreamName());
             writeAndResetBuffer();
             outputHasBeenWritten = true;
         } catch ( IOException e ) {
             throw new RuntimeIOException("Couldn't write file " + getStreamName(), e);
         }
     }
 
-    public static String getVersionLine() {
-        return DEFAULT_VERSION_LINE;
+    private VCFHeaderVersion getOutputVersion(final VCFHeader header) {
+        if (transitionPolicy == VCF42To43VersionTransitionPolicy.DO_NOT_TRANSITION) {
+            // Write pre 4.3 files as 4.2, and 4.3+ files as 4.3
+            return header.getVCFHeaderVersion() != null && header.getVCFHeaderVersion().isAtLeastAsRecentAs(VCFHeaderVersion.VCF4_3)
+                ? header.getVCFHeaderVersion()
+                : VCFHeaderVersion.VCF4_2;
+        } else {
+            // Try to promote to 4.3+
+            return VCFHeader.DEFAULT_VCF_VERSION;
+        }
     }
 
-    public static VCFHeader writeHeader(VCFHeader header,
+    public static VCFHeader writeHeader(final VCFHeader header,
                                         final Writer writer,
-                                        final String versionLine,
+                                        final VCFHeaderVersion version,
                                         final String streamNameForError) {
-
         try {
-            rejectVCFV43Headers(header);
-
-            // Validate that the file version we're writing is version-compatible this header's version.
-            validateHeaderVersion(header, versionLine);
+            VCFHeaderVersion versionToWrite = version;
+            // Validate that the file version we're writing is version-compatible with this header's version.
+            for (final VCFHeaderLine line : header.getMetaDataInSortedOrder() ) {
+                // Skip the fileformat line, because we may be trying to transition versions
+                if ( VCFHeaderVersion.isFormatString(line.getKey()) ) {
+                    continue;
+                }
+                try {
+                    line.validateForVersion(versionToWrite);
+                } catch (final TribbleException e) {
+                    // If 4.3 validation fails for any line, use 4.2
+                    if (versionToWrite == VCFHeader.DEFAULT_VCF_VERSION) {
+                        versionToWrite = VCFHeaderVersion.VCF4_2;
+                    }
+                }
+            }
 
             // The file format field needs to be written first; below any file format lines
             // embedded in the header will be removed
-            writer.write(versionLine + "\n");
+            writer.write(versionToWrite.toHeaderVersionLine() + "\n");
 
             for (final VCFHeaderLine line : header.getMetaDataInSortedOrder() ) {
                 // Remove the fileformat header lines
@@ -201,8 +220,8 @@ public static VCFHeader writeHeader(VCFHeader header,
             // write out the column line
             writer.write(VCFHeader.HEADER_INDICATOR);
             writer.write(header.getHeaderFields().stream()
-                    .map(f -> f.name())
-                    .collect(Collectors.joining(VCFConstants.FIELD_SEPARATOR)).toString());
+                .map(Enum::name)
+                .collect(Collectors.joining(VCFConstants.FIELD_SEPARATOR)));
 
             if ( header.hasGenotypingData() ) {
                 writer.write(VCFConstants.FIELD_SEPARATOR);
@@ -291,21 +310,14 @@ public void add(final VariantContext context) {
 
     @Override
     public void setHeader(final VCFHeader header) {
-        rejectVCFV43Headers(header);
-
         if (outputHasBeenWritten) {
             throw new IllegalStateException("The header cannot be modified after the header or variants have been written to the output stream.");
         }
         this.mHeader = doNotWriteGenotypes ? new VCFHeader(header.getMetaDataInSortedOrder()) : header;
         this.vcfEncoder = new VCFEncoder(this.mHeader, this.allowMissingFieldsInHeader, this.writeFullFormatField);
     }
 
-    // writing vcf v4.3 is not implemented
-    private static void rejectVCFV43Headers(final VCFHeader targetHeader) {
-        if (targetHeader.getVCFHeaderVersion() != null && targetHeader.getVCFHeaderVersion().isAtLeastAsRecentAs(VCFHeaderVersion.VCF4_3)) {
-            throw new IllegalArgumentException(String.format("Writing VCF version %s is not implemented", targetHeader.getVCFHeaderVersion()));
-        }
-
-
+    public void setTransitionPolicy(final VCF42To43VersionTransitionPolicy transitionPolicy) {
+        this.transitionPolicy = transitionPolicy;
     }
 }

src/main/java/htsjdk/variant/variantcontext/writer/VariantContextWriterBuilder.java

@@ -132,6 +132,8 @@ public enum OutputType {
     private boolean createMD5 = Defaults.CREATE_MD5;
     protected EnumSet<Options> options = DEFAULT_OPTIONS.clone();
 
+    private VCF42To43VersionTransitionPolicy vcf42To43VersionTransitionPolicy;
+
     /**
      * Default constructor.  Adds <code>USE_ASYNC_IO</code> to the Options if it is present in Defaults.
      */
@@ -329,6 +331,11 @@ public VariantContextWriterBuilder setOptions(final EnumSet<Options> options) {
         return this;
     }
 
+    public VariantContextWriterBuilder setVCF42to43TransitionPolicy(final VCF42To43VersionTransitionPolicy policy) {
+        this.vcf42To43VersionTransitionPolicy = policy;
+        return this;
+    }
+
     /**
      * Add one option to the set of <code>Options</code> for the <code>VariantContextWriterBuilder</code>, if it's not already present.
      *
@@ -569,20 +576,23 @@ private static boolean isCompressedVCF(final Path outPath) {
     }
 
     private VariantContextWriter createVCFWriter(final Path writerPath, final OutputStream writerStream) {
+        final VCFWriter writer;
         if (idxCreator == null) {
-            return new VCFWriter(writerPath, writerStream, refDict,
+            writer = new VCFWriter(writerPath, writerStream, refDict,
                     options.contains(Options.INDEX_ON_THE_FLY),
                     options.contains(Options.DO_NOT_WRITE_GENOTYPES),
                     options.contains(Options.ALLOW_MISSING_FIELDS_IN_HEADER),
                     options.contains(Options.WRITE_FULL_FORMAT_FIELD));
         }
         else {
-            return new VCFWriter(writerPath, writerStream, refDict, idxCreator,
+            writer = new VCFWriter(writerPath, writerStream, refDict, idxCreator,
                     options.contains(Options.INDEX_ON_THE_FLY),
                     options.contains(Options.DO_NOT_WRITE_GENOTYPES),
                     options.contains(Options.ALLOW_MISSING_FIELDS_IN_HEADER),
                     options.contains(Options.WRITE_FULL_FORMAT_FIELD));
         }
+        writer.setTransitionPolicy(vcf42To43VersionTransitionPolicy);
+        return writer;
     }
 
     private VariantContextWriter createBCFWriter(final Path writerPath, final OutputStream writerStream) {

src/main/java/htsjdk/variant/vcf/VCFCompoundHeaderLine.java

@@ -166,8 +166,13 @@ public String getVersion() {
      *
      * @param vcfTargetVersion the version agains which to validate
      */
+    @Override
     public void validateForVersion(final VCFHeaderVersion vcfTargetVersion) {
         super.validateForVersion(vcfTargetVersion);
+        // Let the 1000 Genomes line through, but only for INFO lines
+        if (this instanceof VCFInfoHeaderLine && getID().equals(VCFConstants.THOUSAND_GENOMES_KEY)) {
+            return;
+        }
         if (vcfTargetVersion.isAtLeastAsRecentAs(VCFHeaderVersion.VCF4_3)) {
             getGenericFields().forEach((k, v) -> {
                 if (!VALID_HEADER_ID_PATTERN.matcher(k).matches()) {

src/main/java/htsjdk/variant/vcf/VCFEncoder.java

@@ -26,10 +26,7 @@
  */
 public class VCFEncoder {
 
-    /**
-     * The encoding used for VCF files: ISO-8859-1. When writing VCF4.3 is implemented, this should change to UTF-8.
-     */
-    public static final Charset VCF_CHARSET = StandardCharsets.ISO_8859_1;
+    public static final Charset VCF_CHARSET = StandardCharsets.UTF_8;
     private static final String QUAL_FORMAT_STRING = "%.2f";
     private static final String QUAL_FORMAT_EXTENSION_TO_TRIM = ".00";
 
@@ -41,6 +38,8 @@ public class VCFEncoder {
 
     private boolean outputTrailingFormatFields = false;
 
+    private VCFTextTransformer vcfTextTransformer;
+
     /**
      * Prepare a VCFEncoder that will encode records appropriate to the given VCF header, optionally
      * allowing missing fields in the header.
@@ -52,6 +51,9 @@ public VCFEncoder(final VCFHeader header, final boolean allowMissingFieldsInHead
         this.header = header;
         this.allowMissingFieldsInHeader = allowMissingFieldsInHeader;
         this.outputTrailingFormatFields = outputTrailingFormatFields;
+        this.vcfTextTransformer = header.getVCFHeaderVersion() != null && header.getVCFHeaderVersion().isAtLeastAsRecentAs(VCFHeaderVersion.VCF4_3)
+            ? new VCFPercentEncodedTextTransformer()
+            : new VCFPassThruTextTransformer();
     }
 
     /**
@@ -242,7 +244,7 @@ String formatVCFField(final Object val) {
             }
             result = sb.toString();
         } else {
-            result = val.toString();
+            result = vcfTextTransformer.encodeText(val.toString());
         }
 
         return result;
@@ -310,7 +312,8 @@ public void addGenotypeData(final VariantContext vc, final Map<Allele, String> a
      * @param vcfoutput VCF output
      * @throws IOException
      */
-    private void appendGenotypeData(final VariantContext vc, final Map<Allele, String> alleleMap, final List<String> genotypeFormatKeys, final Appendable vcfoutput) throws IOException {final int ploidy = vc.getMaxPloidy(2);
+    private void appendGenotypeData(final VariantContext vc, final Map<Allele, String> alleleMap, final List<String> genotypeFormatKeys, final Appendable vcfoutput) throws IOException {
+        final int ploidy = vc.getMaxPloidy(2);
 
         for (final String sample : this.header.getGenotypeSamples()) {
             vcfoutput.append(VCFConstants.FIELD_SEPARATOR);

src/main/java/htsjdk/variant/vcf/VCFHeader.java

@@ -57,7 +57,7 @@
 public class VCFHeader implements Serializable {
     public static final long serialVersionUID = 1L;
     protected static final Log logger = Log.getInstance(VCFHeader.class);
-    public static final VCFHeaderVersion DEFAULT_VCF_VERSION = VCFHeaderVersion.VCF4_2;
+    public static final VCFHeaderVersion DEFAULT_VCF_VERSION = VCFHeaderVersion.VCF4_3;
 
     // the mandatory header fields
     public enum HEADER_FIELDS {