Deflate transfer syntax support (for parse) (#321)

This adds support for Deflated Little Endian Explicit transfer syntax on parsing. Support on writing should come in another change.

This change does this by adding a one way SetDeflate mechanism to the dicomio.Reader. The expectation for now is that this internal API is called once and at the top level of dicom parsing (e.g this doesn't support parsing just some sub-payloads with deflation, though we can make some changes to support that in the future if it's a need).

parse.go

@@ -158,15 +158,22 @@ func NewParser(in io.Reader, bytesToRead int64, frameChannel chan *frame.Frame,
 	implicit := true
 
 	ts, err := p.dataset.FindElementByTag(tag.TransferSyntaxUID)
-	if err != nil {
-		debug.Log("WARN: could not find transfer syntax uid in metadata, proceeding with little endian implicit")
-	} else {
-		bo, implicit, err = uid.ParseTransferSyntaxUID(MustGetStrings(ts.Value)[0])
+	if err == nil {
+		// If we found the transfer syntax, apply it.
+		tsStr := MustGetStrings(ts.Value)[0]
+		bo, implicit, err = uid.ParseTransferSyntaxUID(tsStr)
 		if err != nil {
 			// TODO(suyashkumar): should we attempt to parse with LittleEndian
 			// Implicit here?
 			debug.Log("WARN: could not parse transfer syntax uid in metadata")
 		}
+		if tsStr == uid.DeflatedExplicitVRLittleEndian {
+			p.reader.rawReader.SetDeflate()
+		}
+	} else {
+		// No transfer syntax found, warn the user we're proceeding with the
+		// default Little Endian implicit.
+		debug.Log("WARN: could not find transfer syntax uid in metadata, proceeding with little endian implicit")
 	}
 	p.SetTransferSyntax(bo, implicit)
 
@@ -281,8 +288,8 @@ func SkipPixelData() ParseOption {
 // a PixelData element will be added to the dataset with the
 // PixelDataInfo.IntentionallyUnprocessed = true, and the raw bytes of the
 // entire PixelData element stored in PixelDataInfo.UnprocessedValueData.
-// 
-// In the future, we may be able to extend this functionality to support 
+//
+// In the future, we may be able to extend this functionality to support
 // on-demand processing of elements elsewhere in the library.
 func SkipProcessingPixelDataValue() ParseOption {
 	return func(set *parseOptSet) {

pkg/dicomio/reader.go

@@ -2,6 +2,7 @@ package dicomio
 
 import (
 	"bufio"
+	"compress/flate"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -71,7 +72,16 @@ type Reader interface {
 	// SetCodingSystem sets the charset.CodingSystem to be used when ReadString
 	// is called.
 	SetCodingSystem(cs charset.CodingSystem)
+	// ByteOrder returns the current byte order.
 	ByteOrder() binary.ByteOrder
+	// SetDeflate applies deflate decompression to the underlying reader for all
+	// subsequent reads. This should be set when working with a deflated
+	// transfer syntax. Right now this is expected to be called once when
+	// parsing the top level dicom data, and there is no facility to swap
+	// between deflate and non-deflate reading.
+	// This also sets the current limit to LimitReadUntilEOF, since the original
+	// limits (if any) will be based on uncompressed bytes.
+	SetDeflate()
 }
 
 type reader struct {
@@ -234,6 +244,13 @@ func (r *reader) SetTransferSyntax(bo binary.ByteOrder, implicit bool) {
 	r.implicit = implicit
 }
 
+func (r *reader) SetDeflate() {
+	r.in = bufio.NewReader(flate.NewReader(r.in))
+	// TODO(https://github.com/suyashkumar/dicom/issues/320): consider always
+	// having the top level limit read until EOF.
+	r.limit = LimitReadUntilEOF // needed because original limits may not apply to the deflated reader
+}
+
 func (r *reader) IsImplicit() bool { return r.implicit }
 
 func (r *reader) SetCodingSystem(cs charset.CodingSystem) {

pkg/uid/uid.go

@@ -56,7 +56,7 @@ func ParseTransferSyntaxUID(uid string) (bo binary.ByteOrder, implicit bool, err
 	case ImplicitVRLittleEndian:
 		return binary.LittleEndian, true, nil
 	case DeflatedExplicitVRLittleEndian:
-		fallthrough
+		return binary.LittleEndian, false, nil
 	case ExplicitVRLittleEndian:
 		return binary.LittleEndian, false, nil
 	case ExplicitVRBigEndian:

testdata/data_details.md

@@ -32,6 +32,8 @@ be mentioned in one of them for brevity.
   * Modality: CT
   * Multiple frames
   * Native pixel data
+* [6.dcm](6.dcm)
+  * Deflated Little Endian Transfer Syntax (Explicit VR)
 ### Relevant Citations
 #### For files 1.dcm, 2.dcm:
 ##### Data Citation:
@@ -58,6 +60,6 @@ Desai, S., Baghal, A., Wongsurawat, T., Al-Shukri, S., Gates, K., Farmer, P., Ru
 #### TCIA Citation
 Clark K, Vendt B, Smith K, Freymann J, Kirby J, Koppel P, Moore S, Phillips S, Maffitt D, Pringle M, Tarbox L, Prior F. The Cancer Imaging Archive (TCIA): Maintaining and Operating a Public Information Repository, Journal of Digital Imaging, Volume 26, Number 6, December, 2013, pp 1045-1057. DOI: 10.1007/s10278-013-9622-7
 
-#### File 5.dcm
-This file was sourced from [cornerstone](https://github.com/cornerstonejs/dicomParser/blob/master/testImages/encapsulated/multi-frame/CT0012.explicit_little_endian.dcm) 
-(which is MIT licensed, see the license reproduced in included_licenses.md)
+#### File 5.dcm & 6.dcm
+This file was sourced from cornerstone [5.dcm from here](https://github.com/cornerstonejs/dicomParser/blob/master/testImages/encapsulated/multi-frame/CT0012.explicit_little_endian.dcm), and [6.dcm from here](https://github.com/cornerstonejs/dicomParser/blob/7d2084349bf2bdaffe74021e27b286a6c295ca66/testImages/deflate/image_dfl).
+(Cornerstone is MIT licensed, see the license reproduced in included_licenses.md).

write.go

@@ -26,7 +26,8 @@ var (
 	ErrorUnexpectedValueType = errors.New("Unexpected ValueType")
 	// ErrorUnsupportedBitsPerSample indicates that the BitsPerSample in this
 	// Dataset is not supported when unpacking native PixelData.
-	ErrorUnsupportedBitsPerSample = errors.New("unsupported BitsPerSample value")
+	ErrorUnsupportedBitsPerSample          = errors.New("unsupported BitsPerSample value")
+	errorDeflatedTransferSyntaxUnsupported = errors.New("deflated explicit vr little endian transfer syntax not yet support on write (https://github.com/suyashkumar/dicom/issues/323)")
 )
 
 // Writer is a struct that allows element-by element writing to a DICOM writer.
@@ -261,6 +262,9 @@ func writeMetaElem(w dicomio.Writer, t tag.Tag, ds *Dataset, tagsUsed *map[tag.T
 	if err != nil {
 		return err
 	}
+	if elem.Tag == tag.TransferSyntaxUID && MustGetStrings(elem.Value)[0] == uid.DeflatedExplicitVRLittleEndian {
+		return errorDeflatedTransferSyntaxUnsupported
+	}
 	err = writeElement(w, elem, optSet)
 	if err != nil {
 		return err

write_test.go

@@ -3,6 +3,7 @@ package dicom
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"os"
 	"testing"
 
@@ -27,13 +28,13 @@ import (
 // Write implementation (e.g. it kinda goes both ways and covers Parse too).
 func TestWrite(t *testing.T) {
 	cases := []struct {
-		name          string
-		dataset       Dataset
-		extraElems    []*Element
-		expectedError error
-		opts          []WriteOption
-		parseOpts     []ParseOption
-		cmpOpts       []cmp.Option
+		name       string
+		dataset    Dataset
+		extraElems []*Element
+		wantError  error
+		opts       []WriteOption
+		parseOpts  []ParseOption
+		cmpOpts    []cmp.Option
 	}{
 		{
 			name: "basic types",
@@ -58,7 +59,7 @@ func TestWrite(t *testing.T) {
 					},
 				},
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "private tag",
@@ -70,7 +71,7 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.TransferSyntaxUID, []string{uid.ExplicitVRLittleEndian}),
 				mustNewPrivateElement(tag.Tag{0x0003, 0x0010}, vrraw.ShortText, []string{"some data"}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "sequence (2 Items with 2 values each)",
@@ -120,7 +121,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "sequence (2 Items with 2 values each) - skip vr verification",
@@ -170,8 +171,8 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
-			opts:          []WriteOption{SkipVRVerification()},
+			wantError: nil,
+			opts:      []WriteOption{SkipVRVerification()},
 		},
 		{
 			name: "nested sequences",
@@ -207,7 +208,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "nested sequences - without VR verification",
@@ -243,8 +244,8 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
-			opts:          []WriteOption{SkipVRVerification()},
+			wantError: nil,
+			opts:      []WriteOption{SkipVRVerification()},
 		},
 		{
 			name: "without transfer syntax",
@@ -255,7 +256,7 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.Rows, []int{128}),
 				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
 			}},
-			expectedError: ErrorElementNotFound,
+			wantError: ErrorElementNotFound,
 		},
 		{
 			name: "without transfer syntax with DefaultMissingTransferSyntax",
@@ -267,9 +268,9 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
 			}},
 			// This gets inserted if DefaultMissingTransferSyntax is provided:
-			extraElems:    []*Element{mustNewElement(tag.TransferSyntaxUID, []string{uid.ImplicitVRLittleEndian})},
-			expectedError: nil,
-			opts:          []WriteOption{DefaultMissingTransferSyntax()},
+			extraElems: []*Element{mustNewElement(tag.TransferSyntaxUID, []string{uid.ImplicitVRLittleEndian})},
+			wantError:  nil,
+			opts:       []WriteOption{DefaultMissingTransferSyntax()},
 		},
 		{
 			name: "native PixelData: 8bit",
@@ -299,7 +300,7 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
 				mustNewElement(tag.DimensionIndexPointer, []int{32, 36950}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "native PixelData: 16bit",
@@ -327,7 +328,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "native PixelData: 32bit",
@@ -355,7 +356,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "native PixelData: 2 SamplesPerPixel, 2 frames",
@@ -392,7 +393,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "encapsulated PixelData",
@@ -419,7 +420,7 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
 				mustNewElement(tag.DimensionIndexPointer, []int{32, 36950}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "encapsulated PixelData: multiframe",
@@ -444,7 +445,7 @@ func TestWrite(t *testing.T) {
 				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
 				mustNewElement(tag.DimensionIndexPointer, []int{32, 36950}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "native_PixelData_2samples_2frames_BigEndian",
@@ -481,7 +482,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "native_PixelData_odd_bytes",
@@ -509,7 +510,7 @@ func TestWrite(t *testing.T) {
 					},
 				}),
 			}},
-			expectedError: nil,
+			wantError: nil,
 		},
 		{
 			name: "PixelData with IntentionallyUnprocessed=true",
@@ -526,8 +527,8 @@ func TestWrite(t *testing.T) {
 					IsEncapsulated:           false,
 				}),
 			}},
-			parseOpts:     []ParseOption{SkipProcessingPixelDataValue()},
-			expectedError: nil,
+			parseOpts: []ParseOption{SkipProcessingPixelDataValue()},
+			wantError: nil,
 		},
 		{
 			name: "Native PixelData with IntentionallySkipped=true",
@@ -542,8 +543,8 @@ func TestWrite(t *testing.T) {
 					IsEncapsulated:       false,
 				}),
 			}},
-			parseOpts:     []ParseOption{SkipPixelData()},
-			expectedError: nil,
+			parseOpts: []ParseOption{SkipPixelData()},
+			wantError: nil,
 		},
 		{
 			name: "Encapsulated PixelData with IntentionallySkipped=true",
@@ -558,8 +559,23 @@ func TestWrite(t *testing.T) {
 					IsEncapsulated:       true,
 				})),
 			}},
-			parseOpts:     []ParseOption{SkipPixelData()},
-			expectedError: nil,
+			parseOpts: []ParseOption{SkipPixelData()},
+			wantError: nil,
+		},
+		{
+			name: "deflated transfer syntax returns error",
+			dataset: Dataset{Elements: []*Element{
+				mustNewElement(tag.MediaStorageSOPClassUID, []string{"1.2.840.10008.5.1.4.1.1.1.2"}),
+				mustNewElement(tag.MediaStorageSOPInstanceUID, []string{"1.2.3.4.5.6.7"}),
+				mustNewElement(tag.TransferSyntaxUID, []string{uid.DeflatedExplicitVRLittleEndian}),
+				mustNewElement(tag.BitsAllocated, []int{8}),
+				mustNewElement(tag.FloatingPointValue, []float64{128.10}),
+				setUndefinedLength(mustNewElement(tag.PixelData, PixelDataInfo{
+					IntentionallySkipped: true,
+					IsEncapsulated:       true,
+				})),
+			}},
+			parseOpts: []ParseOption{SkipPixelData()}, wantError: errorDeflatedTransferSyntaxUnsupported,
 		},
 	}
 	for _, tc := range cases {
@@ -568,12 +584,12 @@ func TestWrite(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Unexpected error when creating tempfile: %v", err)
 			}
-			if err = Write(file, tc.dataset, tc.opts...); err != tc.expectedError {
-				t.Fatalf("Write(%v): unexpected error. got: %v, want: %v", tc.dataset, err, tc.expectedError)
+			if err = Write(file, tc.dataset, tc.opts...); !errors.Is(err, tc.wantError) {
+				t.Fatalf("Write(%v): unexpected error. got: %v, want: %v", tc.dataset, err, tc.wantError)
 			}
 			file.Close()
 			// If we expect an error, we do not need to continue to check the value of the written data, so we continue to the next test case.
-			if tc.expectedError != nil {
+			if tc.wantError != nil {
 				return
 			}
 			// Read the data back in and check for equality to the tc.dataset: