Add hashes manifest for streaming archives (#84)

Co-authored-by: Janik Besendorf <janik.besendorf@reporter-ohne-grenzen.de>

Author: besendorf

acquisition/acquisition.go

@@ -148,6 +148,11 @@ func (a *Acquisition) Complete() {
 			}
 		}
 
+		err = a.EncryptedWriter.CreateHashList()
+		if err != nil {
+			log.ErrorExc("Failed to add hashes.csv to encrypted archive", err)
+		}
+
 		// Close the encrypted writer
 		err = a.EncryptedWriter.Close()
 		if err != nil {

acquisition/encrypted_stream.go

@@ -8,7 +8,11 @@ package acquisition
 import (
 	"archive/zip"
 	"bytes"
+	"crypto/sha256"
+	"encoding/csv"
+	"encoding/hex"
 	"fmt"
+	"hash"
 	"io"
 	"os"
 	"path"
@@ -28,6 +32,25 @@ type EncryptedZipWriter struct {
 	zipWriter  *zip.Writer
 	outputPath string
 	closed     bool
+	hashes     []*zipHash
+}
+
+type zipHash struct {
+	name   string
+	hasher hash.Hash
+}
+
+type hashingWriter struct {
+	writer io.Writer
+	hasher hash.Hash
+}
+
+func (hw *hashingWriter) Write(p []byte) (int, error) {
+	n, err := hw.writer.Write(p)
+	if n > 0 {
+		_, _ = hw.hasher.Write(p[:n])
+	}
+	return n, err
 }
 
 // NewEncryptedZipWriter creates a new encrypted zip writer if key.txt exists
@@ -87,6 +110,10 @@ func NewEncryptedZipWriter(uuid string) (*EncryptedZipWriter, error) {
 
 // CreateFile creates a new file in the encrypted zip and returns a writer
 func (ezw *EncryptedZipWriter) CreateFile(name string) (io.Writer, error) {
+	return ezw.createFile(name, true)
+}
+
+func (ezw *EncryptedZipWriter) createFile(name string, trackHash bool) (io.Writer, error) {
 	if err := ezw.checkClosed(); err != nil {
 		return nil, err
 	}
@@ -101,7 +128,25 @@ func (ezw *EncryptedZipWriter) CreateFile(name string) (io.Writer, error) {
 		Modified: time.Now(),
 	}
 
-	return ezw.zipWriter.CreateHeader(header)
+	writer, err := ezw.zipWriter.CreateHeader(header)
+	if err != nil {
+		return nil, err
+	}
+
+	if !trackHash {
+		return writer, nil
+	}
+
+	zipHash := &zipHash{
+		name:   name,
+		hasher: sha256.New(),
+	}
+	ezw.hashes = append(ezw.hashes, zipHash)
+
+	return &hashingWriter{
+		writer: writer,
+		hasher: zipHash.hasher,
+	}, nil
 }
 
 func validateZipEntryName(name string) error {
@@ -151,6 +196,39 @@ func (ezw *EncryptedZipWriter) CreateFileFromReader(name string, src io.Reader)
 	return nil
 }
 
+// CreateHashList adds hashes.csv to the encrypted zip with SHA-256 hashes of
+// the plaintext zip entries written so far.
+func (ezw *EncryptedZipWriter) CreateHashList() error {
+	if err := ezw.checkClosed(); err != nil {
+		return err
+	}
+
+	var buffer bytes.Buffer
+	csvWriter := csv.NewWriter(&buffer)
+	for _, zipHash := range ezw.hashes {
+		if err := csvWriter.Write([]string{
+			zipHash.name,
+			hex.EncodeToString(zipHash.hasher.Sum(nil)),
+		}); err != nil {
+			return fmt.Errorf("failed to write hash entry for %q: %v", zipHash.name, err)
+		}
+	}
+	csvWriter.Flush()
+	if err := csvWriter.Error(); err != nil {
+		return fmt.Errorf("failed to create hash list: %v", err)
+	}
+
+	writer, err := ezw.createFile("hashes.csv", false)
+	if err != nil {
+		return fmt.Errorf("failed to create hashes.csv in zip: %v", err)
+	}
+	if _, err := writer.Write(buffer.Bytes()); err != nil {
+		return fmt.Errorf("failed to write hashes.csv to zip: %v", err)
+	}
+
+	return nil
+}
+
 // CreateFileFromString creates a file with string content in the encrypted zip
 func (ezw *EncryptedZipWriter) CreateFileFromString(name, content string) error {
 	return ezw.CreateFileFromReader(name, strings.NewReader(content))

acquisition/encrypted_stream_test.go

@@ -1,6 +1,97 @@
 package acquisition
 
-import "testing"
+import (
+	"archive/zip"
+	"bytes"
+	"crypto/sha256"
+	"encoding/csv"
+	"encoding/hex"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestCreateHashListTracksPlaintextZipEntries(t *testing.T) {
+	var archive bytes.Buffer
+	ezw := &EncryptedZipWriter{
+		zipWriter: zip.NewWriter(&archive),
+	}
+
+	if err := ezw.CreateFileFromString("first.txt", "first content"); err != nil {
+		t.Fatalf("CreateFileFromString() error = %v", err)
+	}
+
+	writer, err := ezw.CreateFile("stream.bin")
+	if err != nil {
+		t.Fatalf("CreateFile() error = %v", err)
+	}
+	if _, err := writer.Write([]byte("streamed ")); err != nil {
+		t.Fatalf("Write() error = %v", err)
+	}
+	if _, err := writer.Write([]byte("content")); err != nil {
+		t.Fatalf("Write() error = %v", err)
+	}
+
+	if err := ezw.CreateHashList(); err != nil {
+		t.Fatalf("CreateHashList() error = %v", err)
+	}
+	if err := ezw.zipWriter.Close(); err != nil {
+		t.Fatalf("zip Close() error = %v", err)
+	}
+
+	reader, err := zip.NewReader(bytes.NewReader(archive.Bytes()), int64(archive.Len()))
+	if err != nil {
+		t.Fatalf("zip.NewReader() error = %v", err)
+	}
+
+	files := make(map[string]string)
+	for _, file := range reader.File {
+		readCloser, err := file.Open()
+		if err != nil {
+			t.Fatalf("Open(%q) error = %v", file.Name, err)
+		}
+		content, err := io.ReadAll(readCloser)
+		readCloser.Close()
+		if err != nil {
+			t.Fatalf("ReadAll(%q) error = %v", file.Name, err)
+		}
+		files[file.Name] = string(content)
+	}
+
+	records, err := csv.NewReader(strings.NewReader(files["hashes.csv"])).ReadAll()
+	if err != nil {
+		t.Fatalf("ReadAll(hashes.csv) error = %v", err)
+	}
+
+	gotHashes := make(map[string]string)
+	for _, record := range records {
+		if len(record) != 2 {
+			t.Fatalf("hash record has %d fields, want 2: %#v", len(record), record)
+		}
+		gotHashes[record[0]] = record[1]
+	}
+
+	wantHashes := map[string]string{
+		"first.txt":  sha256Hex("first content"),
+		"stream.bin": sha256Hex("streamed content"),
+	}
+	if len(gotHashes) != len(wantHashes) {
+		t.Fatalf("got %d hash records, want %d: %#v", len(gotHashes), len(wantHashes), gotHashes)
+	}
+	for name, wantHash := range wantHashes {
+		if gotHashes[name] != wantHash {
+			t.Fatalf("hash for %q = %q, want %q", name, gotHashes[name], wantHash)
+		}
+	}
+	if _, ok := gotHashes["hashes.csv"]; ok {
+		t.Fatal("hashes.csv should not include a hash record for itself")
+	}
+}
+
+func sha256Hex(content string) string {
+	sum := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(sum[:])
+}
 
 func TestValidateZipEntryName(t *testing.T) {
 	tests := []struct {