|
|
@ -3,7 +3,11 @@ package gzip
|
|
|
|
import (
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"bytes"
|
|
|
|
"compress/gzip"
|
|
|
|
"compress/gzip"
|
|
|
|
|
|
|
|
"crypto/md5"
|
|
|
|
|
|
|
|
"encoding/hex"
|
|
|
|
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io"
|
|
|
|
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
|
|
|
"os"
|
|
|
|
"testing"
|
|
|
|
"testing"
|
|
|
|
)
|
|
|
|
)
|
|
|
@ -158,6 +162,51 @@ func Test_Compressor_CompressFile(t *testing.T) {
|
|
|
|
compareFiles(t, srcFD, dstUncompressedFD)
|
|
|
|
compareFiles(t, srcFD, dstUncompressedFD)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func Test_Compressor_CompressLargeFile(t *testing.T) {
|
|
|
|
|
|
|
|
mb512 := int64(512*1024*1024) + 13
|
|
|
|
|
|
|
|
srcFD := mustOpenTempFile(t)
|
|
|
|
|
|
|
|
_, err := io.CopyN(srcFD, io.LimitReader(rand.New(rand.NewSource(0)), mb512), mb512)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to write random data to source file: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer os.Remove(srcFD.Name())
|
|
|
|
|
|
|
|
defer srcFD.Close()
|
|
|
|
|
|
|
|
// Reset file pointer to beginning
|
|
|
|
|
|
|
|
if _, err := srcFD.Seek(0, 0); err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to seek to beginning of file: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compress it.
|
|
|
|
|
|
|
|
compressor, err := NewCompressor(srcFD, DefaultBufferSize)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to create compressor: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
dstFD := mustOpenTempFile(t)
|
|
|
|
|
|
|
|
defer os.Remove(dstFD.Name())
|
|
|
|
|
|
|
|
defer dstFD.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(dstFD, compressor)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to compress: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Decompress it via actual gzip.
|
|
|
|
|
|
|
|
dstUncompressedFD := mustOpenTempFile(t)
|
|
|
|
|
|
|
|
defer os.Remove(dstUncompressedFD.Name())
|
|
|
|
|
|
|
|
defer dstUncompressedFD.Close()
|
|
|
|
|
|
|
|
dstFD.Seek(0, 0)
|
|
|
|
|
|
|
|
r, err := gzip.NewReader(dstFD)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to create gzip reader: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err = io.Copy(dstUncompressedFD, r)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to decompress: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Compare the files.
|
|
|
|
|
|
|
|
compareFileMD5(t, srcFD.Name(), dstUncompressedFD.Name())
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func mustOpenTempFile(t *testing.T) *os.File {
|
|
|
|
func mustOpenTempFile(t *testing.T) *os.File {
|
|
|
|
t.Helper()
|
|
|
|
t.Helper()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
|
tmpDir := t.TempDir()
|
|
|
@ -201,3 +250,41 @@ func compareFiles(t *testing.T, srcFD, dstFD *os.File) {
|
|
|
|
t.Fatalf("Source data does not match destination data")
|
|
|
|
t.Fatalf("Source data does not match destination data")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func compareFileMD5(t *testing.T, srcPath, dstPath string) {
|
|
|
|
|
|
|
|
t.Helper()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
srcMD5, err := md5sum(srcPath)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to calculate md5sum of source file: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
dstMD5, err := md5sum(dstPath)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
t.Fatalf("Failed to calculate md5sum of destination file: %v", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// compare md5sums
|
|
|
|
|
|
|
|
if srcMD5 != dstMD5 {
|
|
|
|
|
|
|
|
t.Fatal("Source file md5sum does not match destination file md5sum")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func md5sum(path string) (string, error) {
|
|
|
|
|
|
|
|
// open file
|
|
|
|
|
|
|
|
f, err := os.Open(path)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return "", fmt.Errorf("failed to open file: %w", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// create new hash
|
|
|
|
|
|
|
|
h := md5.New()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// copy file to hash
|
|
|
|
|
|
|
|
if _, err := io.Copy(h, f); err != nil {
|
|
|
|
|
|
|
|
return "", fmt.Errorf("failed to copy file to hash: %w", err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// return hex encoded hash
|
|
|
|
|
|
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
|
|
|
|
|
|
}
|
|
|
|