fluidB
/
fluidb-old
1
0
Fork 0
Code Pull Requests Activity

Tighten up tests

Browse Source
master
Philip O'Toole 1 year ago
parent 0309fda396
commit 01db83f29d
2 changed files with 129 additions and 38 deletions
Show Stats Download Patch File Download Diff File

18
command/chunking/dechunker.go
Unescape Escape View File

@ -45,15 +45,17 @@ func (d *Dechunker) WriteChunk(chunk *command.LoadChunkRequest) (bool, error) {
}
d.seqNum = chunk.SequenceNum
buf := bytes.NewBuffer(chunk.Data)
gzw, err := gzip.NewReader(buf)
if err != nil {
return false, fmt.Errorf("failed to create gzip reader: %v", err)
}
defer gzw.Close()
if chunk.Data != nil {
buf := bytes.NewBuffer(chunk.Data)
gzw, err := gzip.NewReader(buf)
if err != nil {
return false, fmt.Errorf("failed to create gzip reader: %v", err)
}
defer gzw.Close()
if _, err := io.Copy(d.file, gzw); err != nil {
return false, fmt.Errorf("failed to write decompressed data to file: %v", err)
if _, err := io.Copy(d.file, gzw); err != nil {
return false, fmt.Errorf("failed to write decompressed data to file: %v", err)
}
}
return chunk.IsLast, nil

149
command/chunking/dechunker_test.go
Unescape Escape View File

@ -3,6 +3,7 @@ package chunking
import (
"bytes"
"compress/gzip"
"crypto/rand"
"fmt"
"io/ioutil"
"os"
@ -13,7 +14,6 @@ import (
)
func Test_SingleChunk(t *testing.T) {
// Define the chunk data.
data := []byte("Hello, World!")
chunk := &command.LoadChunkRequest{
StreamId: "123",
@ -22,20 +22,17 @@ func Test_SingleChunk(t *testing.T) {
Data: mustCompressData(data),
}
// Create a temporary directory for testing.
dir, err := ioutil.TempDir("", "dechunker-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir) // Clean up after the test is done.
defer os.RemoveAll(dir)
// Create the Dechunker.
dechunker, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
}
// Write the chunk to the Dechunker.
isLast, err := dechunker.WriteChunk(chunk)
if err != nil {
t.Fatalf("failed to write chunk: %v", err)
@ -44,7 +41,6 @@ func Test_SingleChunk(t *testing.T) {
t.Errorf("WriteChunk did not return true for isLast")
}
// Close the Dechunker.
filePath, err := dechunker.Close()
if err != nil {
t.Fatalf("failed to close Dechunker: %v", err)
@ -61,7 +57,6 @@ func Test_SingleChunk(t *testing.T) {
}
func Test_MultiChunk(t *testing.T) {
// Define the chunked data.
data1 := []byte("Hello, World!")
chunk1 := &command.LoadChunkRequest{
StreamId: "123",
@ -77,38 +72,85 @@ func Test_MultiChunk(t *testing.T) {
Data: mustCompressData(data2),
}
// Create a temporary directory for testing.
dir, err := ioutil.TempDir("", "dechunker-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir) // Clean up after the test is done.
defer os.RemoveAll(dir)
// Create the Dechunker.
dechunker, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
}
// Write the chunk to the Dechunker.
isLast, err := dechunker.WriteChunk(chunk1)
for _, chunk := range []*command.LoadChunkRequest{chunk1, chunk2} {
isLast, err := dechunker.WriteChunk(chunk)
if err != nil {
t.Fatalf("failed to write chunk: %v", err)
}
if isLast != chunk.IsLast {
t.Errorf("WriteChunk returned wrong value for isLast")
}
}
filePath, err := dechunker.Close()
if err != nil {
t.Fatalf("failed to write chunk: %v", err)
t.Fatalf("failed to close Dechunker: %v", err)
}
// Check the contents of the output file.
got, err := ioutil.ReadFile(filePath)
if err != nil {
t.Fatalf("failed to read output file: %v", err)
}
if isLast {
t.Errorf("WriteChunk returned true for isLast")
if string(got) != string(data1)+string(data2) {
t.Errorf("output file data = %q; want %q", got, string(data1)+string(data2))
}
}
// Write the chunk to the Dechunker.
isLast, err = dechunker.WriteChunk(chunk2)
func Test_MultiChunkNilData(t *testing.T) {
data1 := []byte("Hello, World!")
chunk1 := &command.LoadChunkRequest{
StreamId: "123",
SequenceNum: 1,
IsLast: false,
Data: mustCompressData(data1),
}
data2 := []byte("I'm OK")
chunk2 := &command.LoadChunkRequest{
StreamId: "123",
SequenceNum: 2,
IsLast: false,
Data: mustCompressData(data2),
}
chunk3 := &command.LoadChunkRequest{
StreamId: "123",
SequenceNum: 3,
IsLast: true,
Data: nil,
}
dir, err := ioutil.TempDir("", "dechunker-test")
if err != nil {
t.Fatalf("failed to write chunk: %v", err)
t.Fatalf("failed to create temp dir: %v", err)
}
if !isLast {
t.Errorf("WriteChunk did not return true for isLast")
defer os.RemoveAll(dir)
dechunker, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
}
for _, chunk := range []*command.LoadChunkRequest{chunk1, chunk2, chunk3} {
isLast, err := dechunker.WriteChunk(chunk)
if err != nil {
t.Fatalf("failed to write chunk: %v", err)
}
if isLast != chunk.IsLast {
t.Errorf("WriteChunk returned wrong value for isLast")
}
}
// Close the Dechunker.
filePath, err := dechunker.Close()
if err != nil {
t.Fatalf("failed to close Dechunker: %v", err)
@ -125,7 +167,6 @@ func Test_MultiChunk(t *testing.T) {
}
func Test_UnexpectedStreamID(t *testing.T) {
// Define the original and compressed chunk data.
originalData := []byte("Hello, World!")
compressedData := mustCompressData(originalData)
@ -143,20 +184,17 @@ func Test_UnexpectedStreamID(t *testing.T) {
Data: compressedData,
}
// Create a temporary directory for testing.
dir, err := ioutil.TempDir("", "dechunker-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir) // Clean up after the test is done.
defer os.RemoveAll(dir)
// Create the Dechunker.
dechunker, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
}
// Write the first chunk to the Dechunker.
_, err = dechunker.WriteChunk(chunk1)
if err != nil {
t.Fatalf("failed to write first chunk: %v", err)
@ -173,7 +211,6 @@ func Test_UnexpectedStreamID(t *testing.T) {
}
func Test_ChunksOutOfOrder(t *testing.T) {
// Define the original and compressed chunk data.
originalData := []byte("Hello, World!")
compressedData := mustCompressData(originalData)
@ -191,14 +228,12 @@ func Test_ChunksOutOfOrder(t *testing.T) {
Data: compressedData,
}
// Create a temporary directory for testing.
dir, err := ioutil.TempDir("", "dechunker-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(dir) // Clean up after the test is done.
defer os.RemoveAll(dir)
// Create the Dechunker.
dechunker, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
@ -220,6 +255,60 @@ func Test_ChunksOutOfOrder(t *testing.T) {
}
}
func Test_ReassemblyOfLargeData(t *testing.T) {
dir, err := ioutil.TempDir("", "test-*")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
d, err := NewDechunker(dir)
if err != nil {
t.Fatalf("failed to create Dechunker: %v", err)
}
defer os.Remove(dir)
// Create a large random dataset.
largeData := make([]byte, 2*1024*1024) // 2 MB of data.
if _, err := rand.Read(largeData); err != nil {
t.Fatalf("failed to generate large data: %v", err)
}
// Split the large data into chunks.
numChunks := 16
chunkSize := len(largeData) / numChunks
// Write the chunks to the Dechunker.
for i := 0; i < numChunks; i++ {
start := i * chunkSize
end := start + chunkSize
isLast := i == numChunks-1
if _, err := d.WriteChunk(&command.LoadChunkRequest{
StreamId: "1",
SequenceNum: int64(i + 1),
IsLast: isLast,
Data: mustCompressData(largeData[start:end]),
}); err != nil {
t.Fatalf("failed to write chunk: %v", err)
}
}
outFilePath, err := d.Close()
if err != nil {
t.Fatalf("failed to close Dechunker: %v", err)
}
defer os.Remove(outFilePath)
// The output data should be the same as the original largeData.
outData, err := ioutil.ReadFile(outFilePath)
if err != nil {
t.Fatalf("failed to read output file data: %v", err)
}
if !bytes.Equal(outData, largeData) {
t.Fatalf("output file data does not match original data")
}
}
func mustCompressData(data []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)

Write Preview
Loading…
Cancel
Save