Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: viamrobotics/rdk
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 215ebb8fa1fe08cad87d712d2dd7ee68866152f5
Choose a base ref
..
head repository: viamrobotics/rdk
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: ce4600f7b35e21f03c584a54ff0d92d9e3fc411b
Choose a head ref
353 changes: 62 additions & 291 deletions data/capture_buffer_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package data

import (
"crypto/sha1"
"errors"
"io"
"os"
@@ -658,300 +657,72 @@ func TestCaptureBufferReader(t *testing.T) {
})
}

func BenchmarkChunked(b *testing.B) {
type testCase struct {
name string
data []byte
}
eightKBFilled := make([]byte, 1024*8)
for i := range eightKBFilled {
eightKBFilled[i] = uint8(i % 256)
}

oneMbFilled := make([]byte, 1024*1000)
for i := range eightKBFilled {
oneMbFilled[i] = uint8(i % 256)
}

eightMbFilled := make([]byte, 1024*1000*8)
for i := range eightMbFilled {
eightMbFilled[i] = uint8(i % 256)
}

tcs := []testCase{
{"empty data", []byte{}},
{"small data", []byte("this is a fake image")},
{"8kb empty", make([]byte, 1024*8)},
{"8kb filled", eightKBFilled},
{"1mb empty", make([]byte, 1024*1000)},
{"1mb filled", oneMbFilled},
{"8mb empty", make([]byte, 1024*1000*8)},
{"8mb filled", eightMbFilled},
}

for _, tc := range tcs {
s := sha1.New()
_, err := s.Write(tc.data)
test.That(b, err, test.ShouldBeNil)
expectedHash := s.Sum(nil)
tmpDir := b.TempDir()
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
test.That(b, err, test.ShouldBeNil)

readImageCaptureMetadata := BuildCaptureMetadata(
name.API,
name.ShortName(),
readImage,
additionalParams,
methodParams,
[]string{"my", "tags"},
)

now := time.Now()
timeRequested := timestamppb.New(now.UTC())
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
msg := &v1.SensorData{
Metadata: &v1.SensorMetadata{
TimeRequested: timeRequested,
TimeReceived: timeReceived,
},
Data: &v1.SensorData_Binary{
Binary: tc.data,
},
}

buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))

// Path() is the same as the first paramenter passed to NewCaptureBuffer
test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)

test.That(b, buf.Write(msg), test.ShouldBeNil)
test.That(b, buf.Flush(), test.ShouldBeNil)
dirEntries, err := os.ReadDir(buf.Path())
test.That(b, err, test.ShouldBeNil)
test.That(b, len(dirEntries), test.ShouldEqual, 1)
test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
test.That(b, err, test.ShouldBeNil)
b.Cleanup(func() { test.That(b, f.Close(), test.ShouldBeNil) })

b.ResetTimer()
b.Run("chunked "+tc.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
ret, err := f.Seek(0, io.SeekStart)
test.That(b, err, test.ShouldBeNil)
test.That(b, ret, test.ShouldEqual, 0)
cf2, err := NewCaptureFile(f)
test.That(b, err, test.ShouldBeNil)
test.That(b, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)

var md v1.SensorMetadata
r, err := cf2.BinaryReader(&md)
test.That(b, err, test.ShouldBeNil)
test.That(b, r, test.ShouldNotBeNil)
test.That(b, &md, test.ShouldResemble, msg.GetMetadata())
data := make([]byte, 4064)
h := sha1.New()
for {
n, err := r.Read(data)
if errors.Is(err, io.EOF) {
break
}
test.That(b, err, test.ShouldBeNil)
_, err = h.Write(data[:n])
test.That(b, err, test.ShouldBeNil)
}
actualHash := h.Sum(nil)
test.That(b, actualHash, test.ShouldResemble, expectedHash)
}
})
}
}

func BenchmarkNonChunked(b *testing.B) {
type testCase struct {
name string
data []byte
}
eightKBFilled := make([]byte, 1024*8)
for i := range eightKBFilled {
eightKBFilled[i] = uint8(i % 256)
}

oneMbFilled := make([]byte, 1024*1000)
for i := range eightKBFilled {
oneMbFilled[i] = uint8(i % 256)
}

eightMbFilled := make([]byte, 1024*1000*8)
for i := range eightMbFilled {
eightMbFilled[i] = uint8(i % 256)
}

tcs := []testCase{
{"empty data", []byte{}},
{"small data", []byte("this is a fake image")},
{"8kb empty", make([]byte, 1024*8)},
{"8kb filled", eightKBFilled},
{"1mb empty", make([]byte, 1024*1000)},
{"1mb filled", oneMbFilled},
{"8mb empty", make([]byte, 1024*1000*8)},
{"8mb filled", eightMbFilled},
}

for _, tc := range tcs {
s := sha1.New()
_, err := s.Write(tc.data)
test.That(b, err, test.ShouldBeNil)
expectedHash := s.Sum(nil)
tmpDir := b.TempDir()
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
test.That(b, err, test.ShouldBeNil)

readImageCaptureMetadata := BuildCaptureMetadata(
name.API,
name.ShortName(),
readImage,
additionalParams,
methodParams,
[]string{"my", "tags"},
)

now := time.Now()
timeRequested := timestamppb.New(now.UTC())
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
msg := &v1.SensorData{
Metadata: &v1.SensorMetadata{
TimeRequested: timeRequested,
TimeReceived: timeReceived,
},
Data: &v1.SensorData_Binary{
Binary: tc.data,
},
}

buf := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))

test.That(b, buf.Path(), test.ShouldResemble, tmpDir)
test.That(b, buf.metaData, test.ShouldResemble, readImageCaptureMetadata)

test.That(b, buf.Write(msg), test.ShouldBeNil)
test.That(b, buf.Flush(), test.ShouldBeNil)
dirEntries, err := os.ReadDir(buf.Path())
test.That(b, err, test.ShouldBeNil)
test.That(b, len(dirEntries), test.ShouldEqual, 1)
test.That(b, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
f, err := os.Open(filepath.Join(buf.Path(), dirEntries[0].Name()))
test.That(b, err, test.ShouldBeNil)
b.Cleanup(func() { test.That(b, f.Close(), test.ShouldBeNil) })
b.ResetTimer()
b.Run("non chunked "+tc.name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
ret, err := f.Seek(0, io.SeekStart)
test.That(b, err, test.ShouldBeNil)
test.That(b, ret, test.ShouldEqual, 0)
cf2, err := NewCaptureFile(f)
test.That(b, err, test.ShouldBeNil)
test.That(b, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)

next, err := cf2.ReadNext()
test.That(b, err, test.ShouldBeNil)
test.That(b, next.GetMetadata(), test.ShouldResemble, msg.GetMetadata())
h := sha1.New()
_, err = h.Write(next.GetBinary())
test.That(b, err, test.ShouldBeNil)
actualHash := h.Sum(nil)
test.That(b, actualHash, test.ShouldResemble, expectedHash)
}
})
}
}

func FuzzBinaryReader(f *testing.F) {
eightKBFilled := make([]byte, 1024*8)
for i := range eightKBFilled {
eightKBFilled[i] = uint8(i % 256)
}
func NickTest(t *testing.T) {
tmpDir := t.TempDir()
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
method := readImage
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
tags := []string{"my", "tags"}
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
test.That(t, err, test.ShouldBeNil)

readImageCaptureMetadata := BuildCaptureMetadata(
name.API,
name.ShortName(),
method,
additionalParams,
methodParams,
tags,
)

test.That(t, readImageCaptureMetadata, test.ShouldResemble, &v1.DataCaptureMetadata{
ComponentName: "my-cam",
ComponentType: "rdk:component:camera",
MethodName: readImage,
MethodParameters: methodParams,
Tags: tags,
Type: v1.DataType_DATA_TYPE_BINARY_SENSOR,
FileExtension: ".jpeg",
})

eightMbFilled := make([]byte, 1024*1000*8)
for i := range eightMbFilled {
eightMbFilled[i] = uint8(i % 256)
}
b := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))

tcs := [][]byte{
{},
[]byte("this is a fake image"),
make([]byte, 1024*8),
eightKBFilled,
make([]byte, 1024*1000*8),
}
// Path() is the same as the first paramenter passed to NewCaptureBuffer
test.That(t, b.Path(), test.ShouldResemble, tmpDir)
test.That(t, b.metaData, test.ShouldResemble, readImageCaptureMetadata)

for _, tc := range tcs {
f.Add(tc)
now := time.Now()
timeRequested := timestamppb.New(now.UTC())
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
msg := &v1.SensorData{
Metadata: &v1.SensorMetadata{
TimeRequested: timeRequested,
TimeReceived: timeReceived,
},
Data: &v1.SensorData_Binary{
Binary: []byte("this is a fake image"),
},
}
f.Fuzz(func(t *testing.T, binary []byte) {
tmpDir := t.TempDir()
name := resource.NewName(resource.APINamespaceRDK.WithComponentType("camera"), "my-cam")
additionalParams := map[string]string{"mime_type": rutils.MimeTypeJPEG, "test": "1"}
methodParams, err := rprotoutils.ConvertStringMapToAnyPBMap(additionalParams)
test.That(t, err, test.ShouldBeNil)

readImageCaptureMetadata := BuildCaptureMetadata(
name.API,
name.ShortName(),
readImage,
additionalParams,
methodParams,
[]string{"my", "tags"},
)

now := time.Now()
timeRequested := timestamppb.New(now.UTC())
timeReceived := timestamppb.New(now.Add(time.Millisecond).UTC())
msg := &v1.SensorData{
Metadata: &v1.SensorMetadata{
TimeRequested: timeRequested,
TimeReceived: timeReceived,
},
Data: &v1.SensorData_Binary{
Binary: binary,
},
}

b := NewCaptureBuffer(tmpDir, readImageCaptureMetadata, int64(4*1024))

// Path() is the same as the first paramenter passed to NewCaptureBuffer
test.That(t, b.Path(), test.ShouldResemble, tmpDir)
test.That(t, b.metaData, test.ShouldResemble, readImageCaptureMetadata)

test.That(t, b.Write(msg), test.ShouldBeNil)
test.That(t, b.Flush(), test.ShouldBeNil)
dirEntries, err := os.ReadDir(b.Path())
test.That(t, err, test.ShouldBeNil)
test.That(t, len(dirEntries), test.ShouldEqual, 1)
test.That(t, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
f, err := os.Open(filepath.Join(b.Path(), dirEntries[0].Name()))
test.That(t, err, test.ShouldBeNil)
defer func() { test.That(t, f.Close(), test.ShouldBeNil) }()

cf2, err := NewCaptureFile(f)
test.That(t, err, test.ShouldBeNil)
test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)

var md v1.SensorMetadata
r, err := cf2.BinaryReader(&md)
test.That(t, err, test.ShouldBeNil)
test.That(t, r, test.ShouldNotBeNil)
test.That(t, &md, test.ShouldResemble, msg.GetMetadata())
data, err := io.ReadAll(r)
test.That(t, err, test.ShouldBeNil)
test.That(t, data, test.ShouldResemble, msg.GetBinary())
})
test.That(t, b.Write(msg), test.ShouldBeNil)
test.That(t, b.Flush(), test.ShouldBeNil)
dirEntries, err := os.ReadDir(b.Path())
test.That(t, err, test.ShouldBeNil)
test.That(t, len(dirEntries), test.ShouldEqual, 1)
test.That(t, filepath.Ext(dirEntries[0].Name()), test.ShouldResemble, CompletedCaptureFileExt)
f, err := os.Open(filepath.Join(b.Path(), dirEntries[0].Name()))
test.That(t, err, test.ShouldBeNil)
defer func() { test.That(t, f.Close(), test.ShouldBeNil) }()

cf2, err := NewCaptureFile(f)
test.That(t, err, test.ShouldBeNil)
test.That(t, cf2.ReadMetadata(), test.ShouldResemble, readImageCaptureMetadata)

sd2, err := cf2.ReadNext()
test.That(t, err, test.ShouldBeNil)
test.That(t, sd2, test.ShouldResemble, msg)

_, err = cf2.ReadNext()
test.That(t, err, test.ShouldBeError, io.EOF)
}

//nolint
Loading