diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56b621738..86eee797d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: filters: | build: '**' bench: + - 'test/bench/**' - 'pkg/**' - 'server/**' - 'client/**' diff --git a/test/bench/grpc_bench_test.go b/test/bench/grpc_bench_test.go index c972ed078..bdfa36feb 100644 --- a/test/bench/grpc_bench_test.go +++ b/test/bench/grpc_bench_test.go @@ -25,6 +25,7 @@ import ( "strings" "sync" "testing" + gotime "time" "github.com/stretchr/testify/assert" @@ -35,7 +36,6 @@ import ( "github.com/yorkie-team/yorkie/pkg/document/json" "github.com/yorkie-team/yorkie/pkg/document/presence" "github.com/yorkie-team/yorkie/server" - "github.com/yorkie-team/yorkie/server/backend/database" "github.com/yorkie-team/yorkie/server/logging" "github.com/yorkie-team/yorkie/test/helper" ) @@ -71,14 +71,6 @@ func activeClients(b *testing.B, n int) (clients []*client.Client) { return } -// cleanupClients is a helper function to clean up clients. -func cleanupClients(b *testing.B, clients []*client.Client) { - for _, c := range clients { - assert.NoError(b, c.Deactivate(context.Background())) - assert.NoError(b, c.Close()) - } -} - func benchmarkUpdateAndSync( ctx context.Context, b *testing.B, @@ -99,9 +91,9 @@ func benchmarkUpdateAndSync( } } -func benchmarkUpdateProject(ctx context.Context, b *testing.B, cnt int, adminCli *admin.Client) error { +func benchmarkUpdateProject(ctx context.Context, b *testing.B, cnt int, adminCli *admin.Client, project *types.Project) error { for i := 0; i < cnt; i++ { - name := fmt.Sprintf("name%d", i) + name := fmt.Sprintf("name%d-%d", i, gotime.Now().UnixMilli()) authWebhookURL := fmt.Sprintf("http://authWebhookURL%d", i) var authWebhookMethods []string for _, m := range types.AuthMethods() { @@ -111,7 +103,7 @@ func benchmarkUpdateProject(ctx context.Context, b *testing.B, cnt int, adminCli _, err := adminCli.UpdateProject( ctx, - database.DefaultProjectID.String(), + project.ID.String(), &types.UpdatableProjectFields{ Name: &name, AuthWebhookURL: &authWebhookURL, @@ -197,7 +189,7 @@ func BenchmarkRPC(b *testing.B) { b.Run("client to client via server", func(b *testing.B) { clients := activeClients(b, 2) c1, c2 := clients[0], clients[1] - defer cleanupClients(b, clients) + defer helper.CleanupClients(b, clients) ctx := context.Background() @@ -263,7 +255,7 @@ func BenchmarkRPC(b *testing.B) { func() { clients := activeClients(b, 2) c1, c2 := clients[0], clients[1] - defer cleanupClients(b, clients) + defer helper.CleanupClients(b, clients) ctx := context.Background() doc1 := document.New(helper.TestDocKey(b)) @@ -304,8 +296,11 @@ func BenchmarkRPC(b *testing.B) { defer func() { adminCli.Close() }() ctx := context.Background() + project, err := adminCli.CreateProject(ctx, "admin-cli-test") + assert.NoError(b, err) + for i := 0; i < b.N; i++ { - assert.NoError(b, benchmarkUpdateProject(ctx, b, 500, adminCli)) + assert.NoError(b, benchmarkUpdateProject(ctx, b, 500, adminCli, project)) } }) } diff --git a/test/bench/vv_bench_test.go b/test/bench/vv_bench_test.go new file mode 100644 index 000000000..211d18095 --- /dev/null +++ b/test/bench/vv_bench_test.go @@ -0,0 +1,225 @@ +//go:build bench + +/* + * Copyright 2025 The Yorkie Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bench + +import ( + "context" + "fmt" + "testing" + gotime "time" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + + "github.com/yorkie-team/yorkie/api/converter" + "github.com/yorkie-team/yorkie/client" + "github.com/yorkie-team/yorkie/pkg/document" + "github.com/yorkie-team/yorkie/pkg/document/json" + "github.com/yorkie-team/yorkie/pkg/document/key" + "github.com/yorkie-team/yorkie/pkg/document/presence" + "github.com/yorkie-team/yorkie/server" + "github.com/yorkie-team/yorkie/server/logging" + "github.com/yorkie-team/yorkie/test/helper" +) + +var testServer *server.Yorkie + +func startTestServer(snapshotInterval int64, snapshotThreshold int64) { + config := helper.TestConfig() + config.Backend.SnapshotInterval = snapshotInterval + config.Backend.SnapshotThreshold = snapshotThreshold + + svr, err := server.New(config) + if err != nil { + logging.DefaultLogger().Fatal(err) + } + if err := svr.Start(); err != nil { + logging.DefaultLogger().Fatal(err) + } + testServer = svr +} + +func createDocKey(b *testing.B, i int) key.Key { + return key.Key(fmt.Sprintf("vv-bench-%d-%d", i, gotime.Now().UnixMilli())) +} + +func initializeClientAndDoc( + ctx context.Context, + b *testing.B, + docKey key.Key, +) (*client.Client, *document.Document) { + c, err := client.Dial( + testServer.RPCAddr(), + ) + assert.NoError(b, err) + err = c.Activate(ctx) + assert.NoError(b, err) + d := document.New(docKey) + err = c.Attach(ctx, d) + assert.NoError(b, err) + + return c, d +} + +func initializeClientsAndDocs( + ctx context.Context, + b *testing.B, + n int, + docKey key.Key, +) ([]*client.Client, []*document.Document) { + var clients []*client.Client + var docs []*document.Document + for i := 0; i < n; i++ { + c, d := initializeClientAndDoc(ctx, b, docKey) + clients = append(clients, c) + docs = append(docs, d) + } + return clients, docs +} + +func benchmarkVV( + clientCnt int, + b *testing.B, +) { + for i := 0; i < b.N; i++ { + ctx := context.Background() + docKey := createDocKey(b, i) + + // 1. Activate n clients and attach all clients to the document. + clients, docs := initializeClientsAndDocs(ctx, b, clientCnt, docKey) + c1, cN := clients[0], clients[clientCnt-1] + d1, dN := docs[0], docs[clientCnt-1] + + // 2.Initialize the text. + c1.Sync(ctx) + err := d1.Update(func(root *json.Object, p *presence.Presence) error { + root.SetNewText("text") + return nil + }) + assert.NoError(b, err) + c1.Sync(ctx) + cN.Sync(ctx) + assert.Equal(b, `{"text":[]}`, d1.Marshal()) + assert.Equal(b, `{"text":[]}`, dN.Marshal()) + + // 3. Multi-Client Edit Test + // - With n clients connected + // - c1 performs text edits + // Measurements: + // - ChangePack size + // - Total document(snapshot) size + // - PushPull time + err = d1.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("text").Edit(0, 0, "a") + return nil + }) + assert.NoError(b, err) + assert.Equal(b, `{"text":[{"val":"a"}]}`, d1.Marshal()) + + pack := d1.CreateChangePack() + pbPack, err := converter.ToChangePack(pack) + assert.NoError(b, err) + changePackSize := proto.Size(pbPack) + b.ReportMetric(float64(changePackSize), "1_changepack(bytes)") + + snapshot, err := converter.SnapshotToBytes(d1.RootObject(), d1.AllPresences()) + assert.NoError(b, err) + snapshotSize := len(snapshot) + b.ReportMetric(float64(snapshotSize), "2_snapshot(bytes)") + + start := gotime.Now() + assert.NoError(b, c1.Sync(ctx)) + assert.NoError(b, cN.Sync(ctx)) + duration := gotime.Since(start).Milliseconds() + assert.Equal(b, `{"text":[{"val":"a"}]}`, dN.Marshal()) + b.ReportMetric(float64(duration), "3_pushpull(ms)") + + // 4. All clients detach from the document. + helper.CleanupClients(b, clients) + + // 5. New clients attach to the document. + // Measurements: + // - Attach time (to load the existing document) + start = gotime.Now() + cN1, dN1 := initializeClientAndDoc(ctx, b, docKey) + duration = gotime.Since(start).Milliseconds() + assert.Equal(b, `{"text":[{"val":"a"}]}`, dN1.Marshal()) + b.ReportMetric(float64(duration), "4_attach(ms)") + + cN2, dN2 := initializeClientAndDoc(ctx, b, docKey) + assert.Equal(b, `{"text":[{"val":"a"}]}`, dN2.Marshal()) + + // 6. The new client edits the text. + // Measurements: + // - ChangePack size + // - Total document(snapshot) size + // - PushPull time + err = dN1.Update(func(root *json.Object, p *presence.Presence) error { + root.GetText("text").Edit(0, 0, "b") + return nil + }) + assert.NoError(b, err) + assert.Equal(b, `{"text":[{"val":"b"},{"val":"a"}]}`, dN1.Marshal()) + + pack = dN1.CreateChangePack() + pbPack, err = converter.ToChangePack(pack) + assert.NoError(b, err) + changePackSize = proto.Size(pbPack) + b.ReportMetric(float64(changePackSize), "5_changepack_after_detach(bytes)") + + snapshot, err = converter.SnapshotToBytes(dN1.RootObject(), dN1.AllPresences()) + assert.NoError(b, err) + snapshotSize = len(snapshot) + b.ReportMetric(float64(snapshotSize), "6_snapshot_after_detach(bytes)") + + start = gotime.Now() + assert.NoError(b, cN1.Sync(ctx)) + assert.NoError(b, cN2.Sync(ctx)) + duration = gotime.Since(start).Milliseconds() + assert.Equal(b, `{"text":[{"val":"b"},{"val":"a"}]}`, dN2.Marshal()) + b.ReportMetric(float64(duration), "7_pushpull_after_detach(ms)") + } +} + +func BenchmarkVersionVector(b *testing.B) { + err := logging.SetLogLevel("error") + assert.NoError(b, err) + startTestServer(100000, 100000) + defer func() { + if testServer == nil { + return + } + + if err := testServer.Shutdown(true); err != nil { + logging.DefaultLogger().Error(err) + } + }() + + b.Run("clients 10", func(b *testing.B) { + benchmarkVV(10, b) + }) + + b.Run("clients 100", func(b *testing.B) { + benchmarkVV(100, b) + }) + + b.Run("clients 1000", func(b *testing.B) { + benchmarkVV(1000, b) + }) +} diff --git a/test/helper/helper.go b/test/helper/helper.go index 4137c7321..d8328e195 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -553,3 +553,11 @@ func CreateProjectAndDocuments(t *testing.T, server *server.Yorkie, count int) ( return project, docs } + +// CleanupClients is a helper function to clean up clients. +func CleanupClients(b *testing.B, clients []*client.Client) { + for _, c := range clients { + assert.NoError(b, c.Deactivate(context.Background())) + assert.NoError(b, c.Close()) + } +}