Merge branch 'main' into release

Author: reearth-app[bot]

.github/workflows/deploy_server_nightly.yml

@@ -5,10 +5,6 @@ on:
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
-env:
-  GCP_REGION: us-central1
-  IMAGE: reearth/reearth-visualizer-api:nightly
-  IMAGE_GCP: us-central1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/reearth/reearth-visualizer-api:nightly
 jobs:
   deploy_test:
     runs-on: ubuntu-latest
@@ -20,21 +16,21 @@ jobs:
       - uses: actions/checkout@v4
       - uses: google-github-actions/auth@v2
         with:
-          service_account: ${{ secrets.GCP_SA_EMAIL }}
-          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GC_SA_EMAIL }}
+          workload_identity_provider: ${{ secrets.GC_WORKLOAD_IDENTITY_PROVIDER }}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
       - name: Configure docker
         run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
       - name: docker push
         run: |
-          docker pull $IMAGE
-          docker tag $IMAGE $IMAGE_GCP
-          docker push $IMAGE_GCP
+          docker pull ${{ secrets.SERVER_IMAGE }}
+          docker tag ${{ secrets.SERVER_IMAGE }} ${{ secrets.SERVER_IMAGE_GC }}
+          docker push ${{ secrets.SERVER_IMAGE_GC }}
       - name: Deploy to Cloud Run
         run: |
           gcloud run deploy reearth-visualizer-api \
-            --image $IMAGE_GCP \
-            --region $GCP_REGION \
+            --image ${{ secrets.SERVER_IMAGE_GC }} \
+            --region ${{ secrets.GC_REGION }} \
             --platform managed \
             --quiet

.github/workflows/deploy_web_nightly.yml

@@ -5,10 +5,6 @@ on:
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
-env:
-  GCP_REGION: us-central1
-  IMAGE: reearth/reearth-visualizer-web:nightly
-  IMAGE_GCP: us-central1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/reearth/reearth-visualizer-web:nightly
 jobs:
   deploy_test:
     runs-on: ubuntu-latest
@@ -20,21 +16,21 @@ jobs:
       - uses: actions/checkout@v4
       - uses: google-github-actions/auth@v2
         with:
-          service_account: ${{ secrets.GCP_SA_EMAIL }}
-          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GC_SA_EMAIL }}
+          workload_identity_provider: ${{ secrets.GC_WORKLOAD_IDENTITY_PROVIDER }}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
       - name: Configure docker
         run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
       - name: docker push
         run: |
-          docker pull $IMAGE
-          docker tag $IMAGE $IMAGE_GCP
-          docker push $IMAGE_GCP
+          docker pull ${{ secrets.WEB_IMAGE }}
+          docker tag ${{ secrets.WEB_IMAGE }} ${{ secrets.WEB_IMAGE_GC }}
+          docker push ${{ secrets.WEB_IMAGE_GC }}
       - name: Deploy to Cloud Run
         run: |
           gcloud run deploy reearth-visualizer-web \
-            --image $IMAGE_GCP \
-            --region $GCP_REGION \
+            --image ${{ secrets.WEB_IMAGE_GC }} \
+            --region ${{ secrets.GC_REGION }} \
             --platform managed \
             --quiet

.github/workflows/run_migration.yml

@@ -0,0 +1,39 @@
+name: run-migration
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  migrate:
+    name: Run migration (Nightly)
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      GCP_REGION: us-central1
+      IMAGE: reearth/reearth-visualizer-api:nightly
+      IMAGE_GCP: us-central1-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/reearth/reearth-visualizer-api:nightly
+    steps:
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          service_account: ${{ secrets.GCP_SA_EMAIL }}
+          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+
+      - name: Update Migration Job
+        run: |
+          gcloud run jobs update reearth-visualizer-migration \
+            --image $IMAGE_GCP \
+            --region $GCP_REGION \
+            --quiet
+
+      - name: Execute Migration Job
+        run: |
+          gcloud run jobs execute reearth-visualizer-migration \
+            --region $GCP_REGION \
+            --wait \
+            --quiet

server/Makefile

@@ -19,6 +19,7 @@ help:
 	@echo "  dev               Run the application with hot reloading"
 	@echo "  run-app           Run the application"
 	@echo "  run-db            Run the MongoDB database using Docker Compose"
+	@echo "  run-reset         Initialize the MongoDB data"
 	@echo "  up-gcs            Run the fake-gcs-server using Docker Compose"
 	@echo "  down-gcs          Stop the fake-gcs-server using Docker Compose"
 	@echo "  gql               Generate GraphQL code include dataloader"
@@ -81,6 +82,13 @@ run-app:
 run-db:
 	docker compose -f ../docker-compose.yml up -d reearth-mongo
 
+run-reset:
+	docker stop reearth-visualizer-reearth-mongo-1
+	rm -rf ../mongo
+	rm -rf data
+	make run-db
+	make mockuser
+
 generate: dev-install
 	go generate ./...
 
@@ -101,4 +109,4 @@ schematyper:
 	go run $(SCHEMATYPER) -o $(MANIFEST_DIR)/schema_translation.go --package manifest --prefix Translation ./schemas/plugin_manifest_translation.json
 	go run $(SCHEMATYPER) -o $(MANIFEST_DIR)/schema_gen.go --package manifest ./schemas/plugin_manifest.json
 
-.PHONY: lint test failcheck e2e build dev-install dev run-app run-db gql up-gcs down-gcs mockuser schematyper
+.PHONY: lint test failcheck e2e build dev-install dev run-app run-db run-reset gql up-gcs down-gcs mockuser schematyper

server/e2e/common.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"reflect"
 	"regexp"
 	"strings"
 	"testing"
@@ -31,13 +32,23 @@ import (
 	"golang.org/x/text/language"
 )
 
-type Seeder func(ctx context.Context, r *repo.Container) error
+var (
+	fr                 *gateway.File
+	disabledAuthConfig = &config.Config{
+		Origins: []string{"https://example.com"},
+		AuthSrv: config.AuthSrvConfig{
+			Disabled: true,
+		},
+	}
+)
+
+type Seeder func(ctx context.Context, r *repo.Container, f gateway.File) error
 
 func init() {
 	mongotest.Env = "REEARTH_DB"
 }
 
-func initRepos(t *testing.T, useMongo bool, seeder Seeder) (repos *repo.Container) {
+func initRepos(t *testing.T, useMongo bool, seeder Seeder) (repos *repo.Container, file gateway.File) {
 	ctx := context.Background()
 
 	if useMongo {
@@ -48,18 +59,25 @@ func initRepos(t *testing.T, useMongo bool, seeder Seeder) (repos *repo.Containe
 		repos = memory.New()
 	}
 
+	file = lo.Must(fs.NewFile(afero.NewMemMapFs(), "https://example.com/"))
+	fr = &file
 	if seeder != nil {
-		if err := seeder(ctx, repos); err != nil {
+		if err := seeder(ctx, repos, file); err != nil {
 			t.Fatalf("failed to seed the db: %s", err)
 		}
 	}
 
-	return repos
+	return repos, file
 }
 
 func initGateway() *gateway.Container {
+	if fr == nil {
+		return &gateway.Container{
+			File: lo.Must(fs.NewFile(afero.NewMemMapFs(), "https://example.com/")),
+		}
+	}
 	return &gateway.Container{
-		File: lo.Must(fs.NewFile(afero.NewMemMapFs(), "https://example.com")),
+		File: *fr,
 	}
 }
 
@@ -117,96 +135,51 @@ func StartGQLServerWithRepos(t *testing.T, cfg *config.Config, repos *repo.Conta
 }
 
 func StartGQLServerAndRepos(t *testing.T, seeder Seeder) (*httpexpect.Expect, *accountrepo.Container) {
-	cfg := &config.Config{
-		Origins: []string{"https://example.com"},
-		AuthSrv: config.AuthSrvConfig{
-			Disabled: true,
-		},
-	}
-	repos := initRepos(t, true, seeder)
-	e, _, _ := StartGQLServerWithRepos(t, cfg, repos)
+	repos, _ := initRepos(t, true, seeder)
+	e, _, _ := StartGQLServerWithRepos(t, disabledAuthConfig, repos)
 	return e, repos.AccountRepos()
 }
 
-func initServer(cfg *config.Config, repos *repo.Container, ctx context.Context) (*app.WebServer, *gateway.Container) {
-	gateways := initGateway()
-	return app.NewServer(ctx, &app.ServerConfig{
-		Config:       cfg,
-		Repos:        repos,
-		AccountRepos: repos.AccountRepos(),
-		Gateways:     gateways,
-		Debug:        true,
-	}), gateways
+func startServer(t *testing.T, cfg *config.Config, useMongo bool, seeder Seeder) (*httpexpect.Expect, *repo.Container, *gateway.Container) {
+	repos, _ := initRepos(t, useMongo, seeder)
+	e, gateways, _ := StartGQLServerWithRepos(t, cfg, repos)
+	return e, repos, gateways
 }
 
-func StartServerWithRepos(t *testing.T, cfg *config.Config, repos *repo.Container) (*httpexpect.Expect, *gateway.Container) {
-	t.Helper()
-
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
-
-	ctx := context.Background()
-
-	l, err := net.Listen("tcp", ":0")
-	if err != nil {
-		t.Fatalf("server failed to listen: %v", err)
-	}
-
-	srv, gateways := initServer(cfg, repos, ctx)
-
-	ch := make(chan error)
-	go func() {
-		if err := srv.Serve(l); err != http.ErrServerClosed {
-			ch <- err
-		}
-		close(ch)
-	}()
-	t.Cleanup(func() {
-		if err := srv.Shutdown(context.Background()); err != nil {
-			t.Fatalf("server shutdown: %v", err)
-		}
-
-		if err := <-ch; err != nil {
-			t.Fatalf("server serve: %v", err)
-		}
-	})
-	return httpexpect.Default(t, "http://"+l.Addr().String()), gateways
+func ServerAndRepos(t *testing.T, seeder Seeder) (*httpexpect.Expect, *repo.Container, *gateway.Container) {
+	return startServer(t, disabledAuthConfig, true, seeder)
 }
 
-func StartServerAndRepos(t *testing.T, cfg *config.Config, useMongo bool, seeder Seeder) (*httpexpect.Expect, *repo.Container, *gateway.Container) {
-	repos := initRepos(t, useMongo, seeder)
-	e, gateways := StartServerWithRepos(t, cfg, repos)
-	return e, repos, gateways
+func Server(t *testing.T, seeder Seeder) *httpexpect.Expect {
+	e, _, _ := startServer(t, disabledAuthConfig, true, seeder)
+	return e
 }
 
-func StartServer(t *testing.T, cfg *config.Config, useMongo bool, seeder Seeder) *httpexpect.Expect {
-	e, _, _ := StartServerAndRepos(t, cfg, useMongo, seeder)
+func ServerPingTest(t *testing.T) *httpexpect.Expect {
+	e, _, _ := startServer(t, disabledAuthConfig, false, nil)
 	return e
 }
 
-func Server(t *testing.T, seeder Seeder) *httpexpect.Expect {
+func ServerMockTest(t *testing.T) *httpexpect.Expect {
 	c := &config.Config{
-		Origins: []string{"https://example.com"},
+		Dev:      true,
+		MockAuth: true,
+		Origins:  []string{"https://example.com"},
 		AuthSrv: config.AuthSrvConfig{
 			Disabled: true,
 		},
 	}
-	return StartServer(t, c, true, seeder)
+	e, _, _ := startServer(t, c, true, nil)
+	return e
 }
 
 func ServerLanguage(t *testing.T, lang language.Tag) *httpexpect.Expect {
-	c := &config.Config{
-		Origins: []string{"https://example.com"},
-		AuthSrv: config.AuthSrvConfig{
-			Disabled: true,
-		},
-	}
-	return StartServer(t, c, true,
-		func(ctx context.Context, r *repo.Container) error {
-			return baseSeederWithLang(ctx, r, lang)
+	e, _, _ := startServer(t, disabledAuthConfig, true,
+		func(ctx context.Context, r *repo.Container, f gateway.File) error {
+			return baseSeederWithLang(ctx, r, f, lang)
 		},
 	)
+	return e
 }
 
 type GraphQLRequest struct {
@@ -275,6 +248,7 @@ func RegexpJSONEReadCloser(t *testing.T, actual io.ReadCloser, expected string)
 	actualBuf := new(bytes.Buffer)
 	_, err := actualBuf.ReadFrom(actual)
 	assert.NoError(t, err)
+	// ActualDump(actualBuf)
 	return JSONEqRegexp(t, actualBuf.String(), expected)
 }
 
@@ -284,6 +258,13 @@ func JSONEqRegexpInterface(t *testing.T, actual interface{}, expected string) bo
 	return JSONEqRegexp(t, string(actualBytes), expected)
 }
 
+func JSONEqRegexpValue(t *testing.T, actual *httpexpect.Value, expected string) bool {
+	if actualData, ok := actual.Raw().(map[string]interface{}); ok {
+		return JSONEqRegexpInterface(t, actualData, expected)
+	}
+	return false
+}
+
 func aligningJSON(t *testing.T, str string) string {
 	// Unmarshal and Marshal to make the JSON format consistent
 	var obj interface{}
@@ -295,9 +276,30 @@ func aligningJSON(t *testing.T, str string) string {
 }
 
 func ValueDump(val *httpexpect.Value) {
-	if data, ok := val.Raw().(map[string]interface{}); ok {
+	raw := val.Raw()
+	switch data := raw.(type) {
+	case map[string]interface{}:
 		if text, err := json.MarshalIndent(data, "", "  "); err == nil {
 			fmt.Println(string(text))
 		}
+	case []interface{}:
+		if text, err := json.MarshalIndent(data, "", "  "); err == nil {
+			fmt.Println(string(text))
+		}
+	default:
+		fmt.Println("Unsupported type:", reflect.TypeOf(raw))
+	}
+}
+
+func ActualDump(actual *bytes.Buffer) {
+	var data interface{}
+	if err := json.Unmarshal(actual.Bytes(), &data); err != nil {
+		fmt.Println("Invalid JSON:", err)
+		return
+	}
+	if text, err := json.MarshalIndent(data, "", "  "); err == nil {
+		fmt.Println(string(text))
+	} else {
+		fmt.Println("Failed to format JSON:", err)
 	}
 }