Skip to content

Fetch variable metadata from Spanner #1797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
n-h-diaz merged 8 commits into from
Mar 17, 2026
Merged

Fetch variable metadata from Spanner #1797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
152fc13
Fetch variable metadata from Spanner
n-h-diaz Mar 9, 2026
14fd8b1
allow bulk
n-h-diaz Mar 9, 2026
43b8e9d
gemini
n-h-diaz Mar 9, 2026
79d6c44
Merge branch 'master' into vm
n-h-diaz Mar 11, 2026
dcdd0f4
merge
n-h-diaz Mar 17, 2026
d22e0af
use cache table
n-h-diaz Mar 17, 2026
83f4bd6
Merge branch 'vm' of https://github.com/n-h-diaz/mixer into vm
n-h-diaz Mar 17, 2026
202803a
gemini
n-h-diaz Mar 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions internal/server/spanner/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

"cloud.google.com/go/spanner"
pb "github.com/datacommonsorg/mixer/internal/proto"
v2 "github.com/datacommonsorg/mixer/internal/server/v2"
"github.com/datacommonsorg/mixer/internal/translator/types"
"gopkg.in/yaml.v3"
Expand All @@ -39,6 +40,7 @@ type SpannerClient interface {
ResolveByID(ctx context.Context, nodes []string, in, out string) (map[string][]string, error)
GetEventCollectionDate(ctx context.Context, placeID, eventType string) ([]string, error)
Sparql(ctx context.Context, nodes []types.Node, queries []*types.Query, opts *types.QueryOptions) ([][]string, error)
GetVariableMetadata(ctx context.Context, ids []string) (map[string][]*pb.StatVarSummary_ProvenanceSummary, error)
Id() string
Start()
Close()
Expand Down
7 changes: 5 additions & 2 deletions internal/server/spanner/golden/datasource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,15 @@ func (m *mockSpannerClient) ResolveByID(ctx context.Context, nodes []string, in,
func (m *mockSpannerClient) Sparql(ctx context.Context, nodes []types.Node, queries []*types.Query, opts *types.QueryOptions) ([][]string, error) {
return nil, nil
}
func (m *mockSpannerClient) GetVariableMetadata(ctx context.Context, ids []string) (map[string][]*pb.StatVarSummary_ProvenanceSummary, error) {
return nil, nil
}
func (m *mockSpannerClient) GetEventCollectionDate(ctx context.Context, placeID, eventType string) ([]string, error) {
return nil, nil
}
func (m *mockSpannerClient) Id() string { return "mock" }
func (m *mockSpannerClient) Start() {}
func (m *mockSpannerClient) Close() {}
func (m *mockSpannerClient) Start() {}
func (m *mockSpannerClient) Close() {}

func TestSpannerResolve(t *testing.T) {
client := test.NewSpannerClient()
Expand Down
91 changes: 91 additions & 0 deletions internal/server/spanner/golden/query/get_variable_metadata.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
{
"Count_Household_FamilyHousehold": [
{
"import_name": "CensusACS5YearSurvey",
"series_summary": [
{
"series_key": {
"measurement_method": "CensusACS5yrSurvey"
},
"earliest_date": "2011",
"latest_date": "2023",
"place_type_summary": {
"AdministrativeArea1": {},
"AdministrativeArea2": {},
"AdministrativeArea4": {},
"AdministrativeArea5": {},
"Borough": {},
"CensusCoreBasedStatisticalArea": {},
"CensusCountyDivision": {},
"CensusDivision": {},
"CensusTract": {},
"CensusZipCodeTabulationArea": {},
"City": {},
"CongressionalDistrict": {},
"Country": {},
"County": {},
"ElementarySchoolDistrict": {},
"HighSchoolDistrict": {},
"Neighborhood": {},
"Place": {},
"ProvisionalNode": {},
"SchoolDistrict": {},
"State": {},
"StateComponent": {},
"Town": {},
"Village": {}
},
"observation_count": 2547106,
"time_series_count": 220438
}
],
"observation_count": 2547106,
"time_series_count": 220438
}
],
"Count_Household_HasComputer": [
{
"import_name": "CensusACS5YearSurvey",
"series_summary": [
{
"series_key": {
"measurement_method": "CensusACS5yrSurvey"
},
"earliest_date": "2017",
"latest_date": "2023",
"place_type_summary": {
"AdministrativeArea1": {},
"AdministrativeArea2": {},
"AdministrativeArea4": {},
"AdministrativeArea5": {},
"Borough": {},
"CensusBlockGroup": {},
"CensusCoreBasedStatisticalArea": {},
"CensusCountyDivision": {},
"CensusDivision": {},
"CensusTract": {},
"CensusZipCodeTabulationArea": {},
"City": {},
"CongressionalDistrict": {},
"Country": {},
"County": {},
"ElementarySchoolDistrict": {},
"HighSchoolDistrict": {},
"Neighborhood": {},
"Place": {},
"ProvisionalNode": {},
"SchoolDistrict": {},
"State": {},
"StateComponent": {},
"Town": {},
"Village": {}
},
"observation_count": 2805865,
"time_series_count": 511517
}
],
"observation_count": 2805865,
"time_series_count": 511517
}
]
}
8 changes: 8 additions & 0 deletions internal/server/spanner/golden/query_builder/get_variable_metadata.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
SELECT
key,
TO_JSON_STRING(value) AS value,
FROM
Cache
WHERE
type = 'ProvenanceSummary'
AND key IN ('Count_Household_FamilyHousehold','Count_Household_HasComputer')
12 changes: 12 additions & 0 deletions internal/server/spanner/golden/query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,18 @@ func TestSparqlQuery(t *testing.T) {
}
}

func TestGetVariableMetadataQuery(t *testing.T) {
t.Parallel()

for _, c := range variableMetadataTestCases {
goldenFile := c.golden + ".sql"

runQueryBuilderGoldenTest(t, goldenFile, func(ctx context.Context) (interface{}, error) {
return spanner.GetCacheDataQuery(spanner.TypeProvenanceSummary, c.variables), nil
})
}
}

func TestGetEventCollectionDateQuery(t *testing.T) {
t.Parallel()

Expand Down
10 changes: 10 additions & 0 deletions internal/server/spanner/golden/query_cases_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,16 @@ var sparqlTestCases = []struct {
},
}

var variableMetadataTestCases = []struct {
variables []string
golden string
}{
{
variables: []string{"Count_Household_FamilyHousehold", "Count_Household_HasComputer"},
golden: "get_variable_metadata",
},
}

var eventCollectionDateTestCases = []struct {
placeDcid string
eventType string
Expand Down
17 changes: 17 additions & 0 deletions internal/server/spanner/golden/query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,23 @@ func TestSparql(t *testing.T) {
}
}

func TestGetVariableMetadata(t *testing.T) {
client := test.NewSpannerClient()
if client == nil {
return
}

t.Parallel()

for _, c := range variableMetadataTestCases {
goldenFile := c.golden + ".json"

runQueryGoldenTest(t, goldenFile, func(ctx context.Context) (interface{}, error) {
return client.GetVariableMetadata(ctx, c.variables)
})
}
}

func TestGetEventCollectionDate(t *testing.T) {
client := test.NewSpannerClient()
if client == nil {
Expand Down
7 changes: 7 additions & 0 deletions internal/server/spanner/model.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ import (
"google.golang.org/protobuf/proto"
)

// CacheDataType represents the type of cache data.
type CacheDataType string

const (
TypeProvenanceSummary = "ProvenanceSummary"
)

// Property struct represents a subset of a row in the Edge table.
type Property struct {
SubjectID string `spanner:"subject_id"`
Expand Down
Loading
Loading