Skip to content
348 changes: 274 additions & 74 deletions server/routes/shared_api/metadata.py
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Overall, the implementation looks good to me! This file is getting quite long though. What do you think of pulling out the helper functions into their own file, and leaving just the bp.route() functions here?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Overall, the implementation looks good to me! This file is getting quite long though. What do you think of pulling out the helper functions into their own file, and leaving just the bp.route() functions here? Totally fine to leave this for a followup PR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a good call (and something I was pondering as well). I've put it in as a TODO (along with the other item I flagged) for a separate PR.

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions static/js/tools/download/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -186,10 +186,10 @@ export function Page(props: PagePropType): ReactElement {
}
}

const enrichedFacets = await fetchFacetsWithMetadata(
baseFacets,
dataCommonsClient
);
const enrichedFacets = await fetchFacetsWithMetadata(baseFacets, {
parentPlace: selectedOptions.selectedPlace.dcid,
enclosedPlaceType: selectedOptions.enclosedPlaceType,
});

const sourceSelectorFacetList = [];
for (const sv in enrichedFacets) {
Expand Down
4 changes: 3 additions & 1 deletion static/js/tools/map/compute/facets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ export function useComputeFacetList(chartStore: ChartStore): {
}
}

fetchFacetsWithMetadata(baseFacets, dataCommonsClient)
fetchFacetsWithMetadata(baseFacets, {
entities: data ? Object.keys(data) : [],
})
.then((enrichedMap) => {
if (!enrichedMap[svDcid]) {
setFacetList([]);
Expand Down
6 changes: 4 additions & 2 deletions static/js/tools/scatter/chart_loader.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,11 @@ export function ChartLoader(): ReactElement {
const { x, y, place, display } = useContext(Context);
const cache = useCache();
const chartData = useChartData(cache);

const { facetSelectorMetadata, facetListLoading, facetListError } =
useFacetMetadata(cache?.baseFacets || null);
useFacetMetadata(cache?.baseFacets || null, {
parentPlace: place.value.enclosingPlace.dcid,
enclosedPlaceType: place.value.enclosedPlaceType,
});

const containerRef = useRef<HTMLDivElement>(null);
const embedModalElement = useRef<ChartEmbed>(null);
Expand Down
25 changes: 17 additions & 8 deletions static/js/tools/scatter/compute/facet_metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
import _ from "lodash";
import { useEffect, useState } from "react";

import { WEBSITE_SURFACE } from "../../../shared/constants";
import { getDataCommonsClient } from "../../../utils/data_commons_client";
import { FacetResponse } from "../../../utils/data_fetch_utils";
import { fetchFacetsWithMetadata } from "../../shared/metadata/metadata_fetcher";

Expand All @@ -44,14 +42,23 @@ type FacetMetadataReturn = {
* error state.
*/
export function useFacetMetadata(
baseFacets: FacetResponse | null
baseFacets: FacetResponse | null,
entityContext: {
entities?: string[];
parentPlace?: string;
enclosedPlaceType?: string;
} = {}
): FacetMetadataReturn {
const [facetMetadata, setFacetMetadata] = useState<FacetMetadataReturn>({
facetSelectorMetadata: {},
facetListLoading: !_.isEmpty(baseFacets),
facetListError: false,
});

// Extract the primitive values for safe dependency tracking
const { entities, parentPlace, enclosedPlaceType } = entityContext;
const entitiesString = entities?.join(",");

useEffect(() => {
if (_.isEmpty(baseFacets)) return;

Expand All @@ -64,10 +71,12 @@ export function useFacetMetadata(
}));

try {
const resp = await fetchFacetsWithMetadata(
baseFacets,
getDataCommonsClient(null, WEBSITE_SURFACE)
);
// Pass the extracted values back into the fetcher
const resp = await fetchFacetsWithMetadata(baseFacets, {
entities: entitiesString ? entitiesString.split(",") : undefined,
parentPlace,
enclosedPlaceType,
});

if (cancelled) return;

Expand All @@ -92,7 +101,7 @@ export function useFacetMetadata(
return () => {
cancelled = true;
};
}, [baseFacets]);
}, [baseFacets, entitiesString, parentPlace, enclosedPlaceType]);

return facetMetadata;
}
19 changes: 8 additions & 11 deletions static/js/tools/shared/facet_choice_fetcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import { WEBSITE_SURFACE } from "../../shared/constants";
import { FacetSelectorFacetInfo } from "../../shared/facet_selector/facet_selector";
import { getDataCommonsClient } from "../../utils/data_commons_client";
import { getFacets, getFacetsWithin } from "../../utils/data_fetch_utils";
import { fetchFacetsWithMetadata } from "./metadata/metadata_fetcher";

Expand All @@ -41,17 +40,15 @@ export async function fetchFacetChoices(
placeDcids: string[],
statVars: { dcid: string; name?: string }[]
): Promise<FacetSelectorFacetInfo[]> {
const dataCommonsClient = getDataCommonsClient(null, WEBSITE_SURFACE);
const baseFacets = await getFacets(
"",
placeDcids,
statVars.map((sv) => sv.dcid),
WEBSITE_SURFACE
);
const enrichedFacets = await fetchFacetsWithMetadata(
baseFacets,
dataCommonsClient
);
const enrichedFacets = await fetchFacetsWithMetadata(baseFacets, {
entities: placeDcids,
});
return statVars.map((sv) => ({
dcid: sv.dcid,
name: sv.name || sv.dcid,
Expand All @@ -73,7 +70,6 @@ export async function fetchFacetChoicesWithin(
enclosedPlaceType: string,
statVars: { dcid: string; name?: string; date?: string }[]
): Promise<FacetSelectorFacetInfo[]> {
const dataCommonsClient = getDataCommonsClient(null, WEBSITE_SURFACE);
const facetPromises = statVars.map((sv) =>
getFacetsWithin(
"",
Expand All @@ -85,10 +81,11 @@ export async function fetchFacetChoicesWithin(
)
);
const baseFacets = Object.assign({}, ...(await Promise.all(facetPromises)));
const enrichedFacets = await fetchFacetsWithMetadata(
baseFacets,
dataCommonsClient
);

const enrichedFacets = await fetchFacetsWithMetadata(baseFacets, {
parentPlace,
enclosedPlaceType,
});
return statVars.map((sv) => ({
dcid: sv.dcid,
name: sv.name || sv.dcid,
Expand Down
105 changes: 27 additions & 78 deletions static/js/tools/shared/metadata/metadata_fetcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -290,94 +290,43 @@ async function fetchNodeProperty(
* Fetches and enriches facet metadata with human-readable and supplementary
* information like source names, date ranges, and descriptions.
* @param facets The basic facet response from an API call like getFacets.
* @param dataCommonsClient Client for Data Commons API calls.
* @param entityContext The specific entities or expression used to determine dates.
* @param apiRoot Optional API root URL for requests.
* @returns The facets with enriched StatMetadata.
*/
export async function fetchFacetsWithMetadata(
facets: FacetResponse,
dataCommonsClient: DataCommonsClient,
entityContext: {
entities?: string[];
parentPlace?: string;
enclosedPlaceType?: string;
},
apiRoot = ""
): Promise<FacetResponse> {
const statVars = Object.keys(facets);
if (!statVars.length) return {};

const measurementMethods = new Set<string>();
const units = new Set<string>();
const statVarSet = new Set<string>();
for (const sv in facets) {
statVarSet.add(sv);
for (const facetId in facets[sv]) {
const facet = facets[sv][facetId];
if (facet.measurementMethod) {
measurementMethods.add(facet.measurementMethod);
}
if (facet.unit) {
units.add(facet.unit);
}
}
}

const [provenanceMap, variableData, measurementMethodMap, unitMap] =
await Promise.all([
fetchProvenanceInfo(statVars, facets, undefined, apiRoot),
fetchStatVarProvenanceSummaries(statVars, apiRoot),
fetchNodeProperty(measurementMethods, "description", dataCommonsClient),
fetchNodeProperty(units, "name", dataCommonsClient),
]);

const enrichedFacets: FacetResponse = {};
for (const statVarId of statVars) {
enrichedFacets[statVarId] = {};
for (const facetId in facets[statVarId]) {
const facetInfo = facets[statVarId][facetId];
const newFacetInfo = { ...facetInfo };

if (facetInfo.importName) {
const provenanceId = `dc/base/${facetInfo.importName}`;
const provenanceData = provenanceMap[provenanceId];
if (provenanceData) {
newFacetInfo.sourceName = provenanceData.source?.[0]?.name;
newFacetInfo.provenanceName =
provenanceData.isPartOf?.[0]?.name ||
provenanceData.name?.[0]?.value ||
facetInfo.importName;
}
const seriesList =
variableData[statVarId]?.provenanceSummary?.[provenanceId]
?.seriesSummary;
const matchedSeries = Array.isArray(seriesList)
? matchSeriesByFacet(seriesList, facetInfo)
: undefined;
if (
matchedSeries &&
!DATE_RANGE_SUPPRESSION_PROVENANCES.includes(
newFacetInfo.provenanceName
)
) {
newFacetInfo.dateRangeStart = matchedSeries.earliestDate;
newFacetInfo.dateRangeEnd = matchedSeries.latestDate;
}
}

if (
facetInfo.measurementMethod &&
!MEASUREMENT_METHODS_SUPPRESSION_PROVENANCES.includes(
newFacetInfo.provenanceName
)
) {
newFacetInfo.measurementMethodDescription =
measurementMethodMap[facetInfo.measurementMethod] ||
startCase(facetInfo.measurementMethod.replace(/_/g, " "));
}
if (facetInfo.unit) {
newFacetInfo.unitDisplayName =
unitMap[facetInfo.unit] || facetInfo.unit.replace(/_/g, " ");
}
enrichedFacets[statVarId][facetId] = newFacetInfo;
if (!statVars.length) return facets;

try {
const response = await fetch(`${apiRoot}/api/metadata/facets`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
facets,
statVars,
entities: entityContext.entities,
parentPlace: entityContext.parentPlace,
enclosedPlaceType: entityContext.enclosedPlaceType,
}),
});
if (!response.ok) {
console.error("Failed to enrich facets via API");
return facets;
}
return await response.json();
} catch (e) {
console.error("Error enriching facets:", e);
return facets;
}
return enrichedFacets;
}

/**
Expand Down
7 changes: 3 additions & 4 deletions static/js/tools/timeline/chart.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -466,10 +466,9 @@ class Chart extends Component<ChartPropsType, ChartStateType> {
metadataMap: Record<string, Record<string, StatMetadata>>
): Promise<void> {
try {
const enriched = await fetchFacetsWithMetadata(
metadataMap,
this.dataCommonsClient
);
const enriched = await fetchFacetsWithMetadata(metadataMap, {
entities: Object.keys(this.props.placeNameMap),
});
const facetList = this.getFacetList(statVars, enriched);
this.setState({ facetList, facetListLoading: false });
} catch {
Expand Down
Loading