Skip to content

Commit 9fee844

Browse files
MiquelAdellMiquelAdell
authored
Merge pull request #966 from EyeSeeTea/feature/datastore-support
add dataStore support
2 parents 7b8d1bb + 36e769a commit 9fee844

File tree

20 files changed

+597
-21
lines changed

20 files changed

+597
-21
lines changed

‎src/data/common/D2ApiDataStore.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { D2Api } from "../../types/d2-api";
2+
import { getD2APiFromInstance } from "../../utils/d2-utils";
3+
import { DataStore, DataStoreKey } from "../../domain/metadata/entities/MetadataEntities";
4+
import { promiseMap } from "../../utils/common";
5+
import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata";
6+
import { Instance } from "../../domain/instance/entities/Instance";
7+
8+
export class D2ApiDataStore {
9+
private api: D2Api;
10+
11+
constructor(instance: Instance) {
12+
this.api = getD2APiFromInstance(instance);
13+
}
14+
15+
async getDataStores(filter: { namespaces?: string[] }): Promise<DataStore[]> {
16+
const response = await this.api.request<string[]>({ method: "get", url: "/dataStore" }).getData();
17+
const namespacesWithKeys = await this.getAllKeysFromNamespaces(
18+
filter.namespaces
19+
? DataStoreMetadata.getDataStoreIds(filter.namespaces || []).map(ns => {
20+
const [namespace] = ns.split(DataStoreMetadata.NS_SEPARATOR);
21+
return namespace;
22+
})
23+
: response
24+
);
25+
return namespacesWithKeys;
26+
}
27+
28+
private async getAllKeysFromNamespaces(namespaces: string[]): Promise<DataStore[]> {
29+
const result = await promiseMap<string, DataStore>(namespaces, async namespace => {
30+
const keys = await this.getKeysPaginated([], namespace);
31+
return {
32+
code: namespace,
33+
displayName: namespace,
34+
externalAccess: false,
35+
favorites: [],
36+
id: [namespace, DataStoreMetadata.NS_SEPARATOR].join(""),
37+
keys: keys,
38+
name: namespace,
39+
translations: [],
40+
};
41+
});
42+
return result;
43+
}
44+
45+
private async getKeysPaginated(keysState: DataStoreKey[], namespace: string): Promise<DataStoreKey[]> {
46+
const keyResponse = await this.getKeysByNameSpace(namespace);
47+
return [...keysState, ...keyResponse];
48+
}
49+
50+
private async getKeysByNameSpace(namespace: string): Promise<DataStoreKey[]> {
51+
const response = await this.api
52+
.request<string[]>({
53+
method: "get",
54+
url: `/dataStore/${namespace}`,
55+
// Since v38 we can use the fields parameter to get keys and values in the same request
56+
// Empty fields returns a paginated response
57+
// https://docs.dhis2.org/en/full/develop/dhis-core-version-240/developer-manual.html#query-api
58+
// params: { fields: "", page: page, pageSize: 200 },
59+
})
60+
.getData();
61+
62+
return this.buildArrayDataStoreKey(response, namespace);
63+
}
64+
65+
private buildArrayDataStoreKey(keys: string[], namespace: string): DataStoreKey[] {
66+
return keys.map(key => ({ id: DataStoreMetadata.generateKeyId(namespace, key), displayName: key }));
67+
}
68+
}

‎src/data/data-store/DataStoreMetadataD2Repository.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import _ from "lodash";
2+
import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata";
3+
import { DataStoreMetadataRepository } from "../../domain/data-store/DataStoreMetadataRepository";
4+
import { Instance } from "../../domain/instance/entities/Instance";
5+
import { Stats } from "../../domain/reports/entities/Stats";
6+
import { SynchronizationResult } from "../../domain/reports/entities/SynchronizationResult";
7+
import { promiseMap } from "../../utils/common";
8+
import { StorageDataStoreClient } from "../storage/StorageDataStoreClient";
9+
10+
export class DataStoreMetadataD2Repository implements DataStoreMetadataRepository {
11+
private instance: Instance;
12+
13+
constructor(instance: Instance) {
14+
this.instance = instance;
15+
}
16+
17+
async get(dataStores: DataStoreMetadata[]): Promise<DataStoreMetadata[]> {
18+
const result = await promiseMap(dataStores, async dataStore => {
19+
const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace);
20+
const dataStoreWithValue = this.getValuesByDataStore(dataStoreClient, dataStore);
21+
return dataStoreWithValue;
22+
});
23+
return result;
24+
}
25+
26+
private async getValuesByDataStore(
27+
dataStoreClient: StorageDataStoreClient,
28+
dataStore: DataStoreMetadata
29+
): Promise<DataStoreMetadata> {
30+
const keys = await this.getAllKeys(dataStoreClient, dataStore);
31+
const keyWithValue = await promiseMap(keys, async key => {
32+
const keyValue = await dataStoreClient.getObject(key.id);
33+
return { id: key.id, value: keyValue };
34+
});
35+
const keyInNamespace = _(dataStore.keys).first()?.id;
36+
const sharing = keyInNamespace ? await dataStoreClient.getObjectSharing(keyInNamespace) : undefined;
37+
return new DataStoreMetadata({
38+
namespace: dataStore.namespace,
39+
keys: keyWithValue,
40+
sharing,
41+
});
42+
}
43+
44+
private async getAllKeys(
45+
dataStoreClient: StorageDataStoreClient,
46+
dataStore: DataStoreMetadata
47+
): Promise<DataStoreMetadata["keys"]> {
48+
if (dataStore.keys.length > 0) return dataStore.keys;
49+
const keys = await dataStoreClient.listKeys();
50+
return keys.map(key => ({ id: key, value: "" }));
51+
}
52+
53+
async save(dataStores: DataStoreMetadata[]): Promise<SynchronizationResult> {
54+
const keysIdsToDelete = await this.getKeysToDelete(dataStores);
55+
56+
const resultStats = await promiseMap(dataStores, async dataStore => {
57+
const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace);
58+
const stats = await promiseMap(dataStore.keys, async key => {
59+
const exist = await dataStoreClient.getObject(key.id);
60+
await dataStoreClient.saveObject(key.id, key.value);
61+
if (dataStore.sharing) {
62+
await dataStoreClient.saveObjectSharing(key.id, dataStore.sharing);
63+
}
64+
return exist ? Stats.createOrEmpty({ updated: 1 }) : Stats.createOrEmpty({ imported: 1 });
65+
});
66+
return stats;
67+
});
68+
69+
const deleteStats = await promiseMap(keysIdsToDelete, async keyId => {
70+
const [namespace, key] = keyId.split(DataStoreMetadata.NS_SEPARATOR);
71+
const dataStoreClient = new StorageDataStoreClient(this.instance, namespace);
72+
await dataStoreClient.removeObject(key);
73+
return Stats.createOrEmpty({ deleted: 1 });
74+
});
75+
76+
const allStats = resultStats.flatMap(result => result).concat(deleteStats);
77+
const dataStoreStats = { ...Stats.combine(allStats.map(stat => Stats.create(stat))), type: "DataStore Keys" };
78+
79+
const result: SynchronizationResult = {
80+
date: new Date(),
81+
instance: this.instance,
82+
status: "SUCCESS",
83+
type: "metadata",
84+
stats: dataStoreStats,
85+
typeStats: [dataStoreStats],
86+
};
87+
88+
return result;
89+
}
90+
91+
private async getKeysToDelete(dataStores: DataStoreMetadata[]): Promise<string[]> {
92+
const allKeysToDelete = await promiseMap(dataStores, async dataStore => {
93+
if (dataStore.action === "MERGE") return [];
94+
95+
const existingRecords = await this.get([{ ...dataStore, keys: [] }]);
96+
const existingKeysIds = existingRecords.flatMap(dataStore => {
97+
return dataStore.keys.map(key => DataStoreMetadata.generateKeyId(dataStore.namespace, key.id));
98+
});
99+
100+
const keysIdsToSave = dataStores.flatMap(dataStore => {
101+
return dataStore.keys.map(key => DataStoreMetadata.generateKeyId(dataStore.namespace, key.id));
102+
});
103+
104+
const keysIdsToDelete = existingKeysIds.filter(id => !keysIdsToSave.includes(id));
105+
return keysIdsToDelete;
106+
});
107+
108+
return allKeysToDelete.flat();
109+
}
110+
}

‎src/data/metadata/MetadataD2ApiRepository.ts‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import { debug } from "../../utils/debug";
3838
import { paginate } from "../../utils/pagination";
3939
import { metadataTransformations } from "../transformations/PackageTransformations";
4040
import { D2MetadataUtils } from "./D2MetadataUtils";
41+
import { D2ApiDataStore } from "../common/D2ApiDataStore";
42+
import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata";
4143

4244
export class MetadataD2ApiRepository implements MetadataRepository {
4345
private api: D2Api;
@@ -63,6 +65,8 @@ export class MetadataD2ApiRepository implements MetadataRepository {
6365
): Promise<MetadataPackage<T>> {
6466
const { apiVersion } = this.instance;
6567

68+
const d2ApiDataStore = new D2ApiDataStore(this.instance);
69+
const dataStoreIds = DataStoreMetadata.getDataStoreIds(ids);
6670
const requestFields = typeof fields === "object" ? getFieldsAsString(fields) : fields;
6771
const d2Metadata = await this.getMetadata<D2Model>(ids, requestFields, includeDefaults);
6872

@@ -82,36 +86,61 @@ export class MetadataD2ApiRepository implements MetadataRepository {
8286
metadataTransformations
8387
);
8488

85-
return metadataPackage as T;
89+
const dataStoresMetadata = await this.getDataStoresMetadata(ids);
90+
const responseWithDataStores = { ...metadataPackage, ...dataStoresMetadata } as T;
91+
92+
return responseWithDataStores;
8693
} else {
8794
const metadataPackage = this.transformationRepository.mapPackageFrom(
8895
apiVersion,
8996
d2Metadata,
9097
metadataTransformations
9198
);
9299

93-
return metadataPackage as T;
100+
if (dataStoreIds.length > 0) {
101+
metadataPackage.dataStores = await d2ApiDataStore.getDataStores({ namespaces: ids });
102+
}
103+
const dataStoresMetadata = await this.getDataStoresMetadata(ids);
104+
const responseWithDataStores = { ...metadataPackage, ...dataStoresMetadata } as T;
105+
106+
return responseWithDataStores;
94107
}
95108
}
96109

110+
private async getDataStoresMetadata(ids: Id[]) {
111+
const d2ApiDataStore = new D2ApiDataStore(this.instance);
112+
const dataStoreIds = DataStoreMetadata.getDataStoreIds(ids);
113+
if (dataStoreIds.length === 0) return {};
114+
115+
const dataStores = await d2ApiDataStore.getDataStores({ namespaces: dataStoreIds });
116+
return { dataStores: dataStores };
117+
}
118+
97119
@cache()
98120
public async listMetadata(listParams: ListMetadataParams): Promise<ListMetadataResponse> {
99121
const { type, fields = { $owner: true }, page, pageSize, order, rootJunction, ...params } = listParams;
100122

101123
const filter = this.buildListFilters(params);
102124
const { apiVersion } = this.instance;
103125
const options = { type, fields, filter, order, page, pageSize, rootJunction };
104-
const { objects: baseObjects, pager } = await this.getListPaginated(options);
105-
// Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified.
106-
const objects = _.concat(await this.getParentObjects(listParams), baseObjects);
126+
if (type === "dataStores") {
127+
const d2ApiDataStore = new D2ApiDataStore(this.instance);
128+
const response = await d2ApiDataStore.getDataStores({ namespaces: undefined });
129+
// Hardcoded pagination since DHIS2 does not support pagination for namespaces
130+
return { objects: response, pager: { page: 1, total: response.length, pageSize: 100 } };
131+
} else {
132+
const { objects: baseObjects, pager } = await this.getListPaginated(options);
133+
// Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified.
134+
const objects = _.concat(await this.getParentObjects(listParams), baseObjects);
107135

108-
const metadataPackage = this.transformationRepository.mapPackageFrom(
109-
apiVersion,
110-
{ [type]: objects },
111-
metadataTransformations
112-
);
136+
const metadataPackage = this.transformationRepository.mapPackageFrom(
137+
apiVersion,
138+
{ [type]: objects },
139+
metadataTransformations
140+
);
113141

114-
return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager };
142+
return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager };
143+
}
115144
}
116145

117146
@cache()

‎src/data/storage/StorageDataStoreClient.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ export class StorageDataStoreClient extends StorageClient {
1414
private api: D2Api;
1515
private dataStore: DataStore;
1616

17-
constructor(instance: Instance) {
17+
constructor(instance: Instance, namespace: string = dataStoreNamespace) {
1818
super();
1919
this.api = getD2APiFromInstance(instance);
20-
this.dataStore = this.api.dataStore(dataStoreNamespace);
20+
this.dataStore = this.api.dataStore(namespace);
2121
}
2222

2323
public async getObject<T extends object>(key: string): Promise<T | undefined> {

‎src/domain/common/entities/Struct.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/**
2+
* Base class for typical classes with attributes. Features: create, update.
3+
*
4+
* ```
5+
* class Counter extends Struct<{ id: Id; value: number }>() {
6+
* add(value: number): Counter {
7+
* return this._update({ value: this.value + value });
8+
* }
9+
* }
10+
*
11+
* const counter1 = Counter.create({ id: "some-counter", value: 1 });
12+
* const counter2 = counter1._update({ value: 2 });
13+
* ```
14+
*/
15+
16+
export function Struct<Attrs>() {
17+
abstract class Base {
18+
constructor(_attributes: Attrs) {
19+
Object.assign(this, _attributes);
20+
}
21+
22+
_getAttributes(): Attrs {
23+
const entries = Object.getOwnPropertyNames(this).map(key => [key, (this as any)[key]]);
24+
return Object.fromEntries(entries) as Attrs;
25+
}
26+
27+
protected _update(partialAttrs: Partial<Attrs>): this {
28+
const ParentClass = this.constructor as new (values: Attrs) => typeof this;
29+
return new ParentClass({ ...this._getAttributes(), ...partialAttrs });
30+
}
31+
32+
static create<U extends Base>(this: new (attrs: Attrs) => U, attrs: Attrs): U {
33+
return new this(attrs);
34+
}
35+
}
36+
37+
return Base as {
38+
new (values: Attrs): Attrs & Base;
39+
create: typeof Base["create"];
40+
};
41+
}
42+
43+
const GenericStruct = Struct<unknown>();
44+
45+
export type GenericStructInstance = InstanceType<typeof GenericStruct>;

‎src/domain/common/factories/RepositoryFactory.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
} from "../../aggregated/repositories/AggregatedRepository";
66
import { ConfigRepositoryConstructor } from "../../config/repositories/ConfigRepository";
77
import { CustomDataRepositoryConstructor } from "../../custom-data/repository/CustomDataRepository";
8+
import { DataStoreMetadataRepositoryConstructor } from "../../data-store/DataStoreMetadataRepository";
89
import { EventsRepository, EventsRepositoryConstructor } from "../../events/repositories/EventsRepository";
910
import { FileRepositoryConstructor } from "../../file/repositories/FileRepository";
1011
import { DataSource } from "../../instance/entities/DataSource";
@@ -120,6 +121,11 @@ export class RepositoryFactory {
120121
return this.get<EventsRepositoryConstructor>(Repositories.EventsRepository, [instance]);
121122
}
122123

124+
@cache()
125+
public dataStoreMetadataRepository(instance: Instance) {
126+
return this.get<DataStoreMetadataRepositoryConstructor>(Repositories.DataStoreMetadataRepository, [instance]);
127+
}
128+
123129
@cache()
124130
public teisRepository(instance: Instance): TEIRepository {
125131
return this.get<TEIRepositoryConstructor>(Repositories.TEIsRepository, [instance]);
@@ -209,4 +215,5 @@ export const Repositories = {
209215
MappingRepository: "mappingRepository",
210216
SettingsRepository: "settingsRepository",
211217
SchedulerRepository: "schedulerRepository",
218+
DataStoreMetadataRepository: "dataStoreMetadataRepository",
212219
} as const;

0 commit comments

Comments
 (0)