diff --git a/full-stack-tests/backend/package.json b/full-stack-tests/backend/package.json index 7c95627c371b..308a59c342eb 100644 --- a/full-stack-tests/backend/package.json +++ b/full-stack-tests/backend/package.json @@ -29,7 +29,8 @@ "perftest:metadataPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep PerformanceElementGetMetadata \"./lib/cjs/perftest/ElementCRUD.test.js\"", "perftest:schemaContextPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/SchemaContextIModelDb.test.js\"", "perftest:changesetPerformance": "npm run -s perftest:pre && mocha --timeout=999999999 --grep ChangesetReaderAPI \"./lib/cjs/perftest/ChangesetReader.test.js\"", - "perftest:deleteRelationshipInstances": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/DeleteRelationshipInstances.test.js\"" + "perftest:deleteRelationshipInstances": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/DeleteRelationshipInstances.test.js\"", + "perftest:incrementalLoading": "npm run -s perftest:pre && mocha --timeout=999999999 \"./lib/cjs/perftest/IncrementalLoading.test.js\"" }, "dependencies": { "@azure/storage-blob": "^12.28.0", diff --git a/full-stack-tests/backend/src/perftest/IncrementalLoading.test.ts b/full-stack-tests/backend/src/perftest/IncrementalLoading.test.ts new file mode 100644 index 000000000000..740bde3dec7f --- /dev/null +++ b/full-stack-tests/backend/src/perftest/IncrementalLoading.test.ts @@ -0,0 +1,727 @@ +/*--------------------------------------------------------------------------------------------- +* Copyright (c) Bentley Systems, Incorporated. All rights reserved. +* See LICENSE.md in the project root for license terms and full copyright notice. +*--------------------------------------------------------------------------------------------*/ +import { assert } from "chai"; +import * as path from "path"; +import { IModelHost, IModelJsFs, StandaloneDb } from "@itwin/core-backend"; +import { IModelTestUtils, KnownTestLocations } from "@itwin/core-backend/lib/cjs/test"; +import { IModelIncrementalSchemaLocater } from "@itwin/core-backend/lib/cjs/IModelIncrementalSchemaLocater"; +import { OpenMode, StopWatch } from "@itwin/core-bentley"; +import { ECClass, ECVersion, Schema, SchemaContext, SchemaKey } from "@itwin/ecschema-metadata"; +import { Reporter } from "@itwin/perf-tools"; + +export interface TotalCounts { + totalItemCount: number, + totalClassCount: number, + totalPropertyCount: number, + totalCustomAttrCount: number, +} + +/** + * Options for generating an EC schema. + */ +export interface ClassGenerationOptions { + classCount: number; + propCountperClass?: number; + customAttrCountperClass?: number; + customAttrCountperProperty?: number; + baseClassName?: string; +} + +export interface SchemaGenerationOptions { + schemaName: string; + version: string; + alias?: string; + referenceSchemas?: { schemaName: string; version: string; alias?: string }[]; + entityClasses?: ClassGenerationOptions; + mixins?: ClassGenerationOptions; + relationshipClasses?: ClassGenerationOptions; + structClasses?: ClassGenerationOptions; + customAttributeClasses?: ClassGenerationOptions; + schemaItemCount?: number; +} + +describe("IncrementalLoadingPerformance", async () => { + const configData = await import(path.join(__dirname, "IncrementalLoadingConfig.json")); + const assetDir = path.join(__dirname, "../../../assets"); + + const testSuite = "IncrementalLoadingPerformance"; + const reporter = new Reporter(); + + const testSchemaKey = new SchemaKey("TestSchema", ECVersion.fromString("01.00.00")); + const defaultOptions = { + schemaName: testSchemaKey.name, + version: testSchemaKey.version.toString(), + alias: "ts", + referenceSchemas: [{ schemaName:"BisCore", version:"01.00.14", alias: "bis" }], + }; + let snapshotFile: string; + + function getClassGroups(opts: SchemaGenerationOptions) { + return [ + opts.entityClasses, + opts.structClasses, + opts.mixins, + opts.relationshipClasses, + opts.customAttributeClasses, + ]; + } + + function getTotalClassCount(opts: SchemaGenerationOptions): number { + return getClassGroups(opts).reduce((total, group) => total + (group?.classCount ?? 0), 0); + } + + function getTotalItemCount(opts: SchemaGenerationOptions): number { + const totalClassCount = getTotalClassCount(opts); + + // getItemsXML creates 9 simple schema items per requested count (units, formats, KOQs, etc.) + const simpleItemsPerCount = 9; + const schemaItemCount = opts.schemaItemCount ?? 0; + return totalClassCount + (simpleItemsPerCount * schemaItemCount); + } + + function getTotalPropertyCount(opts: SchemaGenerationOptions): number { + return getClassGroups(opts).reduce((total, group) => { + const count = group?.classCount ?? 0; + const propsPerClass = group?.propCountperClass ?? 0; + return total + (count * propsPerClass); + }, 0); + } + + function getTotalCustomAttributeCount(opts: SchemaGenerationOptions): number { + const normalize = (g?: ClassGenerationOptions) => ({ + count: g?.classCount ?? 0, + props: g?.propCountperClass ?? 0, + attrPerClass: g?.customAttrCountperClass ?? 0, + attrPerProperty: g?.customAttrCountperProperty ?? 0, + }); + + return getClassGroups(opts).reduce((total, group) => { + const { count, props, attrPerClass, attrPerProperty } = normalize(group); + // custom attributes applied to the class itself + those applied to each property of the class + return total + (count * attrPerClass) + (count * props * attrPerProperty); + }, 0); + } + + function getConfigTotals(opts: SchemaGenerationOptions): TotalCounts { + const totalClassCount = getTotalClassCount(opts); + const totalItemCount = getTotalItemCount(opts); + const totalPropertyCount = getTotalPropertyCount(opts); + const totalCustomAttrCount = getTotalCustomAttributeCount(opts); + + return { + totalItemCount, + totalClassCount, + totalPropertyCount, + totalCustomAttrCount, + }; + } + + async function getSchemaTotals(schema?: Schema): Promise { + if (!schema) { + return { totalItemCount: 0, totalClassCount: 0, totalPropertyCount: 0, totalCustomAttrCount: 0 }; + } + + const items = [...schema.getItems()]; + const classItems = items.filter((it): it is ECClass => ECClass.isECClass(it)); + + // Count custom attributes declared on classes + let totalCustomAttrCount = classItems.reduce((acc, cls) => { + return acc + (cls.customAttributes ? [...cls.customAttributes].length : 0); + }, 0); + + // Fetch properties for all classes in parallel and count properties + custom attributes on properties + const properties = await Promise.all(classItems.map(async (cls) => cls.getProperties(true))); + let totalPropertyCount = 0; + for (const props of properties) { + const propertyArray = [...props]; + totalPropertyCount += propertyArray.length; + for (const prop of propertyArray) { + if (prop.customAttributes) { + totalCustomAttrCount += [...prop.customAttributes].length; + } + } + } + + return { + totalItemCount: items.length, + totalClassCount: classItems.length, + totalPropertyCount, + totalCustomAttrCount, + }; + } + + /** + * Generates XML string defining class custom attributes. + * @param opts Options for generating class custom attributes. + * @returns XML string defining class custom attributes. + * @returns + */ + function getCustomAttributesXml(attrPerClass?: number, attrFileName?: string): string { + let customAttrXml: string = ""; + if (attrPerClass && attrPerClass > 0) { + customAttrXml += "\n\t"; + for (let i = 0; i < attrPerClass; ++i) { + customAttrXml += `\n\t `; + } + customAttrXml += "\n\t"; + } + return customAttrXml; + } + + /** + * Generates XML string defining class properties. + * @param opts Options for generating class properties. + * @returns XML string defining class properties. + * @returns + */ + function getPropertiesXml(prefix: string, propsPerClass?: number, attrPerProperty?: number, attrFileName?: string): string { + if (!propsPerClass || propsPerClass <= 0) return ""; + + let propertiesXml: string = ""; + for (let i = 0; i < propsPerClass; ++i) { + propertiesXml += `\n\t` + if (attrPerProperty && attrPerProperty > 0) { + propertiesXml += "\n\t "; + for (let j = 0; j < attrPerProperty; ++j) { + const propName = `Prop${Math.floor(Math.random() * 11)}`; + propertiesXml += `\n\t\t + Old${prefix}_Prop${j} + `; + } + propertiesXml += "\n\t "; + } + propertiesXml += `\n\t`; + } + return propertiesXml; + } + + /** + * Generates XML string defining relationship classes. + * @param opts Options for generating relationship classes. + * @returns XML string defining relationship classes. + */ + function getRelationshipClassesXML(opts?: ClassGenerationOptions, attrFileName?: string): string { + if (!opts || opts.classCount <= 0) return ""; + let classesXml: string = ""; + for (let i = 0; i < opts.classCount; ++i) { + classesXml += `\n `; + classesXml += getCustomAttributesXml(opts.customAttrCountperClass, attrFileName); + classesXml += `\n\t${opts.baseClassName ?? "bis:ElementRefersToElements"}`; + classesXml += `\n\t + + + + + `; + classesXml += getPropertiesXml(`RelationshipTest${i}`, opts.propCountperClass, opts.customAttrCountperProperty, attrFileName); + classesXml += `\n `; + } + return classesXml; + } + + /** + * Generates XML string defining structclasses. + * @param opts Options for generating structclasses. + * @returns XML string defining structclasses. + */ + function getStructClassesXML(opts?: ClassGenerationOptions, attrFileName?: string): string { + if (!opts || opts.classCount <= 0) return ""; + let classesXml: string = ""; + for (let i = 0; i < opts.classCount; ++i) { + classesXml += `\n `; + classesXml += getCustomAttributesXml(opts.customAttrCountperClass, attrFileName); + if (opts.baseClassName !== undefined) { + classesXml += `\n\t${opts.baseClassName}`; + } + classesXml += getPropertiesXml(`StructTest${i}`, opts.propCountperClass, opts.customAttrCountperProperty, attrFileName); + classesXml += `\n `; + } + return classesXml; + } + + /** + * Generates XML string defining entityclasses. + * @param opts Options for generating entityclasses. + * @returns XML string defining entityclasses. + */ + function getEntityClassesXML(opts?: ClassGenerationOptions, attrFileName?: string): string { + if (!opts || opts.classCount <= 0) return ""; + let classesXml: string = ""; + for (let i = 0; i < opts.classCount; ++i) { + classesXml += `\n `; + classesXml += getCustomAttributesXml(opts.customAttrCountperClass, attrFileName); + classesXml += `\n\t${opts.baseClassName ?? "bis:PhysicalElement"}`; + classesXml += getPropertiesXml(`EntityTest${i}`, opts.propCountperClass, opts.customAttrCountperProperty, attrFileName); + classesXml += `\n `; + } + return classesXml; + } + + /** + * Generates XML string defining customattribute classes. + * @param opts Options for generating customattribute classes. + * @returns XML string defining customattribute classes. + */ + function getCustomAttributeClassesXML(opts?: ClassGenerationOptions, attrFileName?: string): string { + if (!opts || opts.classCount <= 0) return ""; + let classesXml: string = ""; + for (let i = 0; i < opts.classCount; ++i) { + classesXml += `\n `; + classesXml += getCustomAttributesXml(opts.customAttrCountperClass, attrFileName); + if (opts.baseClassName !== undefined) { + classesXml += `\n\t${opts.baseClassName}`; + } + classesXml += getPropertiesXml(`CustomAttributeTest${i}`, opts.propCountperClass, opts.customAttrCountperProperty, attrFileName); + classesXml += `\n `; + } + return classesXml; + } + + /** + * Generates XML string defining mixin classes. + * @param opts Options for generating mixin classes. + * @returns XML string defining mixin classes. + */ + function getMixinsXML(opts?: ClassGenerationOptions, attrFileName?: string): string { + if (!opts || opts.classCount <= 0) return ""; + + let classesXml: string = ""; + for (let i = 0; i < opts.classCount; ++i) { + classesXml += `\n + + + bis:Element + `; + if (opts.customAttrCountperClass && opts.customAttrCountperClass > 0) { + for (let j = 0; j < opts.customAttrCountperClass; ++j) { + classesXml += `\n\t `; + } + } + classesXml += `\n\t `; + if (opts.baseClassName !== undefined) { + classesXml += `\n\t${opts.baseClassName}`; + } + classesXml += getPropertiesXml(`IMixinTest${i}`, opts.propCountperClass, opts.customAttrCountperProperty, attrFileName); + classesXml += `\n `; + } + return classesXml; + } + + /** + * Generates XML string defining simple schema items. + * @param opts Options for generating schema items. + * @returns XML string defining schema items. + */ + function getItemsXML(itemsCount?: number): string { + if (!itemsCount || itemsCount <= 0) return ""; + + let itemsXml: string = ""; + // create unit systems + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create property categories + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create phenomena + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create enumerations + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` + + + \n`; + } + // create units + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // creetae inverted units + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create constants + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create formats + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + // create kind of quantities + for (let i = 0; i < itemsCount; ++i) { + itemsXml += ` \n`; + } + return itemsXml; + } + + /** + * Generates schema from options + * @param options Options for generating schema. + * @returns XML string defining schema. + */ + function createSchemaFromOptions(options: SchemaGenerationOptions): string[] { + let attrSchemaName; + const fileNames: string[] = []; + const { + schemaName, + version, + alias, + referenceSchemas, + entityClasses, + mixins, + relationshipClasses, + structClasses, + customAttributeClasses, + schemaItemCount + } = options; + + // Find the max custom attribute counts across all class groups in options + const maxCustomAttrCount = getClassGroups(options).reduce((max, g) => + Math.max(max, (g?.customAttrCountperClass ?? 0), (g?.customAttrCountperProperty ?? 0)), 0); + + if (maxCustomAttrCount > 0) { + attrSchemaName = `${options.schemaName}_CustomAttributes`; + const customAttrSchemaAlias = `${alias ?? schemaName.toLowerCase()}_cs`; + + const customAttrFileName = createSchemaFromOptions({ + schemaName: attrSchemaName, + version: "01.00.01", + alias: customAttrSchemaAlias, + referenceSchemas: [], + customAttributeClasses: { classCount: maxCustomAttrCount, propCountperClass: 10 }, + }); + + (options.referenceSchemas ?? []).push({ schemaName: attrSchemaName, version: '01.00.01', alias: customAttrSchemaAlias }); + fileNames.push(...customAttrFileName); + } + + let xml = `\n`; + if (referenceSchemas && referenceSchemas.length > 0) { + for (const ref of referenceSchemas) { + xml += `\n `; + } + } + xml += getItemsXML(schemaItemCount); + xml += getCustomAttributeClassesXML(customAttributeClasses, attrSchemaName); + xml += getMixinsXML(mixins, attrSchemaName); + xml += getStructClassesXML(structClasses, attrSchemaName); + xml += getEntityClassesXML(entityClasses, attrSchemaName); + xml += getRelationshipClassesXML(relationshipClasses, attrSchemaName); + xml += "\n"; + + const schemaFilePath = path.join(assetDir, `${schemaName}.${version}.ecschema.xml`); + IModelJsFs.writeFileSync(schemaFilePath, xml); + fileNames.push(schemaFilePath); + + return fileNames; + } + + async function waitSchemaLoading(schemaContext: SchemaContext, key: SchemaKey) { + const stopWatch = new StopWatch("", true); + const schema = await schemaContext.getSchema(key); + const stubsTime = stopWatch.current.milliseconds; + + if (schema?.loadingController !== undefined) + await schema.loadingController.wait(); + const totalTime = stopWatch.stop().milliseconds; + + return { schema, stubsTime, totalTime }; + } + + beforeEach(async () => { + if (IModelJsFs.existsSync(assetDir)) + IModelJsFs.removeSync(assetDir); + IModelJsFs.mkdirSync(assetDir); + + await IModelHost.startup(); + + // create an empty imodel + snapshotFile = IModelTestUtils.prepareOutputFile(testSuite, "IncrementalLoading.bim"); + const rootSubject = { name: "TestIModel", description: "Performance tests" }; + const imodel = StandaloneDb.createEmpty(snapshotFile, { rootSubject }); + imodel.saveChanges(); + imodel.close(); + }); + + afterEach(async () => { + if (IModelHost.isValid) { + await IModelHost.shutdown(); + } + }); + + after(() => { + const csvFilePath = path.join(KnownTestLocations.outputDir, testSuite, "PerformanceResults.csv"); + reporter.exportCSV(csvFilePath); + }); + + configData.testCases.increaseClasses.forEach((testCase: SchemaGenerationOptions) => { + it(`Gradually increase the number of classes (${getTotalClassCount(testCase)})`, async () => { + const options: SchemaGenerationOptions = { + ...defaultOptions, + ...testCase, + }; + options.referenceSchemas = [ + ...(defaultOptions.referenceSchemas ?? []), + ...(testCase.referenceSchemas ?? []), + ]; + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + // create schema file(s) + const schemaFileNames = createSchemaFromOptions(options); + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + const { schema, stubsTime, totalTime } = await waitSchemaLoading(schemaContext, testSchemaKey); + assert.isDefined(schema); + + const configTotals = getConfigTotals(options); + const schemaTotals = await getSchemaTotals(schema); + assert.deepEqual(configTotals, schemaTotals); + + reporter.addEntry(testSuite, "IncreaseClasses", "Stubs time", stubsTime, configTotals); + reporter.addEntry(testSuite, "IncreaseClasses", "Total time", totalTime, configTotals); + } finally { + imodel.close(); + } + }); + }); + + configData.testCases.increaseProperties.forEach((testCase: SchemaGenerationOptions) => { + it(`Gradually increase the number of properties (${getTotalPropertyCount(testCase)})`, async () => { + const options: SchemaGenerationOptions = { + ...defaultOptions, + ...testCase, + }; + options.referenceSchemas = [ + ...(defaultOptions.referenceSchemas ?? []), + ...(testCase.referenceSchemas ?? []), + ]; + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + // create schema file(s) + const schemaFileNames = createSchemaFromOptions(options); + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + const { schema, stubsTime, totalTime } = await waitSchemaLoading(schemaContext, testSchemaKey); + assert.isDefined(schema); + + const configTotals = getConfigTotals(options); + const schemaTotals = await getSchemaTotals(schema); + assert.deepEqual(configTotals, schemaTotals); + + reporter.addEntry(testSuite, "IncreaseProperties", "Stubs time", stubsTime, configTotals); + reporter.addEntry(testSuite, "IncreaseProperties", "Total time", totalTime, configTotals); + } finally { + imodel.close(); + } + }); + }); + + configData.testCases.increaseCustomAattributes.forEach((testCase: SchemaGenerationOptions) => { + it(`Gradually increase the number of custom attributes (${getTotalCustomAttributeCount(testCase)})`, async () => { + const options: SchemaGenerationOptions = { + ...defaultOptions, + ...testCase, + }; + options.referenceSchemas = [ + ...(defaultOptions.referenceSchemas ?? []), + ...(testCase.referenceSchemas ?? []), + ]; + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + // create schema file + const schemaFileNames = createSchemaFromOptions(options); + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + const { schema, stubsTime, totalTime } = await waitSchemaLoading(schemaContext, testSchemaKey); + assert.isDefined(schema); + + const configTotals = getConfigTotals(options); + const schemaTotals = await getSchemaTotals(schema); + assert.deepEqual(configTotals, schemaTotals); + + reporter.addEntry(testSuite, "IncreaseCustomAattributes", "Stubs time", stubsTime, configTotals); + reporter.addEntry(testSuite, "IncreaseCustomAattributes", "Total time", totalTime, configTotals); + } finally { + imodel.close(); + } + }); + }); + + configData.testCases.increaseItems.forEach((testCase: SchemaGenerationOptions) => { + it(`Gradually increase the number of all schema items (${getTotalItemCount(testCase)})`, async () => { + const options: SchemaGenerationOptions = { + ...defaultOptions, + ...testCase, + }; + options.referenceSchemas = [ + ...(defaultOptions.referenceSchemas ?? []), + ...(testCase.referenceSchemas ?? []), + ]; + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + // create schema file + const schemaFileNames = createSchemaFromOptions(options); + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + const { schema, stubsTime, totalTime } = await waitSchemaLoading(schemaContext, testSchemaKey); + assert.isDefined(schema); + + const configTotals = getConfigTotals(options); + const schemaTotals = await getSchemaTotals(schema); + assert.deepEqual(configTotals, schemaTotals); + + reporter.addEntry(testSuite, "IncreaseItems", "Stubs time", stubsTime, configTotals); + reporter.addEntry(testSuite, "IncreaseItems", "Total time", totalTime, configTotals); + } + finally { + imodel.close(); + } + }); + }); + + configData.testCases.increaseInheritance[0].forEach((inheritanceLevel: number) => { + it(`Gradually increase the number of base class inheritance (${inheritanceLevel})`, async () => { + const testOpts = configData.testCases.increaseInheritance[1]; + const schemaFileNames: string[] = []; + + const createClassOptionsForLevel = (prevLevel: number, prefix: string, group?: ClassGenerationOptions): ClassGenerationOptions | undefined => { + if (!group) return undefined; + const baseClassName = prevLevel > 0 ? `sch${prevLevel}:${prefix}${prevLevel-1}` : undefined; + return { ...group, baseClassName }; + } + const createSchemaOptionsForLevel = (prevLevel: number, baseOpts: SchemaGenerationOptions): Partial => { + return { + referenceSchemas: [ + ...(prevLevel > 0 ? [{ schemaName: `SimpleSchema${prevLevel}`, version: "01.00.00", alias: `sch${prevLevel}` }]: []), + ...(defaultOptions.referenceSchemas ?? []), + ...(baseOpts.referenceSchemas ?? []), + ], + entityClasses: createClassOptionsForLevel(prevLevel, "EntityTest", baseOpts.entityClasses), + structClasses: createClassOptionsForLevel(prevLevel, "StructTest", baseOpts.structClasses), + relationshipClasses: createClassOptionsForLevel(prevLevel, "RelationshipTest", baseOpts.relationshipClasses), + customAttributeClasses: createClassOptionsForLevel(prevLevel, "CustomAttributeTest", baseOpts.customAttributeClasses), + mixins: createClassOptionsForLevel(prevLevel, "IMixinTest", baseOpts.mixins), + } + } + + for (let i = 1; i <= inheritanceLevel; ++i) { + const schOpts: SchemaGenerationOptions = { + schemaName: `SimpleSchema${i}`, + version: "01.00.00", + alias: `sch${i}`, + ...createSchemaOptionsForLevel(i-1, testOpts), + }; + schemaFileNames.push(...createSchemaFromOptions(schOpts)); + } + + const options: SchemaGenerationOptions = { + ...defaultOptions, + ...createSchemaOptionsForLevel(inheritanceLevel, testOpts), + }; + schemaFileNames.push(...createSchemaFromOptions(options)); + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + const { schema, stubsTime, totalTime } = await waitSchemaLoading(schemaContext, testSchemaKey); + assert.isDefined(schema); + + const configTotals = getConfigTotals(options); + const schemaTotals = await getSchemaTotals(schema); + assert.deepEqual(configTotals, schemaTotals); + + reporter.addEntry(testSuite, "IncreaseInheritance", "Stubs time", stubsTime, { inheritanceLevel, configTotals }); + reporter.addEntry(testSuite, "IncreaseInheritance", "Total time", totalTime, { inheritanceLevel, configTotals }); + } finally { + imodel.close(); + } + }); + }); + + configData.testCases.parallelLoading[0].forEach((schemaCount: number) => { + it(`Gradually increase the number of parallel loading schemas (${schemaCount})`, async () => { + const schemaFileNames: string[] = []; + const schemaKeys: SchemaKey[] = []; + + const testOpts = configData.testCases.parallelLoading[1]; + // prepare schema files and keys + for (let i = 1; i <= schemaCount; ++i) { + const schemaKey = new SchemaKey(`TestSchema${i}`, ECVersion.fromString("01.00.00")); + schemaKeys.push(schemaKey); + + const options: SchemaGenerationOptions = { + schemaName: schemaKey.name, + version: schemaKey.version.toString(), + alias: `ts${i}`, + ...testOpts, + }; + options.referenceSchemas = [ + ...(defaultOptions.referenceSchemas ?? []), + ...(testOpts.referenceSchemas ?? []), + ]; + + schemaFileNames.push(...createSchemaFromOptions(options)); + } + + const imodel = StandaloneDb.openFile(snapshotFile, OpenMode.ReadWrite); + try { + await imodel.importSchemas(schemaFileNames); + imodel.saveChanges(); + + const schemaContext = new SchemaContext(); + const locater = new IModelIncrementalSchemaLocater(imodel, { useMultipleQueries: true }); + schemaContext.addLocater(locater); + + // kick off parallel loads and measure + const promises = schemaKeys.map(async (key) => waitSchemaLoading(schemaContext, key)); + const stopWatch = new StopWatch("", true); + const results = await Promise.all(promises); + const totalTime = stopWatch.stop().milliseconds; + + for (const result of results) { + reporter.addEntry(testSuite, "IncreaseParallelSchemas", "Stubs time", result.stubsTime, { schemaName: result.schema?.fullName, schemaCount }); + reporter.addEntry(testSuite, "IncreaseParallelSchemas", "Total time", result.totalTime, { schemaName: result.schema?.fullName, schemaCount }); + } + reporter.addEntry(testSuite, "IncreaseParallelSchemas", "Overall total time", totalTime, { schemaCount }); + } finally { + imodel.close(); + } + }); + }); +}); diff --git a/full-stack-tests/backend/src/perftest/IncrementalLoadingConfig.json b/full-stack-tests/backend/src/perftest/IncrementalLoadingConfig.json new file mode 100644 index 000000000000..76906bfba22a --- /dev/null +++ b/full-stack-tests/backend/src/perftest/IncrementalLoadingConfig.json @@ -0,0 +1,425 @@ +{ + "testCases": { + "increaseClasses": [ + { + "relationshipClasses": { + "classCount": 100 + }, + "entityClasses": { + "classCount": 500 + }, + "mixins": { + "classCount": 50 + }, + "structClasses": { + "classCount": 100 + }, + "customAttributeClasses": { + "classCount": 250 + } + }, + { + "relationshipClasses": { + "classCount": 200 + }, + "entityClasses": { + "classCount": 1000 + }, + "mixins": { + "classCount": 100 + }, + "structClasses": { + "classCount": 200 + }, + "customAttributeClasses": { + "classCount": 500 + } + }, + { + "relationshipClasses": { + "classCount": 400 + }, + "entityClasses": { + "classCount": 2000 + }, + "mixins": { + "classCount": 200 + }, + "structClasses": { + "classCount": 400 + }, + "customAttributeClasses": { + "classCount": 1000 + } + }, + { + "relationshipClasses": { + "classCount": 800 + }, + "entityClasses": { + "classCount": 4000 + }, + "mixins": { + "classCount": 400 + }, + "structClasses": { + "classCount": 800 + }, + "customAttributeClasses": { + "classCount": 2000 + } + } + ], + "increaseProperties": [ + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 10 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 60 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 5 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 50 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 20 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 20 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 120 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 10 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 100 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 40 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 40 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 240 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 20 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 200 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 80 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 80 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 480 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 40 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 400 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 160 + } + } + ], + "increaseCustomAattributes": [ + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3, + "customAttrCountperClass": 4, + "customAttrCountperProperty": 20 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20, + "customAttrCountperClass": 4, + "customAttrCountperProperty": 20 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10, + "customAttrCountperClass": 4, + "customAttrCountperProperty": 20 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15, + "customAttrCountperClass": 4, + "customAttrCountperProperty": 20 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2, + "customAttrCountperClass": 4, + "customAttrCountperProperty": 20 + } + }, + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3, + "customAttrCountperClass": 8, + "customAttrCountperProperty": 40 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20, + "customAttrCountperClass": 8, + "customAttrCountperProperty": 40 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10, + "customAttrCountperClass": 8, + "customAttrCountperProperty": 40 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15, + "customAttrCountperClass": 8, + "customAttrCountperProperty": 40 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2, + "customAttrCountperClass": 8, + "customAttrCountperProperty": 40 + } + } + ], + "increaseItems": [ + { + "schemaItemCount": 10, + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2, + "customAttrCountperClass": 1, + "customAttrCountperProperty": 5 + } + }, + { + "schemaItemCount": 20, + "relationshipClasses": { + "classCount": 60, + "propCountperClass": 6, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "entityClasses": { + "classCount": 200, + "propCountperClass": 40, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "structClasses": { + "classCount": 40, + "propCountperClass": 20, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "customAttributeClasses": { + "classCount": 60, + "propCountperClass": 30, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + }, + "mixins": { + "classCount": 40, + "propCountperClass": 4, + "customAttrCountperClass": 2, + "customAttrCountperProperty": 10 + } + } + ], + "increaseInheritance": [ + [ + 1, + 2, + 4, + 8, + 16 + ], + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 3 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 20 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 10 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 15 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 2 + } + } + ], + "parallelLoading": [ + [ + 1, + 2, + 4, + 8, + 16 + ], + { + "relationshipClasses": { + "classCount": 30, + "propCountperClass": 10 + }, + "entityClasses": { + "classCount": 100, + "propCountperClass": 60, + "customAttrCountperClass": 1 + }, + "mixins": { + "classCount": 20, + "propCountperClass": 5 + }, + "structClasses": { + "classCount": 20, + "propCountperClass": 50 + }, + "customAttributeClasses": { + "classCount": 30, + "propCountperClass": 20 + } + } + ] + } +} \ No newline at end of file