feat(hub-common): new sites are fully upgraded to the new catalog, old sites can opt into the upgrade

.changeset/fuzzy-lions-own.md

@@ -0,0 +1,5 @@
+---
+"@esri/hub-common": minor
+---
+
+Sites can now be fully upgraded to use the new catalog. New sites created via workspaces will automatically be fully upgraded and won't have any default collections.

packages/common/src/discussions/_internal/DiscussionBusinessRules.ts

@@ -106,10 +106,6 @@ export const DiscussionPermissionPolicies: IPermissionPolicy[] = [
   },
   {
     permission: "hub:discussion:workspace:catalog:events",
-    dependencies: [
-      "hub:discussion:workspace:catalog",
-      "hub:event",
-      "hub:feature:catalogs:edit:advanced",
-    ],
+    dependencies: ["hub:discussion:workspace:catalog", "hub:event"],
   },
 ];

packages/common/src/events/_internal/EventBusinessRules.ts

@@ -105,11 +105,7 @@ export const EventPermissionPolicies: IPermissionPolicy[] = [
   },
   {
     permission: "hub:event:workspace:catalog:events",
-    dependencies: [
-      "hub:event:workspace:catalog",
-      "hub:event",
-      "hub:feature:catalogs:edit:advanced",
-    ],
+    dependencies: ["hub:event:workspace:catalog", "hub:event"],
   },
   {
     permission: "hub:event:manage",

packages/common/src/initiatives/_internal/InitiativeBusinessRules.ts

@@ -165,11 +165,7 @@ export const InitiativePermissionPolicies: IPermissionPolicy[] = [
   },
   {
     permission: "hub:initiative:workspace:catalog:events",
-    dependencies: [
-      "hub:initiative:workspace:catalog",
-      "hub:event",
-      "hub:feature:catalogs:edit:advanced",
-    ],
+    dependencies: ["hub:initiative:workspace:catalog", "hub:event"],
   },
   {
     permission: "hub:initiative:manage",

packages/common/src/permissions/HubPermissionPolicies.ts

@@ -243,39 +243,18 @@ const SystemPermissionPolicies: IPermissionPolicy[] = [
     environments: ["qaext"],
     availability: ["alpha"],
   },
-  /**
-   * Gates advanced editing (e.g. adding new collections, adding
-   * additional scope filters, etc.) in the catalog configuration
-   * experince.
-   *
-   * TODO: Remove the site entity assertion once all catalog
-   * configuration features are supported by sites
-   */
-  {
-    permission: "hub:feature:catalogs:edit:advanced",
-    entityEdit: true,
-    licenses: ["hub-premium"],
-    assertions: [
-      {
-        property: "entity:type",
-        type: "neq",
-        value: "Hub Site Application",
-      },
-    ],
-  },
   /**
    * Gates catalog & collection appearance editing
    * in the catalog configuration experience.
    *
    * TODO: remove environment & availability gating once
    * we are ready to release catalog appearance
-   * configuration. Alternatively, we can remove it entirely
-   * in favor of the "hub:feature:caatalogs:edit:advanced"
-   * permission
+   * configuration.
    */
   {
     permission: "hub:feature:catalogs:edit:appearance",
-    dependencies: ["hub:feature:catalogs:edit:advanced"],
+    entityEdit: true,
+    licenses: ["hub-premium"],
     environments: ["qaext"],
     availability: ["alpha"],
   },

packages/common/src/permissions/_internal/constants.ts

@@ -37,8 +37,6 @@ export const SystemPermissions = [
   "hub:feature:newentityview",
   "hub:feature:history",
   "hub:feature:catalogs", // DEPRECATED - TODO: remove at next major version
-  /** remove once sites support all catalog configuration features */
-  "hub:feature:catalogs:edit:advanced",
   "hub:feature:catalogs:edit:appearance",
   "hub:feature:gallery-card:enterprise",
   "hub:feature:inline-workspace",

packages/common/src/projects/_internal/ProjectBusinessRules.ts

@@ -136,11 +136,7 @@ export const ProjectPermissionPolicies: IPermissionPolicy[] = [
   },
   {
     permission: "hub:project:workspace:catalog:events",
-    dependencies: [
-      "hub:project:workspace:catalog",
-      "hub:event",
-      "hub:feature:catalogs:edit:advanced",
-    ],
+    dependencies: ["hub:project:workspace:catalog", "hub:event"],
   },
   {
     permission: "hub:project:manage",

packages/common/src/sites/HubSites.ts

@@ -24,10 +24,7 @@ import { IHubRequestOptions, IModel } from "../hub-types";
 import { removeDomainsBySiteId } from "./domains/remove-domains-by-site-id";
 import { IHubSearchResult } from "../search/types/IHubSearchResult";
 import { reflectCollectionsToSearchCategories } from "./_internal/reflectCollectionsToSearchCategories";
-import {
-  catalogToLegacy,
-  convertCatalogToLegacyFormat,
-} from "./_internal/convertCatalogToLegacyFormat";
+import { convertCatalogToLegacyFormat } from "./_internal/convertCatalogToLegacyFormat";
 import { convertFeaturesToLegacyCapabilities } from "./_internal/capabilities/convertFeaturesToLegacyCapabilities";
 import { computeLinks } from "./_internal/computeLinks";
 import { handleSubdomainChange } from "./_internal/subdomains";
@@ -40,6 +37,7 @@ import { setProp } from "../objects/set-prop";
 import { mapBy } from "../utils/map-by";
 import { handleDomainChanges } from "./_internal/handleDomainChanges";
 import { AccessLevel } from "../core/types/types";
+import { upgradeSiteSchema } from "./upgrade-site-schema";
 export const HUB_SITE_ITEM_TYPE = "Hub Site Application";
 export const ENTERPRISE_SITE_ITEM_TYPE = "Site Application";
 
@@ -127,9 +125,7 @@ const DEFAULT_SITE_MODEL: IModel = {
     url: "",
   } as IItem,
   data: {
-    catalog: {
-      groups: [],
-    },
+    catalogV2: { schemaVersion: 0 },
     feeds: {},
     values: {
       title: "",
@@ -282,15 +278,21 @@ export async function createSite(
   );
   let model = mapper.entityToStore(site, cloneObject(DEFAULT_SITE_MODEL));
 
-  // At this point `data.catalog` may have become a full IHubCatalog object due to an in-memory
-  // migration. However, we can't persist an IHubCatalog in `data.catalog` without breaking
-  // the application, since most of the app relies on the old catalog structure. As such,
-  // we convert any changes made to the catalog scope into the old format and merge the changes
-  // with the existing structure on the most current model.
-  // TODO: Remove once the application is plumbed to work off an IHubCatalog
-  if (getProp(model, "data.catalog.scopes")) {
-    model.data.catalog = catalogToLegacy(model.data.catalog);
-  }
+  // There's no guarantee that the combination DEFAULT_SITE_MODEL + partialSite
+  // will result in a model with the latest schemaVersion. To prevent issues with
+  // post-processing the catalogV2, we run the upgrade here just in case.
+  model = upgradeSiteSchema(model);
+
+  // TODO: Remove once all sites are fully on catalogV2
+  // Remove the v1 catalog that might be populated during migrations
+  delete model.data.catalog;
+  // Remove default collections and set the v2 catalog
+  const catalogV2 = cloneObject(site.catalog);
+  catalogV2.collections = [];
+  model.data.catalogV2 = catalogV2;
+  // TODO: Remove once all sites are fully on catalogV2. We have to use this flag
+  // to get around some oddities with creating/deploying site templates
+  model.data.useCatalogV2 = true;
 
   // create the backing item
   model = await createModel(
@@ -314,7 +316,7 @@ export async function createSite(
   );
 
   // convert the model into a IHubSite and return
-  return mapper.storeToEntity(updatedModel, {}) as IHubSite;
+  return convertModelToSite(updatedModel, requestOptions);
 }
 
 /**
@@ -377,20 +379,31 @@ export async function updateSite(
     await handleDomainChanges(modelToUpdate, currentModel, requestOptions);
   }
 
-  // Persist the new catalog to new property until the app is fully migrated
+  // Persist the v2 catalog
   modelToUpdate.data.catalogV2 = site.catalog;
-  // Because some old (but critical) application code still uses `data.values.searchCategories`
-  // as the source of truth for collection display configuration, we port all display changes
-  // in `data.catalog.collections` to the search category format.
-  // TODO: Remove once the app no longer relies on `data.values.searchCategories`
-  modelToUpdate = reflectCollectionsToSearchCategories(modelToUpdate);
-  // At this point `data.catalog` has become a full IHubCatalog object due to an in-memory
-  // migration. However, we can't persist an IHubCatalog in `data.catalog` without breaking
-  // the application, since most of the app relies on the old catalog structure. As such,
-  // we convert any changes made to the catalog scope into the old format and merge the changes
-  // with the existing structure on the most current model.
-  // TODO: Remove once the application is plumbed to work off an IHubCatalog
-  modelToUpdate = convertCatalogToLegacyFormat(modelToUpdate, currentModel);
+
+  // Perform backwards compatibility steps if the site is still using the v1 catalog
+  if (site.isCatalogV1Enabled) {
+    // Because some old (but critical) application code still uses `data.values.searchCategories`
+    // as the source of truth for collection display configuration, we port all display changes
+    // in `data.catalog.collections` to the search category format.
+    // TODO: Remove once the app no longer relies on `data.values.searchCategories`
+    modelToUpdate = reflectCollectionsToSearchCategories(modelToUpdate);
+    // We can't persist an IHubCatalog in `data.catalog` without breaking
+    // the application, since most of the app relies on the old catalog structure. As such,
+    // we convert any changes made to the catalog scope into the old format and merge the changes
+    // with the existing structure on the most current model.
+    // TODO: Remove once the application is fully migrated to always use IHubCatalog
+    modelToUpdate = convertCatalogToLegacyFormat(modelToUpdate, currentModel);
+  } else {
+    // ensure the old catalog is removed if the site is not using v1
+    delete modelToUpdate.data.catalog;
+    delete (modelToUpdate.data.values as Record<string, unknown>)
+      .searchCategories;
+    // set the temporary property to indicate the site is fully upgraded to catalogV2.
+    // (only needed to circumvent some oddities around creating / deploying site templates)
+    modelToUpdate.data.useCatalogV2 = true;
+  }
   /**
    * Site capabilities are currently saved as an array on the
    * site.data.values.capabilities. We want to migragte these

packages/common/src/sites/_internal/SiteBusinessRules.ts

@@ -152,7 +152,13 @@ export const SitesPermissionPolicies: IPermissionPolicy[] = [
   {
     permission: "hub:site:workspace:catalog:upgrade",
     dependencies: ["hub:site:workspace:catalog"],
-    availability: ["flag"],
+    assertions: [
+      {
+        property: "entity:isCatalogV1Enabled",
+        type: "eq",
+        value: true,
+      },
+    ],
   },
   {
     permission: "hub:site:workspace:pages",

packages/common/src/sites/_internal/getPropertyMap.ts

@@ -64,7 +64,6 @@ export function getPropertyMap(): IPropertyMap[] {
     entityKey: "isHubHome",
     storeKey: "item.properties.isHubHome",
   });
-  map.push({ entityKey: "catalog", storeKey: "data.catalog" });
 
   map.push({
     entityKey: "features",

packages/common/src/sites/defaults.ts

@@ -66,7 +66,7 @@ export const DEFAULT_SITE_MODEL: IModel = {
     },
   },
   data: {
-    catalog: { schemaVersion: 0 },
+    catalogV2: { schemaVersion: 0 },
     layout: {},
   },
 } as unknown as IModel;

packages/common/test/sites/HubSites.spec.ts

@@ -1,11 +1,4 @@
-import {
-  describe,
-  it,
-  expect,
-  beforeEach,
-  afterEach,
-  vi,
-} from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
 
 // Make ESM namespace export spyable by merging original exports and overriding
 // the specific function we need to spy on. This must be registered before the
@@ -74,6 +67,7 @@ const SITE_DATA = {
         primary: {},
       },
     },
+    searchCategories: [{ key: "components.search.category_tabs.data" }],
   },
 };
 const SITE_MODEL = {
@@ -493,8 +487,9 @@ describe("HubSites:", () => {
       const modelToUpdate = updateModelSpy.mock.calls[0][0];
       expect(modelToUpdate.item.title).toBe(updatedSite.name);
     });
-    it("reflects collection changes to searchCategories", async () => {
+    it("if isCatalogV1Enabled, reflects collection changes to searchCategories", async () => {
       const updatedSite = cloneObject(SITE);
+      updatedSite.isCatalogV1Enabled = true;
       updatedSite.catalog.collections = [
         {
           label: "My Datasets",
@@ -525,8 +520,9 @@ describe("HubSites:", () => {
         },
       ]);
     });
-    it("converts catalog group changes to the old catalog format and stores the new catalog in data.catalogv2", async () => {
+    it("stores the new catalog in data.catalogv2 and (if isCatalogV1Enabled) converts catalog group changes to the old catalog format ", async () => {
       const updatedSite = cloneObject(SITE);
+      updatedSite.isCatalogV1Enabled = true;
       updatedSite.catalog.scopes.item.filters = [
         {
           predicates: [
@@ -554,6 +550,39 @@ describe("HubSites:", () => {
       expect(modelToUpdate.data.catalog).toEqual({ groups: ["9001"] });
       expect(modelToUpdate.data.catalogV2).toEqual(expectedCatalogV2);
     });
+
+    it("deletes old catalog and search categories if upgraded to the v2 catalog", async () => {
+      const updatedSite = cloneObject(SITE);
+      updatedSite.isCatalogV1Enabled = false;
+      updatedSite.catalog.scopes.item.filters = [
+        {
+          predicates: [
+            {
+              group: ["9001"],
+            },
+          ],
+        },
+      ];
+      updatedSite.catalog.scopes.event.filters = [
+        {
+          predicates: [
+            {
+              group: ["1006"],
+            },
+          ],
+        },
+      ];
+      const expectedCatalogV2 = cloneObject(updatedSite.catalog);
+      const chk = await updateSite(updatedSite, MOCK_HUB_REQOPTS);
+
+      expect(chk.id).toBe(GUID);
+      expect(fetchSiteModelSpy).toHaveBeenCalledTimes(1);
+      const modelToUpdate = updateModelSpy.mock.calls[0][0];
+      expect(modelToUpdate.data.catalog).toBeUndefined();
+      expect(modelToUpdate.data.values.searchCategories).toBeUndefined();
+      expect(modelToUpdate.data.useCatalogV2).toBe(true);
+      expect(modelToUpdate.data.catalogV2).toEqual(expectedCatalogV2);
+    });
   });
   describe("createSite:", () => {
     let uniqueDomainSpy: any;
@@ -701,6 +730,7 @@ describe("HubSites:", () => {
               },
             },
           },
+          schemaVersion: 1, // HubSite class sets default to 1 in .toJson()
         };
 
         const chk = await createSite(site, MOCK_HUB_REQOPTS);
@@ -714,7 +744,10 @@ describe("HubSites:", () => {
 
         const modelToUpdate = updateModelSpy.mock.calls[0][0];
         expect(modelToUpdate.data.values.clientId).toBe("FAKE_CLIENT_KEY");
-        expect(modelToUpdate.data.catalog.groups).toContain("9001");
+        expect(modelToUpdate.data.useCatalogV2).toBe(true);
+        expect(modelToUpdate.data.catalogV2).toBeDefined();
+        expect(modelToUpdate.data.catalogV2.collections.length).toBe(0);
+        expect(modelToUpdate.data.catalog).toBeUndefined();
         expect(chk.name).toBe("Special Site");
         expect(chk.url).toBe("https://site.myorg.com");
         expect(chk.culture).toBe("fr-ca");