feat: parse config files as yaml

README.md

@@ -99,30 +99,40 @@ automatically create the specified accounts on startup. If this file is not prov
 Follow the structure of [functionalAccounts.json.minimal.example](/functionalAccounts.json.minimal.example) to create
 your own _functionalAccounts.json_ file.
 
+YAML syntax is also supported. Set `FUNCTIONAL_ACCOUNTS_FILE=functionalAccounts.yaml` in your environment.
+
 ## Frontend configuration and theme
 
 The SciCat backend provides functionality to serve a configuration and theme to a connected frontend. The default files are _frontend.config.json_ and _frontend.theme.json_ located in the `/src/config` directory, locally or in the container. The file names and locations can be configured via the environment variables `FRONTEND_CONFIG_FILE` and `FRONTEND_THEME_FILE`.
 
 Follow the structure of the provided [frontend.config.json](/frontend.config.json) and [frontend.theme.json](/frontend.theme.json) to create your own files.
 
+YAML syntax is also supported. Set `FRONTEND_CONFIG_FILE=frontend.config.yaml` in your environment.
+
 ### Loggers configuration
 
 Providing a file called _loggers.json_ at the root of the project, locally or in the container, and create an external logger class in the `src/loggers/loggingProviders/`directory will automatically create specified one or multiple loggers instances.
 
 The `loggers.json.example` file in the root directory showcases the example of configuration structure for the one or multiple loggers. `logger.service.ts` file contains the configuration handling process logic, and `src/loggers/loggingProviders/grayLogger.ts` includes actual usecase of grayLogger.
 
+YAML syntax is also supported. Set `LOGGERS_CONFIG_FILE=loggers.yaml` in your environment.
+
 ### Proposal types configuration
 
 Providing a file called _proposalTypes.json_ at the root of the project, locally or in the container, will be automatically loaded into the application configuration service under property called `proposalTypes` and used for validation against proposal creation and update.
 
 The `proposalTypes.json.example` file in the root directory showcases the example of configuration structure for proposal types.
 
+YAML syntax is also supported. Set `PROPOSAL_TYPES_FILE=proposalTypes.yaml` in your environment.
+
 ### Dataset types configuration
 
 When providing a file called _datasetTypes.json_ at the root of the project, locally or in the container, it will be automatically loaded into the application configuration service under property called `datasetTypes` and used for validation against dataset creation and update. The types `Raw` and `Derived` are always valid dataset types by default.
 
 The `datasetTypes.json.example` file in the root directory showcases an example of configuration structure for dataset types.
 
+YAML syntax is also supported. Set `DATASET_TYPES_FILE=datasetTypes.yaml` in your environment.
+
 ## Environment variables
 
 Valid environment variables for the .env file. See [.env.example](/.env.example) for examples value formats.
@@ -210,13 +220,13 @@ Valid environment variables for the .env file. See [.env.example](/.env.example)
 | `ES_MAX_RESULT` | number | | Maximum records that can be indexed into Elasticsearch. | 10000 |
 | `ES_FIELDS_LIMIT` | number | | The total number of fields in an index. | 1000 |
 | `ES_REFRESH` | string | | If set to `wait_for`, Elasticsearch will wait till data is inserted into the specified index before returning a response. | false |
-| `FRONTEND_CONFIG_FILE` | string | | The file name for frontend configuration, located in the `/src/config` directory by default. | "./src/config/frontend.config.json" |
-| `FRONTEND_THEME_FILE` | string | | The file name for frontend theme, located in the `/src/config` directory by default. | "./src/config/frontend.theme.json" |
-| `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" |
-| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" |
+| `FRONTEND_CONFIG_FILE` | string | | The file name for frontend configuration (yaml/json), located in the `/src/config` directory by default. | "./src/config/frontend.config.json" |
+| `FRONTEND_THEME_FILE` | string | | The file name for frontend theme (yaml/json), located in the `/src/config` directory by default. | "./src/config/frontend.theme.json" |
+| `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration (yaml/json), located in the project root directory. | "loggers.json" |
+| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration (yaml/json), located in the project root directory. | "proposalTypes.json" |
 | `SWAGGER_PATH` | string | Yes | swaggerPath is the path where the swagger UI will be available. | "explorer"|
 | `MAX_FILE_UPLOAD_SIZE` | string | Yes | Maximum allowed file upload size. | "16mb"|
-| `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts, relative to the project root directory | "functionalAccounts.json"|
+| `FUNCTIONAL_ACCOUNTS_FILE` | string | Yes | The file name for functional accounts (yaml/json), relative to the project root directory | "functionalAccounts.json"|
 | `JOB_CONFIGURATION_FILE` | string | Yes | Path of a job configuration file (conventionally `"jobConfig.yaml"`). If unset, jobs are disabled | |
 | `JOB_DEFAULT_STATUS_CODE` | string | Yes | Default statusCode for new jobs | "jobSubmitted" |
 | `JOB_DEFAULT_STATUS_MESSAGE | string | Yes | Default statusMessage for new jobs | "Job submitted." |

src/common/utils.ts

@@ -1135,6 +1135,16 @@ export function oneOrMore<T>(x: T[] | T): T[] {
   return Array.isArray(x) ? x : [x];
 }
 
+/**
+ * Given a type guard for T, check whether an object is an array of T
+ */
+export function isArrayOf<T>(
+  itemGuard: (item: unknown) => item is T,
+): (arr: unknown) => arr is T[] {
+  return (arr: unknown): arr is T[] =>
+    Array.isArray(arr) && arr.every(itemGuard);
+}
+
 /**
  * Helper function to generate HttpExceptions
  * @param message error message

src/config/configuration.ts

@@ -4,6 +4,7 @@ import localconfiguration from "./localconfiguration";
 import { boolean } from "mathjs";
 import { DEFAULT_PROPOSAL_TYPE } from "src/proposals/schemas/proposal.schema";
 import { DatasetType } from "src/datasets/types/dataset-type.enum";
+import { load } from "js-yaml";
 
 const configuration = () => {
   const accessGroupsStaticValues =
@@ -48,11 +49,11 @@ const configuration = () => {
     modulePath: "./loggingProviders/defaultLogger",
     config: {},
   };
-  const jsonConfigMap: { [key: string]: object | object[] | boolean } = {
+  const jsonConfigMap: { [key: string]: unknown } = {
     datasetTypes: {},
     proposalTypes: {},
   };
-  const jsonConfigFileList: { [key: string]: string } = {
+  const yamlConfigFileList: { [key: string]: string } = {
     frontendConfig:
       process.env.FRONTEND_CONFIG_FILE || "./src/config/frontend.config.json",
     frontendTheme:
@@ -62,12 +63,12 @@ const configuration = () => {
     proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json",
     metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json",
   };
-  Object.keys(jsonConfigFileList).forEach((key) => {
-    const filePath = jsonConfigFileList[key];
+  Object.keys(yamlConfigFileList).forEach((key) => {
+    const filePath = yamlConfigFileList[key];
     if (fs.existsSync(filePath)) {
       const data = fs.readFileSync(filePath, "utf8");
       try {
-        jsonConfigMap[key] = JSON.parse(data);
+        jsonConfigMap[key] = load(data, { filename: filePath });
       } catch (error) {
         console.error(
           "Error json config file parsing " + filePath + " : " + error,
@@ -77,15 +78,19 @@ const configuration = () => {
     }
   });
 
-  // NOTE: Add the default dataset types here
-  Object.assign(jsonConfigMap.datasetTypes, {
-    Raw: DatasetType.Raw,
-    Derived: DatasetType.Derived,
-  });
-  // NOTE: Add the default proposal type here
-  Object.assign(jsonConfigMap.proposalTypes, {
-    DefaultProposal: DEFAULT_PROPOSAL_TYPE,
-  });
+  if (jsonConfigMap.datasetTypes) {
+    // NOTE: Add the default dataset types here
+    Object.assign(jsonConfigMap.datasetTypes, {
+      Raw: DatasetType.Raw,
+      Derived: DatasetType.Derived,
+    });
+  }
+  if (jsonConfigMap.proposalTypes) {
+    // NOTE: Add the default proposal type here
+    Object.assign(jsonConfigMap.proposalTypes, {
+      DefaultProposal: DEFAULT_PROPOSAL_TYPE,
+    });
+  }
 
   const oidcFrontendClients = (() => {
     const clients = ["scicat"];

src/users/dto/create-user.dto.ts

@@ -1,4 +1,5 @@
 import { ApiProperty } from "@nestjs/swagger";
+import { isArrayOf } from "../../common/utils";
 
 export class CreateUserDto {
   @ApiProperty({ required: true })
@@ -19,3 +20,36 @@ export class CreateUserDto {
   @ApiProperty({ required: false })
   readonly authStrategy?: string;
 }
+
+// Type guard for CreateUserDto
+export function isCreateUserDto(obj: unknown): obj is CreateUserDto {
+  if (typeof obj !== "object" || obj === null) return false;
+
+  const allowedKeys = [
+    "username",
+    "email",
+    // optional
+    "password",
+    "role",
+    "global",
+    "authStrategy",
+  ];
+
+  // Check for extra fields
+  const objKeys = Object.keys(obj);
+  if (!objKeys.every((key) => allowedKeys.includes(key))) return false;
+
+  // Type checks
+  if (!("username" in obj) || typeof obj.username !== "string") return false;
+  if (!("email" in obj) || typeof obj.email !== "string") return false;
+  if ("password" in obj && typeof obj.password !== "string") return false;
+  if ("role" in obj && typeof obj.role !== "string") return false;
+  if ("global" in obj && typeof obj.global !== "boolean") return false;
+  if ("authStrategy" in obj && typeof obj.authStrategy !== "string")
+    return false;
+
+  return true;
+}
+
+// Type guard for CreateUserDto[]
+export const isCreateUserDtoArray = isArrayOf(isCreateUserDto);

src/users/users.service.ts

@@ -4,7 +4,7 @@ import { InjectModel } from "@nestjs/mongoose";
 import { genSalt, hash } from "bcrypt";
 import { FilterQuery, Model } from "mongoose";
 import { CreateUserIdentityDto } from "./dto/create-user-identity.dto";
-import { CreateUserDto } from "./dto/create-user.dto";
+import { CreateUserDto, isCreateUserDtoArray } from "./dto/create-user.dto";
 import { RolesService } from "./roles.service";
 import {
   UserIdentity,
@@ -31,6 +31,7 @@ import { UserPayload } from "src/auth/interfaces/userPayload.interface";
 import { AccessGroupService } from "src/auth/access-group-provider/access-group.service";
 import { ReturnedUserDto } from "./dto/returned-user.dto";
 import { UpdateUserDto } from "./dto/update-user.dto";
+import { load } from "js-yaml";
 
 @Injectable()
 export class UsersService implements OnModuleInit {
@@ -55,7 +56,13 @@ export class UsersService implements OnModuleInit {
     const filePath = this.configService.get("functionalAccounts.file");
     if (fs.existsSync(filePath)) {
       const data = fs.readFileSync(filePath, "utf8");
-      functionalAccounts = JSON.parse(data);
+      const parsedData = load(data, { filename: filePath });
+      if (!parsedData || !isCreateUserDtoArray(parsedData)) {
+        throw new Error(
+          `Invalid functional accounts file format. Expected an object but got ${typeof parsedData}`,
+        );
+      }
+      functionalAccounts = parsedData;
     }
 
     if (functionalAccounts && functionalAccounts.length > 0) {