Jrod/dit 10045 reintroduce pull command with ns cli

.github/actions/install-node-dependencies/action.yml

@@ -10,7 +10,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       env:
         cache-name: node_modules-cache
       with:

.github/workflows/required-checks.yml

@@ -1,23 +1,23 @@
 name: "Required Checks"
 
-on:
-  pull_request:
-    branches:
-      - master
+on: pull_request
 
 jobs:
   jest-tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
       - name: Use Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
+
       - uses: ./.github/actions/install-node-dependencies
+
       - name: Jest Tests
         run: |
           npx jest --ci --silent --maxWorkers=1

lib/ditto.ts

@@ -4,7 +4,7 @@ import * as Sentry from "@sentry/node";
 import { version as release } from "../package.json";
 import legacyAppEntry from "./legacy";
 import appEntry from "./src";
-import output from "./src/output";
+import logger from "./src/utils/logger";
 
 // Initialize Sentry
 const environment = process.env.ENV || "development";
@@ -14,7 +14,7 @@ const main = async () => {
   // Check for --legacy flag and run in legacy mode if present
   if (process.argv.includes("--legacy")) {
     console.log(
-      output.warnText(
+      logger.warnText(
         "\nDitto CLI is running in legacy mode. This mode is deprecated and will be removed in a future release.\n"
       )
     );

lib/legacy/init/token.ts

@@ -126,9 +126,8 @@ export const collectAndSaveToken = async (message: string | null = null) => {
     console.log(
       `Thanks for authenticating.  We'll save the key to: ${output.info(
         consts.CONFIG_FILE
-      )}`
+      )}\n`
     );
-    output.nl();
 
     config.saveToken(consts.CONFIG_FILE, consts.API_HOST, token);
     return token;

lib/legacy/utils/promptForProject.ts

@@ -35,7 +35,7 @@ const promptForProject = async ({
   projects,
   limit = 10,
 }: ProjectPromptParams) => {
-  output.nl();
+  output.write("\n");
 
   const choices = projects.map(formatProjectChoice);
   const prompt = new AutoComplete({

lib/src/commands/pull.ts

@@ -1,3 +1,8 @@
+import appContext from "../utils/appContext";
+import formatOutput from "../formatters";
+
 export const pull = async () => {
-  console.log("pull");
+  for (const output of appContext.selectedProjectConfigOutputs) {
+    await formatOutput(output, appContext.projectConfig);
+  }
 };

lib/src/formatters/frameworks/base.ts

@@ -0,0 +1,13 @@
+import OutputFile from "../shared/fileTypes/OutputFile";
+
+export default class BaseFramework {
+  protected format: string;
+
+  constructor(format: string) {
+    this.format = format;
+  }
+
+  process(...args: any[]): OutputFile[] {
+    throw new Error("Not implemented");
+  }
+}

lib/src/formatters/frameworks/i18next.ts

@@ -0,0 +1,94 @@
+import appContext from "../../utils/appContext";
+import JavascriptOutputFile from "../shared/fileTypes/JavascriptOutputFile";
+import OutputFile from "../shared/fileTypes/OutputFile";
+import { applyMixins } from "../shared";
+import javascriptCodegenMixin from "../mixins/javascriptCodegenMixin";
+import JSONOutputFile from "../shared/fileTypes/JSONOutputFile";
+import BaseFramework from "./base";
+
+export default class I18NextFramework extends applyMixins(
+  BaseFramework,
+  javascriptCodegenMixin
+) {
+  process(
+    outputJsonFiles: Record<string, JSONOutputFile<{ variantId: string }>>
+  ) {
+    const outputDir = appContext.projectConfigDir;
+    // Generate Driver file
+
+    const driverFile = new JavascriptOutputFile({
+      filename: "index",
+      path: outputDir,
+    });
+
+    const filesGroupedByVariantId = Object.values(outputJsonFiles).reduce(
+      (acc, file) => {
+        const variantId = file.metadata.variantId;
+        acc[variantId] ??= [];
+        acc[variantId].push(file);
+        return acc;
+      },
+      {} as Record<string, OutputFile[]>
+    );
+
+    driverFile.content += this.generateImportStatements(outputJsonFiles);
+
+    driverFile.content += `\n`;
+
+    driverFile.content += this.generateDefaultExportString(
+      filesGroupedByVariantId
+    );
+
+    return [driverFile];
+  }
+
+  /**
+   * Generates the import statements for the driver file. One import per generated json file.
+   * @param outputJsonFiles - The output json files.
+   * @returns The import statements, stringified.
+   */
+  private generateImportStatements(
+    outputJsonFiles: Record<string, JSONOutputFile<{ variantId: string }>>
+  ) {
+    let importStatements = "";
+    for (const file of Object.values(outputJsonFiles)) {
+      importStatements += this.codegenDefaultImport(
+        this.sanitizeStringForJSVariableName(file.filename),
+        `./${file.filenameWithExtension}`
+      );
+    }
+    return importStatements;
+  }
+
+  /**
+   * Generates the default export for the driver file. By default this is an object with the json imports grouped by variant id.
+   * @param filesGroupedByVariantId - The files grouped by variant id.
+   * @returns The default export, stringified.
+   */
+  private generateDefaultExportString(
+    filesGroupedByVariantId: Record<string, OutputFile[]>
+  ) {
+    const variantIds = Object.keys(filesGroupedByVariantId);
+
+    let defaultExportObjectString = "{\n";
+
+    for (let i = 0; i < variantIds.length; i++) {
+      const variantId = variantIds[i];
+      const files = filesGroupedByVariantId[variantId];
+
+      defaultExportObjectString += `${this.codegenPad(1)}"${variantId}": {\n`;
+      for (const file of files) {
+        defaultExportObjectString += `${this.codegenPad(
+          2
+        )}...${this.sanitizeStringForJSVariableName(file.filename)},\n`;
+      }
+      defaultExportObjectString += `${this.codegenPad(1)}}${
+        i < variantIds.length - 1 ? `,\n` : `\n`
+      }`;
+    }
+
+    defaultExportObjectString += `}`;
+
+    return this.codegenDefaultExport(defaultExportObjectString);
+  }
+}

lib/src/formatters/frameworks/index.ts

@@ -0,0 +1,10 @@
+import I18NextFramework from "./i18next";
+
+export function getFrameworkProcessor(framework: string) {
+  switch (framework) {
+    case "i18next":
+      return new I18NextFramework(framework);
+    default:
+      throw new Error(`Unsupported framework: ${framework}`);
+  }
+}

lib/src/formatters/index.ts

@@ -0,0 +1,18 @@
+import { Output } from "../outputs";
+import { ProjectConfigYAML } from "../services/projectConfig";
+import JSONFormatter from "./json";
+
+export default function handleOutput(
+  output: Output,
+  projectConfig: ProjectConfigYAML
+) {
+  switch (output.format) {
+    case "json":
+      return new JSONFormatter(output, projectConfig).format(
+        output,
+        projectConfig
+      );
+    default:
+      throw new Error(`Unsupported output format: ${output}`);
+  }
+}

lib/src/formatters/json.ts

@@ -0,0 +1,91 @@
+import fetchText, { PullFilters, TextItemsResponse } from "../http/textItems";
+import fetchVariables, { Variable, VariablesResponse } from "../http/variables";
+import BaseFormatter from "./shared/base";
+import OutputFile from "./shared/fileTypes/OutputFile";
+import JSONOutputFile from "./shared/fileTypes/JSONOutputFile";
+import appContext from "../utils/appContext";
+import { applyMixins } from "./shared";
+import { getFrameworkProcessor } from "./frameworks";
+
+type JSONAPIData = {
+  textItems: TextItemsResponse;
+  variablesById: Record<string, Variable>;
+};
+
+export default class JSONFormatter extends applyMixins(
+  BaseFormatter<JSONAPIData>) {
+
+  protected async fetchAPIData() {
+    const filters = this.generatePullFilter();
+    const textItems = await fetchText(filters);
+    const variables = await fetchVariables();
+
+    const variablesById = variables.reduce((acc, variable) => {
+      acc[variable.id] = variable;
+      return acc;
+    }, {} as Record<string, Variable>);
+
+    return { textItems, variablesById };
+  }
+
+  protected async transformAPIData(data: JSONAPIData) {
+    const outputDir = appContext.projectConfigDir;
+
+    let outputJsonFiles: Record<
+      string,
+      JSONOutputFile<{ variantId: string }>
+    > = {};
+
+    const variablesOutputFile = new JSONOutputFile({
+      filename: "variables",
+      path: appContext.projectConfigDir,
+    });
+
+    for (let i = 0; i < data.textItems.length; i++) {
+      const textItem = data.textItems[i];
+
+      const fileName = `${textItem.projectId}___${textItem.variantId || "base"}`;
+
+      outputJsonFiles[fileName] ??= new JSONOutputFile({
+        filename: fileName,
+        path: outputDir,
+        metadata: { variantId: textItem.variantId || "base" },
+      });
+
+
+      outputJsonFiles[fileName].content[textItem.id] = textItem.text;
+      for (const variableId of textItem.variableIds) {
+        const variable = data.variablesById[variableId];
+        variablesOutputFile.content[variableId] = variable.data;
+      }
+    }
+
+    let results: OutputFile[] = [
+      ...Object.values(outputJsonFiles),
+      variablesOutputFile,
+    ]
+
+    if (this.output.framework) {
+      // process framework
+      results.push(...getFrameworkProcessor(this.output.framework).process(outputJsonFiles));
+    }
+
+    return results;
+  }
+
+  private generatePullFilter() {
+    let filters: PullFilters = {
+      projects: this.projectConfig.projects,
+      variants: this.projectConfig.variants,
+    };
+    if (this.output.projects) {
+      filters.projects = this.output.projects;
+    } 
+
+    if (this.output.variants) {
+      filters.variants = this.output.variants;
+    }
+
+    return filters;
+  }
+}

lib/src/formatters/mixins/javascriptCodegenMixin.ts

@@ -0,0 +1,44 @@
+import { Constructor } from "../shared";
+
+interface NamedImport {
+  name: string;
+  alias?: string;
+}
+
+export default function javascriptCodegenMixin<TBase extends Constructor>(
+  Base: TBase
+) {
+  return class JavascriptCodegenHelpers extends Base {
+    protected indentSpaces: number = 2;
+
+    protected sanitizeStringForJSVariableName(str: string) {
+      return str.replace(/[^a-zA-Z0-9]/g, "_");
+    }
+
+    protected codegenNamedImport(modules: NamedImport[], moduleName: string) {
+      const formattedModules = modules
+        .map((m) => {
+          if (m.alias) {
+            return `${m.name} as ${m.alias}`;
+          }
+          return m.name;
+        })
+        .sort()
+        .join(", ");
+
+      return `import { ${formattedModules} } from "${moduleName}";\n`;
+    }
+
+    protected codegenDefaultImport(module: string, moduleName: string) {
+      return `import ${module} from "${moduleName}";\n`;
+    }
+
+    protected codegenDefaultExport(module: string) {
+      return `export default ${module};`;
+    }
+
+    protected codegenPad(depth: number) {
+      return " ".repeat(depth * this.indentSpaces);
+    }
+  };
+}