feat(workspace): workspace-root component (rootDir ".") and the trackAllFiles flag

components/legacy/bit-map/bit-map.spec.ts

@@ -2,7 +2,8 @@ import { expect } from 'chai';
 import { ComponentID } from '@teambit/component-id';
 import { BitId } from '@teambit/legacy-bit-id';
 import { logger } from '@teambit/legacy.logger';
-import { BitMap } from './bit-map';
+import { BitMap, normalizeBitmapContentForVersioning } from './bit-map';
+import { WORKSPACE_ROOT_DIR } from './component-map';
 import { DuplicateRootDir } from './exceptions/duplicate-root-dir';
 
 const getBitmapInstance = async () => {
@@ -90,4 +91,83 @@ describe('BitMap', function () {
       );
     });
   });
+  describe('workspace-root component', () => {
+    const rootComponentParams = {
+      componentId: ComponentID.fromObject({ name: 'ws-root' }, 'my-scope'),
+      files: [{ name: 'README.md', relativePath: 'README.md', test: false }],
+      mainFile: 'README.md',
+      defaultScope: 'my-scope',
+      rootDir: WORKSPACE_ROOT_DIR,
+    };
+    const nestedComponentParams = {
+      componentId: ComponentID.fromObject({ name: 'comp1' }, 'my-scope'),
+      files: [{ name: 'index.js', relativePath: 'index.js', test: false }],
+      mainFile: 'index.js',
+      defaultScope: 'my-scope',
+      rootDir: 'packages/comp1',
+    };
+    it('should allow a rootDir of "." to contain other components, in both add orders', async () => {
+      const rootFirst = await getBitmapInstance();
+      rootFirst.addComponent(rootComponentParams);
+      expect(() => rootFirst.addComponent(nestedComponentParams)).to.not.throw();
+
+      const nestedFirst = await getBitmapInstance();
+      nestedFirst.addComponent(nestedComponentParams);
+      expect(() => nestedFirst.addComponent(rootComponentParams)).to.not.throw();
+    });
+    it('should keep rejecting nesting between two non-root components', async () => {
+      const bitMap = await getBitmapInstance();
+      bitMap.addComponent(nestedComponentParams);
+      expect(() =>
+        bitMap.addComponent({
+          ...nestedComponentParams,
+          componentId: ComponentID.fromObject({ name: 'comp2' }, 'my-scope'),
+          rootDir: 'packages/comp1/nested',
+        })
+      ).to.throw();
+    });
+    it('getNestedRootDirs should return the nested components for the root, and nothing for a leaf', async () => {
+      const bitMap = await getBitmapInstance();
+      bitMap.addComponent(rootComponentParams);
+      bitMap.addComponent(nestedComponentParams);
+      expect(bitMap.getNestedRootDirs(WORKSPACE_ROOT_DIR)).to.deep.equal(['packages/comp1']);
+      expect(bitMap.getNestedRootDirs('packages/comp1')).to.deep.equal([]);
+    });
+  });
+  describe('normalizeBitmapContentForVersioning', () => {
+    const rawBitmap = JSON.stringify(
+      {
+        comp1: {
+          name: 'comp1',
+          scope: 'my-scope',
+          version: '0a14284ddaadde623d5c11f5511594485a14b3c8',
+          defaultScope: 'my-org.demo',
+          mainFile: 'index.ts',
+          rootDir: 'comp1',
+        },
+        '$schema-version': '17.0.0',
+      },
+      null,
+      4
+    );
+    let normalized: string;
+    let parsed: Record<string, any>;
+    before(() => {
+      normalized = normalizeBitmapContentForVersioning(rawBitmap);
+      parsed = JSON.parse(normalized.slice(normalized.indexOf('{')));
+    });
+    it('should empty the fields that change on every snap and export', () => {
+      expect(parsed.comp1.version).to.equal('');
+      expect(parsed.comp1.scope).to.equal('');
+    });
+    it('should keep the durable map intact', () => {
+      expect(parsed.comp1.rootDir).to.equal('comp1');
+      expect(parsed.comp1.mainFile).to.equal('index.ts');
+      expect(parsed.comp1.defaultScope).to.equal('my-org.demo');
+      expect(parsed['$schema-version']).to.equal('17.0.0');
+    });
+    it('should be idempotent, otherwise the root component would never converge', () => {
+      expect(normalizeBitmapContentForVersioning(normalized)).to.equal(normalized);
+    });
+  });
 });

components/legacy/bit-map/bit-map.ts

@@ -28,7 +28,7 @@ import type {
 } from '@teambit/toolbox.path.path';
 import { pathJoinLinux, pathNormalizeToLinux } from '@teambit/toolbox.path.path';
 import type { ComponentMapFile, Config, PathChange } from './component-map';
-import { ComponentMap, getFilesByDir, getGitIgnoreHarmony } from './component-map';
+import { ComponentMap, getFilesByDir, getGitIgnoreHarmony, WORKSPACE_ROOT_DIR } from './component-map';
 import { InvalidBitMap, MissingBitMapComponent } from './exceptions';
 import { DuplicateRootDir } from './exceptions/duplicate-root-dir';
 
@@ -99,15 +99,20 @@ export class BitMap {
 
   /**
    * in case the added component's root-dir is a parent-dir of other components
-   * or other component's root-dir is a parent root-dir of this component, throw an error
+   * or other component's root-dir is a parent root-dir of this component, throw an error.
+   *
+   * the workspace-root component (rootDir ".") is the one exception - it is allowed to contain
+   * other components. its file-set subtracts their root-dirs, so the two never claim the same file.
    */
   private throwForExistingParentDir({ id, rootDir }: ComponentMap) {
+    if (rootDir === WORKSPACE_ROOT_DIR) return;
     const isParentDir = (parent: string, child: string) => {
       const relative = path.relative(parent, child);
       return relative && !relative.startsWith('..');
     };
     this.components.forEach((existingComponentMap) => {
       if (!existingComponentMap.rootDir) return;
+      if (existingComponentMap.rootDir === WORKSPACE_ROOT_DIR) return;
       if (isParentDir(existingComponentMap.rootDir, rootDir)) {
         throw new BitError(
           `unable to add "${id.toString()}", its rootDir ${rootDir} is inside ${
@@ -212,14 +217,34 @@ export class BitMap {
     delete componentsJson[LANE_KEY];
   }
 
+  /**
+   * root-dirs of the components nested inside the given root-dir. their files belong to them, so
+   * the containing component must subtract them from its own file-set.
+   */
+  getNestedRootDirs(rootDir: PathLinuxRelative): PathLinuxRelative[] {
+    return this.components
+      .map((componentMap) => componentMap.rootDir)
+      .filter((nested): nested is PathLinuxRelative => {
+        if (!nested || nested === rootDir) return false;
+        if (rootDir === WORKSPACE_ROOT_DIR) return nested !== WORKSPACE_ROOT_DIR;
+        const relative = path.relative(rootDir, nested);
+        return Boolean(relative) && !relative.startsWith('..');
+      });
+  }
+
   async loadFiles() {
     const gitIgnore = await getGitIgnoreHarmony(this.projectRoot, this.ignoredFiles);
     await Promise.all(
       this.components.map(async (componentMap) => {
         const rootDir = componentMap.rootDir;
         if (!rootDir) return;
         try {
-          componentMap.files = await getFilesByDir(rootDir, this.projectRoot, gitIgnore);
+          componentMap.files = await getFilesByDir(
+            rootDir,
+            this.projectRoot,
+            gitIgnore,
+            this.getNestedRootDirs(rootDir)
+          );
           componentMap.recentlyTracked = true;
         } catch (err: any) {
           componentMap.files = [];
@@ -1019,6 +1044,30 @@ type OutputFileParams = {
   prefixMessage?: string;
 };
 
+/**
+ * the workspace-root component tracks `.bitmap` so a git-free workspace can be restored from the
+ * scope. the `version` and `scope` of every entry change on each snap and export - including the
+ * root component's own entry - so versioning them verbatim would leave that component modified
+ * immediately after every snap, forever, and never converge.
+ *
+ * only the durable part of the map is versioned: which components exist and where they live. the
+ * versions themselves are restored from the component heads on import, which is the correct source
+ * for them anyway.
+ */
+export function normalizeBitmapContentForVersioning(rawContent: string): string {
+  const parsed = json.parse(rawContent, undefined, true) as Record<string, any> | undefined;
+  if (!parsed) return rawContent;
+  Object.keys(parsed).forEach((key) => {
+    const entry = parsed[key];
+    // component entries are objects with a mainFile. skips the schema field (a string) and the
+    // lanes key (an object without a mainFile).
+    if (!entry || typeof entry !== 'object' || Array.isArray(entry) || !('mainFile' in entry)) return;
+    if ('version' in entry) entry.version = '';
+    if ('scope' in entry) entry.scope = '';
+  });
+  return `${AUTO_GENERATED_MSG}${BITMAP_PREFIX_MESSAGE}${JSON.stringify(parsed, null, 4)}`;
+}
+
 async function outputFile({
   filePath,
   content,

components/legacy/bit-map/component-map.ts

@@ -3,7 +3,14 @@ import globby from 'globby';
 import ignore from 'ignore';
 import { pickBy, isNil, sortBy, isEmpty } from 'lodash';
 import type { ComponentID } from '@teambit/component-id';
-import { BIT_MAP, Extensions, PACKAGE_JSON, IGNORE_ROOT_ONLY_LIST } from '@teambit/legacy.constants';
+import {
+  BIT_HIDDEN_DIR,
+  BIT_MAP,
+  DOT_GIT_DIR,
+  Extensions,
+  PACKAGE_JSON,
+  IGNORE_ROOT_ONLY_LIST,
+} from '@teambit/legacy.constants';
 import { ValidationError } from '@teambit/legacy.cli.error';
 import { logger } from '@teambit/legacy.logger';
 import { isValidPath } from '@teambit/legacy.utils';
@@ -26,6 +33,24 @@ import { IgnoredDirectory, ComponentNotFoundInPath } from '@teambit/legacy.consu
 
 export type Config = { [aspectId: string]: Record<string, any> | '-' };
 
+/**
+ * rootDir of a component that owns the workspace root. such a component holds the files that no
+ * other component claims - e.g. the workspace config, CI config, README and license files.
+ * it is the only rootDir allowed to contain other components' root-dirs.
+ */
+export const WORKSPACE_ROOT_DIR = '.';
+
+/**
+ * paths the workspace-root component must never own. relevant only for that component - for any
+ * other component these live outside its root-dir and are never reached by the scan.
+ *
+ * note that `.bitmap` is deliberately NOT here. it is the map of the workspace and a git-free
+ * workspace has to be able to restore it, so the root component tracks it like any other file.
+ * `.bit` (the local object store) and `.git` are the outputs of versioning, not sources, so they
+ * stay excluded.
+ */
+const WORKSPACE_ROOT_IGNORE_LIST = [`${BIT_HIDDEN_DIR}/**`, `${DOT_GIT_DIR}/**`];
+
 export type ComponentMapFile = {
   relativePath: PathLinux;
   /**
@@ -265,13 +290,17 @@ export class ComponentMap {
    * if the component dir has changed since the last tracking, re-scan the component-dir to get the
    * updated list of the files
    */
-  async trackDirectoryChangesHarmony(consumerPath: PathOsBasedAbsolute, ignoredFiles?: string[]): Promise<void> {
+  async trackDirectoryChangesHarmony(
+    consumerPath: PathOsBasedAbsolute,
+    ignoredFiles?: string[],
+    excludeDirs: PathLinux[] = []
+  ): Promise<void> {
     const trackDir = this.rootDir;
     if (!trackDir) {
       return;
     }
     const gitIgnore = await getGitIgnoreHarmony(consumerPath, ignoredFiles);
-    this.files = await getFilesByDir(trackDir, consumerPath, gitIgnore);
+    this.files = await getFilesByDir(trackDir, consumerPath, gitIgnore, excludeDirs);
   }
 
   updateNextVersion(nextVersion: NextVersion) {
@@ -347,10 +376,8 @@ export class ComponentMap {
     if (!isValidPath(this.mainFile)) {
       throw new ValidationError(`${errorMessage} mainFile attribute ${this.mainFile} is invalid`);
     }
-    if (this.rootDir && !isValidPath(this.rootDir)) {
-      throw new ValidationError(`${errorMessage} rootDir attribute ${this.rootDir} is invalid`);
-    }
-    if (this.rootDir && this.rootDir === '.') {
+    // "." is valid - it marks the component that owns the workspace root. see WORKSPACE_ROOT_DIR.
+    if (this.rootDir && this.rootDir !== WORKSPACE_ROOT_DIR && !isValidPath(this.rootDir)) {
       throw new ValidationError(`${errorMessage} rootDir attribute ${this.rootDir} is invalid`);
     }
     if (this.nextVersion && !this.nextVersion.version) {
@@ -382,19 +409,39 @@ if you renamed the mainFile, please re-add the component with the "--main" flag
   }
 }
 
-export async function getFilesByDir(dir: string, consumerPath: string, gitIgnore: any): Promise<ComponentMapFile[]> {
-  const matches = await globby(dir, {
+/**
+ * scan a component's root-dir for its files.
+ *
+ * `excludeDirs` holds the root-dirs of components nested inside `dir`. their files belong to the
+ * nested component, not to this one. this is what makes a workspace-root component (rootDir ".")
+ * possible: it owns every file that no other component claims.
+ */
+export async function getFilesByDir(
+  dir: string,
+  consumerPath: string,
+  gitIgnore: any,
+  excludeDirs: PathLinux[] = []
+): Promise<ComponentMapFile[]> {
+  const isWorkspaceRoot = dir === WORKSPACE_ROOT_DIR;
+  const matches = await globby(isWorkspaceRoot ? '**' : dir, {
     cwd: consumerPath,
     dot: true,
     onlyFiles: true,
     // must ignore node_modules at this stage, although we check for gitignore later on.
     // otherwise, it hurts performance dramatically for components that have node_modules in the comp-dir.
-    ignore: [`${dir}/node_modules/`],
+    ignore: [
+      isWorkspaceRoot ? '**/node_modules/**' : `${dir}/node_modules/`,
+      ...(isWorkspaceRoot ? WORKSPACE_ROOT_IGNORE_LIST : []),
+      ...excludeDirs.map((excludeDir) => `${excludeDir}/**`),
+    ],
   });
   if (!matches.length) throw new ComponentNotFoundInPath(dir);
   const filteredMatches: string[] = gitIgnore.filter(matches);
-  // the path is relative to consumer. remove the rootDir.
-  const relativePathsLinux = filteredMatches.map((match) => pathNormalizeToLinux(match).replace(`${dir}/`, ''));
+  // the path is relative to consumer. remove the rootDir. for the workspace-root component the
+  // paths are already relative to the consumer, so there is nothing to strip.
+  const relativePathsLinux = filteredMatches.map((match) =>
+    isWorkspaceRoot ? pathNormalizeToLinux(match) : pathNormalizeToLinux(match).replace(`${dir}/`, '')
+  );
   const filteredByIgnoredFromRoot = relativePathsLinux.filter((match) => !IGNORE_ROOT_ONLY_LIST.includes(match));
   const bitOrGitIgnore = filteredByIgnoredFromRoot.includes(BIT_IGNORE)
     ? await getBitIgnoreFile(dir)

components/legacy/bit-map/index.ts

@@ -5,6 +5,7 @@ export {
   CURRENT_BITMAP_SCHEMA,
   SCHEMA_FIELD,
   LANE_KEY,
+  normalizeBitmapContentForVersioning,
 } from './bit-map';
 export { MissingBitMapComponent, MissingMainFile, InvalidBitMap } from './exceptions';
 export {
@@ -14,4 +15,5 @@ export {
   Config,
   getIgnoreListHarmony,
   NextVersion,
+  WORKSPACE_ROOT_DIR,
 } from './component-map';

components/legacy/consumer-component/consumer-component.ts

@@ -7,7 +7,13 @@ import { IssuesList } from '@teambit/component-issues';
 import { BitId } from '@teambit/legacy-bit-id';
 import { BitError } from '@teambit/bit-error';
 import type { BuildStatus } from '@teambit/legacy.constants';
-import { getCloudDomain, BIT_WORKSPACE_TMP_DIRNAME, DEFAULT_LANGUAGE, Extensions } from '@teambit/legacy.constants';
+import {
+  getCloudDomain,
+  BIT_MAP,
+  BIT_WORKSPACE_TMP_DIRNAME,
+  DEFAULT_LANGUAGE,
+  Extensions,
+} from '@teambit/legacy.constants';
 import type { Doclet } from '@teambit/semantics.doc-parser';
 import { parser as docsParser } from '@teambit/semantics.doc-parser';
 import { logger } from '@teambit/legacy.logger';
@@ -16,6 +22,7 @@ import type { PathLinux, PathOsBased, PathOsBasedRelative } from '@teambit/toolb
 import { pathNormalizeToLinux } from '@teambit/toolbox.path.path';
 import { sha1 } from '@teambit/toolbox.crypto.sha1';
 import type { ComponentMap } from '@teambit/legacy.bit-map';
+import { normalizeBitmapContentForVersioning, WORKSPACE_ROOT_DIR } from '@teambit/legacy.bit-map';
 import { IgnoredDirectory } from './exceptions/ignored-directory';
 import type { Dist, PackageJsonFile, DataToPersist } from '@teambit/component.sources';
 import { License, SourceFile } from '@teambit/component.sources';
@@ -600,10 +607,19 @@ async function getLoadedFiles(
     logger.error(`rethrowing an error of ${componentMap.noFilesError.message}`);
     throw componentMap.noFilesError;
   }
-  await componentMap.trackDirectoryChangesHarmony(consumer.getPath(), consumer.config.ignoredFiles);
+  await componentMap.trackDirectoryChangesHarmony(
+    consumer.getPath(),
+    consumer.config.ignoredFiles,
+    consumer.bitMap.getNestedRootDirs(componentMap.getRootDir())
+  );
   const sourceFiles = componentMap.files.map((file) => {
     const filePath = path.join(bitDir, file.relativePath);
     const sourceFile = SourceFile.load(filePath, bitDir, consumer.getPath(), { test: file.test || false });
+    // the workspace-root component owns .bitmap. strip the fields that change on every snap so the
+    // component converges instead of being modified again the moment it is snapped.
+    if (componentMap.rootDir === WORKSPACE_ROOT_DIR && file.relativePath === BIT_MAP) {
+      sourceFile.contents = Buffer.from(normalizeBitmapContentForVersioning(sourceFile.contents.toString()));
+    }
     return sourceFile;
   });
   const filePaths = componentMap.getAllFilesPaths();