feat: search NodeTags in files when debug starts

Previously this was launched on start and used ineffective algorithm
with lots of RegExp which allocates too much memory.

Now this part is reworked:

- use simple one-pass algorithm based on string search (indexOf)
- search is launched on debug session starts
- files for parsing searched based on detected PG version (modern
  requires only nodetags.h)

Author: ashenBlade

src/configuration.ts

@@ -3,6 +3,7 @@ import * as vscode from 'vscode';
 import * as vars from './variables';
 import * as utils from './utils';
 import { Log as logger } from './logger';
+import { PghhError } from './error';
 
 export interface VariablesConfiguration {
     /* Array special members */
@@ -59,6 +60,13 @@ export function parseFormatterConfiguration(configFile: unknown): PgindentConfig
 
 export function parseVariablesConfiguration(configFile: unknown): VariablesConfiguration | undefined {
     const parseArrayMember = (obj: unknown): vars.ArraySpecialMemberInfo | undefined => {
+        /* 
+         * {
+         *     "typeName": "parentType",
+         *     "memberName": "memberOfParent",
+         *     "lengthExpr": "expression to get length"
+         * }
+         */
         if (!(typeof obj === 'object' && obj)) {
             return;
         }
@@ -135,6 +143,12 @@ export function parseVariablesConfiguration(configFile: unknown): VariablesConfi
     };
 
     const parseSingleAlias = (obj: unknown): vars.AliasInfo | undefined => {
+        /* 
+         * {
+         *     "alias": "name of alias",
+         *     "type": "real type"
+         * }
+         */
         if (!(typeof obj === 'object' && obj)) {
             return;
         }
@@ -597,7 +611,6 @@ export async function getFormatterConfiguration() {
 
 export async function refreshConfiguration() {
     /* Do not check 'dirtyFlag', because this function must be invoked explicitly */
-
     if (!vscode.workspace.workspaceFolders?.length) {
         return;
     }
@@ -625,6 +638,56 @@ export async function refreshConfiguration() {
     configDirty = false;
 }
 
+async function findConfigurationFile() {
+    if (!vscode.workspace.workspaceFolders?.length) {
+        throw new PghhError('No workspace folders opened');
+    }
+    
+    for (const folder of vscode.workspace.workspaceFolders) {
+        const file = getExtensionConfigFile(folder.uri);
+        if (await utils.fileExists(file)) {
+            return file;
+        }
+    }
+
+    const file = getExtensionConfigFile(vscode.workspace.workspaceFolders[0].uri);
+    const directory = utils.joinPath(file, '..');
+    if (!await utils.directoryExists(directory)) {
+        logger.info('.vscode directory does not exist - creating one at %s', directory.fsPath);
+        await utils.createDirectory(directory);
+    }
+    
+    /* It will be created if necessary */
+    return file;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function mutateConfiguration(mutator: (config: any) => unknown) {
+    const configFile = await findConfigurationFile();
+
+    let contents: string | undefined;    
+    try {
+        contents = await utils.readFile(configFile);
+    } catch (err) {
+        if (!(err instanceof Error && err.name.indexOf('EntryNotFound') !== -1)) {
+            throw err;
+        }
+        contents = undefined;
+    }
+
+    let config;
+    if (contents?.length) {
+        config = JSON.parse(contents);
+    } else {
+        config = undefined;
+    }
+
+    config = mutator(config);
+
+    await utils.writeFile(configFile, JSON.stringify(config, undefined, '    '));
+    configDirty = true;
+}
+
 export function markConfigFileDirty() {
     configDirty = true;
 }

src/extension.ts

@@ -1,10 +1,20 @@
+import * as path from 'path';
 import * as vscode from 'vscode';
+
 import * as utils from './utils';
 import { Features } from './utils';
 import * as vars from './variables';
 import * as dbg from './debugger';
 import * as dap from './dap';
-import { Commands, ExtensionSettingsFileName, PgVariablesViewName, getExtensionConfigFile, markConfigFileDirty, refreshConfiguration, setupVsCodeSettings } from './configuration';
+import { Commands, 
+         ExtensionSettingsFileName,
+         PgVariablesViewName,
+         VsCodeSettings,
+         getExtensionConfigFile,
+         markConfigFileDirty,
+         mutateConfiguration,
+         refreshConfiguration,
+         setupVsCodeSettings } from './configuration';
 import { Log as logger } from './logger';
 import { setupPgConfSupport } from './pgconf';
 import { setupFormatting } from './formatter';
@@ -511,9 +521,163 @@ function setupPgVariablesView(context: vscode.ExtensionContext) {
     /* Setup debugger specific function */
     dbg.setupDebugger(context, pgvars);
 
+    /*
+     * On start try to detect new NodeTags and if they exists, add to NodeVars
+     * and ask user to add them to configuration file.
+     */
+    const disposable = pgvars.onDidDebugStart(async (c) => {
+        /* Run this only once */
+        disposable.dispose();
+        
+        try {
+            await searchNodeTagsWorker(c);
+        } catch (err) {
+            logger.error('could not search for new NodeTags', err);
+        }
+    });
+    
+
     return pgvars;
 }
 
+async function findAllFilesWithNodeTags(folders: readonly vscode.WorkspaceFolder[],
+                                        pgversion: number) {
+    const paths = [];
+    for (const folder of folders) {
+        /* 
+         * Starting from 16 major version NodeTag is autogenerated and stored in nodetags.h.
+         * Avoid parsing 'nodes.h', because it will not give us anything.
+         */
+        let file;
+        if (16_00_00 <= pgversion) {
+            file = utils.getWorkspacePgSrcFile(folder.uri, 'src', 'include', 'nodes', 'nodetags.h');
+        } else {
+            file = utils.getWorkspacePgSrcFile(folder.uri, 'src', 'include', 'nodes', 'nodes.h');
+        }
+
+        if (await utils.fileExists(file)) {
+            paths.push(file);
+        }
+    }
+
+    const customFiles = VsCodeSettings.getCustomNodeTagFiles();
+    if (!customFiles) {
+        return paths;
+    }
+    
+    /* Search custom provided NodeTag files */
+    for (const customFile of customFiles) {
+        let uri;
+        if (path.isAbsolute(customFile)) {
+            uri = vscode.Uri.file(customFile);
+            if (!await utils.fileExists(uri)) {
+                continue;
+            }
+        } else {
+            for (const folder of folders) {
+                uri = utils.getWorkspacePgSrcFile(folder.uri, customFile);
+                if (await utils.fileExists(uri)) {
+                    break;
+                }
+            }
+            if (!uri) {
+                continue;
+            }
+        }
+        
+        paths.push(uri);
+    }
+    
+    return paths;
+}
+
+async function addNodeTagsToConfiguration(nodetags: Set<string>) {
+    logger.info('adding new NodeTags to configuration file');
+    try {
+        await mutateConfiguration((config) => {
+            /* 
+             * Configuration can not exist or it can contain some garbage.
+             * For now I don't know how to perfectly handle this, so
+             * just replace entire field if it's not an array.
+             */
+            if (config === undefined || config === null) {
+                return {
+                    nodetags: Array.from(nodetags),
+                };
+            } else if (typeof config === 'object') {
+                if ('nodetags' in config && Array.isArray(config.nodetags)) {
+                    config.nodetags.push(...Array.from(nodetags));
+                } else {
+                    config.nodetags = Array.from(nodetags);
+                }
+            }
+            return config;
+        });
+    } catch (err) {
+        logger.error('could not add nodetags to configuration file', err);
+    }
+}
+
+/*
+ * Run Worker that will traverse all NodeTag containing files,
+ * parse NodeTags and find which are missing - if find something,
+ * then user is prompted to add them to configuration file.
+ * 
+ * This is quiet CPU intensive operation, so perform in another thread.
+ */
+async function searchNodeTagsWorker(context: vars.ExecContext) {
+    if (!vscode.workspace.workspaceFolders?.length) {
+        return;
+    }
+
+    if (!context.pgversion) {
+        return;
+    }
+
+    /* Find all files containing NodeTags */
+    const paths = await findAllFilesWithNodeTags(
+        vscode.workspace.workspaceFolders, context.pgversion);
+    if (!paths.length) {
+        logger.debug('no NodeTag files found');
+        return;
+    }
+
+    /* Run worker and collect result */
+    const newNodeTags = new Set<string>();
+    for (const path of paths) {
+        const tags = await vars.parseNodeTagsFile(path);
+        if (tags?.size) {
+            for (const t of tags) {
+                newNodeTags.add(t);
+            }
+        }
+    }
+
+    for (const tag of context.nodeVarRegistry.nodeTags) {
+        newNodeTags.delete(tag);
+    }
+
+    if (!newNodeTags.size) {
+        /* No new NodeTags found */
+        return;
+    }
+    
+    for (const tag of newNodeTags) {
+        context.nodeVarRegistry.nodeTags.add(tag);
+    }
+
+    logger.info('found %i new node tags', newNodeTags.size);
+    const answer = await vscode.window.showInformationMessage(
+        `Found ${newNodeTags.size} new NodeTags. ` +
+        `Would you like to add them to configuration file?`,
+        'Yes', 'No');
+    if (answer !== 'Yes') {
+        return;
+    }
+
+    await addNodeTagsToConfiguration(newNodeTags);
+}
+
 function setupConfiguration(context: vscode.ExtensionContext) {
     /* Mark configuration dirty when user changes it - no eager parsing */
     const registerFolderWatcher = (folder: vscode.WorkspaceFolder) => {
@@ -526,7 +690,7 @@ function setupConfiguration(context: vscode.ExtensionContext) {
         configFileWatcher.onDidCreate(markConfigFileDirty, undefined, context.subscriptions);
         configFileWatcher.onDidDelete(markConfigFileDirty, undefined, context.subscriptions);  
     };
-    
+
     if (vscode.workspace.workspaceFolders?.length) {
         vscode.workspace.workspaceFolders.forEach(registerFolderWatcher);
     } else {

src/variables.ts

@@ -436,6 +436,11 @@ export class StepContext {
  * Context of current execution.
  */
 export class ExecContext {
+    /* 
+     * Version number of debugging PG instance
+     */
+    pgversion?: number;
+
     /**
      * Registry about NodeTag variables information
      */
@@ -5404,7 +5409,14 @@ export class PgVariablesViewProvider implements vscode.TreeDataProvider<Variable
         this.context?.step.reset();
         this._onDidChangeTreeData.fire();
     }
-    
+
+    private _onDidDebugStart = new vscode.EventEmitter<ExecContext>();
+    /*
+     * Emitted when debug session started and we return data.
+     * Passes obtained PG version.
+     */
+    readonly onDidDebugStart = this._onDidDebugStart.event;
+
     startDebugging(debug: dbg.GenericDebuggerFacade) {
         this.debug?.dispose();
         this.debug = debug;
@@ -5590,7 +5602,12 @@ export class PgVariablesViewProvider implements vscode.TreeDataProvider<Variable
         }
 
         const context = new ExecContext(this.getDebug(), nodeVars, specialMembers, hashTables);
-        
+        context.pgversion = pgversion;
+
+        if (pgversion) {
+            this._onDidDebugStart.fire(context);
+        }
+
         return context;
     }
 
@@ -5664,3 +5681,51 @@ export class PgVariablesViewProvider implements vscode.TreeDataProvider<Variable
         this._onDidChangeTreeData.dispose();
     }
 }
+
+function isSpace(char: string) {
+    return char === ' ' || char === '\t' || char === '\n';
+}
+
+function isIdentifierChar(char: string) {
+    return    ('a' <= char && char <= 'z') || ('A' <= char && char <= 'Z')
+           || char === '_' /* put it here, because underscore is more common, than digit */
+           || ('0' <= char && char <= '9');
+}
+
+export async function parseNodeTagsFile(file: vscode.Uri) {
+    let content;
+    try {
+        const document = await vscode.workspace.openTextDocument(file);
+        content = document.getText();
+    } catch (error) {
+        logger.error('could not open NodeTags file %s', file.fsPath, error);
+        return;
+    }
+
+    const nodeTags: string[] = [];
+    let prefixIndex = undefined;
+    while ((prefixIndex = content.indexOf('T_', prefixIndex)) !== -1) {
+        /* Check this is start of identifier (not false positive) */
+        if (prefixIndex > 0 && !isSpace(content[prefixIndex - 1])) {
+            prefixIndex += 2;
+            continue;
+        }
+
+        /* Search for end of identifier */
+        let endOfIdent = prefixIndex + 2;
+        while (endOfIdent < content.length && isIdentifierChar(content[endOfIdent])) {
+            endOfIdent++;
+        }
+
+        /* End of file - should not happen */
+        if (content.length <= endOfIdent) {
+            break;
+        }
+
+        const tag = content.substring(prefixIndex + 2, endOfIdent);
+        nodeTags.push(tag);
+        prefixIndex = endOfIdent + 1;
+    }
+    
+    return new Set(nodeTags);
+}