RenameProvider & a lot of supporting infrastructure (#160)

RenameProvider & a lot of supporting infrastructure (#160)

.vscode/launch.json

@@ -15,6 +15,21 @@
 			"outFiles": ["${workspaceRoot}/extension/out/**/*.js"],
             "sourceMaps": true
         },
+        {
+            "name": "Extension Client + Workspace",
+            "type": "extensionHost",
+            "request": "launch",
+            "preLaunchTask": "buildproject",
+            "runtimeExecutable": "${execPath}",
+			"args": [
+                "--extensionDevelopmentPath=${workspaceRoot}",
+                "-g", 
+                "${workspaceFolder}/extension/src/test/SourceFile/renaming",
+                "--disable-extensions"
+            ],
+			"outFiles": ["${workspaceRoot}/extension/out/**/*.js"],
+            "sourceMaps": true
+        },
         {
             "name": "Extension Tests",
             "type": "extensionHost",
@@ -23,7 +38,10 @@
             "runtimeExecutable": "${execPath}",
             "args": [
                 "--extensionDevelopmentPath=${workspaceFolder}",
-                "--extensionTestsPath=${workspaceFolder}/out/test/suite/index"
+                "--extensionTestsPath=${workspaceFolder}/out/test/suite/index",
+                "-g", 
+                "${workspaceFolder}/extension/src/test/SourceFile/renaming",
+                "--disable-extensions"
             ],
             "outFiles": [
                 "${workspaceFolder}/out/test/**/*.js"
@@ -41,7 +59,10 @@
             "runtimeExecutable": "${execPath}",
             "args": [
                 "--extensionDevelopmentPath=${workspaceFolder}",
-                "--extensionTestsPath=${workspaceFolder}/out/test/suite/codeCoverage"
+                "--extensionTestsPath=${workspaceFolder}/out/test/suite/codeCoverage",
+                "-g", 
+                "${workspaceFolder}/extension/src/test/SourceFile/renaming",
+                "--disable-extensions"
             ],
             "outFiles": [
                 "${workspaceFolder}/out/test/**/*.js"

extension/src/commands.ts

@@ -1,12 +1,12 @@
 import * as vscode from "vscode";
 import * as nls from 'vscode-nls';
 import { AutoLispExt } from './extension';
-import { LispContainer } from './format/sexpression';
 import { openWebHelp } from './help/openWebHelp';
 import { generateDocumentationSnippet, getDefunArguments, getDefunAtPosition } from './help/userDocumentation';
 import { showErrorMessage } from './project/projectCommands';
 import { AutolispDefinitionProvider } from './providers/gotoProvider';
-import * as shared from './providers/providerShared';
+import { AutoLispExtPrepareRename, AutoLispExtProvideRenameEdits } from './providers/renameProvider';
+import { SymbolManager } from './symbols';
 
 const localize = nls.loadMessageBundle();
 
@@ -73,4 +73,48 @@ export function registerCommands(context: vscode.ExtensionContext){
 	}));
 
 	AutoLispExt.Subscriptions.push(vscode.languages.registerDefinitionProvider([ 'autolisp', 'lisp'], new AutolispDefinitionProvider()));
+
+	const msgRenameFail = localize("autolispext.providers.rename.failed", "The symbol was invalid for renaming operations");
+	AutoLispExt.Subscriptions.push(vscode.languages.registerRenameProvider(['autolisp', 'lisp'], {
+		prepareRename: function (document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken)
+							    : vscode.ProviderResult<vscode.Range | { range: vscode.Range; placeholder: string; }> 
+		{
+			// Purpose: collect underlying symbol range and feed it as rename popup's initial value
+			try {
+				// offload all meaningful work to something that can be tested.
+				const result = AutoLispExtPrepareRename(document, position);
+				if (!result) {
+					return;
+				}
+				return result;
+			} catch (err) {
+				if (err){
+					showErrorMessage(msgRenameFail, err);
+				}
+			}
+		},
+
+		provideRenameEdits: function (document: vscode.TextDocument, position: vscode.Position, newName: string, token: vscode.CancellationToken)
+						             : vscode.ProviderResult<vscode.WorkspaceEdit> 
+		{
+			// Purpose: only fires if the user provided rename popup with a tangible non-equal value. From here, our
+			//			goal is to find & generate edit information for all valid rename targets within known documents
+			try {
+				// offload all meaningful work to something that can be tested.
+				const result = AutoLispExtProvideRenameEdits(document, position, newName);
+				if (!result) {
+					return;
+				}
+				// subsequent renaming operations could pull outdated cached symbol maps if we don't clear the cache.
+				SymbolManager.invalidateSymbolMapCache();
+				return result;
+			} catch (err) {
+				if (err){
+					showErrorMessage(msgRenameFail, err);
+				}
+			}
+		}
+	}));
+
+
 }

extension/src/documents.ts

@@ -5,12 +5,15 @@ import { AutoLispExt } from './extension';
 import { ProjectTreeProvider  } from "./project/projectTree";
 import * as fs from	'fs-extra';
 import { glob } from 'glob';
+import { DocumentServices } from './services/documentServices';
+import { SymbolManager } from './symbols';
 
 
 enum Origins {
 	OPENED,
 	WSPACE,
-	PROJECT
+	PROJECT,
+	UNKNOWN
 }
 
 
@@ -127,12 +130,18 @@ export class DocumentManager{
 	}
 
 	private normalizePath(path: string): string {
-		return path.replace(/\//g, '\\');
+		return path.replace(/\\/g, '/');
 	}
 
 	private tryUpdateInternal(sources: DocumentSources){
 		if (sources.native && (!sources.internal || !sources.internal.equal(sources.native))) {
-			sources.internal = ReadonlyDocument.getMemoryDocument(sources.native);
+			sources.internal = ReadonlyDocument.getMemoryDocument(sources.native);			
+			if (DocumentServices.hasUnverifiedGlobalizers(sources.internal)) {
+				// symbol mapping actually takes slightly more time than parsing, so the goal is
+				// to keep a persistent representation of anything containing @global exported
+				// variable/function name. Use cases are Rename, Autocomplete and Signature helpers
+				SymbolManager.updateOrCreateSymbolMap(sources.internal, true);
+			}
 		}
 	}
 
@@ -176,7 +185,17 @@ export class DocumentManager{
 	getDocument(nDoc: vscode.TextDocument): ReadonlyDocument {
 		const key = this.documentConsumeOrValidate(nDoc, Origins.OPENED);			
 		return this._cached.get(key)?.internal;
-	}	
+	}
+
+	tryGetDocument(fsPath: string): ReadonlyDocument {
+		// This is something of a hack to query an existing document from a randomly acquired file path.
+		// ultimately, if the LSP file exists, it will return a document, but it may or may not be part
+		// of the "normal sources" and thats why the origin is unknown. More often than not, this will
+		// yield what it already processed in a production setting. However, while running tests this
+		// UNKNOWN version could be the only origin and won't be useable in any "non-testing operations"
+		const key = this.pathConsumeOrValidate(DocumentServices.normalizeFilePath(fsPath), Origins.UNKNOWN);
+		return this._cached.get(key)?.internal;
+	}
 
 	// Gets an array of PRJ ReadonlyDocument references, but verifies the content is based on a vscode.TextDocument's when available
 	private getProjectDocuments(): ReadonlyDocument[] {

extension/src/format/sexpression.ts

@@ -2,11 +2,19 @@ import { closeParenStyle, maximumLineChars, longListFormatStyle, indentSpaces }
 import { isInternalAutoLispOp } from '../completion/autocompletionProvider';
 import { Position, Range } from 'vscode';
 
+// General purpose test for basic known primitive values; Including: T, NIL, Number, (single or multi-line) Strings & Comments
+export const primitiveRegex = /^([\(\)\'\.]|"[\s\S]*"|;[\s\S]*|'?[tT]|'?[nN][iI][lL]|'?-?\d+[eE][+-]?\d+|-?\d+|-?\d+\.\d+)$/;
+const primitiveGlyphs = ['\'', '(', ')', '.', ';']; //, '']; //, null, undefined];
+
 // This interface is intended to let us work in more generic ways without direct context of LispAtom|LispContainer
 export interface ILispFragment {
     symbol: string;
     line: number;
     column: number;
+    flatIndex: number;
+    commentLinks?: Array<number>;
+    hasGlobalFlag?: boolean;
+
 
     readonly body?: LispContainer|undefined;
 
@@ -18,20 +26,33 @@ export interface ILispFragment {
     isComment(): boolean;
     isRightParen(): boolean;
     isLeftParen(): boolean;
+    isPrimitive(): boolean;
     contains(pos: Position): boolean;
     getRange(): Range;
 }
 
 // Represents the most fundamental building blocks of a lisp document
 export class LispAtom implements ILispFragment {
     public symbol: string;
-    public line: number;
-    public column: number;
-
-    constructor(line: number, column: number, sym: string) {
-        this.line = line;
-        this.column = column;
+    protected _line: number;
+    protected _column: number;
+
+    get line(): number { return this._line; }
+    set line(value) { this._line = value; }
+    get column(): number { return this._column; }
+    set column(value) { this._column = value; }
+
+    // These 3 fields exist to support comment driven intelligence. Creation of Symbol mappings is an expensive operation
+    // and these 2 fields prevent ~80% of the required overhead when working in concert with the highly efficient parser
+    public flatIndex: number;
+    public commentLinks?: Array<number>;
+    public hasGlobalFlag?: boolean;
+
+    constructor(line: number, column: number, sym: string, flatIdx = -1) {
+        this._line = line;
+        this._column = column;
         this.symbol = sym;
+        this.flatIndex = flatIdx;
     }
 
 
@@ -99,6 +120,14 @@ export class LispAtom implements ILispFragment {
 		return this.symbol === '(';
     }
 
+    isPrimitive(): boolean {
+        // if (!this['atoms']) {
+        //     return primitiveRegex.test(this.symbol);
+        // }
+        // return false;
+        return primitiveGlyphs.indexOf(this.symbol[0]) > -1
+               || primitiveRegex.test(this.symbol);
+    }
 
     // Returns true if this LispAtom encapsulates the provided Position
     contains(position: Position): boolean {
@@ -131,15 +160,20 @@ export class LispAtom implements ILispFragment {
 export class LispContainer extends LispAtom {
     atoms: Array<ILispFragment>;
     linefeed: string;
+    userSymbols?: Map<string, Array<number>>; // this would only show up on document root LispContainers
 
     // pass through getter for the ILispFragment interface
     get body(): LispContainer { return this; }
+    get line(): number { return this.userSymbols ? 0 : this.getFirstAtom().line; }
+    set line(value) { }   // setter exists only to satisfy contract
+    get column(): number { return this.userSymbols ? 0 : this.getFirstAtom().column; }
+    set column(value) { } // setter exists only to satisfy contract
 
     // LispContainer constructor defaults to a clearly uninitialized state
-    constructor(startIndex: number = -1) {
-        super(startIndex,startIndex,'');
+    constructor(lineFeed: string = '\n') {
+        super(-1,-1,'');
         this.atoms = [];
-        this.linefeed = '\n';
+        this.linefeed = lineFeed;
     }
 
 
@@ -241,9 +275,18 @@ export class LispContainer extends LispAtom {
 
     // Gets a range representing the full LispContainer, especially useful for TextDocument.getText()
     getRange(): Range {
-        const begin: ILispFragment = this.atoms[0];
-        const close: ILispFragment = this.atoms[this.atoms.length -1];
-        return new Range(begin.line, begin.column, close.line, (close.column + close.symbol.length));
+        const begin = this.getFirstAtom().getRange().start;
+        const close = this.getLastAtom().getRange().end;
+        return new Range(begin, close);
+    }
+
+    private getFirstAtom(): ILispFragment {
+        const tail = this.atoms[0];
+        return tail.body?.getFirstAtom() ?? tail;
+    }
+    private getLastAtom(): ILispFragment {
+        const tail = this.atoms[this.atoms.length - 1];
+        return tail.body?.getLastAtom() ?? tail;
     }
 
 
@@ -310,9 +353,68 @@ export class LispContainer extends LispAtom {
         }
         return result;
     }
-}
 
+    // Performance Note: the LispContainer parser takes ~700ms to create the tree from a 12.5mb (20K Lines/1.4M LispAtoms)
+    //                   file. In contrast, the older parsing used on formatting before it received some fixes was taking
+    //                   more than a minute and the revised version is still taking ~7000ms.
+    //                   To flatten the same 12.5mb tree for linear traversal takes ~40ms. The flattening process is
+    //                   considered to be an instantaneous operation even under extreme and highly improbable situations.
+    flatten(into?: Array<LispAtom>): Array<LispAtom> {
+        if (!into) {
+            into= [];
+        }
+        this.atoms.forEach(item => {
+            if (item instanceof LispContainer) {                
+                item.flatten(into);
+            } else if (item instanceof LispAtom) {
+                into.push(item);
+            }
+        });
+        return into;
+    }
 
+
+    // This is only used for verification of the LispContainer parser, but could have future formatting usefullness.
+    // Note1: test files cannot contain lines with only random whitespace or readable character lines that include
+    //        trailing whitespace because a LispContainer has no context to reproduce that. 
+    // Note2: tabs between atoms will be replaced with spaces and thus are not supported in test files.
+    asText(context?: IContainerStringCompilerContext): string {
+        let isRoot = false;
+        if (context === undefined) {
+            context = { result: '', line: 0, column: 0 };
+            isRoot = true;
+        }
+        this.atoms.forEach(item => {
+            while (context.line < item.line) {
+                context.result += this.linefeed;
+                context.line++;
+                context.column = 0;
+            }
+            while (context.column < item.column) {
+                context.result += ' ';
+                context.column++;
+            }
+            if (item instanceof LispContainer) {                
+                item.asText(context);
+            }
+            else {
+                context.result += item.symbol;
+                context.column += item.symbol.length;
+                if (item.isComment() && !item.isLineComment()) {
+                    context.line += item.symbol.split(this.linefeed).length - 1;
+                }
+            }
+        });        
+        return isRoot ? context.result : '';
+    }
+
+}
+
+interface IContainerStringCompilerContext {
+    result: string;
+    line: number;
+    column: number;
+}