Moving browser version to new parser

7 files changed, 653 insertions, 271 deletions

diff --git a/browser-ext/package-lock.json b/browser-ext/package-lock.json
index 878a5e1..d24adb9 100644
--- a/browser-ext/package-lock.json
+++ b/browser-ext/package-lock.json
@@ -10,7 +10,9 @@
 			"license": "MIT",
 			"dependencies": {
 				"vscode-languageserver": "^7.0.0",
-				"vscode-languageserver-textdocument": "^1.0.1"
+				"vscode-languageserver-textdocument": "^1.0.1",
+				"vscode-oniguruma": "^2.0.1",
+				"vscode-textmate": "^9.0.0"
 			},
 			"engines": {
 				"node": "*"
@@ -53,6 +55,16 @@
 			"version": "3.16.0",
 			"resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz",
 			"integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA=="
+		},
+		"node_modules/vscode-oniguruma": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz",
+			"integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ=="
+		},
+		"node_modules/vscode-textmate": {
+			"version": "9.0.0",
+			"resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.0.0.tgz",
+			"integrity": "sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg=="
 		}
 	},
 	"dependencies": {
@@ -87,6 +99,16 @@
 			"version": "3.16.0",
 			"resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz",
 			"integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA=="
+		},
+		"vscode-oniguruma": {
+			"version": "2.0.1",
+			"resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-2.0.1.tgz",
+			"integrity": "sha512-poJU8iHIWnC3vgphJnrLZyI3YdqRlR27xzqDmpPXYzA93R4Gk8z7T6oqDzDoHjoikA2aS82crdXFkjELCdJsjQ=="
+		},
+		"vscode-textmate": {
+			"version": "9.0.0",
+			"resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.0.0.tgz",
+			"integrity": "sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg=="
 		}
 	}
 }
diff --git a/browser-ext/package.json b/browser-ext/package.json
index 7091a32..0fe19c1 100644
--- a/browser-ext/package.json
+++ b/browser-ext/package.json
@@ -13,7 +13,9 @@
 	},
 	"dependencies": {
 		"vscode-languageserver": "^7.0.0",
-		"vscode-languageserver-textdocument": "^1.0.1"
+		"vscode-languageserver-textdocument": "^1.0.1",
+		"vscode-textmate": "^9.0.0",
+		"vscode-oniguruma": "^2.0.1"
 	},
 	"scripts": {}
 }
\ No newline at end of file
diff --git a/browser-ext/src/browserServerMain.ts b/browser-ext/src/browserServerMain.ts
index 55e328d..5d95050 100644
--- a/browser-ext/src/browserServerMain.ts
+++ b/browser-ext/src/browserServerMain.ts
@@ -6,8 +6,8 @@ import { createConnection, BrowserMessageReader, BrowserMessageWriter, SymbolInf
 
 import { Color, ColorInformation, Range, InitializeParams, InitializeResult, ServerCapabilities, TextDocuments, ColorPresentation, TextEdit, TextDocumentIdentifier } from 'vscode-languageserver';
 import { TextDocument } from 'vscode-languageserver-textdocument';
-import { PerlDocument, PerlElem } from "./web-types";
-import { buildNav } from "./web-parse";
+import { ParseType, PerlDocument, PerlElem } from "./web-types";
+import { parseDocument } from "./web-parse";
 
 import { getDefinition } from "./web-navigation";
 import { getSymbols } from "./web-symbols";
@@ -83,15 +83,18 @@ documents.onDidChangeContent(change => {
 
 async function validatePerlDocument(textDocument: TextDocument): Promise<void> {
 	// console.log("Rebuilding symbols for " + textDocument.uri + "");
-    const perlDoc = await buildNav(textDocument);
+    const perlDoc = await parseDocument(textDocument, ParseType.selfNavigation);
     navSymbols.set(textDocument.uri, perlDoc);
     return;
 
 }
 
 
-connection.onDocumentSymbol(params => {
-    return getSymbols(navSymbols, params.textDocument.uri);
+connection.onDocumentSymbol(async params => {
+    let document = documents.get(params.textDocument.uri);
+    // We might  need to async wait for the document to be processed, but I suspect the order is fine
+    if(!document) return;
+    return getSymbols(document, params.textDocument.uri);
 });
 
 // This handler provides the initial list of the completion items.
diff --git a/browser-ext/src/web-navigation.ts b/browser-ext/src/web-navigation.ts
index 960d9f3..623aa02 100644
--- a/browser-ext/src/web-navigation.ts
+++ b/browser-ext/src/web-navigation.ts
@@ -1,13 +1,11 @@
 import {
     DefinitionParams,
     Location,
-    WorkspaceFolder
 } from 'vscode-languageserver/browser';
 import {
     TextDocument
 } from 'vscode-languageserver-textdocument';
 import { PerlDocument, PerlElem, NavigatorSettings } from "./web-types";
-import { realpathSync, existsSync, realpath } from 'fs';
 import { getSymbol, lookupSymbol } from "./web-utils";
 
 
@@ -19,7 +17,7 @@ export function getDefinition(params: DefinitionParams, perlDoc: PerlDocument, t
     if(!symbol) return;
 
     const foundElems = lookupSymbol(perlDoc, symbol, position.line);
-
+    
     if(foundElems.length == 0){
         return;
     }
diff --git a/browser-ext/src/web-parse.ts b/browser-ext/src/web-parse.ts
index 867ff07..f5db7b6 100644
--- a/browser-ext/src/web-parse.ts
+++ b/browser-ext/src/web-parse.ts
@@ -1,264 +1,600 @@
 
-import { PerlDocument, PerlElem, PerlImport, PerlSymbolKind} from "./web-types";
+import { PerlDocument, PerlElem,  PerlSymbolKind, ParseType} from "./web-types";
 import { TextDocument } from 'vscode-languageserver-textdocument';
 
+import vsctm = require('vscode-textmate');
+import oniguruma = require('vscode-oniguruma');
+
+function init_doc (textDocument: TextDocument): PerlDocument {
 
-// Why is this async? It doesn't do anything async yet
-export async function buildNav(textDocument: TextDocument): Promise<PerlDocument> {
 
     let perlDoc: PerlDocument = {
-            elems: new Map(),
-            canonicalElems: new Map(),
-            imported: new Map(),
-            parents: new Map(),
-            filePath: '',
-            uri: textDocument.uri,
-        };
-
-    buildPlTags(textDocument, perlDoc);
-    
+        elems: new Map(),
+        canonicalElems: new Map(),
+        autoloads: new Map(),
+        imported: new Map(),
+        parents: new Map(),
+        filePath: '',
+        uri: textDocument.uri,
+    };
+
     return perlDoc;
 }
 
+type ParserState = {
+    stmt: string;
+    line_number: number;
+    var_continues: boolean;
+    package_name: string;
+    file: string;
+    perlDoc: PerlDocument;
+    parseType: ParseType;
+    codeArray: string[];
+};
+
+type ParseFunc = (state: ParserState) => boolean;
+
+
+
+export async function parseDocument(textDocument: TextDocument, parseType: ParseType ): Promise<PerlDocument> {
+   
+    let parseFunctions: ParseFunc[] = [];
+    if(parseType == ParseType.outline){
+        parseFunctions = [packages, subs, labels, constants, fields, imports, dancer];
+    } else if(parseType == ParseType.selfNavigation){
+        parseFunctions = [knownObj, localVars, packages, subs, labels, constants, fields, imports, autoloads, dancer];
+    }
+
+    let perlDoc = init_doc(textDocument);
+
+    let state: ParserState = {
+        stmt: '',
+        line_number: 0,
+        package_name: '',
+        perlDoc: perlDoc,
+        file: textDocument.uri,
+        var_continues: false,
+        codeArray: await cleanCode(textDocument, perlDoc, parseType),
+        parseType: parseType
+    };
+
+
+    for (state.line_number= 0; state.line_number < state.codeArray.length; state.line_number++) {
+        state.stmt = state.codeArray[state.line_number];
+        // Nothing left? Never mind.
+        if (!state.stmt) {
+            continue;
+        }
+           
+        parseFunctions.some(fn => fn(state));
+    }
+    return perlDoc;    
+}
+
+
+function knownObj(state: ParserState) : boolean{
+    let match;
+    
+        // TODO, allow specifying list of constructor names as config
+        // Declaring an object. Let's store the type
+        // my $constructors = qr/(?:new|connect)/;
+    if( (match = state.stmt.match(/^(?:my|our|local|state)\s+(\$\w+)\s*\=\s*([\w\:]+)\-\>new\s*(?:\((?!.*\)\->)|;)/ )) ||
+        (match = state.stmt.match(/^(?:my|our|local|state)\s+(\$\w+)\s*\=\s*new (\w[\w\:]+)\s*(?:\((?!.*\)\->)|;)/))) {
+        let varName = match[1];
+        let objName = match[2];
+        MakeElem(varName, PerlSymbolKind.LocalVar, objName, state);
+
+        state.var_continues = false; // We skipped ahead of the line here. Why though?
+        return true;
+    } else {
+        return false;
+    }
+}
+
+function localVars (state: ParserState): boolean {
+    
+    // This is a variable declaration if one was started on the previous
+    // line, or if this line starts with my or local
+    let match;
+    if (state.var_continues || (match = state.stmt.match(/^(?:my|our|local|state)\b/))) {
+        // The declaration continues if the line does not end with ;
+        state.var_continues = (!state.stmt.endsWith(";") && !state.stmt.match(/[\)\=\}\{]/));
+        let mod_stmt = state.stmt;
+        // Remove my or local from statement, if present
+        mod_stmt = mod_stmt.replace(/^(my|our|local|state)\s+/, "");
+
+        // Remove any assignment piece
+        mod_stmt = mod_stmt.replace(/\s*=.*/, "");
+
+        // Remove part where sub starts (for signatures). Consider other options here.
+        mod_stmt = mod_stmt.replace(/\s*\}.*/, "");
+
+        // Now find all variable names, i.e. "words" preceded by $, @ or %
+        let vars = mod_stmt.matchAll(/([\$\@\%][\w:]+)\b/g);
+
+        for (let match of vars) {
+            MakeElem(match[1], PerlSymbolKind.LocalVar, '', state);
+        }
+        return true;
+    } 
+    
+    // Lexical loop variables, potentially with labels in front. foreach my $foo
+    else if ((match = state.stmt.match(/^(?:(\w+)\s*:(?!\:))?\s*(?:for|foreach)\s+my\s+(\$[\w]+)\b/))) {
+        if (match[1]) {
+            MakeElem(match[1], PerlSymbolKind.Label, '', state);
+        }
+        MakeElem(match[2], PerlSymbolKind.LocalVar, '', state);
+    }
+    // Lexical match variables if(my ($foo, $bar) ~= ). Optional to detect (my $newstring = $oldstring) =~ s/foo/bar/g;
+    else if ((match = state.stmt.match(/^(?:\}\s*elsif|if|unless|while|until|for)?\s*\(\s*my\b(.*)$/))) {
+        // Remove any assignment piece
+        const mod_stmt = state.stmt.replace(/\s*=.*/, "");
+        let vars = mod_stmt.matchAll(/([\$\@\%][\w]+)\b/g);
+        for (let match of vars) {
+            MakeElem(match[1], PerlSymbolKind.LocalVar, '', state);
+        }
+    }
+
+    // Try-catch exception variables
+    else if ((match = state.stmt.match(/^\}?\s*catch\s*\(\s*(\$\w+)\s*\)\s*\{?$/))) {
+        MakeElem(match[1], PerlSymbolKind.LocalVar, '', state);
+    }
+
+    else {
+        return false;
+    }
+
+    return true;
+}
+
+function packages(state: ParserState) : boolean{
+    // This is a package declaration if the line starts with package
+    let match;
+
+    if ((match = state.stmt.match(/^package\s+([\w:]+)/))) {
+        // Get name of the package
+        state.package_name = match[1];
+        const endLine = PackageEndLine(state);
+        MakeElem(state.package_name, PerlSymbolKind.Package, '', state, endLine);
+    } 
+    
+    // This is a class decoration for Object::Pad, Corinna, or Moops 
+    else if((match = state.stmt.match(/^class\s+([\w:]+)/))){
+        let class_name = match[1];
+        state.package_name = class_name;
+        const endLine = PackageEndLine(state);
+        MakeElem(class_name, PerlSymbolKind.Class, '', state, endLine);
+    }
+    
+    else if((match = state.stmt.match(/^role\s+([\w:]+)/))){
+        const roleName = match[1];
+        // state.package_name = roleName; # Being cautious against changing the package name
+        const endLine = SubEndLine(state);
+        MakeElem(roleName, PerlSymbolKind.Role, '', state, endLine);
+    }
+
+    else {
+        return false;
+    }
+
+    return true;
+}
+
+function subs(state: ParserState) : boolean {
+
+    let match;
+    // This is a sub declaration if the line starts with sub
+    if ((match = state.stmt.match(/^(?:async\s+)?(sub)\s+([\w:]+)(\s+:method)?([^{]*)/)) ||
+            (match = state.stmt.match(/^(?:async\s+)?(method)\s+\$?([\w:]+)()([^{]*)/)) ||
+            (state.perlDoc.imported.has("Function::Parameters") && (match = state.stmt.match(/^(fun)\s+([\w:]+)()([^{]*)/ )))
+            ) {
+        const subName = match[2];
+        const signature = match[4];
+        const kind = (match[1] === 'method' || match[3]) ? PerlSymbolKind.LocalMethod : PerlSymbolKind.LocalSub;
+        const endLine = SubEndLine(state);
+
+        MakeElem(subName, kind, '', state, endLine);
+        // Match the after the sub declaration and before the start of the actual sub for signatures (if any)
+        const vars = signature.matchAll(/([\$\@\%][\w:]+)\b/g);
+
+        // Define subrountine signatures, but exclude prototypes
+        // The declaration continues if the line does not end with ;
+        state.var_continues = !(state.stmt.match(/;$/) || state.stmt.match(/[\)\=\}\{]/));
+
+        for (const matchvar of vars) {
+            MakeElem(matchvar[1], PerlSymbolKind.LocalVar,'', state);
+        }
+    } else {
+        return false;
+    }
+    return true;
+}
+
+function labels(state: ParserState) : boolean {
+
+    let match;
+    // Phaser block
+    if ((match = state.stmt.match(/^(BEGIN|INIT|CHECK|UNITCHECK|END)\s*\{/))) {
+        const phaser = match[1];
+        const endLine = SubEndLine(state);
+
+        MakeElem(phaser, PerlSymbolKind.Phaser, '', state, endLine);
+    }
+
+    // Label line
+    else if ((match = state.stmt.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s*:[^:].*{\s*$/))) {
+        const label = match[1];
+        const endLine = SubEndLine(state);
+
+        MakeElem(label, PerlSymbolKind.Label, '', state, endLine);
+    }
+
+    else {
+        return false;
+    }
+
+    return true;
+}
+
+function constants(state: ParserState) : boolean {
+
+    let match;
+    // Constants. Important because they look like subs (and technically are), so I'll tags them as such 
+    if ((match = state.stmt.match(/^use\s+constant\s+(\w+)\b/))) {
+        MakeElem(match[1], PerlSymbolKind.Constant, '', state);
+        MakeElem("constant", 'u', '', state);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+function fields(state: ParserState) : boolean {
+
+    let match;
+    // Moo/Moose/Object::Pad/Moops/Corinna attributes
+    if ((match = state.stmt.match(/^(?:has|field)(?:\s+|\()["']?([\$@%]?\w+)\b/))) { 
+        const attr = match[1];
+        let type;
+        if(attr.match(/^\w/)){
+            type = PerlSymbolKind.Field;
+            // If you have a locally defined package/class Foo want to reference the attributes as Foo::attr or foo->attr, you need the full path.
+            // Subs don't need this since we find them at compile time. We also find "d" types from imported packages in Inquisitor.pm
+            MakeElem(state.package_name + "::" + attr, PerlSymbolKind.PathedField, '', state);
+        } else {
+            type = PerlSymbolKind.LocalVar;
+        } 
+        // TODO: Define new type. Class variables should probably be shown in the Outline view even though lexical variables are not
+        MakeElem(attr, type, '', state);
+    }
+
+    // Is this captured above?
+    // else if (state.perlDoc.imported.has("Object::Pad") &&
+    //         (match = stmt.match(/^field\s+([\$@%]\w+)\b/))) { //  Object::Pad field
+    //     const attr = match[1];
+    //     MakeElem(attr, PerlSymbolKind.LocalVar, '', file, package_name, line_num, perlDoc);
+    // }
+
+    else if ((state.perlDoc.imported.has("Mars::Class") || state.perlDoc.imported.has("Venus::Class"))
+            && (match = state.stmt.match(/^attr\s+["'](\w+)\b/))) { // Mars attributes
+        const attr = match[1];
+        MakeElem(attr, PerlSymbolKind.Field, '', state);
+        MakeElem(state.package_name + "::" + attr, PerlSymbolKind.PathedField, '', state);
+    }
+
+    else if ((match = state.stmt.match(/^around\s+["']?(\w+)\b/))) { // Moo/Moose overriding subs. 
+        MakeElem(match[1], PerlSymbolKind.LocalSub, '', state);
+    } 
+
+    else {
+        return false;
+    }
+    return true;
+}
+
+function imports(state: ParserState) : boolean {
+
+    let match;
+    if ((match = state.stmt.match(/^use\s+([\w:]+)\b/))) { // Keep track of explicit imports for filtering
+        const importPkg = match[1];
+        MakeElem(importPkg, "u", '', state);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+function autoloads(state: ParserState) : boolean {
+    let match;
+    if ((match = state.stmt.match(/^\$self\->\{\s*(['"]|)_(\w+)\1\s*\}\s*=/))) { // Common paradigm is for autoloaders to basically just point to the class variable
+        const variable = match[2];
+        MakeElem("get_" + variable, PerlSymbolKind.AutoLoadVar, '', state);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+
+function dancer(state: ParserState): boolean {
+    if(!(state.perlDoc.imported.has("Dancer") || state.perlDoc.imported.has("Dancer2") || state.perlDoc.imported.has("Mojolicious::Lite"))) {
+        return false;
+    }
+
+    //const rFilter = /qr\{[^\}]+\}/ ;
+    let match;
+    if( (match = state.stmt.match(/^(?:any|before\_route)\s+\[([^\]]+)\]\s+(?:=>\s*)?(['"])([^"']+)\2\s*=>\s*sub/))) {
+        // Multiple request routing paths
+        let requests = match[1];
+        let route = match[3];
+        // TODO: Put this back
+        requests = requests.replace(/['"\s\n]+/g, "");
+        route = `${requests} ${route}`;
+        const endLine = SubEndLine(state);
+        MakeElem(route, PerlSymbolKind.HttpRoute, '', state, endLine);
+
+        // TODO: I think this is a bug with [^\2] not working
+                                                                                                     // any ['get', 'post'] => '/login' => sub {
+
+    } else if ((match = state.stmt.match(/^(get|any|post|put|patch|delete|del|options|ajax|before_route)\s+(?:[\s\w,\[\]'"]+=>\s*)?(['"])([^'"]+)\2\s*=>\s*sub/))) {
+        // Routing paths
+        let route = match[1] + " " + match[3];
+        const endLine = SubEndLine(state);
+        MakeElem(route, PerlSymbolKind.HttpRoute, '', state, endLine);
+    } else if ((match = state.stmt.match(/^(get|any|post|put|patch|delete|del|options|ajax|before_route)\s+(qr\{[^\}]+\})\s+\s*=>\s*sub/))) {
+        //  Regexp routing paths
+        let route = match[1] + " " + match[2];
+        const endLine = SubEndLine(state);
+        MakeElem(route, PerlSymbolKind.HttpRoute, '', state, endLine);
+    } else if ((match = state.stmt.match(/^(?:hook)\s+(['"]|)(\w+)\1\s*=>\s*sub/))) {
+        // Hooks
+        let hook = match[2];
+        const endLine = SubEndLine(state);
+        MakeElem(hook, PerlSymbolKind.HttpRoute, '', state, endLine);
+    } else {
+        return false;
+    }
+    return true; //  Must've matched
+}
+
+
+async function cleanCode(textDocument: TextDocument, perlDoc: PerlDocument, parseType: ParseType): Promise<string[]> {
+    let code = textDocument.getText();
+
+    const codeArray = code.split("\n");
+    // const offset = textDocument.offsetAt(textDocument.positionAt(0));
+    let codeClean = [];
+
+
+    let commentState: ParserState = {
+        stmt: '',
+        line_number: 0,
+        package_name: '',
+        perlDoc: perlDoc,
+        file: textDocument.uri,
+        var_continues: false,
+        codeArray: codeArray,
+        parseType: parseType
+    };
+
+    for (commentState.line_number=0; commentState.line_number<codeArray.length;commentState.line_number++){
+        commentState.stmt = codeArray[commentState.line_number];
+
+        let match;
+        if (parseType == ParseType.selfNavigation && (match = commentState.stmt.match(/#.*(\$\w+) isa ([\w:]+)\b/))){
+           const pvar = match[1];
+           const typeName = match[2];
+           // TODO: Do I need a file or package here? Canonical variables are weird
+           MakeElem(pvar, PerlSymbolKind.Canonical, typeName, commentState);
+        }
+        
+        let mod_stmt = commentState.stmt;
+        mod_stmt = mod_stmt.replace(/^\s*/, "");
+        mod_stmt = mod_stmt.replace(/\s*$/, "");
+
+        codeClean.push(mod_stmt);
+    }
+
+    // if(parseType == ParseType.outline){
+    //     // If only doing shallow parsing, we don't need to strip {} or find start-end points of subs
+    //     codeClean = await stripCommentsAndQuotes(codeClean);
+    // }
+
+    return codeClean;
+}
+
+ 
+
 
 
-function MakeElem(name: string, type: PerlSymbolKind | 'u' | '1' | '2', typeDetail: string, file: string, pack:string, line:number, perlDoc: PerlDocument) : void{
+function MakeElem(name: string, type: PerlSymbolKind | 'u' | '2',
+    typeDetail: string, state: ParserState, lineEnd: number = 0): void {
 
     if(!name) return; // Don't store empty names (shouldn't happen)
 
-    if (type == '1'){
-        // This object is only intended as the canonicalLookup, not for anything else.
-        return;
+    if(lineEnd == 0){
+        lineEnd = state.line_number
     }
 
     if (type == 'u'){
         // Explictly loaded module. Helpful for focusing autocomplete results
-        perlDoc.imported.set(name, line);
+        state.perlDoc.imported.set(name, state.line_number);
         // if(/\bDBI$/.exec(name)) perlDoc.imported.set(name + "::db", true); // TODO: Build mapping of common constructors to types
         return; // Don't store it as an element
     } 
 
     if (type == '2'){
-        perlDoc.parents.set(name, typeDetail);
+        state.perlDoc.parents.set(name, typeDetail);
         return; // Don't store it as an element
     } 
-
+    
     const newElem: PerlElem = {
         name: name,
         type: type,
         typeDetail: typeDetail,
-        file: file,
-        package: pack,
-        line: line,
-        lineEnd: line,
+        file: state.file,
+        package: state.package_name,
+        line: state.line_number,
+        lineEnd: lineEnd,
         value: "",
     };
 
-    // Move fancy object types into the typeDetail field????
-    if (type.length > 1){
-        // We overwrite, so the last typed element is the canonical one. No reason for this.
-        perlDoc.canonicalElems.set(name, newElem);
+    if (type == '3'){
+        state.perlDoc.autoloads.set(name, newElem);
+        return; // Don't store it as an element
     } 
 
-    let array = perlDoc.elems.get(name) || [];
+
+    if(typeDetail.length > 0){
+        // TODO: The canonicalElems don't need to be PerlElems, they might be just a string.
+        // We overwrite, so the last typed element is the canonical one. No reason for this.
+        state.perlDoc.canonicalElems.set(name, newElem);
+        if (type == '1'){
+            // This object is only intended as the canonicalLookup, not for anything else.    
+            return;
+        } 
+    }
+
+    let array = state.perlDoc.elems.get(name) || [];
     array.push(newElem)
-    perlDoc.elems.set(name, array);
+    state.perlDoc.elems.set(name, array);
 
     return;
 }
 
 
+function SubEndLine (state: ParserState, rFilter: RegExp | null = null) : number {
+ 
+    let pos = 0;
+    let found = false;
+    if(state.parseType != ParseType.outline){
+        return state.line_number;
+    }
 
-function buildPlTags(textDocument: TextDocument, perlDoc: PerlDocument) {
-    const codeArray = cleanCode(textDocument);
-    let sActiveOO: Map<string, boolean> = new Map(); // Keep track of OO frameworks in use to keep down false alarms on field vs has vs attr
-    // Loop through file
-    const file = textDocument.uri;
-    let package_name  = "";
-    let var_continues: boolean = false;
-
-    for (let i = 0; i < codeArray.length; i++) {
-        let line_number = i;
-        let stmt = codeArray[i];
-        // Nothing left? Never mind.
-        if (!stmt) {
-            continue;
-        }
-           
-        // TODO, allow specifying list of constructor names as config
-        // Declaring an object. Let's store the type
-        let match;
-        if ((match = stmt.match(/^(?:my|our|local|state)\s+(\$\w+)\s*\=\s*([\w\:]+)\-\>new\s*(?:\((?!.*\)\->)|;)/ )) ||
-            (match = stmt.match(/^(?:my|our|local|state)\s+(\$\w+)\s*\=\s*new (\w[\w\:]+)\s*(?:\((?!.*\)\->)|;)/))) {
-            let varName = match[1];
-            let objName = match[2];
-            MakeElem(varName, PerlSymbolKind.LocalVar, objName, file, package_name, line_number, perlDoc);
+    for (let i = state.line_number; i < state.codeArray.length; i++) {
+        // Perhaps limit the max depth?
+        let stmt = state.codeArray[i];
 
-            var_continues = false; // We skipped ahead of the line here.
-        }
-        // This is a variable declaration if one was started on the previous
-        // line, or if this line starts with my or local
-        else if (var_continues || (match = stmt.match(/^(?:my|our|local|state)\b/))) {
-            // The declaration continues if the line does not end with ;
-            var_continues = (!stmt.endsWith(";") && !stmt.match(/[\)\=\}\{]/));
-    
-            // Remove my or local from statement, if present
-            stmt = stmt.replace(/^(my|our|local|state)\s+/, "");
-    
-            // Remove any assignment piece
-            stmt = stmt.replace(/\s*=.*/, "");
-    
-            // Remove part where sub starts (for signatures). Consider other options here.
-            stmt = stmt.replace(/\s*\}.*/, "");
-    
-            // Now find all variable names, i.e. "words" preceded by $, @ or %
-            let vars = stmt.matchAll(/([\$\@\%][\w:]+)\b/g);
-    
-            for (let match of vars) {
-                MakeElem(match[1], PerlSymbolKind.LocalVar, '', file, package_name, line_number, perlDoc);
-            }
-        }
-    
-        // Lexical loop variables, potentially with labels in front. foreach my $foo
-        else if ((match = stmt.match(/^(?:(\w+)\s*:(?!\:))?\s*(?:for|foreach)\s+my\s+(\$[\w]+)\b/))) {
-            if (match[1]) {
-                MakeElem(match[1], PerlSymbolKind.Label, '', file, package_name, line_number, perlDoc);
+        
+        if(i == state.line_number){
+            if(rFilter){
+                stmt.replace(rFilter, "");
             }
-            MakeElem(match[2], PerlSymbolKind.LocalVar, '', file, package_name, line_number, perlDoc);
+            // Default argument of empty hash. Other types of hashes may still trip this up
+            stmt.replace(/\$\w+\s*=\s*\{\s*\}/,"");
         }
 
-        // Lexical match variables if(my ($foo, $bar) ~= ). Optional to detect (my $newstring = $oldstring) =~ s/foo/bar/g;
-        else if ((match = stmt.match(/^(?:\}\s*elsif|if|unless|while|until|for)?\s*\(\s*my\b(.*)$/))) {
-            // Remove any assignment piece
-            stmt = stmt.replace(/\s*=.*/, "");
-            let vars = stmt.matchAll(/([\$\@\%][\w]+)\b/g);
-            for (let match of vars) {
-                MakeElem(match[1], PerlSymbolKind.LocalVar, '', file, package_name, line_number, perlDoc);
+        stmt.split('').forEach((char: string) =>  {
+            if(char == '{'){
+                // You may just be finding default function args = {}
+                found = true; 
+                pos++;
+            } else if(char == '}') {
+                pos--;
             }
+        });
+        //  Checking outside the statement is faster, but less accurate
+        if(found && pos == 0){
+            return i;
         }
+    }
+    return state.line_number;
+}
 
-        // This is a package declaration if the line starts with package
-        else if ((match = stmt.match(/^package\s+([\w:]+)/))) {
-            // Get name of the package
-            package_name = match[1];
-            MakeElem(package_name, PerlSymbolKind.Package, '', file, package_name, line_number, perlDoc);
-        }
-
-         // This is a class decoration for Object::Pad, Corinna, or Moops 
-        else if((match = stmt.match(/^class\s+([\w:]+)/))){
-            let class_name = match[1];
-            MakeElem(class_name, PerlSymbolKind.Class, '', file, package_name, line_number, perlDoc);
-        }
 
-        // This is a sub declaration if the line starts with sub
-        else if ((match = stmt.match(/^(?:async\s+)?(sub)\s+([\w:]+)(\s+:method)?([^{]*)/)) ||
-                (match = stmt.match(/^(?:async\s+)?(method)\s+\$?([\w:]+)()([^{]*)/)) ||
-                (sActiveOO.get("Function::Parameters") && (match = stmt.match(/^(fun)\s+([\w:]+)()([^{]*)/ )))
-                ) {
-            const subName = match[2];
-            const signature = match[4];
-            const kind = (match[1] === 'method' || match[3]) ? PerlSymbolKind.LocalMethod : PerlSymbolKind.LocalSub;
-            MakeElem(subName, kind, '', file, package_name, line_number, perlDoc);
-
-            // Match the after the sub declaration and before the start of the actual sub for signatures (if any)
-            const vars = signature.matchAll(/([\$\@\%][\w:]+)\b/g);
-
-            // Define subrountine signatures, but exclude prototypes
-            // The declaration continues if the line does not end with ;
-            var_continues = !(stmt.match(/;$/) || stmt.match(/[\)\=\}\{]/));
-
-            for (const matchvar of vars) {
-                MakeElem(matchvar[1], PerlSymbolKind.LocalVar,'', file, package_name, line_number, perlDoc);
-            }
-        }
-
-        // Phaser block
-        else if ((match = stmt.match(/^(BEGIN|INIT|CHECK|UNITCHECK|END)\s*\{/))) {
-            const phaser = match[1];
-            MakeElem(phaser, PerlSymbolKind.Phaser, '', file, package_name, line_number, perlDoc);
-        }
+function PackageEndLine (state: ParserState)  {
+  
+    if(state.parseType != ParseType.outline){
+        return state.line_number;
+    }
 
-        // Label line
-        else if ((match = stmt.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\s*:[^:].*{\s*$/))) {
-            const label = match[1];
-            MakeElem(label, PerlSymbolKind.Label, '', file, package_name, line_number, perlDoc);
-        }
+    let start_line = state.line_number;
+    if (state.codeArray[start_line].match(/(class|package)[^#]+;/)){
 
-        // Constants. Important because they look like subs (and technically are), so I'll tags them as such 
-        else if ((match = stmt.match(/^use\s+constant\s+(\w+)\b/))) {
-            MakeElem(match[1], PerlSymbolKind.Constant, '', file, package_name, line_number, perlDoc);
-            MakeElem("constant", 'u', '', file, package_name, line_number, perlDoc);
+        // Single line package definition.
+        if (state.codeArray[start_line].match(/{.*(class|package)/)){
+            // Will need to hunt for the end
+        }else if (start_line > 0 && state.codeArray[start_line-1].match(/\{[^}]*$/)){
+            start_line -= 1;
         }
+    }
 
-        // Moo/Moose/Object::Pad/Moops/Corinna attributes
-        else if ((match = stmt.match(/^has(?:\s+|\()["']?([\$@%]?\w+)\b/))) { 
-            const attr = match[1];
-            let type;
-            if(attr.match(/^\w/)){
-                type = PerlSymbolKind.Field;
-                // If you have a locally defined package/class Foo want to reference the attributes as Foo::attr or foo->attr, you need the full path.
-                // Subs don't need this since we find them at compile time. We also find "d" types from imported packages in Inquisitor.pm
-                MakeElem(package_name + "::" + attr, PerlSymbolKind.PathedField, '', file, package_name, line_number, perlDoc);
-            } else {
-                type = PerlSymbolKind.LocalVar;
-            } 
-            // TODO: Define new type. Class variables should probably be shown in the Outline view even though lexical variables are not
-            MakeElem(attr, type, '', file, package_name, line_number, perlDoc);
-        }
+    let pos = 0;
+    let found = false;
+
+    for (let i = start_line; i < state.codeArray.length; i++) {
+        // Perhaps limit the max depth?
+        let stmt = state.codeArray[i];
+        stmt.split('').forEach((char: string) =>  {
+            if(char == '{') {
+                found = true; 
+                pos++;
+            } else if(char == '}') {
+                pos--;
+            }
+        });
 
-        else if (sActiveOO.get("Object::Pad") &&
-                (match = stmt.match(/^field\s+([\$@%]\w+)\b/))) { //  Object::Pad field
-            const attr = match[1];
-            MakeElem(attr, PerlSymbolKind.LocalVar, '', file, package_name, line_number, perlDoc);
+        if(found == false){
+            // If we haven't found the start of the package block, there probably isn't one. 
+            if(stmt.match(/;/) || (i - start_line > 1)){
+                break;
+            }
         }
 
-        else if ((sActiveOO.get("Mars::Class") || sActiveOO.get("Venus::Class"))
-                && (match = stmt.match(/^attr\s+["'](\w+)\b/))) { // Mars attributes
-            const attr = match[1];
-            MakeElem(attr, PerlSymbolKind.Field, '', file, package_name, line_number, perlDoc);
-            MakeElem(package_name + "::" + attr, PerlSymbolKind.PathedField, '', file, package_name, line_number, perlDoc);
+        //  Checking outside the forEach statement is faster, but less accurate
+        if(found && pos == 0){
+            return i;
         }
+    }
 
-        else if ((match = stmt.match(/^around\s+["']?(\w+)\b/))) { // Moo/Moose overriding subs. 
-            MakeElem(match[1], PerlSymbolKind.LocalSub, '', file, package_name, line_number, perlDoc);
-        } 
-        
-        else if ((match = stmt.match(/^use\s+([\w:]+)\b/))) { // Keep track of explicit imports for filtering
-            const importPkg = match[1];
-            MakeElem(importPkg, "u", '', file, package_name, line_number, perlDoc);
-            sActiveOO.set(importPkg, true);
+    for (let i = start_line+1; i < state.codeArray.length; i++) {
+        // TODO: update with class inheritance / version numbers, etc
+        // Although should we do with nested packages/classes? (e.g. Pack A -> Pack B {} -> A)
+        if(state.codeArray[i].match(/^\s*(class|package)\s+([\w:]+)/)){
+            return i-1;
         }
-
     }
-        
-}
-
 
-function cleanCode(textDocument: TextDocument): String[] {
-    const code = textDocument.getText();
-    const codeArray = code.split("\n");
-    const offset = textDocument.offsetAt(textDocument.positionAt(0));
-
-
-    let line_number = -offset;
-
-    let codeClean = [];
+    // If we didn't find an end, run until end of file
+    return state.codeArray.length;
+}
 
-    for (let i=0; i<codeArray.length;i++){
-        line_number++;
+// Leaving some code here, although it doesn't work. 
+// Need to figure out how to load oniguruma with webassembly, plus the perl grammars
+
+// Taken from https://github.com/bolinfest/monaco-tm/blob/908f1ca0cab3e82823cb465108ae86ee2b4ba3fc/src/app.ts#L133C1-L145C2 
+// async function loadVSCodeOnigurumWASM(): Promise<Response | ArrayBuffer> {
+//     const response = await fetch('/node_modules/vscode-oniguruma/release/onig.wasm');
+//     const contentType = response.headers.get('content-type');
+//     if (contentType === 'application/wasm') {
+//       return response;
+//     }
+  
+//     // Using the response directly only works if the server sets the MIME type 'application/wasm'.
+//     // Otherwise, a TypeError is thrown when using the streaming compiler.
+//     // We therefore use the non-streaming compiler :(.
+//     return await response.arrayBuffer();
+//   }
+
+
+// const vscodeOnigurumaLib = loadVSCodeOnigurumWASM().then((wasmBin: Response | ArrayBuffer) => {
+//     return oniguruma.loadWASM(wasmBin);
+//     })
+//     .then(() => {
+//         return {
+//             createOnigScanner(patterns: any) { return new oniguruma.OnigScanner(patterns); },
+//             createOnigString(s: any) { return new oniguruma.OnigString(s); }
+//         };
+//     });
+
+// const registry = new vsctm.Registry({
+//     onigLib: vscodeOnigurumaLib,
+//     loadGrammar: async (scopeName) => {
+//         const grammarResponse= await fetch('https://raw.githubusercontent.com/bscan/PerlNavigator/main/server/perl.tmLanguage.json'); 
+//         const grammar = await grammarResponse.text();
+//         return vsctm.parseRawGrammar(grammar);
+//     }
+// });
 
-        let stmt = codeArray[i];
 
-        if (stmt.match(/^(__END__|__DATA__)\s*$/)) {
-            break;
-        }
-        
-        // Statement will be line with comments, whitespace and POD trimmed
-        stmt = stmt.replace(/^\s*#.*/, "");
-        stmt = stmt.replace(/^\s*/, "");
-        stmt = stmt.replace(/\s*$/, "");
-        codeClean.push(stmt);
-    }
-    return codeClean;
-}
 
diff --git a/browser-ext/src/web-symbols.ts b/browser-ext/src/web-symbols.ts
index 0e89217..d7fc093 100644
--- a/browser-ext/src/web-symbols.ts
+++ b/browser-ext/src/web-symbols.ts
@@ -2,80 +2,93 @@ import {
     SymbolInformation,
     SymbolKind,
     Location,
+    WorkspaceSymbolParams
 } from 'vscode-languageserver/node';
-import { PerlDocument, PerlElem, PerlSymbolKind } from "./web-types";
 
-function waitForDoc (navSymbols: any, uri: string): Promise<PerlDocument> {
-    let retries = 0;
+import {
+    TextDocument
+} from 'vscode-languageserver-textdocument';
+
+import { ParseType, PerlElem, PerlSymbolKind } from "./web-types";
+import { parseDocument } from './web-parse';
+
+export async function getSymbols (textDocument: TextDocument, uri: string ): Promise<SymbolInformation[]> {
     
-    return new Promise((resolve, reject) => {
-        const interval = setInterval(() => {
+    let perlDoc = await parseDocument(textDocument, ParseType.outline);
 
-            if (++retries > 100) { // Wait for 10 seconds looking for the document. 
-                reject("Found no document");
-                clearInterval(interval);
+    let symbols: SymbolInformation[] = [];
+    perlDoc.elems?.forEach((elements: PerlElem[], elemName: string) => {
+        
+        elements.forEach(element => {
+            let kind: SymbolKind;
+            if (element.type == PerlSymbolKind.LocalSub || element.type == PerlSymbolKind.OutlineOnlySub){
+                kind = SymbolKind.Function;
+            } else if (element.type == PerlSymbolKind.LocalMethod){
+                kind = SymbolKind.Method;
+            } else if (element.type == PerlSymbolKind.Package){
+                kind = SymbolKind.Package;
+            } else if (element.type == PerlSymbolKind.Class){
+                kind = SymbolKind.Class;
+            } else if (element.type == PerlSymbolKind.Role){
+                kind = SymbolKind.Interface;
+            } else if (element.type == PerlSymbolKind.Field){
+                kind = SymbolKind.Field;
+            } else if (element.type == PerlSymbolKind.Label){
+                kind = SymbolKind.Key;
+            } else if (element.type == PerlSymbolKind.Phaser){
+                kind = SymbolKind.Event;
+            } else if (element.type == PerlSymbolKind.Constant){
+                kind = SymbolKind.Constant;
+            } else if (element.type == PerlSymbolKind.HttpRoute){
+                kind = SymbolKind.Interface;
+            } else {
+                return;
             }
-            const perlDoc = navSymbols.get(uri);
-
-            if (perlDoc) {
-                resolve(perlDoc);
-                clearInterval(interval);
+            const location: Location = {
+                range: {
+                    start: { line: element.line, character: 0 },
+                    end: { line: element.lineEnd, character: 100 }  
+                },
+                uri: uri
             };
-        }, 100);
+            const newSymbol: SymbolInformation = {
+                kind: kind,
+                location: location,
+                name: elemName
+            }
+
+            symbols.push(newSymbol);
+        }); 
     });
+
+    return symbols;
 }
 
-export function getSymbols (navSymbols: any, uri: string ): Promise<SymbolInformation[]> {
+export function getWorkspaceSymbols (params: WorkspaceSymbolParams, defaultMods:  Map<string, string>): Promise<SymbolInformation[]> {
     
-    return waitForDoc(navSymbols, uri).then((perlDoc) => {
+    return new Promise((resolve, reject) => {
         let symbols: SymbolInformation[] = [];
-        perlDoc.elems?.forEach((elements: PerlElem[], elemName: string) => {
-            
-            elements.forEach(element => {
-                let kind: SymbolKind;
-                if (element.type == PerlSymbolKind.LocalSub){
-                    kind = SymbolKind.Function;
-                } else if (element.type == PerlSymbolKind.LocalMethod){
-                    kind = SymbolKind.Method;
-                } else if (element.type == PerlSymbolKind.Package){
-                    kind = SymbolKind.Package;
-                } else if (element.type == PerlSymbolKind.Class){
-                    kind = SymbolKind.Class;
-                } else if (element.type == PerlSymbolKind.Role){
-                    kind = SymbolKind.Interface;
-                } else if (element.type == PerlSymbolKind.Field){
-                    kind = SymbolKind.Field;
-                } else if (element.type == PerlSymbolKind.Label){
-                    kind = SymbolKind.Key;
-                } else if (element.type == PerlSymbolKind.Phaser){
-                    kind = SymbolKind.Event;
-                } else if (element.type == PerlSymbolKind.Constant){
-                    kind = SymbolKind.Constant;
-                } else {
-                    return;
-                }
+
+        const lcQuery = params.query.toLowerCase();
+        defaultMods.forEach((modUri: string, modName: string) => {
+            if(true){ // Just send the whole list and let the client sort through it with fuzzy search
+            // if(!lcQuery || modName.toLowerCase().startsWith(lcQuery)){ 
+
                 const location: Location = {
                     range: {
-                        start: { line: element.line, character: 0 },
-                        end: { line: element.lineEnd, character: 100 }  
+                        start: { line: 0, character: 0 },
+                        end: { line: 0, character: 100 }  
                     },
-                    uri: uri
+                    uri: modUri
                 };
-                const newSymbol: SymbolInformation = {
-                    kind: kind,
-                    location: location,
-                    name: elemName
-                }
 
-                symbols.push(newSymbol);
-            }); 
+                symbols.push({
+                    name: modName,
+                    kind: SymbolKind.Module,
+                    location: location
+                });
+            }
         });
-
-        return symbols;
-    }).catch((reason)=>{
-        // TODO: Add logging back, but detect STDIO mode first
-        console.log("Failed in getSymbols");
-        console.log(reason);
-        return [];
+        resolve(symbols);
     });
-}
+}
\ No newline at end of file
diff --git a/browser-ext/src/web-types.ts b/browser-ext/src/web-types.ts
index 9515cf8..1eb6122 100644
--- a/browser-ext/src/web-types.ts
+++ b/browser-ext/src/web-types.ts
@@ -33,7 +33,10 @@ export interface NavigatorSettings {
     enableProgress: boolean;
 }
 
-
+export enum ParseType {
+    outline,
+    selfNavigation,
+}
 
 export interface PerlElem {
     name: string,
@@ -55,6 +58,7 @@ export interface PerlImport {
 export interface PerlDocument {
     elems: Map<string, PerlElem[]>;
     canonicalElems: Map<string, PerlElem>;
+    autoloads: Map<string, PerlElem>;
     imported: Map<string, number>;
     parents: Map<string, string>;
     filePath: string;
@@ -72,6 +76,7 @@ export interface CompletionPrefix {
     charEnd: number,
 }
 
+
 export enum PerlSymbolKind {
     Module       = "m",
     Package      = "p",
@@ -88,9 +93,12 @@ export enum PerlSymbolKind {
     Constant     = "n",
     Label        = "l",
     Phaser       = "e",
-    Canonical    = "1",
-    // UseStatement = "u", // Reserved: used in pltags, but removed before symbol assignment.
+    Canonical    = "1", // 2 and 3 are also reserved
+    // UseStatement = "u" . Reserved: used in pltags, but removed before symbol assignment.
     ImportedVar  = "c",
     ImportedHash = "h",
+    HttpRoute    = "g",
+    OutlineOnlySub = "j",
+    AutoLoadVar  = "3"
 }