Merge pull request #2821 from skarph/master

custom luadoc generation

11 files changed, 827 insertions, 539 deletions

diff --git a/changelog.md b/changelog.md
index 2f0ea8f7..dd5dd5c4 100644
--- a/changelog.md
+++ b/changelog.md
@@ -2,6 +2,8 @@
 
 ## Unreleased
 <!-- Add all new changes here. They will be moved under a version at release -->
+* `NEW` Custom documentation exporter
+* `NEW` Setting: `Lua.docScriptPath`: Path to a script that overrides `cli.doc.export`, allowing user-specified documentation exporting.
 
 ## 3.10.5
 `2024-8-19`
diff --git a/locale/en-us/script.lua b/locale/en-us/script.lua
index cf2fbe8e..9c9163ae 100644
--- a/locale/en-us/script.lua
+++ b/locale/en-us/script.lua
@@ -656,10 +656,10 @@ CLI_DOC_INITING   =
 'Loading documents ...'
 CLI_DOC_DONE      =
 [[
-Document exporting completed!
-Raw data: {}
-Markdown(example): {}
+Documentation exported:
 ]]
+CLI_DOC_WORKING   =
+'Building docs...'
 
 TYPE_ERROR_ENUM_GLOBAL_DISMATCH =
 'Type `{child}` cannot match enumeration type of `{parent}`'
diff --git a/locale/pt-br/script.lua b/locale/pt-br/script.lua
index 50568aeb..e763fb6c 100644
--- a/locale/pt-br/script.lua
+++ b/locale/pt-br/script.lua
@@ -654,12 +654,10 @@ CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
 'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
 CLI_DOC_INITING   = -- TODO: need translate!
 'Loading documents ...'
-CLI_DOC_DONE      = -- TODO: need translate!
-[[
-Document exporting completed!
-Raw data: {}
-Markdown(example): {}
-]]
+CLI_DOC_DONE      =
+'Documentos exportados:'
+CLI_DOC_WORKING   =
+'Construindo docs...'
 
 TYPE_ERROR_ENUM_GLOBAL_DISMATCH = -- TODO: need translate!
 'Type `{child}` cannot match enumeration type of `{parent}`'
diff --git a/locale/zh-cn/script.lua b/locale/zh-cn/script.lua
index 9cea601a..561bb27e 100644
--- a/locale/zh-cn/script.lua
+++ b/locale/zh-cn/script.lua
@@ -654,12 +654,10 @@ CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
 'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
 CLI_DOC_INITING   =
 '加载文档 ...'
-CLI_DOC_DONE      =
-[[
-文档导出完成！
-原始数据: {}
-Markdown(演示用): {}
-]]
+CLI_DOC_DONE      = -- TODO: need translate!
+'文档导出完成！'
+CLI_DOC_WORKING   =
+'正在生成文档...'
 
 TYPE_ERROR_ENUM_GLOBAL_DISMATCH =
 '类型 `{child}` 无法匹配 `{parent}` 的枚举类型'
diff --git a/locale/zh-tw/script.lua b/locale/zh-tw/script.lua
index 1feaf2ad..bab26ed8 100644
--- a/locale/zh-tw/script.lua
+++ b/locale/zh-tw/script.lua
@@ -655,12 +655,9 @@ CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
 CLI_DOC_INITING   = -- TODO: need translate!
 'Loading documents ...'
 CLI_DOC_DONE      = -- TODO: need translate!
-[[
-Document exporting completed!
-Raw data: {}
-Markdown(example): {}
-]]
-
+'Document exporting completed!'
+CLI_DOC_WORKING   =
+'正在產生文件...'
 TYPE_ERROR_ENUM_GLOBAL_DISMATCH = -- TODO: need translate!
 'Type `{child}` cannot match enumeration type of `{parent}`'
 TYPE_ERROR_ENUM_GENERIC_UNSUPPORTED = -- TODO: need translate!
diff --git a/script/cli/doc.lua b/script/cli/doc.lua
deleted file mode 100644
index c413d354..00000000
--- a/script/cli/doc.lua
+++ /dev/null
@@ -1,464 +0,0 @@
-local lclient  = require 'lclient'
-local furi     = require 'file-uri'
-local ws       = require 'workspace'
-local files    = require 'files'
-local util     = require 'utility'
-local jsonb    = require 'json-beautify'
-local lang     = require 'language'
-local define   = require 'proto.define'
-local config   = require 'config.config'
-local await    = require 'await'
-local vm       = require 'vm'
-local guide    = require 'parser.guide'
-local getDesc  = require 'core.hover.description'
-local getLabel = require 'core.hover.label'
-local doc2md   = require 'cli.doc2md'
-local progress = require 'progress'
-local fs       = require 'bee.filesystem'
-
-local export = {}
-
----@async
-local function packObject(source, mark)
-    if type(source) ~= 'table' then
-        return source
-    end
-    if not mark then
-        mark = {}
-    end
-    if mark[source] then
-        return
-    end
-    mark[source] = true
-    local new = {}
-    if (#source > 0 and next(source, #source) == nil)
-    or source.type == 'funcargs' then
-        new = {}
-        for i = 1, #source do
-            new[i] = packObject(source[i], mark)
-        end
-    else
-        for k, v in pairs(source) do
-            if k == 'type'
-            or k == 'name'
-            or k == 'start'
-            or k == 'finish'
-            or k == 'types' then
-                new[k] = packObject(v, mark)
-            end
-        end
-        if source.type == 'function' then
-            new['args'] = packObject(source.args, mark)
-            local _, _, max = vm.countReturnsOfFunction(source)
-            if max > 0 then
-                new.returns = {}
-                for i = 1, max do
-                    local rtn = vm.getReturnOfFunction(source, i)
-                    new.returns[i] = packObject(rtn)
-                end
-            end
-            new['view'] = getLabel(source, source.parent.type == 'setmethod')
-        end
-        if source.type == 'local'
-        or source.type == 'self' then
-            new['name'] = source[1]
-        end
-        if source.type == 'function.return' then
-            new['desc'] = source.comment and getDesc(source.comment)
-            new['rawdesc'] = source.comment and getDesc(source.comment, true)
-        end
-        if source.type == 'doc.type.table' then
-            new['fields'] = packObject(source.fields, mark)
-        end
-        if source.type == 'doc.field.name'
-        or source.type == 'doc.type.arg.name' then
-            new['[1]'] = packObject(source[1], mark)
-            new['view'] = source[1]
-        end
-        if source.type == 'doc.type.function' then
-            new['args'] = packObject(source.args, mark)
-            if source.returns then
-                new['returns'] = packObject(source.returns, mark)
-            end
-        end
-        if source.bindDocs then
-            new['desc'] = getDesc(source)
-            new['rawdesc'] = getDesc(source, true)
-        end
-        new['view'] = new['view'] or vm.getInfer(source):view(ws.rootUri)
-    end
-    return new
-end
-
----@async
-local function getExtends(source)
-    if source.type == 'doc.class' then
-        if not source.extends then
-            return nil
-        end
-        return packObject(source.extends)
-    end
-    if source.type == 'doc.alias' then
-        if not source.extends then
-            return nil
-        end
-        return packObject(source.extends)
-    end
-end
-
----@async
----@param global vm.global
----@param results table
-local function collectTypes(global, results)
-    if guide.isBasicType(global.name) then
-        return
-    end
-    local result = {
-        name    = global.name,
-        type    = 'type',
-        desc    = nil,
-        rawdesc = nil,
-        defines = {},
-        fields  = {},
-    }
-    for _, set in ipairs(global:getSets(ws.rootUri)) do
-        local uri = guide.getUri(set)
-        if files.isLibrary(uri) then
-            goto CONTINUE
-        end
-        result.defines[#result.defines+1] = {
-            type    = set.type,
-            file    = guide.getUri(set),
-            start   = set.start,
-            finish  = set.finish,
-            extends = getExtends(set),
-        }
-        result.desc = result.desc or getDesc(set)
-        result.rawdesc = result.rawdesc or getDesc(set, true)
-        ::CONTINUE::
-    end
-    if #result.defines == 0 then
-        return
-    end
-    table.sort(result.defines, function (a, b)
-        if a.file ~= b.file then
-            return a.file < b.file
-        end
-        return a.start < b.start
-    end)
-    results[#results+1] = result
-    ---@async
-    ---@diagnostic disable-next-line: not-yieldable
-    vm.getClassFields(ws.rootUri, global, vm.ANY, function (source)
-        if source.type == 'doc.field' then
-            ---@cast source parser.object
-            if files.isLibrary(guide.getUri(source)) then
-                return
-            end
-            local field = {}
-            result.fields[#result.fields+1] = field
-            if source.field.type == 'doc.field.name' then
-                field.name = source.field[1]
-            else
-                field.name = ('[%s]'):format(vm.getInfer(source.field):view(ws.rootUri))
-            end
-            field.type    = source.type
-            field.file    = guide.getUri(source)
-            field.start   = source.start
-            field.finish  = source.finish
-            field.desc    = getDesc(source)
-            field.rawdesc = getDesc(source, true)
-            field.extends = packObject(source.extends)
-            field.visible = vm.getVisibleType(source)
-            return
-        end
-        if source.type == 'setfield'
-        or source.type == 'setmethod' then
-            ---@cast source parser.object
-            if files.isLibrary(guide.getUri(source)) then
-                return
-            end
-            local field = {}
-            result.fields[#result.fields+1] = field
-            field.name    = (source.field or source.method)[1]
-            field.type    = source.type
-            field.file    = guide.getUri(source)
-            field.start   = source.start
-            field.finish  = source.finish
-            field.desc    = getDesc(source)
-            field.rawdesc = getDesc(source, true)
-            field.extends = packObject(source.value)
-            field.visible = vm.getVisibleType(source)
-            if vm.isAsync(source, true) then
-                field.async = true
-            end
-            local depr = vm.getDeprecated(source)
-            if (depr and not depr.versions) then
-                field.deprecated = true
-            end
-            return
-        end
-        if source.type == 'tableindex' then
-            ---@cast source parser.object
-            if source.index.type ~= 'string' then
-                return
-            end
-            if files.isLibrary(guide.getUri(source)) then
-                return
-            end
-            local field = {}
-            result.fields[#result.fields+1] = field
-            field.name    = source.index[1]
-            field.type    = source.type
-            field.file    = guide.getUri(source)
-            field.start   = source.start
-            field.finish  = source.finish
-            field.desc    = getDesc(source)
-            field.rawdesc = getDesc(source, true)
-            field.extends = packObject(source.value)
-            field.visible = vm.getVisibleType(source)
-            return
-        end
-    end)
-    table.sort(result.fields, function (a, b)
-        if a.name ~= b.name then
-            return a.name < b.name
-        end
-        if a.file ~= b.file then
-            return a.file < b.file
-        end
-        return a.start < b.start
-    end)
-end
-
----@async
----@param global vm.global
----@param results table
-local function collectVars(global, results)
-    local result = {
-        name    = global:getCodeName(),
-        type    = 'variable',
-        desc    = nil,
-        defines = {},
-    }
-    for _, set in ipairs(global:getSets(ws.rootUri)) do
-        if set.type == 'setglobal'
-        or set.type == 'setfield'
-        or set.type == 'setmethod'
-        or set.type == 'setindex' then
-            result.defines[#result.defines+1] = {
-                type    = set.type,
-                file    = guide.getUri(set),
-                start   = set.start,
-                finish  = set.finish,
-                extends = packObject(set.value),
-            }
-            result.desc = result.desc or getDesc(set)
-            result.rawdesc = result.rawdesc or getDesc(set, true)
-            result.defines[#result.defines].extends['desc'] = getDesc(set)
-            result.defines[#result.defines].extends['rawdesc'] = getDesc(set, true)
-            if vm.isAsync(set, true) then
-                result.defines[#result.defines].extends['async'] = true
-            end
-            local depr = vm.getDeprecated(set)
-            if (depr and not depr.versions) then
-                result.defines[#result.defines].extends['deprecated'] = true
-            end
-        end
-    end
-    if #result.defines == 0 then
-        return
-    end
-    table.sort(result.defines, function (a, b)
-        if a.file ~= b.file then
-            return a.file < b.file
-        end
-        return a.start < b.start
-    end)
-    results[#results+1] = result
-end
-
----Add config settings to JSON output.
----@param results table
-local function collectConfig(results)
-    local result = {
-        name = 'LuaLS',
-        type = 'luals.config',
-        DOC = fs.absolute(fs.path(DOC)):string(),
-        defines = {},
-        fields = {}
-    }
-    results[#results+1] = result
-end
-
----@async
----@param callback fun(i, max)
-function export.export(outputPath, callback)
-    local results = {}
-    local globals = vm.getAllGlobals()
-
-    collectConfig(results)
-    local max = 0
-    for _ in pairs(globals) do
-        max = max + 1
-    end
-    local i = 0
-    for _, global in pairs(globals) do
-        if global.cate == 'variable' then
-            collectVars(global, results)
-        elseif global.cate == 'type' then
-            collectTypes(global, results)
-        end
-        i = i + 1
-        callback(i, max)
-    end
-
-    table.sort(results, function (a, b)
-        return a.name < b.name
-    end)
-
-    local docPath = outputPath .. '/doc.json'
-    jsonb.supportSparseArray = true
-    util.saveFile(docPath, jsonb.beautify(results))
-
-    local mdPath = doc2md.buildMD(outputPath)
-    return docPath, mdPath
-end
-
-function export.getDocOutputPath()
-    local doc_output_path = ''
-    if type(DOC_OUT_PATH) == 'string' then
-        doc_output_path = fs.absolute(fs.path(DOC_OUT_PATH)):string()
-    elseif DOC_OUT_PATH == true then
-        doc_output_path = fs.current_path():string()
-    else
-        doc_output_path = LOGPATH
-    end
-    return doc_output_path
-end
-
----@async
----@param outputPath string
-function export.makeDoc(outputPath)
-    ws.awaitReady(ws.rootUri)
-
-    local expandAlias = config.get(ws.rootUri, 'Lua.hover.expandAlias')
-    config.set(ws.rootUri, 'Lua.hover.expandAlias', false)
-    local _ <close> = function ()
-        config.set(ws.rootUri, 'Lua.hover.expandAlias', expandAlias)
-    end
-
-    await.sleep(0.1)
-
-    local prog <close> = progress.create(ws.rootUri, '正在生成文档...', 0)
-    local docPath, mdPath = export.export(outputPath, function (i, max)
-        prog:setMessage(('%d/%d'):format(i, max))
-        prog:setPercentage((i) / max * 100)
-    end)
-
-    return docPath, mdPath
-end
-
-
----Find file 'doc.json'.
----@return fs.path
-local function findDocJson()
-    local doc_json_path
-    if type(DOC_UPDATE) == 'string' then
-        doc_json_path = fs.absolute(fs.path(DOC_UPDATE)) .. '/doc.json'
-    else
-        doc_json_path = fs.current_path() .. '/doc.json'
-    end
-    if fs.exists(doc_json_path) then
-        return doc_json_path
-    else
-        error(string.format('Error: File "%s" not found.', doc_json_path))
-    end
-end
-
----@return string # path of 'doc.json'
----@return string # path to be documented
-local function getPathDocUpdate()
-    local doc_json_path = findDocJson()
-    local ok, doc_path = pcall(
-        function ()
-            local json = require('json')
-            local json_file = io.open(doc_json_path:string(), 'r'):read('*all')
-            local json_data = json.decode(json_file)
-            for _, section in ipairs(json_data) do
-                if section.type == 'luals.config' then
-                    return section.DOC
-                end
-            end
-    end)
-    if ok then
-        local doc_json_dir = doc_json_path:string():gsub('/doc.json', '')
-        return doc_json_dir, doc_path
-    else
-        error(string.format('Error: Cannot update "%s".', doc_json_path .. '/doc.json'))
-    end
-end
-
-function export.runCLI()
-    lang(LOCALE)
-
-    if DOC_UPDATE then
-        DOC_OUT_PATH, DOC = getPathDocUpdate()
-    end
-
-    if type(DOC) ~= 'string' then
-        print(lang.script('CLI_CHECK_ERROR_TYPE', type(DOC)))
-        return
-    end
-
-    local rootUri = furi.encode(fs.absolute(fs.path(DOC)):string())
-    if not rootUri then
-        print(lang.script('CLI_CHECK_ERROR_URI', DOC))
-        return
-    end
-
-    print('root uri = ' .. rootUri)
-
-    util.enableCloseFunction()
-
-    local lastClock = os.clock()
-
-    ---@async
-    lclient():start(function (client)
-        client:registerFakers()
-
-        client:initialize {
-            rootUri = rootUri,
-        }
-
-        io.write(lang.script('CLI_DOC_INITING'))
-
-        config.set(nil, 'Lua.diagnostics.enable', false)
-        config.set(nil, 'Lua.hover.expandAlias', false)
-
-        ws.awaitReady(rootUri)
-        await.sleep(0.1)
-
-        local docPath, mdPath = export.export(export.getDocOutputPath(), function (i, max)
-            if os.clock() - lastClock > 0.2 then
-                lastClock = os.clock()
-                local output = '\x0D'
-                            .. ('>'):rep(math.ceil(i / max * 20))
-                            .. ('='):rep(20 - math.ceil(i / max * 20))
-                            .. ' '
-                            .. ('0'):rep(#tostring(max) - #tostring(i))
-                            .. tostring(i) .. '/' .. tostring(max)
-                io.write(output)
-            end
-        end)
-
-        io.write('\x0D')
-
-        print(lang.script('CLI_DOC_DONE'
-            , ('[%s](%s)'):format(files.normalize(docPath), furi.encode(docPath))
-            , ('[%s](%s)'):format(files.normalize(mdPath),  furi.encode(mdPath))
-        ))
-    end)
-end
-
-return export
diff --git a/script/cli/doc/export.lua b/script/cli/doc/export.lua
new file mode 100644
index 00000000..5a8c3239
--- /dev/null
+++ b/script/cli/doc/export.lua
@@ -0,0 +1,354 @@
+local ws       = require 'workspace'
+local vm       = require 'vm'
+local guide    = require 'parser.guide'
+
+local getDesc  = require 'core.hover.description'
+local getLabel = require 'core.hover.label'
+local jsonb    = require 'json-beautify'
+local util     = require 'utility'
+local markdown = require 'provider.markdown'
+
+---@alias doctype
+---| 'doc.alias'
+---| 'doc.class'
+---| 'doc.field'
+---| 'doc.field.name'
+---| 'doc.type.arg.name'
+---| 'doc.type.function'
+---| 'doc.type.table'
+---| 'funcargs'
+---| 'function'
+---| 'function.return'
+---| 'global.type'
+---| 'global.variable'
+---| 'local'
+---| 'luals.config'
+---| 'self'
+---| 'setfield'
+---| 'setglobal'
+---| 'setindex'
+---| 'setmethod'
+---| 'tableindex'
+---| 'type'
+
+---@class docUnion broadest possible collection of exported docs, these are never all together.
+---@field [1] string in name when table, always the same as view
+---@field args docUnion[] list of argument docs passed to function
+---@field async boolean has @async tag
+---@field defines docUnion[] list of places where this is doc is defined and how its defined there
+---@field deprecated boolean has @deprecated tag
+---@field desc string code commentary
+---@field extends string | docUnion ? what type this 'is'. string:<Parent_Class> for type: 'type', docUnion for type: 'function', string<primative> for other type 's
+---@field fields docUnion[] class's fields
+---@field file string path to where this token is defined
+---@field finish [integer, integer] 0-indexed [line, column] position of end of token
+---@field name string canonical name
+---@field rawdesc string same as desc, but may have other things for types doc.retun andr doc.param (unused?)
+---@field returns docUnion | docUnion[] list of docs for return values. if singluar, then always {type: 'undefined'}? might be a bug.
+---@field start [integer, integer] 0-indexed [line, column] position of start of token
+---@field type doctype role that this token plays in documentation. different from the 'type'/'class' this token is
+---@field types docUnion[] type union? unclear. seems to be related to alias, maybe
+---@field view string full method name, class, basal type, or unknown. in name table same as [1]
+---@field visible 'package'|'private'|'protected'|'public' visibilty tag
+
+local export = {}
+
+function export.getLocalPath(uri)
+    --remove uri root (and prefix)
+    local local_file_uri = uri
+    local i, j = local_file_uri:find(DOC)
+    if not j then
+        return '[FORIEGN]'..uri
+    end
+    return local_file_uri:sub( j + 1 )
+end
+
+function export.positionOf(rowcol)
+    return type(rowcol) == 'table' and guide.positionOf(rowcol[1], rowcol[2]) or -1
+end
+
+function export.sortDoc(a,b)
+    if a.name ~= b.name then
+        return a.name < b.name
+    end
+
+    if a.file ~= b.file then
+        return a.file < b.file
+    end
+
+    return export.positionOf(a.start) < export.positionOf(b.start)
+end
+
+
+--- recursively generate documentation all parser objects downstream of `source`
+---@async
+---@param source parser.object | vm.global
+---@param has_seen table? keeps track of visited nodes in documentation tree
+---@return docUnion | [docUnion] | string | number | boolean | nil 
+function export.documentObject(source, has_seen)
+    --is this a primative type? then we dont need to process it.
+    if type(source) ~= 'table' then return source end
+
+    --set up/check recursion
+    if not has_seen then has_seen = {} end
+    if has_seen[source] then
+        return nil
+    end
+    has_seen[source] = true
+
+    --is this an array type? then process each array item and collect it
+    if (#source > 0 and next(source, #source) == nil) then
+        local objs = {} --make a pure numerical array
+        for i, child in ipairs(source) do
+            objs[i] = export.documentObject(child, has_seen)
+        end
+        return objs
+    end
+
+    --if neither, then this is a singular docUnion
+    local obj = export.makeDocObject['INIT'](source, has_seen)
+
+    --check if this source has a type (no type sources are usually autogen'd anon functions's return values that are not explicitly stated)
+    if not obj.type then return obj end
+
+    local res = export.makeDocObject[obj.type](source, obj, has_seen)
+    if res == false then
+        return nil
+    end
+    return res or obj
+end
+
+---Switch statement table. functions can be overriden by user file.
+---@table
+export.makeDocObject = setmetatable({}, {__index = function(t, k)
+    return function()
+        --print('DocError: no type "'..k..'"')
+    end
+end})
+
+export.makeDocObject['INIT'] = function(source, has_seen)
+    ---@as docUnion
+    local ok, desc = pcall(getDesc, source)
+    local rawok, rawdesc = pcall(getDesc, source, true)
+    return {
+        type = source.cate or source.type,
+        name = export.documentObject((source.getCodeName and source:getCodeName()) or source.name, has_seen),
+        start = source.start and {guide.rowColOf(source.start)},
+        finish = source.finish and {guide.rowColOf(source.finish)},
+        types = export.documentObject(source.types, has_seen),
+        view = vm.getInfer(source):view(ws.rootUri),
+        desc = ok and desc or nil,
+        rawdesc = rawok and rawdesc or nil,
+    }
+end
+
+export.makeDocObject['doc.alias'] = function(source, obj, has_seen)
+
+end
+
+export.makeDocObject['doc.field'] = function(source, obj, has_seen)
+    if source.field.type == 'doc.field.name' then
+        obj.name = source.field[1]
+    else
+        obj.name = ('[%s]'):format(vm.getInfer(source.field):view(ws.rootUri))
+    end
+    obj.file = export.getLocalPath(guide.getUri(source))
+    obj.extends = source.extends and export.documentObject(source.extends, has_seen) --check if bug?
+    obj.async = vm.isAsync(source, true) and true or false --if vm.isAsync(set, true) then result.defines[#result.defines].extends['async'] = true end
+    obj.deprecated = vm.getDeprecated(source) and true or false --  if (depr and not depr.versions) the result.defines[#result.defines].extends['deprecated'] = true end
+    obj.visible = vm.getVisibleType(source)
+end
+
+export.makeDocObject['doc.class'] = function(source, obj, has_seen)
+    local extends = source.extends or source.value --doc.class or other
+    local field = source.field or source.method 
+    obj.name = type(field) == 'table' and field[1] or nil
+    obj.file = export.getLocalPath(guide.getUri(source))
+    obj.extends = extends and export.documentObject(extends, has_seen)
+    obj.async = vm.isAsync(source, true) and true or false
+    obj.deprecated = vm.getDeprecated(source) and true or false
+    obj.visible = vm.getVisibleType(source)
+end
+
+export.makeDocObject['doc.field.name'] = function(source, obj, has_seen)
+    obj['[1]'] = export.documentObject(source[1], has_seen)
+    obj.view = source[1]
+end
+
+export.makeDocObject['doc.type.arg.name'] = export.makeDocObject['doc.field.name']
+
+export.makeDocObject['doc.type.function'] = function(source, obj, has_seen)
+    obj.args = export.documentObject(source.args, has_seen)
+    obj.returns = export.documentObject(source.returns, has_seen)
+end
+
+export.makeDocObject['doc.type.table'] = function(source, obj, has_seen)
+    obj.fields = export.documentObject(source.fields, has_seen)
+end
+
+export.makeDocObject['funcargs'] = function(source, obj, has_seen)
+    local objs = {} --make a pure numerical array
+    for i, child in ipairs(source) do
+        objs[i] = export.documentObject(child, has_seen)
+    end
+    return objs
+end
+
+export.makeDocObject['function'] = function(source, obj, has_seen)
+    obj.args = export.documentObject(source.args, has_seen)
+    obj.view = getLabel(source, source.parent.type == 'setmethod')
+    local _, _, max = vm.countReturnsOfFunction(source)
+    if max > 0 then obj.returns = {} end
+    for i = 1, max do
+        obj.returns[i] = export.documentObject(vm.getReturnOfFunction(source, i), has_seen) --check if bug?
+    end
+end
+
+export.makeDocObject['function.return'] = function(source, obj, has_seen)
+    obj.desc = source.comment and getDesc(source.comment)
+    obj.rawdesc = source.comment and getDesc(source.comment, true)
+end
+
+export.makeDocObject['local'] = function(source, obj, has_seen)
+    obj.name = source[1]
+end
+
+export.makeDocObject['luals.config'] = function(source, obj, has_seen)
+
+end
+
+export.makeDocObject['self'] = export.makeDocObject['local']
+
+export.makeDocObject['setfield'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setglobal'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setindex'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setmethod'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['tableindex'] = function(source, obj, has_seen)
+    obj.name = source.index[1]
+end
+
+export.makeDocObject['type'] = function(source, obj, has_seen)
+    if export.makeDocObject['variable'](source, obj, has_seen) == false then
+        return false
+    end
+    obj.fields = {}
+    vm.getClassFields(ws.rootUri, source, vm.ANY, function (next_source, mark)
+        if next_source.type == 'doc.field'
+        or next_source.type == 'setfield'
+        or next_source.type == 'setmethod'
+        or next_source.type == 'tableindex'
+        then
+            table.insert(obj.fields, export.documentObject(next_source, has_seen))
+        end
+    end)
+    table.sort(obj.fields, export.sortDoc)
+end
+
+export.makeDocObject['variable'] = function(source, obj, has_seen)
+    obj.defines = {}
+    for _, set in ipairs(source:getSets(ws.rootUri)) do
+        if set.type == 'setglobal'
+        or set.type == 'setfield'
+        or set.type == 'setmethod'
+        or set.type == 'setindex'
+        or set.type == 'doc.alias'
+        or set.type == 'doc.class'
+        then
+            table.insert(obj.defines, export.documentObject(set, has_seen))
+        end
+    end
+    if #obj.defines == 0 then return false end
+    table.sort(obj.defines, export.sortDoc)
+end
+
+---gathers the globals that are to be exported in documentation
+---@async
+---@return table globals
+function export.gatherGlobals()
+    local all_globals = vm.getAllGlobals()
+    local globals = {}
+    for _, g in pairs(all_globals) do
+        table.insert(globals, g)
+    end
+    return globals
+end
+
+---builds a lua table of based on `globals` and their elements
+---@async
+---@param globals table
+---@param callback fun(i, max)
+function export.makeDocs(globals, callback)
+    local docs = {}
+
+    for i, global in ipairs(globals) do
+        table.insert(docs, export.documentObject(global))
+        callback(i, #globals)
+    end
+
+    table.sort(docs, export.sortDoc)
+
+    return docs
+end
+
+---takes the table from `makeDocs`, serializes it, and exports it
+---@async
+---@param docs table
+---@param outputDir string
+---@return boolean ok, string[] outputPaths, (string|nil)[]? errs
+function export.serializeAndExport(docs, outputDir)
+    local jsonPath = outputDir .. '/doc.json'
+    local mdPath = outputDir .. '/doc.md'
+
+    --export to json
+    local old_jsonb_supportSparseArray = jsonb.supportSparseArray
+    jsonb.supportSparseArray = true
+    local jsonOk, jsonErr = util.saveFile(jsonPath, jsonb.beautify(docs))
+    jsonb.supportSparseArray = old_jsonb_supportSparseArray
+
+
+    --export to markdown
+    local md  = markdown()
+    for _, class in ipairs(docs) do
+        md:add('md', '# ' .. class.name)
+        md:emptyLine()
+        md:add('md', class.desc)
+        md:emptyLine()
+        if class.defines then
+            for _, define in ipairs(class.defines) do
+                if define.extends then
+                    md:add('lua', define.extends.view)
+                    md:emptyLine()
+                end
+            end
+        end
+        if class.fields then
+            local mark = {}
+            for _, field in ipairs(class.fields) do
+                if not mark[field.name] then
+                    mark[field.name] = true
+                    md:add('md', '## ' .. field.name)
+                    md:emptyLine()
+                    md:add('lua', field.extends.view)
+                    md:emptyLine()
+                    md:add('md', field.desc)
+                    md:emptyLine()
+                end
+            end
+        end
+        md:splitLine()
+    end
+    local mdOk, mdErr = util.saveFile(mdPath, md:string())
+
+    --error checking save file
+    if( not (jsonOk and mdOk) ) then
+        return false, {jsonPath, mdPath}, {jsonErr, mdErr}
+    end
+
+    return true, {jsonPath, mdPath}
+end
+
+return export
\ No newline at end of file
diff --git a/script/cli/doc/init.lua b/script/cli/doc/init.lua
new file mode 100644
index 00000000..78a16e9e
--- /dev/null
+++ b/script/cli/doc/init.lua
@@ -0,0 +1,243 @@
+local lclient  = require 'lclient'
+local furi     = require 'file-uri'
+local ws       = require 'workspace'
+local files    = require 'files'
+local util     = require 'utility'
+local lang     = require 'language'
+local config   = require 'config.config'
+local await    = require 'await'
+local progress = require 'progress'
+local fs       = require 'bee.filesystem'
+
+local doc = {}
+
+---Find file 'doc.json'.
+---@return fs.path
+local function findDocJson()
+    local doc_json_path
+    if type(DOC_UPDATE) == 'string' then
+        doc_json_path = fs.absolute(fs.path(DOC_UPDATE)) .. '/doc.json'
+    else
+        doc_json_path = fs.current_path() .. '/doc.json'
+    end
+    if fs.exists(doc_json_path) then
+        return doc_json_path
+    else
+        error(string.format('Error: File "%s" not found.', doc_json_path))
+    end
+end
+
+---@return string # path of 'doc.json'
+---@return string # path to be documented
+local function getPathDocUpdate()
+    local doc_json_path = findDocJson()
+    local ok, doc_path = pcall(
+        function ()
+            local json = require('json')
+            local json_file = io.open(doc_json_path:string(), 'r'):read('*all')
+            local json_data = json.decode(json_file)
+            for _, section in ipairs(json_data) do
+                if section.type == 'luals.config' then
+                    return section.DOC
+                end
+            end
+    end)
+    if ok then
+        local doc_json_dir = doc_json_path:string():gsub('/doc.json', '')
+        return doc_json_dir, doc_path
+    else
+        error(string.format('Error: Cannot update "%s".', doc_json_path .. '/doc.json'))
+    end
+end
+
+---clones a module and assigns any internal upvalues pointing to the module to the new clone
+---useful for sandboxing
+---@param tbl table module to be cloned
+---@return table module_clone the cloned module
+local function reinstantiateModule(tbl, _new_module, _old_module, _has_seen)
+    _old_module = _old_module or tbl --remember old module only at root
+    _has_seen = _has_seen or {} --remember visited indecies
+    if(type(tbl) == 'table') then
+        if _has_seen[tbl] then return _has_seen[tbl] end
+        local clone = {}
+        _has_seen[tbl] = true
+        for key, value in pairs(tbl) do
+            clone[key] = reinstantiateModule(value, _new_module or clone, _old_module, _has_seen)
+        end
+        setmetatable(clone, getmetatable(tbl))
+        return clone
+    elseif(type(tbl) == 'function') then
+        local func = tbl
+        if _has_seen[func] then return _has_seen[func] end --copy function pointers instead of building clones
+        local upvalues = {}
+        local i = 1
+        while true do
+            local label, value = debug.getupvalue(func, i)
+            if not value then break end
+            upvalues[i] = value == _old_module and _new_module or value
+            i = i + 1
+        end
+        local new_func = load(string.dump(func))--, 'function@reinstantiateModule()', 'b', _ENV)
+        for index, upvalue in ipairs(upvalues) do
+            debug.setupvalue(new_func, index, upvalue)
+        end
+        _has_seen[func] = new_func
+        return new_func
+    else
+        return tbl
+    end
+end
+
+--these modules need to be loaded by the time this function is created
+--im leaving them here since this is a pretty strange function that might get moved somewhere else later
+--so make sure to bring these with you!
+require 'workspace'
+require 'vm'
+require 'parser.guide'
+require 'core.hover.description'
+require 'core.hover.label'
+require 'json-beautify'
+require 'utility'
+require 'provider.markdown'
+
+---Gets config file's doc gen overrides.
+---@return table dirty_module clone of the export module modified by user buildscript
+local function injectBuildScript()
+    local sub_path = config.get(ws.rootUri, 'Lua.docScriptPath')
+    local module = reinstantiateModule( ( require 'cli.doc.export' ) )
+    --if default, then no build script modifications
+    if sub_path == '' then
+        return module
+    end
+    local resolved_path = fs.absolute(fs.path(DOC)):string() .. sub_path
+    local f <close> = io.open(resolved_path, 'r')
+    if not f then
+        error('could not open config file at '..tostring(resolved_path))
+    end
+    --include all `require`s in script.cli.doc.export in enviroment
+    --NOTE: allows access to the global enviroment!
+    local data, err = loadfile(resolved_path, 't', setmetatable({
+            export = module,
+
+            ws       = require 'workspace',
+            vm       = require 'vm',
+            guide    = require 'parser.guide',
+            getDesc  = require 'core.hover.description',
+            getLabel = require 'core.hover.label',
+            jsonb    = require 'json-beautify',
+            util     = require 'utility',
+            markdown = require 'provider.markdown'
+        },
+        {__index = _G}))
+    if err or not data then
+       error(err, 0)
+    end
+    data()
+    return module
+end
+
+---runtime call for documentation exporting
+---@async
+---@param outputPath string
+function doc.makeDoc(outputPath)
+    ws.awaitReady(ws.rootUri)
+
+    local expandAlias = config.get(ws.rootUri, 'Lua.hover.expandAlias')
+    config.set(ws.rootUri, 'Lua.hover.expandAlias', false)
+    local _ <close> = function ()
+        config.set(ws.rootUri, 'Lua.hover.expandAlias', expandAlias)
+    end
+
+    await.sleep(0.1)
+
+    -- ready --
+
+    local prog <close> = progress.create(ws.rootUri, lang.script('CLI_DOC_WORKING'), 0)
+
+    local dirty_export = injectBuildScript()
+
+    local globals = dirty_export.gatherGlobals()
+
+    local docs = dirty_export.makeDocs(globals, function (i, max)
+        prog:setMessage(('%d/%d'):format(i, max))
+        prog:setPercentage((i) / max * 100)
+    end)
+
+    local ok, outPaths, err = dirty_export.serializeAndExport(docs, outputPath)
+    if not ok then
+        error(err)
+    end
+
+    return table.unpack(outPaths)
+end
+
+---CLI call for documentation (parameter '--DOC=...' is passed to server)
+function doc.runCLI()
+    lang(LOCALE)
+
+    if DOC_UPDATE then
+        DOC_OUT_PATH, DOC = getPathDocUpdate()
+    end
+
+    if type(DOC) ~= 'string' then
+        print(lang.script('CLI_CHECK_ERROR_TYPE', type(DOC)))
+        return
+    end
+
+    local rootUri = furi.encode(fs.absolute(fs.path(DOC)):string())
+    if not rootUri then
+        print(lang.script('CLI_CHECK_ERROR_URI', DOC))
+        return
+    end
+
+    print('root uri = ' .. rootUri)
+
+    util.enableCloseFunction()
+
+    local lastClock = os.clock()
+
+    ---@async
+    lclient():start(function (client)
+        client:registerFakers()
+
+        client:initialize {
+            rootUri = rootUri,
+        }
+        io.write(lang.script('CLI_DOC_INITING'))
+
+        config.set(nil, 'Lua.diagnostics.enable', false)
+        config.set(nil, 'Lua.hover.expandAlias', false)
+
+        ws.awaitReady(rootUri)
+        await.sleep(0.1)
+
+        --ready--
+
+        local dirty_export = injectBuildScript()
+
+        local globals = dirty_export.gatherGlobals()
+
+        local docs = dirty_export.makeDocs(globals, function (i, max)
+            if os.clock() - lastClock > 0.2 then
+                lastClock = os.clock()
+                local output = '\x0D'
+                            .. ('>'):rep(math.ceil(i / max * 20))
+                            .. ('='):rep(20 - math.ceil(i / max * 20))
+                            .. ' '
+                            .. ('0'):rep(#tostring(max) - #tostring(i))
+                            .. tostring(i) .. '/' .. tostring(max)
+                io.write(output)
+            end
+        end)
+        io.write('\x0D')
+
+        local ok, outPaths, err = dirty_export.serializeAndExport(docs, DOC_OUT_PATH)
+        print(lang.script('CLI_DOC_DONE'))
+        for i, path in ipairs(outPaths) do
+            local this_err = (type(err) == 'table') and err[i] or nil
+            print(this_err or files.normalize(path))
+        end
+    end)
+end
+
+return doc
\ No newline at end of file
diff --git a/script/cli/doc2md.lua b/script/cli/doc2md.lua
deleted file mode 100644
index 70c1b2a0..00000000
--- a/script/cli/doc2md.lua
+++ /dev/null
@@ -1,53 +0,0 @@
--- This is an example of how to process the generated `doc.json` file.
--- You can use it to generate a markdown file or a html file.
-
-local jsonc    = require 'jsonc'
-local util     = require 'utility'
-local markdown = require 'provider.markdown'
-
-local export = {}
-
-function export.buildMD(outputPath)
-    local doc = jsonc.decode_jsonc(util.loadFile(outputPath .. '/doc.json'))
-    local md  = markdown()
-
-    assert(type(doc) == 'table')
-
-    for _, class in ipairs(doc) do
-        md:add('md', '# ' .. class.name)
-        md:emptyLine()
-        md:add('md', class.desc)
-        md:emptyLine()
-        if class.defines then
-            for _, define in ipairs(class.defines) do
-                if define.extends then
-                    md:add('lua', define.extends.view)
-                    md:emptyLine()
-                end
-            end
-        end
-        if class.fields then
-            local mark = {}
-            for _, field in ipairs(class.fields) do
-                if not mark[field.name] then
-                    mark[field.name] = true
-                    md:add('md', '## ' .. field.name)
-                    md:emptyLine()
-                    md:add('lua', field.extends.view)
-                    md:emptyLine()
-                    md:add('md', field.desc)
-                    md:emptyLine()
-                end
-            end
-        end
-        md:splitLine()
-    end
-
-    local mdPath = outputPath .. '/doc.md'
-
-    util.saveFile(mdPath, md:string())
-
-    return mdPath
-end
-
-return export
diff --git a/script/config/template.lua b/script/config/template.lua
index ee7dde37..6d691b0a 100644
--- a/script/config/template.lua
+++ b/script/config/template.lua
@@ -408,6 +408,8 @@ local template = {
                                                 'glob',
                                                 'lua',
                                             },
+    --testma
+    ["Lua.docScriptPath"]                   = Type.String,
     -- VSCode
     ["Lua.addonManager.enable"]             = Type.Boolean >> true,
     ['files.associations']                  = Type.Hash(Type.String, Type.String),
diff --git a/script/vm/compiler.lua b/script/vm/compiler.lua
index e0cb54c7..5f3317b9 100644
--- a/script/vm/compiler.lua
+++ b/script/vm/compiler.lua
@@ -513,6 +513,217 @@ function vm.getClassFields(suri, object, key, pushResult)
     searchGlobal(object)
 end
 
+---for exporting, only gets unique, noninherited fields
+---@param suri uri
+---@param object vm.global
+---@param key string|number|integer|boolean|vm.global|vm.ANY
+---@param pushResult fun(field: vm.object, isMark?: boolean, discardParentFields?: boolean)
+function vm.getSimpleClassFields(suri, object, key, pushResult)
+    local mark = {}
+    local function searchClass(class, searchedFields, discardParentFields)
+        local name = class.name
+        if mark[name] then
+            return
+        end
+        mark[name] = true
+        searchedFields = searchedFields or {}
+        searchedFields[1] = searchedFields[1] or {}
+        searchedFields[name] = searchedFields[name] or {}
+        local function uniqueOrOverrideField(fieldKey)
+            if(class == object) then
+                --search only this class's tree if end of branch
+                return not searchedFields[name][fieldKey]
+            else
+                --search whole tree
+                return not searchedFields[1][fieldKey]
+            end
+        end
+
+        local hasFounded = {}
+        local function copyToSearched()
+            for fieldKey in pairs(hasFounded) do
+                searchedFields[name][fieldKey] = true
+                searchedFields[1][fieldKey] = true
+                hasFounded[fieldKey] = nil
+            end
+        end
+
+        local sets = class:getSets(suri)
+        --go fully up the class tree first and exhaust it all
+        for _, set in ipairs(sets) do
+            if set.type == 'doc.class' then
+                -- look into extends(if field not found)
+                if not searchedFields[key] and set.extends then
+                    for _, extend in ipairs(set.extends) do
+                        if extend.type == 'doc.extends.name' then
+                            local extendType = vm.getGlobal('type', extend[1])
+                            if extendType then
+                                pushResult(extendType, true, false)
+                                searchClass(extendType, searchedFields, true)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+        copyToSearched()
+
+        for _, set in ipairs(sets) do
+            if set.type == 'doc.class' then
+                -- check ---@field
+                for _, field in ipairs(set.fields) do
+                    local fieldKey = guide.getKeyName(field)
+                    if fieldKey then
+                        -- ---@field x boolean -> class.x
+                        if key == vm.ANY
+                        or fieldKey == key then
+                            if uniqueOrOverrideField(fieldKey) then
+                                pushResult(field, true, discardParentFields)
+                                hasFounded[fieldKey] = true
+                            end
+                        end
+                        goto CONTINUE
+                    end
+                    if key == vm.ANY then
+                        pushResult(field, true, discardParentFields)
+                        goto CONTINUE
+                    end
+                    if hasFounded[key] then
+                        goto CONTINUE
+                    end
+                    local keyType = type(key)
+                    if keyType == 'table' then
+                        -- ---@field [integer] boolean -> class[integer]
+                        local fieldNode = vm.compileNode(field.field)
+                        if vm.isSubType(suri, key.name, fieldNode) then
+                            local nkey = '|' .. key.name
+                            if uniqueOrOverrideField(nkey) then
+                                pushResult(field, true, discardParentFields)
+                                hasFounded[nkey] = true
+                            end
+                        end
+                    else
+                        local keyObject
+                        if keyType == 'number' then
+                            if math.tointeger(key) then
+                                keyObject = { type = 'integer', [1] = key }
+                            else
+                                keyObject = { type = 'number', [1] = key }
+                            end
+                        elseif keyType == 'boolean'
+                        or     keyType == 'string' then
+                            keyObject = { type = keyType, [1] = key }
+                        end
+                        if keyObject and field.field.type ~= 'doc.field.name' then
+                            -- ---@field [integer] boolean -> class[1]
+                            local fieldNode = vm.compileNode(field.field)
+                            if vm.isSubType(suri, keyObject, fieldNode) then
+                                local nkey = '|' .. keyType
+                                if uniqueOrOverrideField(nkey) then
+                                    pushResult(field, true, discardParentFields)
+                                    hasFounded[nkey] = true
+                                end
+                            end
+                        end
+                    end
+                    ::CONTINUE::
+                end
+            end
+        end
+        copyToSearched()
+
+        for _, set in ipairs(sets) do
+            if set.type == 'doc.class' then
+                -- check local field and global field
+                if uniqueOrOverrideField(key) and set.bindSource then
+                    local src = set.bindSource
+                    if src.value and src.value.type == 'table' then
+                        searchFieldSwitch('table', suri, src.value, key, function (field)
+                            local fieldKey = guide.getKeyName(field)
+                            if fieldKey then
+                                if uniqueOrOverrideField(fieldKey)
+                                and guide.isAssign(field) then
+                                    hasFounded[fieldKey] = true
+                                    pushResult(field, true, discardParentFields)
+                                end
+                            end
+                        end)
+                    end
+                    if  src.value
+                    and src.value.type == 'select'
+                    and src.value.vararg.type == 'call' then
+                        local func = src.value.vararg.node
+                        local args = src.value.vararg.args
+                        if  func.special == 'setmetatable'
+                        and args
+                        and args[1]
+                        and args[1].type == 'table' then
+                            searchFieldSwitch('table', suri, args[1], key, function (field)
+                                local fieldKey = guide.getKeyName(field)
+                                if fieldKey then
+                                    if uniqueOrOverrideField(fieldKey)
+                                    and guide.isAssign(field) then
+                                        hasFounded[fieldKey] = true
+                                        pushResult(field, true, discardParentFields)
+                                    end
+                                end
+                            end)
+                        end
+                    end
+                end
+            end
+        end
+        copyToSearched()
+
+        for _, set in ipairs(sets) do
+            if set.type == 'doc.class' then
+                if uniqueOrOverrideField(key) and set.bindSource then
+                    local src = set.bindSource
+                    searchFieldSwitch(src.type, suri, src, key, function (field)
+                        local fieldKey = guide.getKeyName(field)
+                        if fieldKey and uniqueOrOverrideField(fieldKey) then
+                            if  uniqueOrOverrideField(fieldKey)
+                            and guide.isAssign(field)
+                            and field.value then
+                                if  vm.getVariableID(field)
+                                and vm.getVariableID(field) == vm.getVariableID(field.value) then
+                                elseif vm.getGlobalNode(src)
+                                and    vm.getGlobalNode(src) == vm.getGlobalNode(field.value) then
+                                else
+                                    hasFounded[fieldKey] = true
+                                end
+                                pushResult(field, true, discardParentFields)
+                            end
+                        end
+                    end)
+                end
+            end
+        end
+        copyToSearched()
+    end
+
+    local function searchGlobal(class)
+        if class.cate == 'type' and class.name == '_G' then
+            if key == vm.ANY then
+                local sets = vm.getGlobalSets(suri, 'variable')
+                for _, set in ipairs(sets) do
+                    pushResult(set)
+                end
+            elseif type(key) == 'string' then
+                local global = vm.getGlobal('variable', key)
+                if global then
+                    for _, set in ipairs(global:getSets(suri)) do
+                        pushResult(set)
+                    end
+                end
+            end
+        end
+    end
+
+    searchClass(object)
+    searchGlobal(object)
+end
+
 ---@param func  parser.object
 ---@param index integer
 ---@return (parser.object|vm.generic)?
@@ -2100,4 +2311,4 @@ function vm.compileNode(source)
     local node = vm.getNode(source)
     ---@cast node -?
     return node
-end
+end
\ No newline at end of file