2 files changed, 597 insertions, 0 deletions
diff --git a/script/cli/doc/export.lua b/script/cli/doc/export.lua
new file mode 100644
index 00000000..5a8c3239
--- /dev/null
+++ b/script/cli/doc/export.lua
@@ -0,0 +1,354 @@
+local ws       = require 'workspace'
+local vm       = require 'vm'
+local guide    = require 'parser.guide'
+
+local getDesc  = require 'core.hover.description'
+local getLabel = require 'core.hover.label'
+local jsonb    = require 'json-beautify'
+local util     = require 'utility'
+local markdown = require 'provider.markdown'
+
+---@alias doctype
+---| 'doc.alias'
+---| 'doc.class'
+---| 'doc.field'
+---| 'doc.field.name'
+---| 'doc.type.arg.name'
+---| 'doc.type.function'
+---| 'doc.type.table'
+---| 'funcargs'
+---| 'function'
+---| 'function.return'
+---| 'global.type'
+---| 'global.variable'
+---| 'local'
+---| 'luals.config'
+---| 'self'
+---| 'setfield'
+---| 'setglobal'
+---| 'setindex'
+---| 'setmethod'
+---| 'tableindex'
+---| 'type'
+
+---@class docUnion broadest possible collection of exported docs, these are never all together.
+---@field [1] string in name when table, always the same as view
+---@field args docUnion[] list of argument docs passed to function
+---@field async boolean has @async tag
+---@field defines docUnion[] list of places where this is doc is defined and how its defined there
+---@field deprecated boolean has @deprecated tag
+---@field desc string code commentary
+---@field extends string | docUnion ? what type this 'is'. string:<Parent_Class> for type: 'type', docUnion for type: 'function', string<primative> for other type 's
+---@field fields docUnion[] class's fields
+---@field file string path to where this token is defined
+---@field finish [integer, integer] 0-indexed [line, column] position of end of token
+---@field name string canonical name
+---@field rawdesc string same as desc, but may have other things for types doc.retun andr doc.param (unused?)
+---@field returns docUnion | docUnion[] list of docs for return values. if singluar, then always {type: 'undefined'}? might be a bug.
+---@field start [integer, integer] 0-indexed [line, column] position of start of token
+---@field type doctype role that this token plays in documentation. different from the 'type'/'class' this token is
+---@field types docUnion[] type union? unclear. seems to be related to alias, maybe
+---@field view string full method name, class, basal type, or unknown. in name table same as [1]
+---@field visible 'package'|'private'|'protected'|'public' visibilty tag
+
+local export = {}
+
+function export.getLocalPath(uri)
+    --remove uri root (and prefix)
+    local local_file_uri = uri
+    local i, j = local_file_uri:find(DOC)
+    if not j then
+        return '[FORIEGN]'..uri
+    end
+    return local_file_uri:sub( j + 1 )
+end
+
+function export.positionOf(rowcol)
+    return type(rowcol) == 'table' and guide.positionOf(rowcol[1], rowcol[2]) or -1
+end
+
+function export.sortDoc(a,b)
+    if a.name ~= b.name then
+        return a.name < b.name
+    end
+
+    if a.file ~= b.file then
+        return a.file < b.file
+    end
+
+    return export.positionOf(a.start) < export.positionOf(b.start)
+end
+
+
+--- recursively generate documentation all parser objects downstream of `source`
+---@async
+---@param source parser.object | vm.global
+---@param has_seen table? keeps track of visited nodes in documentation tree
+---@return docUnion | [docUnion] | string | number | boolean | nil 
+function export.documentObject(source, has_seen)
+    --is this a primative type? then we dont need to process it.
+    if type(source) ~= 'table' then return source end
+
+    --set up/check recursion
+    if not has_seen then has_seen = {} end
+    if has_seen[source] then
+        return nil
+    end
+    has_seen[source] = true
+
+    --is this an array type? then process each array item and collect it
+    if (#source > 0 and next(source, #source) == nil) then
+        local objs = {} --make a pure numerical array
+        for i, child in ipairs(source) do
+            objs[i] = export.documentObject(child, has_seen)
+        end
+        return objs
+    end
+
+    --if neither, then this is a singular docUnion
+    local obj = export.makeDocObject['INIT'](source, has_seen)
+
+    --check if this source has a type (no type sources are usually autogen'd anon functions's return values that are not explicitly stated)
+    if not obj.type then return obj end
+
+    local res = export.makeDocObject[obj.type](source, obj, has_seen)
+    if res == false then
+        return nil
+    end
+    return res or obj
+end
+
+---Switch statement table. functions can be overriden by user file.
+---@table
+export.makeDocObject = setmetatable({}, {__index = function(t, k)
+    return function()
+        --print('DocError: no type "'..k..'"')
+    end
+end})
+
+export.makeDocObject['INIT'] = function(source, has_seen)
+    ---@as docUnion
+    local ok, desc = pcall(getDesc, source)
+    local rawok, rawdesc = pcall(getDesc, source, true)
+    return {
+        type = source.cate or source.type,
+        name = export.documentObject((source.getCodeName and source:getCodeName()) or source.name, has_seen),
+        start = source.start and {guide.rowColOf(source.start)},
+        finish = source.finish and {guide.rowColOf(source.finish)},
+        types = export.documentObject(source.types, has_seen),
+        view = vm.getInfer(source):view(ws.rootUri),
+        desc = ok and desc or nil,
+        rawdesc = rawok and rawdesc or nil,
+    }
+end
+
+export.makeDocObject['doc.alias'] = function(source, obj, has_seen)
+
+end
+
+export.makeDocObject['doc.field'] = function(source, obj, has_seen)
+    if source.field.type == 'doc.field.name' then
+        obj.name = source.field[1]
+    else
+        obj.name = ('[%s]'):format(vm.getInfer(source.field):view(ws.rootUri))
+    end
+    obj.file = export.getLocalPath(guide.getUri(source))
+    obj.extends = source.extends and export.documentObject(source.extends, has_seen) --check if bug?
+    obj.async = vm.isAsync(source, true) and true or false --if vm.isAsync(set, true) then result.defines[#result.defines].extends['async'] = true end
+    obj.deprecated = vm.getDeprecated(source) and true or false --  if (depr and not depr.versions) the result.defines[#result.defines].extends['deprecated'] = true end
+    obj.visible = vm.getVisibleType(source)
+end
+
+export.makeDocObject['doc.class'] = function(source, obj, has_seen)
+    local extends = source.extends or source.value --doc.class or other
+    local field = source.field or source.method 
+    obj.name = type(field) == 'table' and field[1] or nil
+    obj.file = export.getLocalPath(guide.getUri(source))
+    obj.extends = extends and export.documentObject(extends, has_seen)
+    obj.async = vm.isAsync(source, true) and true or false
+    obj.deprecated = vm.getDeprecated(source) and true or false
+    obj.visible = vm.getVisibleType(source)
+end
+
+export.makeDocObject['doc.field.name'] = function(source, obj, has_seen)
+    obj['[1]'] = export.documentObject(source[1], has_seen)
+    obj.view = source[1]
+end
+
+export.makeDocObject['doc.type.arg.name'] = export.makeDocObject['doc.field.name']
+
+export.makeDocObject['doc.type.function'] = function(source, obj, has_seen)
+    obj.args = export.documentObject(source.args, has_seen)
+    obj.returns = export.documentObject(source.returns, has_seen)
+end
+
+export.makeDocObject['doc.type.table'] = function(source, obj, has_seen)
+    obj.fields = export.documentObject(source.fields, has_seen)
+end
+
+export.makeDocObject['funcargs'] = function(source, obj, has_seen)
+    local objs = {} --make a pure numerical array
+    for i, child in ipairs(source) do
+        objs[i] = export.documentObject(child, has_seen)
+    end
+    return objs
+end
+
+export.makeDocObject['function'] = function(source, obj, has_seen)
+    obj.args = export.documentObject(source.args, has_seen)
+    obj.view = getLabel(source, source.parent.type == 'setmethod')
+    local _, _, max = vm.countReturnsOfFunction(source)
+    if max > 0 then obj.returns = {} end
+    for i = 1, max do
+        obj.returns[i] = export.documentObject(vm.getReturnOfFunction(source, i), has_seen) --check if bug?
+    end
+end
+
+export.makeDocObject['function.return'] = function(source, obj, has_seen)
+    obj.desc = source.comment and getDesc(source.comment)
+    obj.rawdesc = source.comment and getDesc(source.comment, true)
+end
+
+export.makeDocObject['local'] = function(source, obj, has_seen)
+    obj.name = source[1]
+end
+
+export.makeDocObject['luals.config'] = function(source, obj, has_seen)
+
+end
+
+export.makeDocObject['self'] = export.makeDocObject['local']
+
+export.makeDocObject['setfield'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setglobal'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setindex'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['setmethod'] = export.makeDocObject['doc.class']
+
+export.makeDocObject['tableindex'] = function(source, obj, has_seen)
+    obj.name = source.index[1]
+end
+
+export.makeDocObject['type'] = function(source, obj, has_seen)
+    if export.makeDocObject['variable'](source, obj, has_seen) == false then
+        return false
+    end
+    obj.fields = {}
+    vm.getClassFields(ws.rootUri, source, vm.ANY, function (next_source, mark)
+        if next_source.type == 'doc.field'
+        or next_source.type == 'setfield'
+        or next_source.type == 'setmethod'
+        or next_source.type == 'tableindex'
+        then
+            table.insert(obj.fields, export.documentObject(next_source, has_seen))
+        end
+    end)
+    table.sort(obj.fields, export.sortDoc)
+end
+
+export.makeDocObject['variable'] = function(source, obj, has_seen)
+    obj.defines = {}
+    for _, set in ipairs(source:getSets(ws.rootUri)) do
+        if set.type == 'setglobal'
+        or set.type == 'setfield'
+        or set.type == 'setmethod'
+        or set.type == 'setindex'
+        or set.type == 'doc.alias'
+        or set.type == 'doc.class'
+        then
+            table.insert(obj.defines, export.documentObject(set, has_seen))
+        end
+    end
+    if #obj.defines == 0 then return false end
+    table.sort(obj.defines, export.sortDoc)
+end
+
+---gathers the globals that are to be exported in documentation
+---@async
+---@return table globals
+function export.gatherGlobals()
+    local all_globals = vm.getAllGlobals()
+    local globals = {}
+    for _, g in pairs(all_globals) do
+        table.insert(globals, g)
+    end
+    return globals
+end
+
+---builds a lua table of based on `globals` and their elements
+---@async
+---@param globals table
+---@param callback fun(i, max)
+function export.makeDocs(globals, callback)
+    local docs = {}
+
+    for i, global in ipairs(globals) do
+        table.insert(docs, export.documentObject(global))
+        callback(i, #globals)
+    end
+
+    table.sort(docs, export.sortDoc)
+
+    return docs
+end
+
+---takes the table from `makeDocs`, serializes it, and exports it
+---@async
+---@param docs table
+---@param outputDir string
+---@return boolean ok, string[] outputPaths, (string|nil)[]? errs
+function export.serializeAndExport(docs, outputDir)
+    local jsonPath = outputDir .. '/doc.json'
+    local mdPath = outputDir .. '/doc.md'
+
+    --export to json
+    local old_jsonb_supportSparseArray = jsonb.supportSparseArray
+    jsonb.supportSparseArray = true
+    local jsonOk, jsonErr = util.saveFile(jsonPath, jsonb.beautify(docs))
+    jsonb.supportSparseArray = old_jsonb_supportSparseArray
+
+
+    --export to markdown
+    local md  = markdown()
+    for _, class in ipairs(docs) do
+        md:add('md', '# ' .. class.name)
+        md:emptyLine()
+        md:add('md', class.desc)
+        md:emptyLine()
+        if class.defines then
+            for _, define in ipairs(class.defines) do
+                if define.extends then
+                    md:add('lua', define.extends.view)
+                    md:emptyLine()
+                end
+            end
+        end
+        if class.fields then
+            local mark = {}
+            for _, field in ipairs(class.fields) do
+                if not mark[field.name] then
+                    mark[field.name] = true
+                    md:add('md', '## ' .. field.name)
+                    md:emptyLine()
+                    md:add('lua', field.extends.view)
+                    md:emptyLine()
+                    md:add('md', field.desc)
+                    md:emptyLine()
+                end
+            end
+        end
+        md:splitLine()
+    end
+    local mdOk, mdErr = util.saveFile(mdPath, md:string())
+
+    --error checking save file
+    if( not (jsonOk and mdOk) ) then
+        return false, {jsonPath, mdPath}, {jsonErr, mdErr}
+    end
+
+    return true, {jsonPath, mdPath}
+end
+
+return export
+\ No newline at end of file
diff --git a/script/cli/doc/init.lua b/script/cli/doc/init.lua
new file mode 100644
index 00000000..78a16e9e
--- /dev/null
+++ b/script/cli/doc/init.lua
@@ -0,0 +1,243 @@
+local lclient  = require 'lclient'
+local furi     = require 'file-uri'
+local ws       = require 'workspace'
+local files    = require 'files'
+local util     = require 'utility'
+local lang     = require 'language'
+local config   = require 'config.config'
+local await    = require 'await'
+local progress = require 'progress'
+local fs       = require 'bee.filesystem'
+
+local doc = {}
+
+---Find file 'doc.json'.
+---@return fs.path
+local function findDocJson()
+    local doc_json_path
+    if type(DOC_UPDATE) == 'string' then
+        doc_json_path = fs.absolute(fs.path(DOC_UPDATE)) .. '/doc.json'
+    else
+        doc_json_path = fs.current_path() .. '/doc.json'
+    end
+    if fs.exists(doc_json_path) then
+        return doc_json_path
+    else
+        error(string.format('Error: File "%s" not found.', doc_json_path))
+    end
+end
+
+---@return string # path of 'doc.json'
+---@return string # path to be documented
+local function getPathDocUpdate()
+    local doc_json_path = findDocJson()
+    local ok, doc_path = pcall(
+        function ()
+            local json = require('json')
+            local json_file = io.open(doc_json_path:string(), 'r'):read('*all')
+            local json_data = json.decode(json_file)
+            for _, section in ipairs(json_data) do
+                if section.type == 'luals.config' then
+                    return section.DOC
+                end
+            end
+    end)
+    if ok then
+        local doc_json_dir = doc_json_path:string():gsub('/doc.json', '')
+        return doc_json_dir, doc_path
+    else
+        error(string.format('Error: Cannot update "%s".', doc_json_path .. '/doc.json'))
+    end
+end
+
+---clones a module and assigns any internal upvalues pointing to the module to the new clone
+---useful for sandboxing
+---@param tbl table module to be cloned
+---@return table module_clone the cloned module
+local function reinstantiateModule(tbl, _new_module, _old_module, _has_seen)
+    _old_module = _old_module or tbl --remember old module only at root
+    _has_seen = _has_seen or {} --remember visited indecies
+    if(type(tbl) == 'table') then
+        if _has_seen[tbl] then return _has_seen[tbl] end
+        local clone = {}
+        _has_seen[tbl] = true
+        for key, value in pairs(tbl) do
+            clone[key] = reinstantiateModule(value, _new_module or clone, _old_module, _has_seen)
+        end
+        setmetatable(clone, getmetatable(tbl))
+        return clone
+    elseif(type(tbl) == 'function') then
+        local func = tbl
+        if _has_seen[func] then return _has_seen[func] end --copy function pointers instead of building clones
+        local upvalues = {}
+        local i = 1
+        while true do
+            local label, value = debug.getupvalue(func, i)
+            if not value then break end
+            upvalues[i] = value == _old_module and _new_module or value
+            i = i + 1
+        end
+        local new_func = load(string.dump(func))--, 'function@reinstantiateModule()', 'b', _ENV)
+        for index, upvalue in ipairs(upvalues) do
+            debug.setupvalue(new_func, index, upvalue)
+        end
+        _has_seen[func] = new_func
+        return new_func
+    else
+        return tbl
+    end
+end
+
+--these modules need to be loaded by the time this function is created
+--im leaving them here since this is a pretty strange function that might get moved somewhere else later
+--so make sure to bring these with you!
+require 'workspace'
+require 'vm'
+require 'parser.guide'
+require 'core.hover.description'
+require 'core.hover.label'
+require 'json-beautify'
+require 'utility'
+require 'provider.markdown'
+
+---Gets config file's doc gen overrides.
+---@return table dirty_module clone of the export module modified by user buildscript
+local function injectBuildScript()
+    local sub_path = config.get(ws.rootUri, 'Lua.docScriptPath')
+    local module = reinstantiateModule( ( require 'cli.doc.export' ) )
+    --if default, then no build script modifications
+    if sub_path == '' then
+        return module
+    end
+    local resolved_path = fs.absolute(fs.path(DOC)):string() .. sub_path
+    local f <close> = io.open(resolved_path, 'r')
+    if not f then
+        error('could not open config file at '..tostring(resolved_path))
+    end
+    --include all `require`s in script.cli.doc.export in enviroment
+    --NOTE: allows access to the global enviroment!
+    local data, err = loadfile(resolved_path, 't', setmetatable({
+            export = module,
+
+            ws       = require 'workspace',
+            vm       = require 'vm',
+            guide    = require 'parser.guide',
+            getDesc  = require 'core.hover.description',
+            getLabel = require 'core.hover.label',
+            jsonb    = require 'json-beautify',
+            util     = require 'utility',
+            markdown = require 'provider.markdown'
+        },
+        {__index = _G}))
+    if err or not data then
+       error(err, 0)
+    end
+    data()
+    return module
+end
+
+---runtime call for documentation exporting
+---@async
+---@param outputPath string
+function doc.makeDoc(outputPath)
+    ws.awaitReady(ws.rootUri)
+
+    local expandAlias = config.get(ws.rootUri, 'Lua.hover.expandAlias')
+    config.set(ws.rootUri, 'Lua.hover.expandAlias', false)
+    local _ <close> = function ()
+        config.set(ws.rootUri, 'Lua.hover.expandAlias', expandAlias)
+    end
+
+    await.sleep(0.1)
+
+    -- ready --
+
+    local prog <close> = progress.create(ws.rootUri, lang.script('CLI_DOC_WORKING'), 0)
+
+    local dirty_export = injectBuildScript()
+
+    local globals = dirty_export.gatherGlobals()
+
+    local docs = dirty_export.makeDocs(globals, function (i, max)
+        prog:setMessage(('%d/%d'):format(i, max))
+        prog:setPercentage((i) / max * 100)
+    end)
+
+    local ok, outPaths, err = dirty_export.serializeAndExport(docs, outputPath)
+    if not ok then
+        error(err)
+    end
+
+    return table.unpack(outPaths)
+end
+
+---CLI call for documentation (parameter '--DOC=...' is passed to server)
+function doc.runCLI()
+    lang(LOCALE)
+
+    if DOC_UPDATE then
+        DOC_OUT_PATH, DOC = getPathDocUpdate()
+    end
+
+    if type(DOC) ~= 'string' then
+        print(lang.script('CLI_CHECK_ERROR_TYPE', type(DOC)))
+        return
+    end
+
+    local rootUri = furi.encode(fs.absolute(fs.path(DOC)):string())
+    if not rootUri then
+        print(lang.script('CLI_CHECK_ERROR_URI', DOC))
+        return
+    end
+
+    print('root uri = ' .. rootUri)
+
+    util.enableCloseFunction()
+
+    local lastClock = os.clock()
+
+    ---@async
+    lclient():start(function (client)
+        client:registerFakers()
+
+        client:initialize {
+            rootUri = rootUri,
+        }
+        io.write(lang.script('CLI_DOC_INITING'))
+
+        config.set(nil, 'Lua.diagnostics.enable', false)
+        config.set(nil, 'Lua.hover.expandAlias', false)
+
+        ws.awaitReady(rootUri)
+        await.sleep(0.1)
+
+        --ready--
+
+        local dirty_export = injectBuildScript()
+
+        local globals = dirty_export.gatherGlobals()
+
+        local docs = dirty_export.makeDocs(globals, function (i, max)
+            if os.clock() - lastClock > 0.2 then
+                lastClock = os.clock()
+                local output = '\x0D'
+                            .. ('>'):rep(math.ceil(i / max * 20))
+                            .. ('='):rep(20 - math.ceil(i / max * 20))
+                            .. ' '
+                            .. ('0'):rep(#tostring(max) - #tostring(i))
+                            .. tostring(i) .. '/' .. tostring(max)
+                io.write(output)
+            end
+        end)
+        io.write('\x0D')
+
+        local ok, outPaths, err = dirty_export.serializeAndExport(docs, DOC_OUT_PATH)
+        print(lang.script('CLI_DOC_DONE'))
+        for i, path in ipairs(outPaths) do
+            local this_err = (type(err) == 'table') and err[i] or nil
+            print(this_err or files.normalize(path))
+        end
+    end)
+end
+
+return doc
+\ No newline at end of file