local subprocess = require 'bee.subprocess' local method = require 'method' local thread = require 'bee.thread' local async = require 'async' local rpc = require 'rpc' local parser = require 'parser' local core = require 'core' local lang = require 'language' local updateTimer= require 'timer' local buildVM = require 'vm' local sourceMgr = require 'vm.source' local localMgr = require 'vm.local' local valueMgr = require 'vm.value' local chainMgr = require 'vm.chain' local functionMgr= require 'vm.function' local listMgr = require 'vm.list' local emmyMgr = require 'emmy.manager' local config = require 'config' local task = require 'task' local files = require 'files' local uric = require 'uri' local capability = require 'capability' local plugin = require 'plugin' local ErrorCodes = { -- Defined by JSON RPC ParseError = -32700, InvalidRequest = -32600, MethodNotFound = -32601, InvalidParams = -32602, InternalError = -32603, serverErrorStart = -32099, serverErrorEnd = -32000, ServerNotInitialized = -32002, UnknownErrorCode = -32001, -- Defined by the protocol. RequestCancelled = -32800, } local CachedVM = setmetatable({}, {__mode = 'kv'}) ---@class LSP local mt = {} mt.__index = mt ---@type files mt._files = nil function mt:_callMethod(name, params) local optional if name:sub(1, 2) == '$/' then name = name:sub(3) optional = true end local f = method[name] if f then local clock = os.clock() local suc, res = xpcall(f, debug.traceback, self, params) local passed = os.clock() - clock if passed > 0.2 then log.debug(('Task [%s] takes [%.3f]sec.'):format(name, passed)) end if suc then return res else local ok, r = pcall(table.dump, params) local dump = ok and r or '' log.debug(('Task [%s] failed, params: %s'):format( name, dump )) log.error(res) if res:find 'not enough memory' then self:restartDueToMemoryLeak() end return nil, { code = ErrorCodes.InternalError, message = r .. '\n' .. res, } end end if optional then return nil else return nil, { code = ErrorCodes.MethodNotFound, message = 'MethodNotFound', } end end function mt:responseProto(id, response, err) local container = table.container() if err then container.error = err else container.result = response end rpc:response(id, container) end function mt:_doProto(proto) local id = proto.id local name = proto.method local params = proto.params local response, err = self:_callMethod(name, params) if not id then return end if type(response) == 'function' then response(function (final) self:responseProto(id, final) end) else self:responseProto(id, response, err) end end function mt:clearDiagnostics(uri) rpc:notify('textDocument/publishDiagnostics', { uri = uri, diagnostics = {}, }) self._needDiagnostics[uri] = nil end ---@param uri uri ---@param compiled table ---@param mode string ---@return boolean function mt:needCompile(uri, compiled, mode) self._needDiagnostics[uri] = true if self._needCompile[uri] then return false end if not compiled then compiled = {} end if compiled[uri] then return false end self._needCompile[uri] = compiled if mode == 'child' then table.insert(self._needCompile, uri) else table.insert(self._needCompile, 1, uri) end return true end function mt:isNeedCompile(uri) return self._needCompile[uri] end function mt:isWaitingCompile() if self._needCompile[1] then return true else return false end end ---@param uri uri ---@param version integer ---@param text string function mt:saveText(uri, version, text) self._lastLoadedVM = uri self._files:save(uri, text, version) self:needCompile(uri) end ---@param uri uri function mt:isDeadText(uri) return self._files:isDead(uri) end ---@param uri uri ---@return boolean function mt:isLua(uri) if not self.workspace then return true end local path = self.workspace:absolutePathByUri(uri) if not path then return false end if self.workspace:isLuaFile(path) then return true end return false end function mt:isIgnored(uri) if not self.workspace then return true end if not self.workspace.gitignore then return true end local path = self.workspace:relativePathByUri(uri) if not path then return true end if self.workspace.gitignore(path:string()) then return true end return false end ---@param uri uri ---@param version integer ---@param text string function mt:open(uri, version, text) if not self:isLua(uri) then return end self:saveText(uri, version, text) self._files:open(uri, text) end ---@param uri uri function mt:close(uri) self._files:close(uri) if self._files:isLibrary(uri) then return end if not self:isLua(uri) or self:isIgnored(uri) then self:removeText(uri) end end ---@param uri uri ---@return boolean function mt:isOpen(uri) return self._files:isOpen(uri) end ---@param uri uri ---@param path path ---@param text string function mt:checkReadFile(uri, path, text) if not text then log.debug('No file: ', path) return false end local size = #text / 1000.0 if size > config.config.workspace.preloadFileSize then log.info(('Skip large file, size: %.3f KB: %s'):format(size, uri)) return false end if self:getCachedFileCount() >= config.config.workspace.maxPreload then if not self._hasShowHitMaxPreload then self._hasShowHitMaxPreload = true rpc:notify('window/showMessage', { type = 3, message = lang.script('MWS_MAX_PRELOAD', config.config.workspace.maxPreload), }) end return false end return true end ---@param uri uri ---@param path path ---@param buf string ---@param compiled table function mt:readText(uri, path, buf, compiled) if self._files:get(uri) then log.debug('Read failed due to duplicate:', uri) return end if not self:isLua(uri) then log.debug('Read failed due to not lua:', uri) return end if not self._files:isOpen() and self:isIgnored(uri) then log.debug('Read failed due to ignored:', uri) return end local text = buf or io.load(path) if not self._files:isOpen() and not self:checkReadFile(uri, path, text) then log.debug('Read failed due to check failed:', uri) return end self._files:save(uri, text, 0) self:needCompile(uri, compiled) end ---@param uri uri ---@param path path ---@param buf string ---@param compiled table function mt:readLibrary(uri, path, buf, compiled) if not self:isLua(uri) then return end if not self:checkReadFile(uri, path, buf) then return end self._files:save(uri, buf, 0) self._files:setLibrary(uri) self:needCompile(uri, compiled) self:clearDiagnostics(uri) end ---@param uri uri function mt:removeText(uri) self._files:remove(uri) self:compileVM(uri) end function mt:getCachedFileCount() return self._files:count() end function mt:reCompile() if self.global then self.global:remove() end if self.chain then self.chain:remove() end if self.emmy then self.emmy:remove() end local compiled = {} self._files:clearVM() for _, obj in pairs(listMgr.list) do if obj.type == 'source' or obj.type == 'function' then obj:kill() end end self.global = core.global(self) self.chain = chainMgr() self.emmy = emmyMgr() self.globalValue = nil self._compileTask:remove() self._needCompile = {} local n = 0 for uri in self._files:eachFile() do self:needCompile(uri, compiled) n = n + 1 end log.debug('reCompile:', n, self._files:count()) self:_testMemory('skip') end function mt:reDiagnostic() for uri in self._files:eachFile() do self:clearDiagnostics(uri) self._needDiagnostics[uri] = true end end function mt:clearAllFiles() for uri in self._files:eachFile() do self:clearDiagnostics(uri) end self._files:clear() end ---@param uri uri function mt:loadVM(uri) local file = self._files:get(uri) if not file then return nil end if uri ~= self._lastLoadedVM then self:needCompile(uri) end if self._compileTask and not self._compileTask:isRemoved() and self._compileTask:get 'uri' == uri then self._compileTask:fastForward() else self:compileVM(uri) end if file:getVM() then self._lastLoadedVM = uri end return file:getVM(), file:getLines() end function mt:_markCompiled(uri, compiled) local newCompiled = self._needCompile[uri] if newCompiled then newCompiled[uri] = true self._needCompile[uri] = nil end for i, u in ipairs(self._needCompile) do if u == uri then table.remove(self._needCompile, i) break end end if newCompiled == compiled then return compiled end if not compiled then compiled = {} end for k, v in pairs(newCompiled) do compiled[k] = v end return compiled end ---@param file file ---@return table function mt:compileAst(file) local ast, err, comments = parser:parse(file:getText(), 'lua', config.config.runtime.version) file.comments = comments if ast then file:setAstErr(err) else if type(err) == 'string' then local message = lang.script('PARSER_CRASH', err) log.debug(message) rpc:notify('window/showMessage', { type = 3, message = lang.script('PARSER_CRASH', err:match '%.lua%:%d+%:(.+)' or err), }) if message:find 'not enough memory' then self:restartDueToMemoryLeak() end end end return ast end ---@param file file ---@param uri uri function mt:_clearChainNode(file, uri) for pUri in file:eachParent() do local parent = self._files:get(pUri) if parent then parent:removeChild(uri) end end end ---@param file file ---@param compiled table function mt:_compileChain(file, compiled) if not compiled then compiled = {} end for uri in file:eachChild() do self:needCompile(uri, compiled, 'child') end for uri in file:eachParent() do self:needCompile(uri, compiled, 'parent') end end function mt:_compileGlobal(compiled) local uris = self.global:getAllUris() for _, uri in ipairs(uris) do self:needCompile(uri, compiled, 'global') end end function mt:_clearGlobal(uri) self.global:clearGlobal(uri) end function mt:_hasSetGlobal(uri) return self.global:hasSetGlobal(uri) end ---@param uri uri function mt:compileVM(uri) local file = self._files:get(uri) if not file then self:_markCompiled(uri) return nil end local compiled = self._needCompile[uri] if not compiled then return nil end file:removeVM() local clock = os.clock() local ast = self:compileAst(file) local version = file:getVersion() local astCost = os.clock() - clock if astCost > 0.1 then log.warn(('Compile Ast[%s] takes [%.3f] sec, size [%.3f]kb'):format(uri, astCost, #file:getText() / 1000)) end file:clearOldText() self:_clearChainNode(file, uri) self:_clearGlobal(uri) local clock = os.clock() local vm, err = buildVM(ast, self, uri, file:getText()) if vm then CachedVM[vm] = true end if self:isDeadText(uri) or file:isRemoved() or version ~= file:getVersion() then if vm then vm:remove() end return nil end if self._needCompile[uri] then self:_markCompiled(uri, compiled) self._needDiagnostics[uri] = true else if vm then vm:remove() end return nil end file:saveVM(vm, version, os.clock() - clock) local clock = os.clock() local lines = parser:lines(file:getText(), 'utf8') local lineCost = os.clock() - clock file:saveLines(lines, lineCost) if file:getVMCost() > 0.2 then log.debug(('Compile VM[%s] takes: %.3f sec'):format(uri, file:getVMCost())) end if not vm then error(err) end self:_compileChain(file, compiled) if self:_hasSetGlobal(uri) then self:_compileGlobal(compiled) end return file end ---@param uri uri function mt:doDiagnostics(uri) if not config.config.diagnostics.enable then self._needDiagnostics[uri] = nil return end if not self._needDiagnostics[uri] then return end local name = 'textDocument/publishDiagnostics' local file = self._files:get(uri) if not file or file:isRemoved() or not file:getVM() or file:getVM():isRemoved() or self._files:isLibrary(uri) then self._needDiagnostics[uri] = nil self:clearDiagnostics(uri) return end local data = { uri = uri, vm = file:getVM(), lines = file:getLines(), version = file:getVM():getVersion(), } local res = self:_callMethod(name, data) if self:isDeadText(uri) then return end if file:getVM():getVersion() ~= data.version then return end if self._needDiagnostics[uri] then self._needDiagnostics[uri] = nil else return end if res then rpc:notify(name, { uri = uri, diagnostics = res, }) else self:clearDiagnostics(uri) end end ---@param uri uri ---@return file function mt:getFile(uri) return self._files:get(uri) end ---@param uri uri ---@return VM ---@return table ---@return string function mt:getVM(uri) local file = self._files:get(uri) if not file then return nil end return file:getVM(), file:getLines(), file:getText() end ---@param uri uri ---@return string ---@return string function mt:getText(uri) local file = self._files:get(uri) if not file then return nil end return file:getText(), file:getOldText() end function mt:getComments(uri) local file = self._files:get(uri) if not file then return nil end return file:getComments() end ---@param uri uri ---@return table function mt:getAstErrors(uri) local file = self._files:get(uri) if not file then return nil end return file:getAstErr() end ---@param child uri ---@param parent uri function mt:compileChain(child, parent) local parentFile = self._files:get(parent) local childFile = self._files:get(child) if not parentFile or not childFile then return end if parentFile == childFile then return end parentFile:addChild(child) childFile:addParent(parent) end function mt:checkWorkSpaceComplete() if self._hasCheckedWorkSpaceComplete then return end self._hasCheckedWorkSpaceComplete = true if self.workspace:isComplete() then return end self._needShowComplete = true rpc:notify('window/showMessage', { type = 3, message = lang.script.MWS_NOT_COMPLETE, }) end function mt:_createCompileTask() if not self:isWaitingCompile() and not next(self._needDiagnostics) then if self._needShowComplete then self._needShowComplete = nil rpc:notify('window/showMessage', { type = 3, message = lang.script.MWS_COMPLETE, }) end end self._compileTask = task(function () self:doDiagnostics(self._lastLoadedVM) local uri = self._needCompile[1] if uri then self._compileTask:set('uri', uri) pcall(function () self:compileVM(uri) end) else uri = next(self._needDiagnostics) if uri then self:doDiagnostics(uri) end end end) end function mt:_doCompileTask() if not self._compileTask or self._compileTask:isRemoved() then self:_createCompileTask() end while true do local res = self._compileTask:step() if res == 'stop' then self._compileTask:remove() break end if self._compileTask:isRemoved() then break end end self:_loadProto() end function mt:_loadProto() while true do local ok, proto = self._proto:pop() if not ok then break end if proto.method then self:_doProto(proto) else rpc:recieve(proto) end end end function mt:restartDueToMemoryLeak() rpc:requestWait('window/showMessageRequest', { type = 3, message = lang.script('DEBUG_MEMORY_LEAK', '[Lua]'), actions = { { title = lang.script.DEBUG_RESTART_NOW, } } }, function () os.exit(true) end) ac.wait(5, function () os.exit(true) end) end function mt:reScanFiles() if not self.workspace then return end log.debug('reScanFiles') self:clearAllFiles() self.workspace:scanFiles() end function mt:onUpdateConfig(updated, other) local oldConfig = table.deepCopy(config.config) local oldOther = table.deepCopy(config.other) config:setConfig(updated, other) local newConfig = config.config local newOther = config.other if not table.equal(oldConfig.runtime, newConfig.runtime) then local library = require 'core.library' library.reload() self:reCompile() end if not table.equal(oldConfig.diagnostics, newConfig.diagnostics) then log.debug('reDiagnostic') self:reDiagnostic() end if newConfig.completion.enable then capability.completion.enable() else capability.completion.disable() end if not table.equal(oldConfig.plugin, newConfig.plugin) then plugin.load(self.workspace) end if not table.equal(oldConfig.workspace, newConfig.workspace) or not table.equal(oldConfig.plugin, newConfig.plugin) or not table.equal(oldOther.associations, newOther.associations) or not table.equal(oldOther.exclude, newOther.exclude) then self:reScanFiles() end end function mt:_testMemory(skipDead) local clock = os.clock() collectgarbage() log.debug('collectgarbage: ', ('%.3f'):format(os.clock() - clock)) local clock = os.clock() local cachedVM = 0 local cachedSource = 0 local cachedFunction = 0 for _, file in self._files:eachFile() do local vm = file:getVM() if vm and not vm:isRemoved() then cachedVM = cachedVM + 1 cachedSource = cachedSource + #vm.sources cachedFunction = cachedFunction + #vm.funcs end end local aliveVM = 0 local deadVM = 0 for vm in pairs(CachedVM) do if vm:isRemoved() then deadVM = deadVM + 1 else aliveVM = aliveVM + 1 end end local alivedSource = 0 local deadSource = 0 for _, id in pairs(sourceMgr.watch) do if listMgr.get(id) then alivedSource = alivedSource + 1 else deadSource = deadSource + 1 end end local alivedFunction = 0 local deadFunction = 0 for _, id in pairs(functionMgr.watch) do if listMgr.get(id) then alivedFunction = alivedFunction + 1 else deadFunction = deadFunction + 1 end end local totalLocal = 0 for _ in pairs(localMgr.watch) do totalLocal = totalLocal + 1 end local totalValue = 0 local deadValue = 0 for value in pairs(valueMgr.watch) do totalValue = totalValue + 1 if not value:getSource() then deadValue = deadValue + 1 end end local totalEmmy = self.emmy:count() local mem = collectgarbage 'count' local threadInfo = async.info local threadBuf = {} for i, count in ipairs(threadInfo) do if count then threadBuf[i] = ('#%03d Mem: [%.3f]kb'):format(i, count) else threadBuf[i] = ('#%03d Mem: '):format(i) end end log.debug(('\n\z State\n\z Main Mem: [%.3f]kb\n\z %s\n\z -------------------\n\z CachedVM: [%d]\n\z AlivedVM: [%d]\n\z DeadVM: [%d]\n\z -------------------\n\z CachedSrc: [%d]\n\z AlivedSrc: [%d]\n\z DeadSrc: [%d]\n\z -------------------\n\z CachedFunc:[%d]\n\z AlivedFunc:[%d]\n\z DeadFunc: [%d]\n\z -------------------\n\z TotalVal: [%d]\n\z DeadVal: [%d]\n\z -------------------\n\z TotalLoc: [%d]\n\z TotalEmmy: [%d]\n\z'):format( mem, table.concat(threadBuf, '\n'), cachedVM, aliveVM, deadVM, cachedSource, alivedSource, deadSource, cachedFunction, alivedFunction, deadFunction, totalValue, deadValue, totalLocal, totalEmmy )) log.debug('test memory: ', ('%.3f'):format(os.clock() - clock)) if deadValue / totalValue >= 0.5 and not skipDead then self:_testFindDeadValues() end end function mt:_testFindDeadValues() if self._testHasFoundDeadValues then return end self._testHasFoundDeadValues = true log.debug('Start find dead values, may takes few seconds...') local mark = {} local stack = {} local count = 0 local clock = os.clock() local function push(info) stack[#stack+1] = info end local function pop() stack[#stack] = nil end local function showStack(uri) count = count + 1 log.debug(uri, table.concat(stack, '->')) end local function scan(name, tbl) if count > 100 or os.clock() - clock > 5.0 then return end if type(tbl) ~= 'table' then return end if mark[tbl] then return end mark[tbl] = true if tbl.type then push(('%s<%s>'):format(name, tbl.type)) else push(name) end if tbl.type == 'value' then if not tbl:getSource() then showStack(tbl.uri) end elseif tbl.type == 'files' then for k, v in tbl:eachFile() do scan(k, v) end else for k, v in pairs(tbl) do scan(k, v) end end pop() end scan('root', self._files) log.debug('Finish...') end function mt:onTick() self:_loadProto() self:_doCompileTask() if (os.clock() - self._clock >= 60 and not self:isWaitingCompile()) or (os.clock() - self._clock >= 300) then self._clock = os.clock() self:_testMemory() end end function mt:listen() subprocess.filemode(io.stdin, 'b') subprocess.filemode(io.stdout, 'b') io.stdin:setvbuf 'no' io.stdout:setvbuf 'no' local _, out = async.run 'proto' self._proto = out local timerClock = 0.0 while true do local startClock = os.clock() async.onTick() self:onTick() local delta = os.clock() - timerClock local suc, err = xpcall(updateTimer, log.error, delta) if not suc then io.stderr:write(err) io.stderr:flush() end timerClock = os.clock() local passedClock = os.clock() - startClock if passedClock > 0.1 then thread.sleep(0.0) else thread.sleep(0.001) end end end return function () local session = setmetatable({ _needCompile = {}, _needDiagnostics = {}, _clock = -100, _version = 0, _files = files(), }, mt) session.global = core.global(session) session.chain = chainMgr() session.emmy = emmyMgr() return session end