Merge branch 'master' of https://github.com/sumneko/lua-language-server into pt-br-locale-support

16 files changed, 306 insertions, 54 deletions

diff --git a/.luarc.json b/.luarc.json
index 1efff2fc..2a2fcd2f 100644
--- a/.luarc.json
+++ b/.luarc.json
@@ -1,29 +1,43 @@
 {
     "$schema": "https://raw.githubusercontent.com/sumneko/vscode-lua/master/setting/schema.json",
-    "diagnostics.disable": [
-        "close-non-object"
-    ],
-    "runtime.version": "Lua 5.4",
-    "workspace.maxPreload": 1600,
-    "workspace.preloadFileSize": 1000,
-    "diagnostics.neededFileStatus": {
-        "undefined-field": "Any"
+    
+    "color": {
+        "mode": "SemanticEnhanced"
     },
-    "workspace.ignoreDir": [
-        "locale/",
-        "libs/",
-        "3rd",
-        ".vscode",
-        "meta"
-    ],
-    "workspace.library": [
-        "script/meta",
-        "bin"
-    ],
-    "runtime.path": [
-        "?.lua",
-        "script/?.lua",
-        "script/?/init.lua"
-    ],
-    "Lua.color.mode": "SemanticEnhanced"
+    "diagnostics": {
+        "disable": [
+            "close-non-object"
+        ],
+        "neededFileStatus": {
+            "undefined-field": "Any"
+        },
+        "ignoredFiles": "Opened",
+        "libraryFiles": "Opened"
+    },
+    "runtime": {
+        "version": "Lua 5.4",
+        "path": [
+            "?.lua",
+            "script/?.lua",
+            "script/?/init.lua"
+        ]
+    },
+    "workspace": {
+        "maxPreload": 1600,
+        "preloadFileSize": 1000,
+        "ignoreDir": [
+            "locale/",
+            "libs/",
+            "3rd",
+            ".vscode",
+            "meta"
+        ],
+        "library": [
+            "script/meta",
+            "bin"
+        ]
+    },
+    "hint": {
+        "enable": true
+    }
 }
diff --git a/changelog.md b/changelog.md
index 796cc74c..3d9fcf9f 100644
--- a/changelog.md
+++ b/changelog.md
@@ -3,6 +3,8 @@
 ## 2.4.6
 * `FIX` [#744](https://github.com/sumneko/lua-language-server/issues/744)
 * `FIX` [#748](https://github.com/sumneko/lua-language-server/issues/748)
+* `FIX` [#749](https://github.com/sumneko/lua-language-server/issues/749)
+* `FIX` [#752](https://github.com/sumneko/lua-language-server/issues/752)
 
 ## 2.4.5
 `2021-10-18`
diff --git a/locale/en-us/meta.lua b/locale/en-us/meta.lua
index ffb4fee7..acbb510c 100644
--- a/locale/en-us/meta.lua
+++ b/locale/en-us/meta.lua
@@ -47,7 +47,7 @@ The order in which the indices are enumerated is not specified, *even for numeri
 
 The behavior of `next` is undefined if, during the traversal, you assign any value to a non-existent field in the table. You may however modify existing fields. In particular, you may set existing fields to nil.
 ]]
-paris               = [[
+pairs               = [[
 If `t` has a metamethod `__pairs`, calls it with t as argument and returns the first three results from the call.
 
 Otherwise, returns three values: the $next function, the table `t`, and `nil`, so that the construction
diff --git a/locale/zh-cn/meta.lua b/locale/zh-cn/meta.lua
index a054a464..c57a435c 100644
--- a/locale/zh-cn/meta.lua
+++ b/locale/zh-cn/meta.lua
@@ -47,7 +47,7 @@ next                = [[
 
 当在遍历过程中你给表中并不存在的域赋值， `next` 的行为是未定义的。 然而你可以去修改那些已存在的域。 特别指出，你可以清除一些已存在的域。
 ]]
-paris               = [[
+pairs               = [[
 如果 `t` 有元方法 `__pairs`， 以 `t` 为参数调用它，并返回其返回的前三个值。
 
 否则，返回三个值：`next` 函数， 表 `t`，以及 `nil`。 因此以下代码
diff --git a/meta/template/basic.lua b/meta/template/basic.lua
index c92f0a2a..5665dc59 100644
--- a/meta/template/basic.lua
+++ b/meta/template/basic.lua
@@ -132,7 +132,7 @@ function module(name, ...) end
 ---@return V
 function next(table, index) end
 
----#DES 'paris'
+---#DES 'pairs'
 ---@generic T: table, K, V
 ---@param t T
 ---@return fun(table: table<K, V>, index?: K):K, V
diff --git a/script/config/config.lua b/script/config/config.lua
index 92d77a5c..461ec260 100644
--- a/script/config/config.lua
+++ b/script/config/config.lua
@@ -152,7 +152,7 @@ local Template = {
                                                 "?/init.lua",
                                             },
     ['Lua.runtime.special']                 = Type.Hash(Type.String, Type.String),
-    ['Lua.runtime.meta']                    = Type.String >> '${version} ${language}',
+    ['Lua.runtime.meta']                    = Type.String >> '${version} ${language} ${encoding}',
     ['Lua.runtime.unicodeName']             = Type.Boolean,
     ['Lua.runtime.nonstandardSymbol']       = Type.Hash(Type.String, Type.Boolean, ';'),
     ['Lua.runtime.plugin']                  = Type.String,
diff --git a/script/core/infer.lua b/script/core/infer.lua
index 56ab2b5b..ab297209 100644
--- a/script/core/infer.lua
+++ b/script/core/infer.lua
@@ -583,7 +583,7 @@ function m.hasType(source, tp, mark)
     end
     if tp == 'function' then
         for infer in pairs(infers) do
-            if infer ~= 0 and infer:sub(1, 4) == 'fun(' then
+            if infer ~= CACHE and infer:sub(1, 4) == 'fun(' then
                 return true
             end
         end
diff --git a/script/encoder/ansi.lua b/script/encoder/ansi.lua
new file mode 100644
index 00000000..1016e668
--- /dev/null
+++ b/script/encoder/ansi.lua
@@ -0,0 +1,24 @@
+local platform = require 'bee.platform'
+local unicode
+
+if platform.OS == 'Windows' then
+    unicode = require 'bee.unicode'
+end
+
+local m = {}
+
+function m.decode(text)
+    if not unicode then
+        return text
+    end
+    return unicode.a2u(text)
+end
+
+function m.encode(text)
+    if not unicode then
+        return text
+    end
+    return unicode.u2a(text)
+end
+
+return m
diff --git a/script/encoder/init.lua b/script/encoder/init.lua
new file mode 100644
index 00000000..f33f5a1d
--- /dev/null
+++ b/script/encoder/init.lua
@@ -0,0 +1,106 @@
+local ansi    = require 'encoder.ansi'
+local utf16le = require 'encoder.utf16le'
+local utf16be = require 'encoder.utf16be'
+
+---@alias encoder.encoding '"utf8"'|'"utf16"'|'"utf16le"'|'"utf16be"'
+
+local m = {}
+
+---@param encoding encoder.encoding
+---@param s string
+---@param i integer
+---@param j integer
+function m.len(encoding, s, i, j)
+    i = i or 1
+    j = j or #s
+    if encoding == 'utf16'
+    or encoding == 'utf16' then
+        local us = utf16le.encode(s:sub(i, j))
+        return #us // 2
+    end
+    if encoding == 'utf16be' then
+        local us = utf16be.encode(s:sub(i, j))
+        return #us // 2
+    end
+    if encoding == 'utf8' then
+        return utf8.len(s, i, j, true)
+    end
+    log.error('Unsupport len encoding:', encoding)
+    return j - i + 1
+end
+
+---@param encoding encoder.encoding
+---@param s string
+---@param n integer
+---@param i integer
+function m.offset(encoding, s, n, i)
+    i = i or 1
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        local line = s:match('[^\r\n]*', i)
+        if not line:find '[\x80-\xff]' then
+            return n + i - 1
+        end
+        local us = utf16le.encode(line)
+        local os = utf16le.decode(us:sub(1, n * 2 - 2))
+        return #os + i
+    end
+    if encoding == 'utf16be' then
+        local line = s:match('[^\r\n]*', i)
+        if not line:find '[\x80-\xff]' then
+            return n + i - 1
+        end
+        local us = utf16be.encode(line)
+        local os = utf16be.decode(us:sub(1, n * 2 - 2))
+        return #os + i
+    end
+    if encoding == 'utf8' then
+        return utf8.offset(s, n, i)
+    end
+    log.error('Unsupport offset encoding:', encoding)
+    return n + i - 1
+end
+
+---@param encoding encoder.encoding
+---@param text string
+---@return string
+function m.encode(encoding, text)
+    if encoding == 'utf8' then
+        return text
+    end
+    if encoding == 'ansi' then
+        return ansi.encode(text)
+    end
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        return utf16le.encode(text)
+    end
+    if encoding == 'utf16be' then
+        return utf16be.encode(text)
+    end
+    log.error('Unsupport encode encoding:', encoding)
+    return text
+end
+
+---@param encoding encoder.encoding
+---@param text string
+---@return string
+function m.decode(encoding, text)
+    if encoding == 'utf8' then
+        return text
+    end
+    if encoding == 'ansi' then
+        return ansi.decode(text)
+    end
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        return utf16le.decode(text)
+    end
+    if encoding == 'utf16be' then
+        return utf16be.decode(text)
+    end
+    log.error('Unsupport encode encoding:', encoding)
+    return text
+end
+
+return m
diff --git a/script/encoder/utf16be.lua b/script/encoder/utf16be.lua
new file mode 100644
index 00000000..179a676d
--- /dev/null
+++ b/script/encoder/utf16be.lua
@@ -0,0 +1,46 @@
+local function tochar(code)
+    return string.char((code >> 8) & 0xFF, code & 0xFF)
+end
+
+local function tobyte(s, i)
+    local h, l = string.byte(s, i, i+1)
+    return (h << 8) | l
+end
+
+local function char(code)
+    if code <= 0xffff then
+        return tochar(code)
+    end
+    code = code - 0x10000
+    return tochar(0xD800 + (code >> 10))..tochar(0xDC00 + (code & 0x3FF))
+end
+
+local m = {}
+
+function m.encode(s)
+    local r = {}
+    for _, c in utf8.codes(s) do
+        r[#r+1] = char(c)
+    end
+    return table.concat(r)
+end
+
+function m.decode(s)
+    local r = {}
+    local i = 1
+    while i < #s do
+        local code1 = tobyte(s, i)
+        if code1 < 0xD800 then
+            r[#r+1] = utf8.char(code1)
+        else
+            i = i + 2
+            local code2 = tobyte(s, i)
+            local code = 0x10000 + ((code1 - 0xD800) << 10) + ((code2 - 0xDC00) & 0x3FF)
+            r[#r+1] = utf8.char(code)
+        end
+        i = i + 2
+    end
+    return table.concat(r)
+end
+
+return m
diff --git a/script/encoder/utf16le.lua b/script/encoder/utf16le.lua
new file mode 100644
index 00000000..cfbc33c2
--- /dev/null
+++ b/script/encoder/utf16le.lua
@@ -0,0 +1,46 @@
+local function tochar(code)
+    return string.char(code & 0xFF, (code >> 8) & 0xFF)
+end
+
+local function tobyte(s, i)
+    local l, h = string.byte(s, i, i+1)
+    return (h << 8) | l
+end
+
+local function char(code)
+    if code <= 0xffff then
+        return tochar(code)
+    end
+    code = code - 0x10000
+    return tochar(0xD800 + (code >> 10))..tochar(0xDC00 + (code & 0x3FF))
+end
+
+local m = {}
+
+function m.encode(s)
+    local r = {}
+    for _, c in utf8.codes(s) do
+        r[#r+1] = char(c)
+    end
+    return table.concat(r)
+end
+
+function m.decode(s)
+    local r = {}
+    local i = 1
+    while i < #s do
+        local code1 = tobyte(s, i)
+        if code1 < 0xD800 then
+            r[#r+1] = utf8.char(code1)
+        else
+            i = i + 2
+            local code2 = tobyte(s, i)
+            local code = 0x10000 + ((code1 - 0xD800) << 10) + ((code2 - 0xDC00) & 0x3FF)
+            r[#r+1] = utf8.char(code)
+        end
+        i = i + 2
+    end
+    return table.concat(r)
+end
+
+return m
diff --git a/script/files.lua b/script/files.lua
index b367f09c..4d649b29 100644
--- a/script/files.lua
+++ b/script/files.lua
@@ -10,12 +10,7 @@ local util     = require 'utility'
 local guide    = require 'parser.guide'
 local smerger  = require 'string-merger'
 local progress = require "progress"
-
-local unicode
-
-if platform.OS == 'Windows' then
-    unicode  = require 'bee.unicode'
-end
+local encoder  = require 'encoder'
 
 ---@class files
 local m = {}
@@ -144,10 +139,9 @@ function m.setText(uri, text, isTrust, instance)
     if file.trusted and not isTrust then
         return
     end
-    if not isTrust and unicode then
-        if config.get 'Lua.runtime.fileEncoding' == 'ansi' then
-            text = unicode.a2u(text)
-        end
+    if not isTrust then
+        local encoding = config.get 'Lua.runtime.fileEncoding'
+        text = encoder.decode(encoding, text)
     end
     if file.originText == text then
         return
diff --git a/script/fs-utility.lua b/script/fs-utility.lua
index 2f5d404e..f4e55716 100644
--- a/script/fs-utility.lua
+++ b/script/fs-utility.lua
@@ -579,4 +579,12 @@ function m.scanDirectory(dir, callback)
     end
 end
 
+function m.listDirectory(dir)
+    if dir.type == 'dummy' then
+        return dir:listDirectory()
+    else
+        return fs.pairs(dir)
+    end
+end
+
 return m
diff --git a/script/library.lua b/script/library.lua
index f4511ee6..0911750e 100644
--- a/script/library.lua
+++ b/script/library.lua
@@ -10,6 +10,7 @@ local define  = require "proto.define"
 local files   = require 'files'
 local await   = require 'await'
 local timer   = require 'timer'
+local encoder = require 'encoder'
 
 local m = {}
 
@@ -200,11 +201,13 @@ local function initBuiltIn()
     if not m.inited then
         return
     end
-    local langID  = lang.id
-    local version = config.get 'Lua.runtime.version'
+    local langID   = lang.id
+    local version  = config.get 'Lua.runtime.version'
+    local encoding = config.get 'Lua.runtime.fileEncoding'
     local metaPath = fs.path(METAPATH) / config.get 'Lua.runtime.meta':gsub('%$%{(.-)%}', {
         version  = version,
         language = langID,
+        encoding = encoding,
     })
 
     local metaLang = loadMetaLocale('en-US')
@@ -233,6 +236,7 @@ local function initBuiltIn()
         local metaDoc = compileSingleMetaDoc(fsu.loadFile(libPath), metaLang, status)
         if metaDoc then
             local outPath = metaPath / libName
+            encoder.encode(encoding, metaDoc)
             out:saveFile(libName, metaDoc)
             m.metaPaths[#m.metaPaths+1] = outPath:string()
         end
diff --git a/script/proto/converter.lua b/script/proto/converter.lua
index 1c8e20fc..cf6331f1 100644
--- a/script/proto/converter.lua
+++ b/script/proto/converter.lua
@@ -1,5 +1,9 @@
-local guide = require 'parser.guide'
-local files = require 'files'
+local guide   = require 'parser.guide'
+local files   = require 'files'
+local encoder = require 'encoder'
+
+-- TODO
+local offsetEncoding = 'utf16'
 
 local m = {}
 
@@ -16,7 +20,7 @@ local function rawPackPosition(uri, pos)
                 local start = lineOffset
                 local finish = lineOffset + col - 1
                 if start <= #text and finish <= #text then
-                    col = utf8.len(text, lineOffset, lineOffset + col - 1, true)
+                    col = encoder.len(offsetEncoding, text, lineOffset, lineOffset + col - 1)
                 end
             else
                 col = 0
@@ -41,7 +45,7 @@ local function diffedPackPosition(uri, pos)
         if text then
             local lineOffset  = originLines[row]
             local finalOffset = math.min(lineOffset + col - 1, #text + 1)
-            col = utf8.len(text, lineOffset, finalOffset, true)
+            col = encoder.len(offsetEncoding, text, lineOffset, finalOffset)
         end
     end
     return {
@@ -68,7 +72,7 @@ local function rawUnpackPosition(uri, position)
         local text  = files.getText(uri)
         if state and text then
             local lineOffset = state.lines[row]
-            local textOffset = utf8.offset(text, col + 1, lineOffset)
+            local textOffset = encoder.offset(offsetEncoding, text, col + 1, lineOffset)
             if textOffset and lineOffset then
                 col = textOffset - lineOffset
             end
@@ -85,7 +89,7 @@ local function diffedUnpackPosition(uri, position)
         local text  = files.getOriginText(uri)
         if text then
             local lineOffset = originLines[row]
-            local textOffset = utf8.offset(text, col + 1, lineOffset)
+            local textOffset = encoder.offset(offsetEncoding, text, col + 1, lineOffset)
             if textOffset and lineOffset then
                 col = textOffset - lineOffset
             end
diff --git a/script/text-merger.lua b/script/text-merger.lua
index ba7cff58..afd3343f 100644
--- a/script/text-merger.lua
+++ b/script/text-merger.lua
@@ -1,5 +1,9 @@
-local files = require 'files'
-local util  = require 'utility'
+local files   = require 'files'
+local util    = require 'utility'
+local encoder = require 'encoder'
+
+-- TODO
+local offsetEncoding = 'utf16'
 
 local function splitRows(text)
     local rows = {}
@@ -14,14 +18,14 @@ local function getLeft(text, char)
         return ''
     end
     local left
-    local length = util.utf8Len(text)
+    local length = encoder.len(offsetEncoding, text)
 
     if char == 0 then
         left = ''
     elseif char >= length then
         left = text
     else
-        left = text:sub(1, utf8.offset(text, char + 1) - 1)
+        left = text:sub(1, encoder.offset(offsetEncoding, text, char + 1) - 1)
     end
 
     return left
@@ -32,14 +36,14 @@ local function getRight(text, char)
         return ''
     end
     local right
-    local length = util.utf8Len(text)
+    local length = encoder.len(offsetEncoding, text)
 
     if char == 0 then
         right = text
     elseif char >= length then
         right = ''
     else
-        right = text:sub(utf8.offset(text, char + 1))
+        right = text:sub(encoder.offset(offsetEncoding, text, char + 1))
     end
 
     return right