stash

6 files changed, 137 insertions, 23 deletions

diff --git a/script/encoder/init.lua b/script/encoder/init.lua
index 152193e7..f33f5a1d 100644
--- a/script/encoder/init.lua
+++ b/script/encoder/init.lua
@@ -1,5 +1,106 @@
-return {
-    ansi    = require 'encoder.ansi',
-    utf16le = require 'encoder.utf16le',
-    utf16be = require 'encoder.utf16be'
-}
+local ansi    = require 'encoder.ansi'
+local utf16le = require 'encoder.utf16le'
+local utf16be = require 'encoder.utf16be'
+
+---@alias encoder.encoding '"utf8"'|'"utf16"'|'"utf16le"'|'"utf16be"'
+
+local m = {}
+
+---@param encoding encoder.encoding
+---@param s string
+---@param i integer
+---@param j integer
+function m.len(encoding, s, i, j)
+    i = i or 1
+    j = j or #s
+    if encoding == 'utf16'
+    or encoding == 'utf16' then
+        local us = utf16le.encode(s:sub(i, j))
+        return #us // 2
+    end
+    if encoding == 'utf16be' then
+        local us = utf16be.encode(s:sub(i, j))
+        return #us // 2
+    end
+    if encoding == 'utf8' then
+        return utf8.len(s, i, j, true)
+    end
+    log.error('Unsupport len encoding:', encoding)
+    return j - i + 1
+end
+
+---@param encoding encoder.encoding
+---@param s string
+---@param n integer
+---@param i integer
+function m.offset(encoding, s, n, i)
+    i = i or 1
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        local line = s:match('[^\r\n]*', i)
+        if not line:find '[\x80-\xff]' then
+            return n + i - 1
+        end
+        local us = utf16le.encode(line)
+        local os = utf16le.decode(us:sub(1, n * 2 - 2))
+        return #os + i
+    end
+    if encoding == 'utf16be' then
+        local line = s:match('[^\r\n]*', i)
+        if not line:find '[\x80-\xff]' then
+            return n + i - 1
+        end
+        local us = utf16be.encode(line)
+        local os = utf16be.decode(us:sub(1, n * 2 - 2))
+        return #os + i
+    end
+    if encoding == 'utf8' then
+        return utf8.offset(s, n, i)
+    end
+    log.error('Unsupport offset encoding:', encoding)
+    return n + i - 1
+end
+
+---@param encoding encoder.encoding
+---@param text string
+---@return string
+function m.encode(encoding, text)
+    if encoding == 'utf8' then
+        return text
+    end
+    if encoding == 'ansi' then
+        return ansi.encode(text)
+    end
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        return utf16le.encode(text)
+    end
+    if encoding == 'utf16be' then
+        return utf16be.encode(text)
+    end
+    log.error('Unsupport encode encoding:', encoding)
+    return text
+end
+
+---@param encoding encoder.encoding
+---@param text string
+---@return string
+function m.decode(encoding, text)
+    if encoding == 'utf8' then
+        return text
+    end
+    if encoding == 'ansi' then
+        return ansi.decode(text)
+    end
+    if encoding == 'utf16'
+    or encoding == 'utf16le' then
+        return utf16le.decode(text)
+    end
+    if encoding == 'utf16be' then
+        return utf16be.decode(text)
+    end
+    log.error('Unsupport encode encoding:', encoding)
+    return text
+end
+
+return m
diff --git a/script/files.lua b/script/files.lua
index 566b012c..4d649b29 100644
--- a/script/files.lua
+++ b/script/files.lua
@@ -140,9 +140,8 @@ function m.setText(uri, text, isTrust, instance)
         return
     end
     if not isTrust then
-        if config.get 'Lua.runtime.fileEncoding' == 'ansi' then
-            text = encoder.ansi.decode(text)
-        end
+        local encoding = config.get 'Lua.runtime.fileEncoding'
+        text = encoder.decode(encoding, text)
     end
     if file.originText == text then
         return
diff --git a/script/fs-utility.lua b/script/fs-utility.lua
index 2f5d404e..f4e55716 100644
--- a/script/fs-utility.lua
+++ b/script/fs-utility.lua
@@ -579,4 +579,12 @@ function m.scanDirectory(dir, callback)
     end
 end
 
+function m.listDirectory(dir)
+    if dir.type == 'dummy' then
+        return dir:listDirectory()
+    else
+        return fs.pairs(dir)
+    end
+end
+
 return m
diff --git a/script/library.lua b/script/library.lua
index 66b26b0d..0911750e 100644
--- a/script/library.lua
+++ b/script/library.lua
@@ -236,9 +236,7 @@ local function initBuiltIn()
         local metaDoc = compileSingleMetaDoc(fsu.loadFile(libPath), metaLang, status)
         if metaDoc then
             local outPath = metaPath / libName
-            if encoding == 'ansi' then
-                metaDoc = encoder.ansi.encode(metaDoc)
-            end
+            encoder.encode(encoding, metaDoc)
             out:saveFile(libName, metaDoc)
             m.metaPaths[#m.metaPaths+1] = outPath:string()
         end
diff --git a/script/proto/converter.lua b/script/proto/converter.lua
index 1c8e20fc..cf6331f1 100644
--- a/script/proto/converter.lua
+++ b/script/proto/converter.lua
@@ -1,5 +1,9 @@
-local guide = require 'parser.guide'
-local files = require 'files'
+local guide   = require 'parser.guide'
+local files   = require 'files'
+local encoder = require 'encoder'
+
+-- TODO
+local offsetEncoding = 'utf16'
 
 local m = {}
 
@@ -16,7 +20,7 @@ local function rawPackPosition(uri, pos)
                 local start = lineOffset
                 local finish = lineOffset + col - 1
                 if start <= #text and finish <= #text then
-                    col = utf8.len(text, lineOffset, lineOffset + col - 1, true)
+                    col = encoder.len(offsetEncoding, text, lineOffset, lineOffset + col - 1)
                 end
             else
                 col = 0
@@ -41,7 +45,7 @@ local function diffedPackPosition(uri, pos)
         if text then
             local lineOffset  = originLines[row]
             local finalOffset = math.min(lineOffset + col - 1, #text + 1)
-            col = utf8.len(text, lineOffset, finalOffset, true)
+            col = encoder.len(offsetEncoding, text, lineOffset, finalOffset)
         end
     end
     return {
@@ -68,7 +72,7 @@ local function rawUnpackPosition(uri, position)
         local text  = files.getText(uri)
         if state and text then
             local lineOffset = state.lines[row]
-            local textOffset = utf8.offset(text, col + 1, lineOffset)
+            local textOffset = encoder.offset(offsetEncoding, text, col + 1, lineOffset)
             if textOffset and lineOffset then
                 col = textOffset - lineOffset
             end
@@ -85,7 +89,7 @@ local function diffedUnpackPosition(uri, position)
         local text  = files.getOriginText(uri)
         if text then
             local lineOffset = originLines[row]
-            local textOffset = utf8.offset(text, col + 1, lineOffset)
+            local textOffset = encoder.offset(offsetEncoding, text, col + 1, lineOffset)
             if textOffset and lineOffset then
                 col = textOffset - lineOffset
             end
diff --git a/script/text-merger.lua b/script/text-merger.lua
index ba7cff58..afd3343f 100644
--- a/script/text-merger.lua
+++ b/script/text-merger.lua
@@ -1,5 +1,9 @@
-local files = require 'files'
-local util  = require 'utility'
+local files   = require 'files'
+local util    = require 'utility'
+local encoder = require 'encoder'
+
+-- TODO
+local offsetEncoding = 'utf16'
 
 local function splitRows(text)
     local rows = {}
@@ -14,14 +18,14 @@ local function getLeft(text, char)
         return ''
     end
     local left
-    local length = util.utf8Len(text)
+    local length = encoder.len(offsetEncoding, text)
 
     if char == 0 then
         left = ''
     elseif char >= length then
         left = text
     else
-        left = text:sub(1, utf8.offset(text, char + 1) - 1)
+        left = text:sub(1, encoder.offset(offsetEncoding, text, char + 1) - 1)
     end
 
     return left
@@ -32,14 +36,14 @@ local function getRight(text, char)
         return ''
     end
     local right
-    local length = util.utf8Len(text)
+    local length = encoder.len(offsetEncoding, text)
 
     if char == 0 then
         right = text
     elseif char >= length then
         right = ''
     else
-        right = text:sub(utf8.offset(text, char + 1))
+        right = text:sub(encoder.offset(offsetEncoding, text, char + 1))
     end
 
     return right