patch 7.4.852

Problem:    On MS-Windows console Vim uses ANSI APIs for keyboard input and
            console output, it cannot input/output Unicode characters.
Solution:   Use Unicode APIs for console I/O. (Ken Takata, Yasuhiro Matsumoto)

3 files changed, 114 insertions, 89 deletions

diff --git a/src/os_win32.c b/src/os_win32.c
index 8f97c3bc2..7695e938c 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -213,8 +213,8 @@ static void standout(void);
 static void standend(void);
 static void visual_bell(void);
 static void cursor_visible(BOOL fVisible);
-static BOOL write_chars(LPCSTR pchBuf, DWORD cchToWrite);
-static char_u tgetch(int *pmodifiers, char_u *pch2);
+static DWORD write_chars(char_u *pchBuf, DWORD cbToWrite);
+static WCHAR tgetch(int *pmodifiers, WCHAR *pch2);
 static void create_conin(void);
 static int s_cursor_visible = TRUE;
 static int did_create_conin = FALSE;
@@ -265,15 +265,15 @@ read_console_input(
     if (!win8_or_later)
     {
 	if (nLength == -1)
-	    return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
-	return ReadConsoleInput(hInput, lpBuffer, 1, &dwEvents);
+	    return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
+	return ReadConsoleInputW(hInput, lpBuffer, 1, &dwEvents);
     }
 
     if (s_dwMax == 0)
     {
 	if (nLength == -1)
-	    return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
-	if (!ReadConsoleInput(hInput, s_irCache, IRSIZE, &dwEvents))
+	    return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
+	if (!ReadConsoleInputW(hInput, s_irCache, IRSIZE, &dwEvents))
 	    return FALSE;
 	s_dwIndex = 0;
 	s_dwMax = dwEvents;
@@ -868,9 +868,9 @@ static const struct
 #endif
 
 #if defined(__GNUC__) && !defined(__MINGW32__)  && !defined(__CYGWIN__)
-# define AChar AsciiChar
+# define UChar UnicodeChar
 #else
-# define AChar uChar.AsciiChar
+# define UChar uChar.UnicodeChar
 #endif
 
 /* The return code indicates key code size. */
@@ -889,12 +889,12 @@ win32_kbd_patch_key(
 
     if (s_iIsDead == 2)
     {
-	pker->AChar = (CHAR) awAnsiCode[1];
+	pker->UChar = (WCHAR) awAnsiCode[1];
 	s_iIsDead = 0;
 	return 1;
     }
 
-    if (pker->AChar != 0)
+    if (pker->UChar != 0)
 	return 1;
 
     vim_memset(abKeystate, 0, sizeof (abKeystate));
@@ -909,7 +909,7 @@ win32_kbd_patch_key(
     }
 
     /* Clear any pending dead keys */
-    ToAscii(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 0);
+    ToUnicode(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 2, 0);
 
     if (uMods & SHIFT_PRESSED)
 	abKeystate[VK_SHIFT] = 0x80;
@@ -922,11 +922,11 @@ win32_kbd_patch_key(
 	    abKeystate[VK_MENU] = abKeystate[VK_RMENU] = 0x80;
     }
 
-    s_iIsDead = ToAscii(pker->wVirtualKeyCode, pker->wVirtualScanCode,
-			abKeystate, awAnsiCode, 0);
+    s_iIsDead = ToUnicode(pker->wVirtualKeyCode, pker->wVirtualScanCode,
+			abKeystate, awAnsiCode, 2, 0);
 
     if (s_iIsDead > 0)
-	pker->AChar = (CHAR) awAnsiCode[0];
+	pker->UChar = (WCHAR) awAnsiCode[0];
 
     return s_iIsDead;
 }
@@ -953,8 +953,8 @@ static BOOL g_fJustGotFocus = FALSE;
     static BOOL
 decode_key_event(
     KEY_EVENT_RECORD	*pker,
-    char_u		*pch,
-    char_u		*pch2,
+    WCHAR		*pch,
+    WCHAR		*pch2,
     int			*pmodifiers,
     BOOL		fDoPost)
 {
@@ -982,7 +982,7 @@ decode_key_event(
     }
 
     /* special cases */
-    if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->AChar == NUL)
+    if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->UChar == NUL)
     {
 	/* Ctrl-6 is Ctrl-^ */
 	if (pker->wVirtualKeyCode == '6')
@@ -1044,7 +1044,7 @@ decode_key_event(
 	*pch = NUL;
     else
     {
-	*pch = (i > 0) ? pker->AChar : NUL;
+	*pch = (i > 0) ? pker->UChar : NUL;
 
 	if (pmodifiers != NULL)
 	{
@@ -1436,7 +1436,7 @@ WaitForChar(long msec)
     DWORD	    dwNow = 0, dwEndTime = 0;
     INPUT_RECORD    ir;
     DWORD	    cRecords;
-    char_u	    ch, ch2;
+    WCHAR	    ch, ch2;
 
     if (msec > 0)
 	/* Wait until the specified time has elapsed. */
@@ -1523,7 +1523,7 @@ WaitForChar(long msec)
 #ifdef FEAT_MBYTE_IME
 		/* Windows IME sends two '\n's with only one 'ENTER'.  First:
 		 * wVirtualKeyCode == 13. second: wVirtualKeyCode == 0 */
-		if (ir.Event.KeyEvent.uChar.UnicodeChar == 0
+		if (ir.Event.KeyEvent.UChar == 0
 			&& ir.Event.KeyEvent.wVirtualKeyCode == 13)
 		{
 		    read_console_input(g_hConIn, &ir, 1, &cRecords);
@@ -1586,10 +1586,10 @@ create_conin(void)
 /*
  * Get a keystroke or a mouse event
  */
-    static char_u
-tgetch(int *pmodifiers, char_u *pch2)
+    static WCHAR
+tgetch(int *pmodifiers, WCHAR *pch2)
 {
-    char_u ch;
+    WCHAR ch;
 
     for (;;)
     {
@@ -1658,11 +1658,6 @@ mch_inchar(
 #define TYPEAHEADLEN 20
     static char_u   typeahead[TYPEAHEADLEN];	/* previously typed bytes. */
     static int	    typeaheadlen = 0;
-#ifdef FEAT_MBYTE
-    static char_u   *rest = NULL;	/* unconverted rest of previous read */
-    static int	    restlen = 0;
-    int		    unconverted;
-#endif
 
     /* First use any typeahead that was kept because "buf" was too small. */
     if (typeaheadlen > 0)
@@ -1761,38 +1756,11 @@ mch_inchar(
 	else
 #endif
 	{
-	    char_u	ch2 = NUL;
+	    WCHAR	ch2 = NUL;
 	    int		modifiers = 0;
 
 	    c = tgetch(&modifiers, &ch2);
 
-#ifdef FEAT_MBYTE
-	    /* stolen from fill_input_buf() in ui.c */
-	    if (rest != NULL)
-	    {
-		/* Use remainder of previous call, starts with an invalid
-		 * character that may become valid when reading more. */
-		if (restlen > TYPEAHEADLEN - typeaheadlen)
-		    unconverted = TYPEAHEADLEN - typeaheadlen;
-		else
-		    unconverted = restlen;
-		mch_memmove(typeahead + typeaheadlen, rest, unconverted);
-		if (unconverted == restlen)
-		{
-		    vim_free(rest);
-		    rest = NULL;
-		}
-		else
-		{
-		    restlen -= unconverted;
-		    mch_memmove(rest, rest + unconverted, restlen);
-		}
-		typeaheadlen += unconverted;
-	    }
-	    else
-		unconverted = 0;
-#endif
-
 	    if (typebuf_changed(tb_change_cnt))
 	    {
 		/* "buf" may be invalid now if a client put something in the
@@ -1816,27 +1784,36 @@ mch_inchar(
 		int	n = 1;
 		int     conv = FALSE;
 
-		typeahead[typeaheadlen] = c;
-		if (ch2 != NUL)
-		{
-		    typeahead[typeaheadlen + 1] = 3;
-		    typeahead[typeaheadlen + 2] = ch2;
-		    n += 2;
-		}
 #ifdef FEAT_MBYTE
-		/* Only convert normal characters, not special keys.  Need to
-		 * convert before applying ALT, otherwise mapping <M-x> breaks
-		 * when 'tenc' is set. */
-		if (input_conv.vc_type != CONV_NONE
-						&& (ch2 == NUL || c != K_NUL))
+		if (ch2 == NUL)
 		{
-		    conv = TRUE;
-		    typeaheadlen -= unconverted;
-		    n = convert_input_safe(typeahead + typeaheadlen,
-				n + unconverted, TYPEAHEADLEN - typeaheadlen,
-				rest == NULL ? &rest : NULL, &restlen);
+		    int	    i;
+		    char_u  *p;
+		    WCHAR   ch[2];
+
+		    ch[0] = c;
+		    if (c >= 0xD800 && c <= 0xDBFF)	/* High surrogate */
+		    {
+			ch[1] = tgetch(&modifiers, &ch2);
+			n++;
+		    }
+		    p = utf16_to_enc(ch, &n);
+		    if (p != NULL)
+		    {
+			for (i = 0; i < n; i++)
+			    typeahead[typeaheadlen + i] = p[i];
+			vim_free(p);
+		    }
 		}
+		else
 #endif
+		    typeahead[typeaheadlen] = c;
+		if (ch2 != NUL)
+		{
+		    typeahead[typeaheadlen + n] = 3;
+		    typeahead[typeaheadlen + n + 1] = (char_u)ch2;
+		    n += 2;
+		}
 
 		if (conv)
 		{
@@ -5366,27 +5343,73 @@ cursor_visible(BOOL fVisible)
 
 
 /*
- * write `cchToWrite' characters in `pchBuf' to the screen
- * Returns the number of characters actually written (at least one).
+ * write `cbToWrite' bytes in `pchBuf' to the screen
+ * Returns the number of bytes actually written (at least one).
  */
-    static BOOL
+    static DWORD
 write_chars(
-    LPCSTR pchBuf,
-    DWORD  cchToWrite)
+    char_u *pchBuf,
+    DWORD  cbToWrite)
 {
     COORD coord = g_coord;
     DWORD written;
 
-    FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cchToWrite,
-				coord, &written);
-    /* When writing fails or didn't write a single character, pretend one
-     * character was written, otherwise we get stuck. */
-    if (WriteConsoleOutputCharacter(g_hConOut, pchBuf, cchToWrite,
-				coord, &written) == 0
-	    || written == 0)
-	written = 1;
+#ifdef FEAT_MBYTE
+    if (enc_codepage >= 0 && (int)GetACP() != enc_codepage)
+    {
+	static WCHAR	*unicodebuf = NULL;
+	static int	unibuflen = 0;
+	int		length;
+	DWORD		n, cchwritten, cells;
 
-    g_coord.X += (SHORT) written;
+	length = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite, 0, 0);
+	if (unicodebuf == NULL || length > unibuflen)
+	{
+	    vim_free(unicodebuf);
+	    unicodebuf = (WCHAR *)lalloc(length * sizeof(WCHAR), FALSE);
+	    unibuflen = length;
+	}
+	MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite,
+			    unicodebuf, unibuflen);
+
+	cells = mb_string2cells(pchBuf, cbToWrite);
+	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cells,
+				    coord, &written);
+	/* When writing fails or didn't write a single character, pretend one
+	 * character was written, otherwise we get stuck. */
+	if (WriteConsoleOutputCharacterW(g_hConOut, unicodebuf, length,
+		    coord, &cchwritten) == 0
+		|| cchwritten == 0)
+	    cchwritten = 1;
+
+	if (cchwritten == length)
+	{
+	    written = cbToWrite;
+	    g_coord.X += (SHORT)cells;
+	}
+	else
+	{
+	    char_u *p = pchBuf;
+	    for (n = 0; n < cchwritten; n++)
+		mb_cptr_adv(p);
+	    written = p - pchBuf;
+	    g_coord.X += (SHORT)mb_string2cells(pchBuf, written);
+	}
+    }
+    else
+#endif
+    {
+	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cbToWrite,
+				    coord, &written);
+	/* When writing fails or didn't write a single character, pretend one
+	 * character was written, otherwise we get stuck. */
+	if (WriteConsoleOutputCharacter(g_hConOut, (LPCSTR)pchBuf, cbToWrite,
+		    coord, &written) == 0
+		|| written == 0)
+	    written = 1;
+
+	g_coord.X += (SHORT) written;
+    }
 
     while (g_coord.X > g_srScrollRegion.Right)
     {
diff --git a/src/ui.c b/src/ui.c
index 7bbf2838e..e0e54d924 100644
--- a/src/ui.c
+++ b/src/ui.c
@@ -42,7 +42,7 @@ ui_write(s, len)
     /* Don't output anything in silent mode ("ex -s") unless 'verbose' set */
     if (!(silent_mode && p_verbose == 0))
     {
-#ifdef FEAT_MBYTE
+#if defined(FEAT_MBYTE) && !defined(WIN3264)
 	char_u	*tofree = NULL;
 
 	if (output_conv.vc_type != CONV_NONE)
@@ -56,7 +56,7 @@ ui_write(s, len)
 
 	mch_write(s, len);
 
-#ifdef FEAT_MBYTE
+#if defined(FEAT_MBYTE) && !defined(WIN3264)
 	if (output_conv.vc_type != CONV_NONE)
 	    vim_free(tofree);
 #endif
diff --git a/src/version.c b/src/version.c
index 559f0c1ab..2aa380747 100644
--- a/src/version.c
+++ b/src/version.c
@@ -742,6 +742,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    852,
+/**/
     851,
 /**/
     850,