core: do not display non printable chars, fix function utf8_char_size_screen

Now the function utf8_char_size_screen can return -1 when the char is not
printable.

It has a specific behavior for some chars:

- U+0009: value of option weechat.look.tab_width
- U+0001 to U+001F (except U+0009): 1
- U+00AD (soft hyphen): -1
- U+200B (zero width space): -1

3 files changed, 139 insertions, 49 deletions

diff --git a/tests/unit/core/test-core-command.cpp b/tests/unit/core/test-core-command.cpp
index 75216ebbf..7293b9f50 100644
--- a/tests/unit/core/test-core-command.cpp
+++ b/tests/unit/core/test-core-command.cpp
@@ -151,9 +151,8 @@ TEST(CoreCommand, Debug)
 {
     const char *command_debug_unicode =
         "/debug unicode "
-        "\u00AD"  /* soft hyphen */
         "\u00E9"  /* é */
-        "\u200B"  /* zero width space */
+        "\u26C4"  /* ⛄ (snowman without snow) */
         "";
 
     /* test command "/debug list" */
@@ -215,9 +214,8 @@ TEST(CoreCommand, Debug)
 
     /* test command "/debug unicode" */
     WEE_CMD_CORE(command_debug_unicode);
-    WEE_CHECK_MSG_CORE("  \"\u00AD\" (U+00AD, 173, 0xC2 0xAD): 2 / 1, 1 / 1, 1, 1, 1");
     WEE_CHECK_MSG_CORE("  \"\u00E9\" (U+00E9, 233, 0xC3 0xA9): 2 / 1, 1 / 1, 1, 1, 1");
-    WEE_CHECK_MSG_CORE("  \"\u200B\" (U+200B, 8203, 0xE2 0x80 0x8B): 3 / 1, 1 / 0, 0, 0, 0");
+    WEE_CHECK_MSG_CORE("  \"\u26C4\" (U+26C4, 9924, 0xE2 0x9B 0x84): 3 / 1, 1 / 2, 2, 2, 2");
 
     /* test command "/debug windows" */
     /* TODO: write tests */
diff --git a/tests/unit/core/test-core-utf8.cpp b/tests/unit/core/test-core-utf8.cpp
index 2bcb6f3f0..26f42bde2 100644
--- a/tests/unit/core/test-core-utf8.cpp
+++ b/tests/unit/core/test-core-utf8.cpp
@@ -45,6 +45,30 @@ extern "C"
     }
 
 /*
+ * delete:
+ *   []
+ *   U+007F (127)
+ *   UTF-8: 1 byte = 0x7F
+ */
+#define UNICODE_DELETE "\u007f"
+
+/*
+ * next line:
+ *   []
+ *   U+0085 (133)
+ *   UTF-8: 2 bytes = 0xC2 0x85
+ */
+#define UNICODE_NEXT_LINE "\u0085"
+
+/*
+ * private use two:
+ *   [’]
+ *   U+0092 (146)
+ *   UTF-8: 2 bytes = 0xC2 0X92
+ */
+#define UNICODE_PRIVATE_USE_TWO "\u0092"
+
+/*
  * soft hyphen:
  *   [­]
  *   U+00AD (173)
@@ -474,6 +498,9 @@ TEST(CoreUtf8, Size)
     LONGS_EQUAL(2, utf8_char_size ("ë"));
     LONGS_EQUAL(3, utf8_char_size ("€"));
     LONGS_EQUAL(1, utf8_char_size ("\x01"));
+    LONGS_EQUAL(1, utf8_char_size (UNICODE_DELETE));
+    LONGS_EQUAL(2, utf8_char_size (UNICODE_NEXT_LINE));
+    LONGS_EQUAL(2, utf8_char_size (UNICODE_PRIVATE_USE_TWO));
     LONGS_EQUAL(2, utf8_char_size (UNICODE_SOFT_HYPHEN));
     LONGS_EQUAL(3, utf8_char_size (UNICODE_ZERO_WIDTH_SPACE));
     LONGS_EQUAL(3, utf8_char_size (UNICODE_SNOWMAN));
@@ -495,13 +522,16 @@ TEST(CoreUtf8, Size)
     LONGS_EQUAL(1, utf8_char_size_screen ("ë"));
     LONGS_EQUAL(1, utf8_char_size_screen ("€"));
     LONGS_EQUAL(1, utf8_char_size_screen ("\x01"));
-    LONGS_EQUAL(1, utf8_char_size_screen (UNICODE_SOFT_HYPHEN));
-    LONGS_EQUAL(0, utf8_char_size_screen (UNICODE_ZERO_WIDTH_SPACE));
+    LONGS_EQUAL(-1, utf8_char_size_screen (UNICODE_DELETE));
+    LONGS_EQUAL(-1, utf8_char_size_screen (UNICODE_NEXT_LINE));
+    LONGS_EQUAL(-1, utf8_char_size_screen (UNICODE_PRIVATE_USE_TWO));
+    LONGS_EQUAL(-1, utf8_char_size_screen (UNICODE_SOFT_HYPHEN));
+    LONGS_EQUAL(-1, utf8_char_size_screen (UNICODE_ZERO_WIDTH_SPACE));
     LONGS_EQUAL(2, utf8_char_size_screen (UNICODE_SNOWMAN));
     LONGS_EQUAL(2, utf8_char_size_screen (UNICODE_CJK_YELLOW));
     LONGS_EQUAL(2, utf8_char_size_screen (UNICODE_HAN_CHAR));
     /* ë as iso-8859-15: invalid UTF-8 */
-    LONGS_EQUAL(1, utf8_char_size_screen ("\xeb"));
+    LONGS_EQUAL(-1, utf8_char_size_screen ("\xeb"));
     /* ël as iso-8859-15: invalid UTF-8 */
     LONGS_EQUAL(1, utf8_char_size_screen ("\xebl"));
     /* ëlm as iso-8859-15: invalid UTF-8 */
@@ -517,6 +547,9 @@ TEST(CoreUtf8, Size)
     LONGS_EQUAL(1, utf8_strlen ("€"));
     LONGS_EQUAL(1, utf8_strlen ("\x01"));
     LONGS_EQUAL(4, utf8_strlen (UTF8_NOEL_VALID));
+    LONGS_EQUAL(1, utf8_strlen (UNICODE_DELETE));
+    LONGS_EQUAL(1, utf8_strlen (UNICODE_NEXT_LINE));
+    LONGS_EQUAL(1, utf8_strlen (UNICODE_PRIVATE_USE_TWO));
     LONGS_EQUAL(1, utf8_strlen (UNICODE_SOFT_HYPHEN));
     LONGS_EQUAL(1, utf8_strlen (UNICODE_ZERO_WIDTH_SPACE));
     LONGS_EQUAL(1, utf8_strlen (UNICODE_SNOWMAN));
@@ -537,13 +570,18 @@ TEST(CoreUtf8, Size)
     LONGS_EQUAL(1, utf8_strlen_screen ("A"));
     LONGS_EQUAL(1, utf8_strlen_screen ("ë"));
     LONGS_EQUAL(1, utf8_strlen_screen ("€"));
-    LONGS_EQUAL(1, utf8_strlen_screen ("\x7f"));
     LONGS_EQUAL(1, utf8_strlen_screen ("\x01"));
     LONGS_EQUAL(4, utf8_strlen_screen (UTF8_NOEL_VALID));
     LONGS_EQUAL(4, utf8_strlen_screen ("abc\x01"));
     LONGS_EQUAL(8, utf8_strlen_screen ("a" "\x01" UTF8_NOEL_VALID "\x02" "b"));
-    LONGS_EQUAL(1, utf8_strlen_screen (UNICODE_SOFT_HYPHEN));
-    LONGS_EQUAL(5, utf8_strlen_screen ("a" "\x01" UNICODE_SOFT_HYPHEN "\x02" "b"));
+    LONGS_EQUAL(0, utf8_strlen_screen (UNICODE_DELETE));
+    LONGS_EQUAL(4, utf8_strlen_screen ("a" "\x01" UNICODE_DELETE "\x02" "b"));
+    LONGS_EQUAL(0, utf8_strlen_screen (UNICODE_NEXT_LINE));
+    LONGS_EQUAL(4, utf8_strlen_screen ("a" "\x01" UNICODE_NEXT_LINE "\x02" "b"));
+    LONGS_EQUAL(0, utf8_strlen_screen (UNICODE_PRIVATE_USE_TWO));
+    LONGS_EQUAL(4, utf8_strlen_screen ("a" "\x01" UNICODE_PRIVATE_USE_TWO "\x02" "b"));
+    LONGS_EQUAL(0, utf8_strlen_screen (UNICODE_SOFT_HYPHEN));
+    LONGS_EQUAL(4, utf8_strlen_screen ("a" "\x01" UNICODE_SOFT_HYPHEN "\x02" "b"));
     LONGS_EQUAL(0, utf8_strlen_screen (UNICODE_ZERO_WIDTH_SPACE));
     LONGS_EQUAL(4, utf8_strlen_screen ("a" "\x01" UNICODE_ZERO_WIDTH_SPACE "\x02" "b"));
     LONGS_EQUAL(2, utf8_strlen_screen (UNICODE_SNOWMAN));
diff --git a/tests/unit/gui/test-gui-chat.cpp b/tests/unit/gui/test-gui-chat.cpp
index e359595e3..f5c8ddfaa 100644
--- a/tests/unit/gui/test-gui-chat.cpp
+++ b/tests/unit/gui/test-gui-chat.cpp
@@ -26,10 +26,9 @@ extern "C"
 #include <string.h>
 #include "src/gui/gui-buffer.h"
 #include "src/gui/gui-chat.h"
+#include "src/gui/gui-color.h"
 #include "src/gui/gui-line.h"
 #include "src/gui/gui-window.h"
-
-extern int gui_chat_char_size_screen (const char *utf_char);
 }
 
 #define WEE_GET_WORD_INFO(__result_word_start_offset,                   \
@@ -75,54 +74,38 @@ TEST(GuiChat, PrefixBuild)
 
 /*
  * Tests functions:
- *   gui_chat_utf_char_valid
- */
-
-TEST(GuiChat, UtfCharValid)
-{
-    LONGS_EQUAL(0, gui_chat_utf_char_valid (NULL));
-    LONGS_EQUAL(0, gui_chat_utf_char_valid (""));
-
-    LONGS_EQUAL(0, gui_chat_utf_char_valid ("\x01"));
-    LONGS_EQUAL(0, gui_chat_utf_char_valid ("\x1F"));
-
-    LONGS_EQUAL(0, gui_chat_utf_char_valid ("\x92"));
-    LONGS_EQUAL(0, gui_chat_utf_char_valid ("\x7F"));
-
-    LONGS_EQUAL(1, gui_chat_utf_char_valid ("\x93"));
-    LONGS_EQUAL(1, gui_chat_utf_char_valid ("\x80"));
-
-    LONGS_EQUAL(1, gui_chat_utf_char_valid ("abc"));
-}
-
-/*
- * Tests functions:
- *   gui_chat_char_size_screen
- */
-
-TEST(GuiChat, CharSizeScreen)
-{
-    LONGS_EQUAL(0, gui_chat_char_size_screen (NULL));
-
-    LONGS_EQUAL(1, gui_chat_char_size_screen ("\x01"));
-
-    LONGS_EQUAL(1, gui_chat_char_size_screen ("no\xc3\xabl"));
-    LONGS_EQUAL(2, gui_chat_char_size_screen ("\xe2\xbb\xa9"));
-}
-
-/*
- * Tests functions:
  *   gui_chat_strlen
  */
 
 TEST(GuiChat, Strlen)
 {
+    char string[128];
+
     LONGS_EQUAL(0, gui_chat_strlen (NULL));
     LONGS_EQUAL(0, gui_chat_strlen (""));
 
+    /* soft hyphen */
+    LONGS_EQUAL(1, gui_chat_strlen ("\u00ad"));
+
+    /* zero width space */
+    LONGS_EQUAL(1, gui_chat_strlen ("\u200b"));
+
+    /* next line (non printable char */
+    LONGS_EQUAL(1, gui_chat_strlen ("\u0085"));
+
     LONGS_EQUAL(3, gui_chat_strlen ("abc"));
     LONGS_EQUAL(4, gui_chat_strlen ("no\xc3\xabl"));
     LONGS_EQUAL(1, gui_chat_strlen ("\xe2\xbb\xa9"));
+
+    /* "é" + color + "à" */
+    snprintf (string, sizeof (string),
+              "é%sà", gui_color_get_custom ("red"));
+    LONGS_EQUAL(2, gui_chat_strlen (string));
+
+    /* "a" + soft hyphen + color + zero width space + "b" */
+    snprintf (string, sizeof (string),
+              "a" "\u00ad" "%s" "\u200b" "b", gui_color_get_custom ("red"));
+    LONGS_EQUAL(4, gui_chat_strlen (string));
 }
 
 /*
@@ -132,12 +115,33 @@ TEST(GuiChat, Strlen)
 
 TEST(GuiChat, StrlenScreen)
 {
+    char string[128];
+
     LONGS_EQUAL(0, gui_chat_strlen_screen (NULL));
     LONGS_EQUAL(0, gui_chat_strlen_screen (""));
 
+    /* soft hyphen */
+    LONGS_EQUAL(0, gui_chat_strlen_screen ("\u00ad"));
+
+    /* zero width space */
+    LONGS_EQUAL(0, gui_chat_strlen_screen ("\u200b"));
+
+    /* next line (non printable char) */
+    LONGS_EQUAL(0, gui_chat_strlen_screen ("\u0085"));
+
     LONGS_EQUAL(3, gui_chat_strlen_screen ("abc"));
     LONGS_EQUAL(4, gui_chat_strlen_screen ("no\xc3\xabl"));
     LONGS_EQUAL(2, gui_chat_strlen_screen ("\xe2\xbb\xa9"));
+
+    /* "é" + color + "à" */
+    snprintf (string, sizeof (string),
+              "é%sà", gui_color_get_custom ("red"));
+    LONGS_EQUAL(2, gui_chat_strlen_screen (string));
+
+    /* "a" + soft hyphen + color + zero width space + "b" */
+    snprintf (string, sizeof (string),
+              "a" "\u00ad" "%s" "\u200b" "b", gui_color_get_custom ("red"));
+    LONGS_EQUAL(2, gui_chat_strlen_screen (string));
 }
 
 /*
@@ -150,6 +154,8 @@ TEST(GuiChat, StringAddOffset)
     const char *str_empty = "";
     const char *str_noel = "no\xc3\xabl";
     const char *str_other = "A\xe2\xbb\xa9Z";
+    const char *str_soft_hyphen = "A" "\u00ad" "Z";
+    const char *str_zero_width_space = "A" "\u200b" "Z";
 
     POINTERS_EQUAL(NULL, gui_chat_string_add_offset (NULL, -1));
     POINTERS_EQUAL(NULL, gui_chat_string_add_offset (NULL, 0));
@@ -174,6 +180,20 @@ TEST(GuiChat, StringAddOffset)
     POINTERS_EQUAL(str_other + 5, gui_chat_string_add_offset (str_other, 3));
     POINTERS_EQUAL(str_other + 5, gui_chat_string_add_offset (str_other, 4));
     POINTERS_EQUAL(str_other + 5, gui_chat_string_add_offset (str_other, 5));
+
+    POINTERS_EQUAL(str_soft_hyphen, gui_chat_string_add_offset (str_soft_hyphen, -1));
+    POINTERS_EQUAL(str_soft_hyphen, gui_chat_string_add_offset (str_soft_hyphen, 0));
+    POINTERS_EQUAL(str_soft_hyphen + 1, gui_chat_string_add_offset (str_soft_hyphen, 1));
+    POINTERS_EQUAL(str_soft_hyphen + 3, gui_chat_string_add_offset (str_soft_hyphen, 2));
+    POINTERS_EQUAL(str_soft_hyphen + 4, gui_chat_string_add_offset (str_soft_hyphen, 3));
+    POINTERS_EQUAL(str_soft_hyphen + 4, gui_chat_string_add_offset (str_soft_hyphen, 4));
+
+    POINTERS_EQUAL(str_zero_width_space, gui_chat_string_add_offset (str_zero_width_space, -1));
+    POINTERS_EQUAL(str_zero_width_space, gui_chat_string_add_offset (str_zero_width_space, 0));
+    POINTERS_EQUAL(str_zero_width_space + 1, gui_chat_string_add_offset (str_zero_width_space, 1));
+    POINTERS_EQUAL(str_zero_width_space + 4, gui_chat_string_add_offset (str_zero_width_space, 2));
+    POINTERS_EQUAL(str_zero_width_space + 5, gui_chat_string_add_offset (str_zero_width_space, 3));
+    POINTERS_EQUAL(str_zero_width_space + 5, gui_chat_string_add_offset (str_zero_width_space, 4));
 }
 
 /*
@@ -186,6 +206,8 @@ TEST(GuiChat, StringAddOffsetScreen)
     const char *str_empty = "";
     const char *str_noel = "no\xc3\xabl";
     const char *str_other = "A\xe2\xbb\xa9Z";
+    const char *str_soft_hyphen = "A" "\u00ad" "Z";
+    const char *str_zero_width_space = "A" "\u200b" "Z";
 
     POINTERS_EQUAL(NULL, gui_chat_string_add_offset_screen (NULL, -1));
     POINTERS_EQUAL(NULL, gui_chat_string_add_offset_screen (NULL, 0));
@@ -210,6 +232,18 @@ TEST(GuiChat, StringAddOffsetScreen)
     POINTERS_EQUAL(str_other + 4, gui_chat_string_add_offset_screen (str_other, 3));
     POINTERS_EQUAL(str_other + 5, gui_chat_string_add_offset_screen (str_other, 4));
     POINTERS_EQUAL(str_other + 5, gui_chat_string_add_offset_screen (str_other, 5));
+
+    POINTERS_EQUAL(str_soft_hyphen, gui_chat_string_add_offset_screen (str_soft_hyphen, -1));
+    POINTERS_EQUAL(str_soft_hyphen, gui_chat_string_add_offset_screen (str_soft_hyphen, 0));
+    POINTERS_EQUAL(str_soft_hyphen + 3, gui_chat_string_add_offset_screen (str_soft_hyphen, 1));
+    POINTERS_EQUAL(str_soft_hyphen + 4, gui_chat_string_add_offset_screen (str_soft_hyphen, 2));
+    POINTERS_EQUAL(str_soft_hyphen + 4, gui_chat_string_add_offset_screen (str_soft_hyphen, 3));
+
+    POINTERS_EQUAL(str_zero_width_space, gui_chat_string_add_offset_screen (str_zero_width_space, -1));
+    POINTERS_EQUAL(str_zero_width_space, gui_chat_string_add_offset_screen (str_zero_width_space, 0));
+    POINTERS_EQUAL(str_zero_width_space + 4, gui_chat_string_add_offset_screen (str_zero_width_space, 1));
+    POINTERS_EQUAL(str_zero_width_space + 5, gui_chat_string_add_offset_screen (str_zero_width_space, 2));
+    POINTERS_EQUAL(str_zero_width_space + 5, gui_chat_string_add_offset_screen (str_zero_width_space, 3));
 }
 
 /*
@@ -242,6 +276,16 @@ TEST(GuiChat, StringRealPos)
     LONGS_EQUAL(0, gui_chat_string_real_pos ("\xe2\xbb\xa9", 0, 1));
     LONGS_EQUAL(0, gui_chat_string_real_pos ("\xe2\xbb\xa9", 1, 1));
     LONGS_EQUAL(3, gui_chat_string_real_pos ("\xe2\xbb\xa9", 2, 1));
+
+    /* soft hyphen */
+    LONGS_EQUAL(0, gui_chat_string_real_pos ("A" "\u00ad" "Z", 0, 0));
+    LONGS_EQUAL(3, gui_chat_string_real_pos ("A" "\u00ad" "Z", 1, 0));
+    LONGS_EQUAL(4, gui_chat_string_real_pos ("A" "\u00ad" "Z", 2, 0));
+
+    /* zero width space */
+    LONGS_EQUAL(0, gui_chat_string_real_pos ("A" "\u200b" "Z", 0, 1));
+    LONGS_EQUAL(4, gui_chat_string_real_pos ("A" "\u200b" "Z", 1, 1));
+    LONGS_EQUAL(5, gui_chat_string_real_pos ("A" "\u200b" "Z", 2, 1));
 }
 
 /*
@@ -270,6 +314,16 @@ TEST(GuiChat, StringPos)
     LONGS_EQUAL(0, gui_chat_string_pos ("\xe2\xbb\xa9", 0));
     LONGS_EQUAL(1, gui_chat_string_pos ("\xe2\xbb\xa9", 1));
     LONGS_EQUAL(1, gui_chat_string_pos ("\xe2\xbb\xa9", 2));
+
+    /* soft hyphen */
+    LONGS_EQUAL(0, gui_chat_string_pos ("A" "\u00ad" "Z", 0));
+    LONGS_EQUAL(1, gui_chat_string_pos ("A" "\u00ad" "Z", 1));
+    LONGS_EQUAL(2, gui_chat_string_pos ("A" "\u00ad" "Z", 2));
+
+    /* zero width space */
+    LONGS_EQUAL(0, gui_chat_string_pos ("A" "\u200b" "Z", 0));
+    LONGS_EQUAL(1, gui_chat_string_pos ("A" "\u200b" "Z", 1));
+    LONGS_EQUAL(2, gui_chat_string_pos ("A" "\u200b" "Z", 2));
 }
 
 /*