LibWeb+LibSyntax: Implement nested syntax highlighters

And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.

2 files changed, 40 insertions, 13 deletions

diff --git a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp
index df66bede63..fe899e5940 100644
--- a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp
+++ b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp
@@ -6,6 +6,7 @@
  */
 
 #include <AK/Debug.h>
+#include <LibJS/SyntaxHighlighter.h>
 #include <LibWeb/HTML/Parser/HTMLTokenizer.h>
 #include <LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h>
 
@@ -18,16 +19,17 @@ enum class AugmentedTokenKind : u32 {
     CloseTag,
     Comment,
     Doctype,
+    __Count,
 };
 
-bool SyntaxHighlighter::is_identifier(void* token) const
+bool SyntaxHighlighter::is_identifier(u64 token) const
 {
     if (!token)
         return false;
     return false;
 }
 
-bool SyntaxHighlighter::is_navigatable(void*) const
+bool SyntaxHighlighter::is_navigatable(u64) const
 {
     return false;
 }
@@ -35,8 +37,8 @@ bool SyntaxHighlighter::is_navigatable(void*) const
 void SyntaxHighlighter::rehighlight(Palette const& palette)
 {
     dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(HTML::SyntaxHighlighter) starting rehighlight");
-    (void)palette;
     auto text = m_client->get_text();
+    clear_nested_token_pairs();
 
     Vector<GUI::TextDocumentSpan> spans;
     auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, AugmentedTokenKind kind) {
@@ -51,7 +53,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
                 { end_line, end_column },
             },
             move(attributes),
-            (void*)kind,
+            static_cast<u64>(kind),
             false);
     };
 
@@ -61,6 +63,9 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
         Javascript,
         CSS,
     } state { State::HTML };
+    StringBuilder substring_builder;
+    GUI::TextPosition substring_start_position;
+
     for (;;) {
         auto token = tokenizer.next_token();
         if (!token.has_value() || token.value().is_end_of_file())
@@ -71,19 +76,41 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
             if (token->tag_name() == "script"sv) {
                 tokenizer.switch_to(HTMLTokenizer::State::ScriptData);
                 state = State::Javascript;
+                substring_start_position = { token->end_position().line, token->end_position().column };
             } else if (token->tag_name() == "style"sv) {
                 tokenizer.switch_to(HTMLTokenizer::State::RAWTEXT);
                 state = State::CSS;
+                substring_start_position = { token->end_position().line, token->end_position().column };
             }
         } else if (token->is_end_tag()) {
             if (token->tag_name().is_one_of("script"sv, "style"sv)) {
                 if (state == State::Javascript) {
-                    // FIXME: Highlight javascript code here instead.
+                    Syntax::ProxyHighlighterClient proxy_client {
+                        *m_client,
+                        substring_start_position,
+                        static_cast<u64>(AugmentedTokenKind::__Count) + first_free_token_kind_serial_value(),
+                        substring_builder.string_view()
+                    };
+                    {
+                        JS::SyntaxHighlighter highlighter;
+                        highlighter.attach(proxy_client);
+                        highlighter.rehighlight(palette);
+                        highlighter.detach();
+                        register_nested_token_pairs(proxy_client.corrected_token_pairs(highlighter.matching_token_pairs()));
+                    }
+
+                    spans.append(proxy_client.corrected_spans());
+                    substring_builder.clear();
                 } else if (state == State::CSS) {
                     // FIXME: Highlight CSS code here instead.
+                    substring_builder.clear();
                 }
                 state = State::HTML;
             }
+        } else if (state != State::HTML) {
+            VERIFY(token->is_character());
+            substring_builder.append_code_point(token->code_point());
+            continue;
         }
 
         size_t token_start_offset = token->is_end_tag() ? 1 : 0;
@@ -135,7 +162,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
         dbgln("(HTML::SyntaxHighlighter) list of all spans:");
         for (auto& span : spans)
-            dbgln("{}, {}", span.range, span.attributes.color);
+            dbgln("{}, {} - {}", span.range, span.attributes.color, span.data);
         dbgln("(HTML::SyntaxHighlighter) end of list");
     }
 
@@ -145,16 +172,16 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
     m_client->do_update();
 }
 
-Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
+Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
 {
     static Vector<MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ (void*)AugmentedTokenKind::OpenTag, (void*)AugmentedTokenKind::CloseTag });
+        pairs.append({ static_cast<u64>(AugmentedTokenKind::OpenTag), static_cast<u64>(AugmentedTokenKind::CloseTag) });
     }
     return pairs;
 }
 
-bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
+bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
 {
     return token0 == token1;
 }
diff --git a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h
index 6d412c940a..79f38c3d98 100644
--- a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h
+++ b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h
@@ -15,15 +15,15 @@ public:
     SyntaxHighlighter() = default;
     virtual ~SyntaxHighlighter() override = default;
 
-    virtual bool is_identifier(void*) const override;
-    virtual bool is_navigatable(void*) const override;
+    virtual bool is_identifier(u64) const override;
+    virtual bool is_navigatable(u64) const override;
 
     virtual Syntax::Language language() const override { return Syntax::Language::HTML; }
     virtual void rehighlight(Palette const&) override;
 
 protected:
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
-    virtual bool token_types_equal(void*, void*) const override;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
+    virtual bool token_types_equal(u64, u64) const override;
 
     size_t m_line { 1 };
     size_t m_column { 0 };