LibWeb+LibSyntax: Implement nested syntax highlighters

And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.

2 files changed, 98 insertions, 10 deletions

diff --git a/Userland/Libraries/LibSyntax/Highlighter.cpp b/Userland/Libraries/LibSyntax/Highlighter.cpp
index c80ee33104..df5dbeed5f 100644
--- a/Userland/Libraries/LibSyntax/Highlighter.cpp
+++ b/Userland/Libraries/LibSyntax/Highlighter.cpp
@@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair()
         Backward,
     };
 
-    auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional<size_t> {
+    auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional<size_t> {
         size_t nesting_level = 0;
         bool forward = direction == Direction::Forward;
 
@@ -130,4 +130,19 @@ void Highlighter::cursor_did_change()
     highlight_matching_token_pair();
 }
 
+Vector<Highlighter::MatchingTokenPair> Highlighter::matching_token_pairs() const
+{
+    auto own_pairs = matching_token_pairs_impl();
+    own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size());
+    for (auto& nested_pair : m_nested_token_pairs)
+        own_pairs.append(nested_pair);
+    return own_pairs;
+}
+
+void Highlighter::register_nested_token_pairs(Vector<MatchingTokenPair> pairs)
+{
+    for (auto& pair : pairs)
+        m_nested_token_pairs.set(pair);
+}
+
 }
diff --git a/Userland/Libraries/LibSyntax/Highlighter.h b/Userland/Libraries/LibSyntax/Highlighter.h
index de0b6bdc2b..7a2541f96e 100644
--- a/Userland/Libraries/LibSyntax/Highlighter.h
+++ b/Userland/Libraries/LibSyntax/Highlighter.h
@@ -41,26 +41,30 @@ public:
     virtual void rehighlight(const Palette&) = 0;
     virtual void highlight_matching_token_pair();
 
-    virtual bool is_identifier(void*) const { return false; };
-    virtual bool is_navigatable(void*) const { return false; };
+    virtual bool is_identifier(u64) const { return false; };
+    virtual bool is_navigatable(u64) const { return false; };
 
     void attach(HighlighterClient&);
     void detach();
     void cursor_did_change();
 
+    struct MatchingTokenPair {
+        u64 open;
+        u64 close;
+    };
+    Vector<MatchingTokenPair> matching_token_pairs() const;
+
 protected:
     Highlighter() { }
 
     // FIXME: This should be WeakPtr somehow
     HighlighterClient* m_client { nullptr };
 
-    struct MatchingTokenPair {
-        void* open;
-        void* close;
-    };
-
-    virtual Vector<MatchingTokenPair> matching_token_pairs() const = 0;
-    virtual bool token_types_equal(void*, void*) const = 0;
+    virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const = 0;
+    virtual bool token_types_equal(u64, u64) const = 0;
+    void register_nested_token_pairs(Vector<MatchingTokenPair>);
+    void clear_nested_token_pairs() { m_nested_token_pairs.clear(); }
+    size_t first_free_token_kind_serial_value() const { return m_nested_token_pairs.size(); }
 
     struct BuddySpan {
         int index { -1 };
@@ -69,6 +73,75 @@ protected:
 
     bool m_has_brace_buddies { false };
     BuddySpan m_brace_buddies[2];
+    HashTable<MatchingTokenPair> m_nested_token_pairs;
+};
+
+class ProxyHighlighterClient final : public Syntax::HighlighterClient {
+public:
+    ProxyHighlighterClient(Syntax::HighlighterClient& client, GUI::TextPosition start, u64 nested_kind_start_value, StringView source)
+        : m_document(client.get_document())
+        , m_text(source)
+        , m_start(start)
+        , m_nested_kind_start_value(nested_kind_start_value)
+    {
+    }
+
+    Vector<GUI::TextDocumentSpan> corrected_spans() const
+    {
+        Vector<GUI::TextDocumentSpan> spans { m_spans };
+        for (auto& entry : spans) {
+            entry.range.start() = {
+                entry.range.start().line() + m_start.line(),
+                entry.range.start().line() == 0 ? entry.range.start().column() + m_start.column() : entry.range.start().column(),
+            };
+            entry.range.end() = {
+                entry.range.end().line() + m_start.line(),
+                entry.range.end().line() == 0 ? entry.range.end().column() + m_start.column() : entry.range.end().column(),
+            };
+            if (entry.data != (u64)-1)
+                entry.data += m_nested_kind_start_value;
+        }
+
+        return spans;
+    }
+
+    Vector<Syntax::Highlighter::MatchingTokenPair> corrected_token_pairs(Vector<Syntax::Highlighter::MatchingTokenPair> pairs) const
+    {
+        for (auto& pair : pairs) {
+            pair.close += m_nested_kind_start_value;
+            pair.open += m_nested_kind_start_value;
+        }
+        return pairs;
+    }
+
+private:
+    virtual Vector<GUI::TextDocumentSpan>& spans() override { return m_spans; }
+    virtual const Vector<GUI::TextDocumentSpan>& spans() const override { return m_spans; }
+    virtual void set_span_at_index(size_t index, GUI::TextDocumentSpan span) override { m_spans.at(index) = move(span); }
+
+    virtual String highlighter_did_request_text() const override { return m_text; }
+    virtual void highlighter_did_request_update() override { }
+    virtual GUI::TextDocument& highlighter_did_request_document() override { return m_document; }
+    virtual GUI::TextPosition highlighter_did_request_cursor() const override { return {}; }
+    virtual void highlighter_did_set_spans(Vector<GUI::TextDocumentSpan> spans) override { m_spans = move(spans); }
+
+    Vector<GUI::TextDocumentSpan> m_spans;
+    GUI::TextDocument& m_document;
+    StringView m_text;
+    GUI::TextPosition m_start;
+    u64 m_nested_kind_start_value { 0 };
 };
 
 }
+
+template<>
+struct AK::Traits<Syntax::Highlighter::MatchingTokenPair> : public AK::GenericTraits<Syntax::Highlighter::MatchingTokenPair> {
+    static unsigned hash(Syntax::Highlighter::MatchingTokenPair const& pair)
+    {
+        return pair_int_hash(u64_hash(pair.open), u64_hash(pair.close));
+    }
+    static bool equals(Syntax::Highlighter::MatchingTokenPair const& a, Syntax::Highlighter::MatchingTokenPair const& b)
+    {
+        return a.open == b.open && a.close == b.close;
+    }
+};