LibJS: Make scoping follow the spec

Before this we used an ad-hoc combination of references and 'variables'
stored in a hashmap. This worked in most cases but is not spec like.
Additionally hoisting, dynamically naming functions and scope analysis
was not done properly.

This patch fixes all of that by:
  - Implement BindingInitialization for destructuring assignment.
  - Implementing a new ScopePusher which tracks the lexical and var
    scoped declarations. This hoists functions to the top level if no
    lexical declaration name overlaps. Furthermore we do checking of
    redeclarations in the ScopePusher now requiring less checks all over
    the place.
  - Add methods for parsing the directives and statement lists instead
    of having that code duplicated in multiple places. This allows
    declarations to pushed to the appropriate scope more easily.
  - Remove the non spec way of storing 'variables' in
    DeclarativeEnvironment and make Reference follow the spec instead of
    checking both the bindings and 'variables'.
  - Remove all scoping related things from the Interpreter. And instead
    use environments as specified by the spec. This also includes fixing
    that NativeFunctions did not produce a valid FunctionEnvironment
    which could cause issues with callbacks and eval. All
    FunctionObjects now have a valid NewFunctionEnvironment
    implementation.
  - Remove execute_statements from Interpreter and instead use
    ASTNode::execute everywhere this simplifies AST.cpp as you no longer
    need to worry about which method to call.
  - Make ScopeNodes setup their own environment. This uses four
    different methods specified by the spec
    {Block, Function, Eval, Global}DeclarationInstantiation with the
    annexB extensions.
  - Implement and use NamedEvaluation where specified.

Additionally there are fixes to things exposed by these changes to eval,
{for, for-in, for-of} loops and assignment.

Finally it also fixes some tests in test-js which where passing before
but not now that we have correct behavior :^).

1 files changed, 248 insertions, 66 deletions

diff --git a/Userland/Libraries/LibJS/Runtime/ECMAScriptFunctionObject.cpp b/Userland/Libraries/LibJS/Runtime/ECMAScriptFunctionObject.cpp
index df3fd9e1d0..94bd59e967 100644
--- a/Userland/Libraries/LibJS/Runtime/ECMAScriptFunctionObject.cpp
+++ b/Userland/Libraries/LibJS/Runtime/ECMAScriptFunctionObject.cpp
@@ -11,6 +11,7 @@
 #include <LibJS/Bytecode/Generator.h>
 #include <LibJS/Bytecode/Interpreter.h>
 #include <LibJS/Interpreter.h>
+#include <LibJS/Runtime/AbstractOperations.h>
 #include <LibJS/Runtime/Array.h>
 #include <LibJS/Runtime/ECMAScriptFunctionObject.h>
 #include <LibJS/Runtime/Error.h>
@@ -107,86 +108,269 @@ void ECMAScriptFunctionObject::visit_edges(Visitor& visitor)
     }
 }
 
-FunctionEnvironment* ECMAScriptFunctionObject::create_environment(FunctionObject& function_being_invoked)
+// 9.1.2.4 NewFunctionEnvironment ( F, newTarget ), https://tc39.es/ecma262/#sec-newfunctionenvironment
+FunctionEnvironment* ECMAScriptFunctionObject::new_function_environment(Object* new_target)
 {
-    HashMap<FlyString, Variable> variables;
+    auto* environment = heap().allocate<FunctionEnvironment>(global_object(), m_environment);
+    environment->set_function_object(*this);
+    if (this_mode() == ThisMode::Lexical) {
+        environment->set_this_binding_status(FunctionEnvironment::ThisBindingStatus::Lexical);
+    }
+
+    environment->set_new_target(new_target ? new_target : js_undefined());
+    return environment;
+}
+
+// 10.2.11 FunctionDeclarationInstantiation ( func, argumentsList ), https://tc39.es/ecma262/#sec-functiondeclarationinstantiation
+ThrowCompletionOr<void> ECMAScriptFunctionObject::function_declaration_instantiation(Interpreter* interpreter)
+{
+    auto& vm = this->vm();
+
+    auto& callee_context = vm.running_execution_context();
+
+    // Needed to extract declarations and functions
+    ScopeNode const* scope_body = nullptr;
+    if (is<ScopeNode>(*m_ecmascript_code))
+        scope_body = static_cast<ScopeNode const*>(m_ecmascript_code.ptr());
+
+    bool has_parameter_expressions = false;
+
+    // FIXME: Maybe compute has duplicates at parse time? (We need to anyway since it's an error in some cases)
+
+    bool has_duplicates = false;
+    HashTable<FlyString> parameter_names;
     for (auto& parameter : m_formal_parameters) {
+        if (parameter.default_value)
+            has_parameter_expressions = true;
+
         parameter.binding.visit(
-            [&](const FlyString& name) { variables.set(name, { js_undefined(), DeclarationKind::Var }); },
-            [&](const NonnullRefPtr<BindingPattern>& binding) {
-                binding->for_each_bound_name([&](const auto& name) {
-                    variables.set(name, { js_undefined(), DeclarationKind::Var });
+            [&](FlyString const& name) {
+                if (parameter_names.set(name) != AK::HashSetResult::InsertedNewEntry)
+                    has_duplicates = true;
+            },
+            [&](NonnullRefPtr<BindingPattern> const& pattern) {
+                if (pattern->contains_expression())
+                    has_parameter_expressions = true;
+
+                pattern->for_each_bound_name([&](auto& name) {
+                    if (parameter_names.set(name) != AK::HashSetResult::InsertedNewEntry)
+                        has_duplicates = true;
                 });
             });
     }
 
-    if (is<ScopeNode>(ecmascript_code())) {
-        for (auto& declaration : static_cast<const ScopeNode&>(ecmascript_code()).variables()) {
-            for (auto& declarator : declaration.declarations()) {
-                declarator.target().visit(
-                    [&](const NonnullRefPtr<Identifier>& id) {
-                        variables.set(id->string(), { js_undefined(), declaration.declaration_kind() });
-                    },
-                    [&](const NonnullRefPtr<BindingPattern>& binding) {
-                        binding->for_each_bound_name([&](const auto& name) {
-                            variables.set(name, { js_undefined(), declaration.declaration_kind() });
-                        });
-                    });
-            }
+    auto needs_argument_object = this_mode() != ThisMode::Lexical;
+    if (parameter_names.contains(vm.names.arguments.as_string()))
+        needs_argument_object = false;
+
+    HashTable<FlyString> function_names;
+    Vector<FunctionDeclaration const&> functions_to_initialize;
+
+    if (scope_body) {
+        scope_body->for_each_var_function_declaration_in_reverse_order([&](FunctionDeclaration const& function) {
+            if (function_names.set(function.name()) == AK::HashSetResult::InsertedNewEntry)
+                functions_to_initialize.append(function);
+        });
+
+        auto arguments_name = vm.names.arguments.as_string();
+
+        if (!has_parameter_expressions && function_names.contains(arguments_name))
+            needs_argument_object = false;
+
+        if (!has_parameter_expressions && needs_argument_object) {
+            scope_body->for_each_lexically_declared_name([&](auto const& name) {
+                if (name == arguments_name)
+                    needs_argument_object = false;
+                return IterationDecision::Continue;
+            });
         }
     }
 
-    auto* environment = heap().allocate<FunctionEnvironment>(global_object(), m_environment, move(variables));
-    environment->set_function_object(static_cast<ECMAScriptFunctionObject&>(function_being_invoked));
-    if (m_is_arrow_function) {
-        environment->set_this_binding_status(FunctionEnvironment::ThisBindingStatus::Lexical);
-        if (is<FunctionEnvironment>(m_environment))
-            environment->set_new_target(static_cast<FunctionEnvironment*>(m_environment)->new_target());
+    Environment* environment;
+
+    if (is_strict_mode() || !has_parameter_expressions) {
+        environment = callee_context.lexical_environment;
+    } else {
+        environment = new_declarative_environment(*callee_context.lexical_environment);
+        VERIFY(callee_context.variable_environment == callee_context.lexical_environment);
+        callee_context.lexical_environment = environment;
     }
-    return environment;
+
+    for (auto const& parameter_name : parameter_names) {
+        if (environment->has_binding(parameter_name))
+            continue;
+
+        environment->create_mutable_binding(global_object(), parameter_name, false);
+        if (has_duplicates)
+            environment->initialize_binding(global_object(), parameter_name, js_undefined());
+        VERIFY(!vm.exception());
+    }
+
+    if (needs_argument_object) {
+        Object* arguments_object;
+        if (is_strict_mode() || !has_simple_parameter_list())
+            arguments_object = create_unmapped_arguments_object(global_object(), vm.running_execution_context().arguments);
+        else
+            arguments_object = create_mapped_arguments_object(global_object(), *this, formal_parameters(), vm.running_execution_context().arguments, *environment);
+
+        if (is_strict_mode())
+            environment->create_immutable_binding(global_object(), vm.names.arguments.as_string(), false);
+        else
+            environment->create_mutable_binding(global_object(), vm.names.arguments.as_string(), false);
+
+        environment->initialize_binding(global_object(), vm.names.arguments.as_string(), arguments_object);
+        parameter_names.set(vm.names.arguments.as_string());
+    }
+
+    // We now treat parameterBindings as parameterNames.
+
+    // The spec makes an iterator here to do IteratorBindingInitialization but we just do it manually
+    auto& execution_context_arguments = vm.running_execution_context().arguments;
+
+    for (size_t i = 0; i < m_formal_parameters.size(); ++i) {
+        auto& parameter = m_formal_parameters[i];
+        parameter.binding.visit(
+            [&](auto const& param) {
+                Value argument_value;
+                if (parameter.is_rest) {
+                    auto* array = Array::create(global_object(), 0);
+                    for (size_t rest_index = i; rest_index < execution_context_arguments.size(); ++rest_index)
+                        array->indexed_properties().append(execution_context_arguments[rest_index]);
+                    argument_value = move(array);
+                } else if (i < execution_context_arguments.size() && !execution_context_arguments[i].is_undefined()) {
+                    argument_value = execution_context_arguments[i];
+                } else if (parameter.default_value) {
+                    // FIXME: Support default arguments in the bytecode world!
+                    if (interpreter)
+                        argument_value = parameter.default_value->execute(*interpreter, global_object());
+                    if (vm.exception())
+                        return;
+                } else {
+                    argument_value = js_undefined();
+                }
+
+                Environment* used_environment = has_duplicates ? nullptr : environment;
+
+                if constexpr (IsSame<FlyString const&, decltype(param)>) {
+                    Reference reference = vm.resolve_binding(param, used_environment);
+                    if (vm.exception())
+                        return;
+                    // Here the difference from hasDuplicates is important
+                    if (has_duplicates)
+                        reference.put_value(global_object(), argument_value);
+                    else
+                        reference.initialize_referenced_binding(global_object(), argument_value);
+                } else if (IsSame<NonnullRefPtr<BindingPattern> const&, decltype(param)>) {
+                    // Here the difference from hasDuplicates is important
+                    auto result = vm.binding_initialization(param, argument_value, used_environment, global_object());
+                    if (result.is_error())
+                        return;
+                }
+
+                if (vm.exception())
+                    return;
+            });
+
+        if (auto* exception = vm.exception())
+            return throw_completion(exception->value());
+    }
+
+    Environment* var_environment;
+
+    HashTable<FlyString> instantiated_var_names;
+
+    if (!has_parameter_expressions) {
+        if (scope_body) {
+            scope_body->for_each_var_declared_name([&](auto const& name) {
+                if (!parameter_names.contains(name) && instantiated_var_names.set(name) == AK::HashSetResult::InsertedNewEntry) {
+                    environment->create_mutable_binding(global_object(), name, false);
+                    environment->initialize_binding(global_object(), name, js_undefined());
+                }
+            });
+        }
+        var_environment = environment;
+    } else {
+        var_environment = new_declarative_environment(*environment);
+        callee_context.variable_environment = var_environment;
+
+        if (scope_body) {
+            scope_body->for_each_var_declared_name([&](auto const& name) {
+                if (instantiated_var_names.set(name) != AK::HashSetResult::InsertedNewEntry)
+                    return IterationDecision::Continue;
+                var_environment->create_mutable_binding(global_object(), name, false);
+
+                Value initial_value;
+                if (!parameter_names.contains(name) || function_names.contains(name))
+                    initial_value = js_undefined();
+                else
+                    initial_value = environment->get_binding_value(global_object(), name, false);
+
+                var_environment->initialize_binding(global_object(), name, initial_value);
+
+                return IterationDecision::Continue;
+            });
+        }
+    }
+
+    // B.3.2.1 Changes to FunctionDeclarationInstantiation, https://tc39.es/ecma262/#sec-web-compat-functiondeclarationinstantiation
+    if (!m_strict && scope_body) {
+        scope_body->for_each_function_hoistable_with_annexB_extension([&](FunctionDeclaration& function_declaration) {
+            auto& function_name = function_declaration.name();
+            if (parameter_names.contains(function_name))
+                return IterationDecision::Continue;
+            // The spec says 'initializedBindings' here but that does not exist and it then adds it to 'instantiatedVarNames' so it probably means 'instantiatedVarNames'.
+            if (!instantiated_var_names.contains(function_name) && function_name != vm.names.arguments.as_string()) {
+                var_environment->create_mutable_binding(global_object(), function_name, false);
+                VERIFY(!vm.exception());
+                var_environment->initialize_binding(global_object(), function_name, js_undefined());
+                instantiated_var_names.set(function_name);
+            }
+
+            function_declaration.set_should_do_additional_annexB_steps();
+            return IterationDecision::Continue;
+        });
+    }
+
+    Environment* lex_environment;
+
+    if (!is_strict_mode())
+        lex_environment = new_declarative_environment(*var_environment);
+    else
+        lex_environment = var_environment;
+
+    callee_context.lexical_environment = lex_environment;
+
+    if (!scope_body)
+        return {};
+
+    scope_body->for_each_lexically_scoped_declaration([&](Declaration const& declaration) {
+        declaration.for_each_bound_name([&](auto const& name) {
+            if (declaration.is_constant_declaration())
+                lex_environment->create_immutable_binding(global_object(), name, true);
+            else
+                lex_environment->create_mutable_binding(global_object(), name, false);
+            return IterationDecision::Continue;
+        });
+    });
+
+    VERIFY(!vm.exception());
+
+    for (auto& declaration : functions_to_initialize) {
+        auto* function = ECMAScriptFunctionObject::create(global_object(), declaration.name(), declaration.body(), declaration.parameters(), declaration.function_length(), lex_environment, declaration.kind(), declaration.is_strict_mode());
+        var_environment->set_mutable_binding(global_object(), declaration.name(), function, false);
+    }
+
+    return {};
 }
 
 Value ECMAScriptFunctionObject::execute_function_body()
 {
     auto& vm = this->vm();
-
-    Interpreter* ast_interpreter = nullptr;
     auto* bytecode_interpreter = Bytecode::Interpreter::current();
 
-    auto prepare_arguments = [&] {
-        auto& execution_context_arguments = vm.running_execution_context().arguments;
-        for (size_t i = 0; i < m_formal_parameters.size(); ++i) {
-            auto& parameter = m_formal_parameters[i];
-            parameter.binding.visit(
-                [&](const auto& param) {
-                    Value argument_value;
-                    if (parameter.is_rest) {
-                        auto* array = Array::create(global_object(), 0);
-                        for (size_t rest_index = i; rest_index < execution_context_arguments.size(); ++rest_index)
-                            array->indexed_properties().append(execution_context_arguments[rest_index]);
-                        argument_value = move(array);
-                    } else if (i < execution_context_arguments.size() && !execution_context_arguments[i].is_undefined()) {
-                        argument_value = execution_context_arguments[i];
-                    } else if (parameter.default_value) {
-                        // FIXME: Support default arguments in the bytecode world!
-                        if (!bytecode_interpreter)
-                            argument_value = parameter.default_value->execute(*ast_interpreter, global_object());
-                        if (vm.exception())
-                            return;
-                    } else {
-                        argument_value = js_undefined();
-                    }
-
-                    vm.assign(param, argument_value, global_object(), true, vm.lexical_environment());
-                });
-
-            if (vm.exception())
-                return;
-        }
-    };
-
     if (bytecode_interpreter) {
-        prepare_arguments();
+        // FIXME: pass something to evaluate default arguments with
+        TRY_OR_DISCARD(function_declaration_instantiation(nullptr));
         if (!m_bytecode_executable.has_value()) {
             m_bytecode_executable = Bytecode::Generator::generate(m_ecmascript_code, m_kind == FunctionKind::Generator);
             auto& passes = JS::Bytecode::Interpreter::optimization_pipeline();
@@ -206,7 +390,7 @@ Value ECMAScriptFunctionObject::execute_function_body()
     } else {
         VERIFY(m_kind != FunctionKind::Generator);
         OwnPtr<Interpreter> local_interpreter;
-        ast_interpreter = vm.interpreter_if_exists();
+        Interpreter* ast_interpreter = vm.interpreter_if_exists();
 
         if (!ast_interpreter) {
             local_interpreter = Interpreter::create_with_existing_realm(*realm());
@@ -215,11 +399,9 @@ Value ECMAScriptFunctionObject::execute_function_body()
 
         VM::InterpreterExecutionScope scope(*ast_interpreter);
 
-        prepare_arguments();
-        if (vm.exception())
-            return {};
+        TRY_OR_DISCARD(function_declaration_instantiation(ast_interpreter));
 
-        return ast_interpreter->execute_statement(global_object(), m_ecmascript_code, ScopeType::Function);
+        return m_ecmascript_code->execute(*ast_interpreter, global_object());
     }
 }