LibX86: Split up the ModRM and SIB bytes into multiple members

This will allow adding extra bits with REX prefixes

2 files changed, 60 insertions, 118 deletions

diff --git a/Userland/Libraries/LibX86/Instruction.cpp b/Userland/Libraries/LibX86/Instruction.cpp
index f0a5e4bd16..6fca0585e2 100644
--- a/Userland/Libraries/LibX86/Instruction.cpp
+++ b/Userland/Libraries/LibX86/Instruction.cpp
@@ -1371,79 +1371,35 @@ String MemoryOrRegisterReference::to_string_a16() const
     return String::formatted("{}{}", base, displacement_string);
 }
 
-static String sib_to_string(u8 rm, u8 sib)
+String MemoryOrRegisterReference::sib_to_string(ProcessorMode) const
 {
     String scale;
     String index;
     String base;
-    switch (sib & 0xC0) {
-    case 0x00:;
-        break;
-    case 0x40:
-        scale = "*2";
-        break;
-    case 0x80:
-        scale = "*4";
-        break;
-    case 0xC0:
-        scale = "*8";
-        break;
-    }
-    switch ((sib >> 3) & 0x07) {
-    case 0:
-        index = "eax";
+    switch (m_sib_scale) {
+    case 0:;
         break;
     case 1:
-        index = "ecx";
+        scale = "*2";
         break;
     case 2:
-        index = "edx";
+        scale = "*4";
         break;
     case 3:
-        index = "ebx";
-        break;
-    case 4:
-        break;
-    case 5:
-        index = "ebp";
-        break;
-    case 6:
-        index = "esi";
-        break;
-    case 7:
-        index = "edi";
+        scale = "*8";
         break;
     }
-    switch (sib & 0x07) {
-    case 0:
-        base = "eax";
-        break;
-    case 1:
-        base = "ecx";
-        break;
-    case 2:
-        base = "edx";
-        break;
-    case 3:
-        base = "ebx";
-        break;
-    case 4:
-        base = "esp";
-        break;
-    case 6:
-        base = "esi";
-        break;
-    case 7:
-        base = "edi";
-        break;
-    default: // 5
-        switch ((rm >> 6) & 3) {
+    if (m_sib_index != 4)
+        index = register_name(RegisterIndex32(m_sib_index));
+    if (m_sib_base == 5) {
+        switch (m_reg) {
         case 1:
         case 2:
             base = "ebp";
             break;
         }
-        break;
+    } else {
+        base = register_name(RegisterIndex32(m_sib_base));
     }
     StringBuilder builder;
     if (base.is_empty()) {
@@ -1467,35 +1423,17 @@ String MemoryOrRegisterReference::to_string_a64() const
     bool has_displacement = false;
     switch (mod()) {
     case 0b00:
-        has_displacement = rm() == 5;
+        has_displacement = m_rm == 5;
         break;
     case 0b01:
     case 0b10:
         has_displacement = true;
     }
-    if (m_has_sib && (m_sib & 7) == 5)
+    if (m_has_sib && m_sib_base == 5)
         has_displacement = true;
 
     String base;
-    switch (rm()) {
-    case 0:
-        base = "rax";
-        break;
-    case 1:
-        base = "rcx";
-        break;
-    case 2:
-        base = "rdx";
-        break;
-    case 3:
-        base = "rbx";
-        break;
-    case 6:
-        base = "rsi";
-        break;
-    case 7:
-        base = "rdi";
-        break;
+    switch (m_rm) {
     case 5:
         if (mod() == 0)
             base = String::formatted("{:#08x}", m_displacement32);
@@ -1503,8 +1441,10 @@ String MemoryOrRegisterReference::to_string_a64() const
             base = "rbp";
         break;
     case 4:
-        base = sib_to_string(m_rm_byte, m_sib);
+        base = sib_to_string(ProcessorMode::Long);
         break;
+    default:
+        base = register_name(RegisterIndex32(m_rm));
     }
 
     if (!has_displacement)
@@ -1529,29 +1469,11 @@ String MemoryOrRegisterReference::to_string_a32() const
     case 0b10:
         has_displacement = true;
     }
-    if (m_has_sib && (m_sib & 7) == 5)
+    if (m_has_sib && m_sib_base == 5)
         has_displacement = true;
 
     String base;
-    switch (rm()) {
-    case 0:
-        base = "eax";
-        break;
-    case 1:
-        base = "ecx";
-        break;
-    case 2:
-        base = "edx";
-        break;
-    case 3:
-        base = "ebx";
-        break;
-    case 6:
-        base = "esi";
-        break;
-    case 7:
-        base = "edi";
-        break;
+    switch (m_rm) {
     case 5:
         if (mod() == 0)
             base = String::formatted("{:x}", m_displacement32);
@@ -1559,8 +1481,10 @@ String MemoryOrRegisterReference::to_string_a32() const
             base = "ebp";
         break;
     case 4:
-        base = sib_to_string(m_rm_byte, m_sib);
+        base = sib_to_string(ProcessorMode::Protected);
         break;
+    default:
+        base = register_name(RegisterIndex32(m_rm));
     }
 
     if (!has_displacement)
diff --git a/Userland/Libraries/LibX86/Instruction.h b/Userland/Libraries/LibX86/Instruction.h
index 547a2e6c48..7d0121eb3b 100644
--- a/Userland/Libraries/LibX86/Instruction.h
+++ b/Userland/Libraries/LibX86/Instruction.h
@@ -54,6 +54,11 @@ enum class AddressSize : u8 {
     Size64,
 };
 
+enum class ProcessorMode : u8 {
+    Protected,
+    Long,
+};
+
 enum IsLockPrefixAllowed {
     LockPrefixNotAllowed = 0,
     LockPrefixAllowed
@@ -448,6 +453,7 @@ public:
     String to_string_fpu80(Instruction const&) const;
     String to_string_mm(Instruction const&) const;
     String to_string_xmm(Instruction const&) const;
+    String sib_to_string(ProcessorMode) const;
 
     bool is_register() const { return m_register_index != 0x7f; }
 
@@ -458,9 +464,10 @@ public:
     FpuRegisterIndex reg_fpu() const { return static_cast<FpuRegisterIndex>(register_index()); }
 
     // helpers to get the parts by name as in the spec
-    u8 mod() const { return m_rm_byte >> 6; }
-    u8 reg() const { return m_rm_byte >> 3 & 0b111; }
-    u8 rm() const { return m_rm_byte & 0b111; }
+    u8 mod() const { return m_mod; }
+    u8 reg() const { return m_reg; }
+    u8 rm() const { return m_rm; }
+    u8 modrm_byte() const { return (m_mod << 6) | ((m_reg & 7) << 3) | (m_rm & 7); }
 
     template<typename CPU, typename T>
     void write8(CPU&, Instruction const&, T);
@@ -518,8 +525,12 @@ private:
         u16 m_displacement16;
     };
 
-    u8 m_rm_byte { 0 };
-    u8 m_sib { 0 };
+    u8 m_mod : 2 { 0 };
+    u8 m_reg : 4 { 0 };
+    u8 m_rm : 4 { 0 };
+    u8 m_sib_scale : 2 { 0 };
+    u8 m_sib_index : 4 { 0 };
+    u8 m_sib_base : 4 { 0 };
     u8 m_displacement_bytes { 0 };
     u8 m_register_index : 7 { 0x7f };
     bool m_has_sib : 1 { false };
@@ -556,8 +567,8 @@ public:
     String mnemonic() const;
 
     u8 op() const { return m_op; }
-    u8 modrm_byte() const { return m_modrm.m_rm_byte; }
-    u8 slash() const { return (modrm_byte() >> 3) & 7; }
+    u8 modrm_byte() const { return m_modrm.modrm_byte(); }
+    u8 slash() const { return m_modrm.reg() & 7; }
 
     u8 imm8() const { return m_imm1; }
     u16 imm16() const { return m_imm1; }
@@ -708,12 +719,12 @@ ALWAYS_INLINE LogicalAddress MemoryOrRegisterReference::resolve32(const CPU& cpu
 template<typename CPU>
 ALWAYS_INLINE u32 MemoryOrRegisterReference::evaluate_sib(const CPU& cpu, SegmentRegister& default_segment) const
 {
-    u32 scale_shift = m_sib >> 6;
+    u32 scale_shift = m_sib_scale;
     u32 index = 0;
-    switch ((m_sib >> 3) & 0x07) {
+    switch (m_sib_index) {
     case 0 ... 3:
     case 5 ... 7:
-        index = cpu.const_gpr32((RegisterIndex32)((m_sib >> 3) & 0x07)).value();
+        index = cpu.const_gpr32((RegisterIndex32)m_sib_index).value();
         break;
     case 4:
         index = 0;
@@ -721,10 +732,10 @@ ALWAYS_INLINE u32 MemoryOrRegisterReference::evaluate_sib(const CPU& cpu, Segmen
     }
 
     u32 base = m_displacement32;
-    switch (m_sib & 0x07) {
+    switch (m_sib_base) {
     case 0 ... 3:
     case 6 ... 7:
-        base += cpu.const_gpr32((RegisterIndex32)(m_sib & 0x07)).value();
+        base += cpu.const_gpr32((RegisterIndex32)m_sib_base).value();
         break;
     case 4:
         default_segment = SegmentRegister::SS;
@@ -1046,7 +1057,10 @@ ALWAYS_INLINE Instruction::Instruction(InstructionStreamType& stream, OperandSiz
 template<typename InstructionStreamType>
 ALWAYS_INLINE void MemoryOrRegisterReference::decode(InstructionStreamType& stream, AddressSize address_size)
 {
-    m_rm_byte = stream.read8();
+    u8 mod_rm_byte = stream.read8();
+    m_mod = mod_rm_byte >> 6;
+    m_reg = (mod_rm_byte >> 3) & 7;
+    m_rm = mod_rm_byte & 7;
 
     if (address_size == AddressSize::Size32) {
         decode32(stream);
@@ -1106,10 +1120,12 @@ ALWAYS_INLINE void MemoryOrRegisterReference::decode16(InstructionStreamType&)
 template<typename InstructionStreamType>
 ALWAYS_INLINE void MemoryOrRegisterReference::decode32(InstructionStreamType& stream)
 {
-    switch (mod()) {
+    switch (m_mod) {
     case 0b00:
-        if (rm() == 5)
+        if (m_rm == 5) {
             m_displacement_bytes = 4;
+            return;
+        }
         break;
     case 0b01:
         m_displacement_bytes = 1;
@@ -1122,10 +1138,13 @@ ALWAYS_INLINE void MemoryOrRegisterReference::decode32(InstructionStreamType& st
         return;
     }
 
-    m_has_sib = rm() == 4;
+    m_has_sib = m_rm == 4;
     if (m_has_sib) {
-        m_sib = stream.read8();
-        if ((m_sib & 0x07) == 5) {
+        u8 sib_byte = stream.read8();
+        m_sib_scale = sib_byte >> 6;
+        m_sib_index = (sib_byte >> 3) & 7;
+        m_sib_base = sib_byte & 7;
+        if (m_sib_base == 5) {
             switch (mod()) {
             case 0b00:
                 m_displacement_bytes = 4;
@@ -1138,7 +1157,6 @@ ALWAYS_INLINE void MemoryOrRegisterReference::decode32(InstructionStreamType& st
                 break;
             default:
                 VERIFY_NOT_REACHED();
-                break;
             }
         }
     }