target/i386: Reject ModRM.reg == 7 early when decoding 0xFE/0xFF (group 4 and 5) instructions

According to both table A-6 in Volume 2 of Intel 64 and IA-32 Architectures
Software Developer's Manual and table A-6 in Volume 3 of AMD64 Architecture
Programmer's Manual, the opcode for ModRM.reg == 7 is reserved for both group 4
(0xFE) and 5 (0xFF) instructions.

Although Unicorn already rejects ModRM.reg == 7 early with group 4 (0xFE)
instructions, that is not the case with group 5 (0xFF) instructions.
The latter case is eventually handled in the `default:` block of `switch(op)`,
but by the time that code is reached, some micro-ops are already generated.
These may include a memory load micro-op. As a result, an invalid memory access
error may be produced instead of an expected invalid instruction error.

Fix this bug by rejecting group 4 and 5 instructions with ModRM.reg == 7
before generating any micro-operations.

Signed-off-by: Andrey Polivoda <apolivodaa433@gmail.com>

Author: unknown-1-0

qemu/target/i386/translate.c

@@ -5493,7 +5493,7 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
-        if (op >= 2 && b == 0xfe) {
+        if (op == 7 || (op >= 2 && b == 0xfe)) {
             goto unknown_op;
         }
         if (CODE64(s)) {

tests/unit/test_x86.c

@@ -2618,6 +2618,23 @@ static void test_x86_aas_flags(void)
     OK(uc_close(uc));
 }
 
+static void test_x86_group_5_modrm_reg_7(void)
+{
+    uc_engine *uc;
+    char code[] = {
+        0xff, (7 << 3), 0x01, 0x02, 0x03, 0x04
+    };
+
+    uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_64, code, sizeof(code));
+
+    uint64_t rax = code_start + code_len + 0x100;
+    OK(uc_reg_write(uc, UC_X86_REG_RAX, &rax));
+    uc_assert_err(UC_ERR_INSN_INVALID,
+            uc_emu_start(uc, code_start, code_start + sizeof(code), 0, 0));
+
+    OK(uc_close(uc));
+}
+
 TEST_LIST = {
     {"test_x86_in", test_x86_in},
     {"test_x86_out", test_x86_out},
@@ -2695,4 +2712,5 @@ TEST_LIST = {
     {"test_x86_mem_hooks_pc_guarantee", test_x86_mem_hooks_pc_guarantee},
     {"test_x86_aaa_flags", test_x86_aaa_flags},
     {"test_x86_aas_flags", test_x86_aas_flags},
+    {"test_x86_group_5_modrm_reg_7", test_x86_group_5_modrm_reg_7},
     {NULL, NULL}};