diff options
Diffstat (limited to 'tcg/riscv/tcg-target.c.inc')
| -rw-r--r-- | tcg/riscv/tcg-target.c.inc | 1377 |
1 files changed, 906 insertions, 471 deletions
diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index dc8d8f1de2..639363039b 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -27,6 +27,7 @@ * THE SOFTWARE. */ +#include "../tcg-ldst.c.inc" #include "../tcg-pool.c.inc" #ifdef CONFIG_DEBUG_TCG @@ -68,7 +69,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { static const int tcg_target_reg_alloc_order[] = { /* Call saved registers */ - /* TCG_REG_S0 reservered for TCG_AREG0 */ + /* TCG_REG_S0 reserved for TCG_AREG0 */ TCG_REG_S1, TCG_REG_S2, TCG_REG_S3, @@ -112,40 +113,40 @@ static const int tcg_target_call_iarg_regs[] = { TCG_REG_A7, }; -static const int tcg_target_call_oarg_regs[] = { - TCG_REG_A0, - TCG_REG_A1, -}; +#ifndef have_zbb +bool have_zbb; +#endif +#if defined(__riscv_arch_test) && defined(__riscv_zba) +# define have_zba true +#else +static bool have_zba; +#endif +#if defined(__riscv_arch_test) && defined(__riscv_zicond) +# define have_zicond true +#else +static bool have_zicond; +#endif + +static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) +{ + tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); + tcg_debug_assert(slot >= 0 && slot <= 1); + return TCG_REG_A0 + slot; +} #define TCG_CT_CONST_ZERO 0x100 #define TCG_CT_CONST_S12 0x200 #define TCG_CT_CONST_N12 0x400 #define TCG_CT_CONST_M12 0x800 +#define TCG_CT_CONST_J12 0x1000 -#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) -/* - * For softmmu, we need to avoid conflicts with the first 5 - * argument registers to call the helper. Some of these are - * also used for the tlb lookup. - */ -#ifdef CONFIG_SOFTMMU -#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_A0, 5) -#else -#define SOFTMMU_RESERVE_REGS 0 -#endif +#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) - -static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) -{ - if (TCG_TARGET_REG_BITS == 32) { - return sextract32(val, pos, len); - } else { - return sextract64(val, pos, len); - } -} +#define sextreg sextract64 /* test if a constant matches the constraint */ -static bool tcg_target_const_match(int64_t val, TCGType type, int ct) +static bool tcg_target_const_match(int64_t val, int ct, + TCGType type, TCGCond cond, int vece) { if (ct & TCG_CT_CONST) { return 1; @@ -153,13 +154,33 @@ static bool tcg_target_const_match(int64_t val, TCGType type, int ct) if ((ct & TCG_CT_CONST_ZERO) && val == 0) { return 1; } - if ((ct & TCG_CT_CONST_S12) && val == sextreg(val, 0, 12)) { + /* + * Sign extended from 12 bits: [-0x800, 0x7ff]. + * Used for most arithmetic, as this is the isa field. + */ + if ((ct & TCG_CT_CONST_S12) && val >= -0x800 && val <= 0x7ff) { + return 1; + } + /* + * Sign extended from 12 bits, negated: [-0x7ff, 0x800]. + * Used for subtraction, where a constant must be handled by ADDI. + */ + if ((ct & TCG_CT_CONST_N12) && val >= -0x7ff && val <= 0x800) { return 1; } - if ((ct & TCG_CT_CONST_N12) && -val == sextreg(-val, 0, 12)) { + /* + * Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff]. + * Used by addsub2 and movcond, which may need the negative value, + * and requires the modified constant to be representable. + */ + if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { return 1; } - if ((ct & TCG_CT_CONST_M12) && val >= -0xfff && val <= 0xfff) { + /* + * Inverse of sign extended from 12 bits: ~[-0x800, 0x7ff]. + * Used to map ANDN back to ANDI, etc. + */ + if ((ct & TCG_CT_CONST_J12) && ~val >= -0x800 && ~val <= 0x7ff) { return 1; } return 0; @@ -219,7 +240,6 @@ typedef enum { OPC_XOR = 0x4033, OPC_XORI = 0x4013, -#if TCG_TARGET_REG_BITS == 64 OPC_ADDIW = 0x1b, OPC_ADDW = 0x3b, OPC_DIVUW = 0x200503b, @@ -234,25 +254,37 @@ typedef enum { OPC_SRLIW = 0x501b, OPC_SRLW = 0x503b, OPC_SUBW = 0x4000003b, -#else - /* Simplify code throughout by defining aliases for RV32. */ - OPC_ADDIW = OPC_ADDI, - OPC_ADDW = OPC_ADD, - OPC_DIVUW = OPC_DIVU, - OPC_DIVW = OPC_DIV, - OPC_MULW = OPC_MUL, - OPC_REMUW = OPC_REMU, - OPC_REMW = OPC_REM, - OPC_SLLIW = OPC_SLLI, - OPC_SLLW = OPC_SLL, - OPC_SRAIW = OPC_SRAI, - OPC_SRAW = OPC_SRA, - OPC_SRLIW = OPC_SRLI, - OPC_SRLW = OPC_SRL, - OPC_SUBW = OPC_SUB, -#endif OPC_FENCE = 0x0000000f, + OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ + + /* Zba: Bit manipulation extension, address generation */ + OPC_ADD_UW = 0x0800003b, + + /* Zbb: Bit manipulation extension, basic bit manipulation */ + OPC_ANDN = 0x40007033, + OPC_CLZ = 0x60001013, + OPC_CLZW = 0x6000101b, + OPC_CPOP = 0x60201013, + OPC_CPOPW = 0x6020101b, + OPC_CTZ = 0x60101013, + OPC_CTZW = 0x6010101b, + OPC_ORN = 0x40006033, + OPC_REV8 = 0x6b805013, + OPC_ROL = 0x60001033, + OPC_ROLW = 0x6000103b, + OPC_ROR = 0x60005033, + OPC_RORW = 0x6000503b, + OPC_RORI = 0x60005013, + OPC_RORIW = 0x6000501b, + OPC_SEXT_B = 0x60401013, + OPC_SEXT_H = 0x60501013, + OPC_XNOR = 0x40004033, + OPC_ZEXT_H = 0x0800403b, + + /* Zicond: integer conditional operations */ + OPC_CZERO_EQZ = 0x0e005033, + OPC_CZERO_NEZ = 0x0e007033, } RISCVInsn; /* @@ -389,7 +421,7 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) { int i; for (i = 0; i < count; ++i) { - p[i] = encode_i(OPC_ADDI, TCG_REG_ZERO, TCG_REG_ZERO, 0); + p[i] = OPC_NOP; } } @@ -483,7 +515,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_target_long lo, hi, tmp; int shift, ret; - if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { + if (type == TCG_TYPE_I32) { val = (int32_t)val; } @@ -494,7 +526,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, } hi = val - lo; - if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { + if (val == (int32_t)val) { tcg_out_opc_upper(s, OPC_LUI, rd, hi); if (lo != 0) { tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); @@ -502,7 +534,6 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, return; } - /* We can only be here if TCG_TARGET_REG_BITS != 32 */ tmp = tcg_pcrel_diff(s, (void *)val); if (tmp == (int32_t)tmp) { tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); @@ -544,6 +575,18 @@ static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); } +static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) +{ + return false; +} + +static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, + tcg_target_long imm) +{ + /* This function is only used for passing structs by reference. */ + g_assert_not_reached(); +} + static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) { tcg_out_opc_imm(s, OPC_ANDI, ret, arg, 0xff); @@ -551,26 +594,42 @@ static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); - tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); + if (have_zbb) { + tcg_out_opc_reg(s, OPC_ZEXT_H, ret, arg, TCG_REG_ZERO); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); + } } static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); - tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); + if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, ret, arg, TCG_REG_ZERO); + } else { + tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); + tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); + } } -static void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg) +static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); - tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); + if (have_zbb) { + tcg_out_opc_imm(s, OPC_SEXT_B, ret, arg, 0); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); + } } -static void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg) +static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) { - tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); - tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); + if (have_zbb) { + tcg_out_opc_imm(s, OPC_SEXT_H, ret, arg, 0); + } else { + tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); + tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); + } } static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) @@ -578,13 +637,30 @@ static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); } +static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + if (ret != arg) { + tcg_out_ext32s(s, ret, arg); + } +} + +static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_ext32u(s, ret, arg); +} + +static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg) +{ + tcg_out_ext32s(s, ret, arg); +} + static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, TCGReg addr, intptr_t offset) { intptr_t imm12 = sextreg(offset, 0, 12); if (offset != imm12) { - intptr_t diff = offset - (uintptr_t)s->code_ptr; + intptr_t diff = tcg_pcrel_diff(s, (void *)offset); if (addr == TCG_REG_ZERO && diff == (int32_t)diff) { imm12 = sextreg(diff, 0, 12); @@ -622,15 +698,15 @@ static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_LW : OPC_LD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2) { - bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); - tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); + RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SW : OPC_SD; + tcg_out_ldst(s, insn, arg, arg1, arg2); } static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, @@ -686,9 +762,15 @@ static void tcg_out_addsub2(TCGContext *s, if (cbl) { tcg_out_opc_imm(s, opc_addi, rl, al, bl); tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, rl, bl); - } else if (rl == al && rl == bl) { + } else if (al == bl) { + /* + * If the input regs overlap, this is a simple doubling + * and carry-out is the input msb. This special case is + * required when the output reg overlaps the input, + * but we might as well use it always. + */ tcg_out_opc_imm(s, OPC_SLTI, TCG_REG_TMP0, al, 0); - tcg_out_opc_reg(s, opc_addi, rl, al, bl); + tcg_out_opc_reg(s, opc_add, rl, al, al); } else { tcg_out_opc_reg(s, opc_add, rl, al, bl); tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, @@ -731,64 +813,309 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, tcg_out_opc_branch(s, op, arg1, arg2, 0); } -static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg arg1, TCGReg arg2) +#define SETCOND_INV TCG_TARGET_NB_REGS +#define SETCOND_NEZ (SETCOND_INV << 1) +#define SETCOND_FLAGS (SETCOND_INV | SETCOND_NEZ) + +static int tcg_out_setcond_int(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, tcg_target_long arg2, bool c2) { + int flags = 0; + switch (cond) { - case TCG_COND_EQ: - tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1); + case TCG_COND_EQ: /* -> NE */ + case TCG_COND_GE: /* -> LT */ + case TCG_COND_GEU: /* -> LTU */ + case TCG_COND_GT: /* -> LE */ + case TCG_COND_GTU: /* -> LEU */ + cond = tcg_invert_cond(cond); + flags ^= SETCOND_INV; break; - case TCG_COND_NE: - tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); - tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret); - break; - case TCG_COND_LT: - tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); - break; - case TCG_COND_GE: - tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + default: break; + } + + switch (cond) { case TCG_COND_LE: - tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); - break; - case TCG_COND_GT: - tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); + case TCG_COND_LEU: + /* + * If we have a constant input, the most efficient way to implement + * LE is by adding 1 and using LT. Watch out for wrap around for LEU. + * We don't need to care for this for LE because the constant input + * is constrained to signed 12-bit, and 0x800 is representable in the + * temporary register. + */ + if (c2) { + if (cond == TCG_COND_LEU) { + /* unsigned <= -1 is true */ + if (arg2 == -1) { + tcg_out_movi(s, TCG_TYPE_REG, ret, !(flags & SETCOND_INV)); + return ret; + } + cond = TCG_COND_LTU; + } else { + cond = TCG_COND_LT; + } + tcg_debug_assert(arg2 <= 0x7ff); + if (++arg2 == 0x800) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP0, arg2); + arg2 = TCG_REG_TMP0; + c2 = false; + } + } else { + TCGReg tmp = arg2; + arg2 = arg1; + arg1 = tmp; + cond = tcg_swap_cond(cond); /* LE -> GE */ + cond = tcg_invert_cond(cond); /* GE -> LT */ + flags ^= SETCOND_INV; + } break; - case TCG_COND_LTU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); + default: break; - case TCG_COND_GEU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + } + + switch (cond) { + case TCG_COND_NE: + flags |= SETCOND_NEZ; + if (!c2) { + tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2); + } else if (arg2 == 0) { + ret = arg1; + } else { + tcg_out_opc_imm(s, OPC_XORI, ret, arg1, arg2); + } break; - case TCG_COND_LEU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); - tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); + + case TCG_COND_LT: + if (c2) { + tcg_out_opc_imm(s, OPC_SLTI, ret, arg1, arg2); + } else { + tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); + } break; - case TCG_COND_GTU: - tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); + + case TCG_COND_LTU: + if (c2) { + tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, arg2); + } else { + tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); + } break; + default: - g_assert_not_reached(); - break; - } + g_assert_not_reached(); + } + + return ret | flags; } -static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, - TCGReg bl, TCGReg bh, TCGLabel *l) +static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, tcg_target_long arg2, bool c2) { - /* todo */ - g_assert_not_reached(); + int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2); + + if (tmpflags != ret) { + TCGReg tmp = tmpflags & ~SETCOND_FLAGS; + + switch (tmpflags & SETCOND_FLAGS) { + case SETCOND_INV: + /* Intermediate result is boolean: simply invert. */ + tcg_out_opc_imm(s, OPC_XORI, ret, tmp, 1); + break; + case SETCOND_NEZ: + /* Intermediate result is zero/non-zero: test != 0. */ + tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp); + break; + case SETCOND_NEZ | SETCOND_INV: + /* Intermediate result is zero/non-zero: test == 0. */ + tcg_out_opc_imm(s, OPC_SLTIU, ret, tmp, 1); + break; + default: + g_assert_not_reached(); + } + } } -static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, - TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) +static void tcg_out_negsetcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg arg1, tcg_target_long arg2, bool c2) { - /* todo */ - g_assert_not_reached(); + int tmpflags; + TCGReg tmp; + + /* For LT/GE comparison against 0, replicate the sign bit. */ + if (c2 && arg2 == 0) { + switch (cond) { + case TCG_COND_GE: + tcg_out_opc_imm(s, OPC_XORI, ret, arg1, -1); + arg1 = ret; + /* fall through */ + case TCG_COND_LT: + tcg_out_opc_imm(s, OPC_SRAI, ret, arg1, TCG_TARGET_REG_BITS - 1); + return; + default: + break; + } + } + + tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2); + tmp = tmpflags & ~SETCOND_FLAGS; + + /* If intermediate result is zero/non-zero: test != 0. */ + if (tmpflags & SETCOND_NEZ) { + tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, tmp); + tmp = ret; + } + + /* Produce the 0/-1 result. */ + if (tmpflags & SETCOND_INV) { + tcg_out_opc_imm(s, OPC_ADDI, ret, tmp, -1); + } else { + tcg_out_opc_reg(s, OPC_SUB, ret, TCG_REG_ZERO, tmp); + } +} + +static void tcg_out_movcond_zicond(TCGContext *s, TCGReg ret, TCGReg test_ne, + int val1, bool c_val1, + int val2, bool c_val2) +{ + if (val1 == 0) { + if (c_val2) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val2); + val2 = TCG_REG_TMP1; + } + tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, val2, test_ne); + return; + } + + if (val2 == 0) { + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1); + val1 = TCG_REG_TMP1; + } + tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, val1, test_ne); + return; + } + + if (c_val2) { + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP1, val1 - val2); + } else { + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val1, -val2); + } + tcg_out_opc_reg(s, OPC_CZERO_EQZ, ret, TCG_REG_TMP1, test_ne); + tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val2); + return; + } + + if (c_val1) { + tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_TMP1, val2, -val1); + tcg_out_opc_reg(s, OPC_CZERO_NEZ, ret, TCG_REG_TMP1, test_ne); + tcg_out_opc_imm(s, OPC_ADDI, ret, ret, val1); + return; + } + + tcg_out_opc_reg(s, OPC_CZERO_NEZ, TCG_REG_TMP1, val2, test_ne); + tcg_out_opc_reg(s, OPC_CZERO_EQZ, TCG_REG_TMP0, val1, test_ne); + tcg_out_opc_reg(s, OPC_OR, ret, TCG_REG_TMP0, TCG_REG_TMP1); +} + +static void tcg_out_movcond_br1(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, TCGReg cmp2, + int val, bool c_val) +{ + RISCVInsn op; + int disp = 8; + + tcg_debug_assert((unsigned)cond < ARRAY_SIZE(tcg_brcond_to_riscv)); + op = tcg_brcond_to_riscv[cond].op; + tcg_debug_assert(op != 0); + + if (tcg_brcond_to_riscv[cond].swap) { + tcg_out_opc_branch(s, op, cmp2, cmp1, disp); + } else { + tcg_out_opc_branch(s, op, cmp1, cmp2, disp); + } + if (c_val) { + tcg_out_opc_imm(s, OPC_ADDI, ret, TCG_REG_ZERO, val); + } else { + tcg_out_opc_imm(s, OPC_ADDI, ret, val, 0); + } +} + +static void tcg_out_movcond_br2(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, TCGReg cmp2, + int val1, bool c_val1, + int val2, bool c_val2) +{ + TCGReg tmp; + + /* TCG optimizer reorders to prefer ret matching val2. */ + if (!c_val2 && ret == val2) { + cond = tcg_invert_cond(cond); + tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val1, c_val1); + return; + } + + if (!c_val1 && ret == val1) { + tcg_out_movcond_br1(s, cond, ret, cmp1, cmp2, val2, c_val2); + return; + } + + tmp = (ret == cmp1 || ret == cmp2 ? TCG_REG_TMP1 : ret); + if (c_val1) { + tcg_out_movi(s, TCG_TYPE_REG, tmp, val1); + } else { + tcg_out_mov(s, TCG_TYPE_REG, tmp, val1); + } + tcg_out_movcond_br1(s, cond, tmp, cmp1, cmp2, val2, c_val2); + tcg_out_mov(s, TCG_TYPE_REG, ret, tmp); +} + +static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret, + TCGReg cmp1, int cmp2, bool c_cmp2, + TCGReg val1, bool c_val1, + TCGReg val2, bool c_val2) +{ + int tmpflags; + TCGReg t; + + if (!have_zicond && (!c_cmp2 || cmp2 == 0)) { + tcg_out_movcond_br2(s, cond, ret, cmp1, cmp2, + val1, c_val1, val2, c_val2); + return; + } + + tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, cmp1, cmp2, c_cmp2); + t = tmpflags & ~SETCOND_FLAGS; + + if (have_zicond) { + if (tmpflags & SETCOND_INV) { + tcg_out_movcond_zicond(s, ret, t, val2, c_val2, val1, c_val1); + } else { + tcg_out_movcond_zicond(s, ret, t, val1, c_val1, val2, c_val2); + } + } else { + cond = tmpflags & SETCOND_INV ? TCG_COND_EQ : TCG_COND_NE; + tcg_out_movcond_br2(s, cond, ret, t, TCG_REG_ZERO, + val1, c_val1, val2, c_val2); + } +} + +static void tcg_out_cltz(TCGContext *s, TCGType type, RISCVInsn insn, + TCGReg ret, TCGReg src1, int src2, bool c_src2) +{ + tcg_out_opc_imm(s, insn, ret, src1, 0); + + if (!c_src2 || src2 != (type == TCG_TYPE_I32 ? 32 : 64)) { + /* + * The requested zero result does not match the insn, so adjust. + * Note that constraints put 'ret' in a new register, so the + * computation above did not clobber either 'src1' or 'src2'. + */ + tcg_out_movcond(s, TCG_COND_EQ, ret, src1, 0, true, + src2, c_src2, ret, false); + } } static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) @@ -801,24 +1128,23 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) if (offset == sextreg(offset, 0, 20)) { /* short jump: -2097150 to 2097152 */ tcg_out_opc_jump(s, OPC_JAL, link, offset); - } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { + } else if (offset == (int32_t)offset) { /* long jump: -2147483646 to 2147483648 */ tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); ret = reloc_call(s->code_ptr - 2, arg); tcg_debug_assert(ret == true); - } else if (TCG_TARGET_REG_BITS == 64) { + } else { /* far jump: 64-bit */ tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); tcg_target_long base = (tcg_target_long)arg - imm; tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); - } else { - g_assert_not_reached(); } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, + const TCGHelperInfo *info) { tcg_out_call_int(s, arg, false); } @@ -846,58 +1172,6 @@ static void tcg_out_mb(TCGContext *s, TCGArg a0) * Load/store and TLB */ -#if defined(CONFIG_SOFTMMU) -#include "../tcg-ldst.c.inc" - -/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, - * TCGMemOpIdx oi, uintptr_t ra) - */ -static void * const qemu_ld_helpers[8] = { - [MO_UB] = helper_ret_ldub_mmu, - [MO_SB] = helper_ret_ldsb_mmu, -#ifdef HOST_WORDS_BIGENDIAN - [MO_UW] = helper_be_lduw_mmu, - [MO_SW] = helper_be_ldsw_mmu, - [MO_UL] = helper_be_ldul_mmu, -#if TCG_TARGET_REG_BITS == 64 - [MO_SL] = helper_be_ldsl_mmu, -#endif - [MO_Q] = helper_be_ldq_mmu, -#else - [MO_UW] = helper_le_lduw_mmu, - [MO_SW] = helper_le_ldsw_mmu, - [MO_UL] = helper_le_ldul_mmu, -#if TCG_TARGET_REG_BITS == 64 - [MO_SL] = helper_le_ldsl_mmu, -#endif - [MO_Q] = helper_le_ldq_mmu, -#endif -}; - -/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, - * uintxx_t val, TCGMemOpIdx oi, - * uintptr_t ra) - */ -static void * const qemu_st_helpers[4] = { - [MO_8] = helper_ret_stb_mmu, -#ifdef HOST_WORDS_BIGENDIAN - [MO_16] = helper_be_stw_mmu, - [MO_32] = helper_be_stl_mmu, - [MO_64] = helper_be_stq_mmu, -#else - [MO_16] = helper_le_stw_mmu, - [MO_32] = helper_le_stl_mmu, - [MO_64] = helper_le_stq_mmu, -#endif -}; - -/* We don't support oversize guests */ -QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); - -/* We expect to use a 12-bit negative offset from ENV. */ -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); -QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); - static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) { tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); @@ -905,92 +1179,19 @@ static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) tcg_debug_assert(ok); } -static void tcg_out_tlb_load(TCGContext *s, TCGReg addrl, - TCGReg addrh, TCGMemOpIdx oi, - tcg_insn_unit **label_ptr, bool is_load) +bool tcg_target_has_memory_bswap(MemOp memop) { - MemOp opc = get_memop(oi); - unsigned s_bits = opc & MO_SIZE; - unsigned a_bits = get_alignment_bits(opc); - tcg_target_long compare_mask; - int mem_index = get_mmuidx(oi); - int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); - int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); - int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); - TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; - - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); - - tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addrl, - TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); - - /* Load the tlb comparator and the addend. */ - tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, - is_load ? offsetof(CPUTLBEntry, addr_read) - : offsetof(CPUTLBEntry, addr_write)); - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, - offsetof(CPUTLBEntry, addend)); - - /* We don't support unaligned accesses. */ - if (a_bits < s_bits) { - a_bits = s_bits; - } - /* Clear the non-page, non-alignment bits from the address. */ - compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); - if (compare_mask == sextreg(compare_mask, 0, 12)) { - tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, compare_mask); - } else { - tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); - tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); - } - - /* Compare masked address with the TLB entry. */ - label_ptr[0] = s->code_ptr; - tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); - - /* TLB Hit - translate address using addend. */ - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, TCG_REG_TMP0, addrl); - addrl = TCG_REG_TMP0; - } - tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); + return false; } -static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi, - TCGType ext, - TCGReg datalo, TCGReg datahi, - TCGReg addrlo, TCGReg addrhi, - void *raddr, tcg_insn_unit **label_ptr) -{ - TCGLabelQemuLdst *label = new_ldst_label(s); - - label->is_ld = is_ld; - label->oi = oi; - label->type = ext; - label->datalo_reg = datalo; - label->datahi_reg = datahi; - label->addrlo_reg = addrlo; - label->addrhi_reg = addrhi; - label->raddr = tcg_splitwx_to_rx(raddr); - label->label_ptr[0] = label_ptr[0]; -} +/* We have three temps, we might as well expose them. */ +static const TCGLdstHelperParam ldst_helper_param = { + .ntmp = 3, .tmp = { TCG_REG_TMP0, TCG_REG_TMP1, TCG_REG_TMP2 } +}; static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; - MemOp opc = get_memop(oi); - TCGReg a0 = tcg_target_call_iarg_regs[0]; - TCGReg a1 = tcg_target_call_iarg_regs[1]; - TCGReg a2 = tcg_target_call_iarg_regs[2]; - TCGReg a3 = tcg_target_call_iarg_regs[3]; - - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } + MemOp opc = get_memop(l->oi); /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { @@ -998,13 +1199,9 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* call load helper */ - tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); - tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); - tcg_out_movi(s, TCG_TYPE_PTR, a2, oi); - tcg_out_movi(s, TCG_TYPE_PTR, a3, (tcg_target_long)l->raddr); - - tcg_out_call(s, qemu_ld_helpers[opc & MO_SSIZE]); - tcg_out_mov(s, (opc & MO_SIZE) == MO_64, l->datalo_reg, a0); + tcg_out_ld_helper_args(s, l, &ldst_helper_param); + tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false); + tcg_out_ld_helper_ret(s, l, true, &ldst_helper_param); tcg_out_goto(s, l->raddr); return true; @@ -1012,19 +1209,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) { - TCGMemOpIdx oi = l->oi; - MemOp opc = get_memop(oi); - MemOp s_bits = opc & MO_SIZE; - TCGReg a0 = tcg_target_call_iarg_regs[0]; - TCGReg a1 = tcg_target_call_iarg_regs[1]; - TCGReg a2 = tcg_target_call_iarg_regs[2]; - TCGReg a3 = tcg_target_call_iarg_regs[3]; - TCGReg a4 = tcg_target_call_iarg_regs[4]; - - /* We don't support oversize guests */ - if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { - g_assert_not_reached(); - } + MemOp opc = get_memop(l->oi); /* resolve label address */ if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { @@ -1032,112 +1217,196 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) } /* call store helper */ - tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); - tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); - tcg_out_mov(s, TCG_TYPE_PTR, a2, l->datalo_reg); - switch (s_bits) { - case MO_8: - tcg_out_ext8u(s, a2, a2); - break; - case MO_16: - tcg_out_ext16u(s, a2, a2); - break; - default: - break; - } - tcg_out_movi(s, TCG_TYPE_PTR, a3, oi); - tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr); - - tcg_out_call(s, qemu_st_helpers[opc & MO_SIZE]); + tcg_out_st_helper_args(s, l, &ldst_helper_param); + tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); tcg_out_goto(s, l->raddr); return true; } -#endif /* CONFIG_SOFTMMU */ -static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, - TCGReg base, MemOp opc, bool is_64) +/* We expect to use a 12-bit negative offset from ENV. */ +#define MIN_TLB_MASK_TABLE_OFS -(1 << 11) + +/* + * For system-mode, perform the TLB load and compare. + * For user-mode, perform any required alignment tests. + * In both cases, return a TCGLabelQemuLdst structure if the slow path + * is required and fill in @h with the host address for the fast path. + */ +static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, + TCGReg addr_reg, MemOpIdx oi, + bool is_ld) +{ + TCGType addr_type = s->addr_type; + TCGLabelQemuLdst *ldst = NULL; + MemOp opc = get_memop(oi); + TCGAtomAlign aa; + unsigned a_mask; + + aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false); + a_mask = (1u << aa.align) - 1; + + if (tcg_use_softmmu) { + unsigned s_bits = opc & MO_SIZE; + unsigned s_mask = (1u << s_bits) - 1; + int mem_index = get_mmuidx(oi); + int fast_ofs = tlb_mask_table_ofs(s, mem_index); + int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); + int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); + int compare_mask; + TCGReg addr_adj; + + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs); + + tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr_reg, + s->page_bits - CPU_TLB_ENTRY_BITS); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); + + /* + * For aligned accesses, we check the first byte and include the + * alignment bits within the address. For unaligned access, we + * check that we don't cross pages using the address of the last + * byte of the access. + */ + addr_adj = addr_reg; + if (a_mask < s_mask) { + addr_adj = TCG_REG_TMP0; + tcg_out_opc_imm(s, addr_type == TCG_TYPE_I32 ? OPC_ADDIW : OPC_ADDI, + addr_adj, addr_reg, s_mask - a_mask); + } + compare_mask = s->page_mask | a_mask; + if (compare_mask == sextreg(compare_mask, 0, 12)) { + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_adj, compare_mask); + } else { + tcg_out_movi(s, addr_type, TCG_REG_TMP1, compare_mask); + tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_adj); + } + + /* Load the tlb comparator and the addend. */ + QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN); + tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2, + is_ld ? offsetof(CPUTLBEntry, addr_read) + : offsetof(CPUTLBEntry, addr_write)); + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, + offsetof(CPUTLBEntry, addend)); + + /* Compare masked address with the TLB entry. */ + ldst->label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); + + /* TLB Hit - translate address using addend. */ + if (addr_type != TCG_TYPE_I32) { + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, addr_reg, TCG_REG_TMP2); + } else if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, TCG_REG_TMP0, + addr_reg, TCG_REG_TMP2); + } else { + tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg); + tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, + TCG_REG_TMP0, TCG_REG_TMP2); + } + *pbase = TCG_REG_TMP0; + } else { + TCGReg base; + + if (a_mask) { + ldst = new_ldst_label(s); + ldst->is_ld = is_ld; + ldst->oi = oi; + ldst->addrlo_reg = addr_reg; + + /* We are expecting alignment max 7, so we can always use andi. */ + tcg_debug_assert(a_mask == sextreg(a_mask, 0, 12)); + tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); + + ldst->label_ptr[0] = s->code_ptr; + tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); + } + + if (guest_base != 0) { + base = TCG_REG_TMP0; + if (addr_type != TCG_TYPE_I32) { + tcg_out_opc_reg(s, OPC_ADD, base, addr_reg, + TCG_GUEST_BASE_REG); + } else if (have_zba) { + tcg_out_opc_reg(s, OPC_ADD_UW, base, addr_reg, + TCG_GUEST_BASE_REG); + } else { + tcg_out_ext32u(s, base, addr_reg); + tcg_out_opc_reg(s, OPC_ADD, base, base, TCG_GUEST_BASE_REG); + } + } else if (addr_type != TCG_TYPE_I32) { + base = addr_reg; + } else { + base = TCG_REG_TMP0; + tcg_out_ext32u(s, base, addr_reg); + } + *pbase = base; + } + + return ldst; +} + +static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, + TCGReg base, MemOp opc, TCGType type) { /* Byte swapping is left to middle-end expansion. */ tcg_debug_assert((opc & MO_BSWAP) == 0); switch (opc & (MO_SSIZE)) { case MO_UB: - tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LBU, val, base, 0); break; case MO_SB: - tcg_out_opc_imm(s, OPC_LB, lo, base, 0); + tcg_out_opc_imm(s, OPC_LB, val, base, 0); break; case MO_UW: - tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); + tcg_out_opc_imm(s, OPC_LHU, val, base, 0); break; case MO_SW: - tcg_out_opc_imm(s, OPC_LH, lo, base, 0); + tcg_out_opc_imm(s, OPC_LH, val, base, 0); break; case MO_UL: - if (TCG_TARGET_REG_BITS == 64 && is_64) { - tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); + if (type == TCG_TYPE_I64) { + tcg_out_opc_imm(s, OPC_LWU, val, base, 0); break; } /* FALLTHRU */ case MO_SL: - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - break; - case MO_Q: - /* Prefer to load from offset 0 first, but allow for overlap. */ - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_imm(s, OPC_LD, lo, base, 0); - } else if (lo != base) { - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - } else { - tcg_out_opc_imm(s, OPC_LW, hi, base, 4); - tcg_out_opc_imm(s, OPC_LW, lo, base, 0); - } + tcg_out_opc_imm(s, OPC_LW, val, base, 0); + break; + case MO_UQ: + tcg_out_opc_imm(s, OPC_LD, val, base, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - TCGMemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#endif - TCGReg base = TCG_REG_TMP0; - - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); - add_qemu_ldst_label(s, 1, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); -#else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; - } - if (guest_base != 0) { - tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); + TCGLabelQemuLdst *ldst; + TCGReg base; + + ldst = prepare_host_addr(s, &base, addr_reg, oi, true); + tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), data_type); + + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data_reg; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } - tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); -#endif } -static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, +static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, TCGReg base, MemOp opc) { /* Byte swapping is left to middle-end expansion. */ @@ -1145,66 +1414,81 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, switch (opc & (MO_SSIZE)) { case MO_8: - tcg_out_opc_store(s, OPC_SB, base, lo, 0); + tcg_out_opc_store(s, OPC_SB, base, val, 0); break; case MO_16: - tcg_out_opc_store(s, OPC_SH, base, lo, 0); + tcg_out_opc_store(s, OPC_SH, base, val, 0); break; case MO_32: - tcg_out_opc_store(s, OPC_SW, base, lo, 0); + tcg_out_opc_store(s, OPC_SW, base, val, 0); break; case MO_64: - if (TCG_TARGET_REG_BITS == 64) { - tcg_out_opc_store(s, OPC_SD, base, lo, 0); - } else { - tcg_out_opc_store(s, OPC_SW, base, lo, 0); - tcg_out_opc_store(s, OPC_SW, base, hi, 4); - } + tcg_out_opc_store(s, OPC_SD, base, val, 0); break; default: g_assert_not_reached(); } } -static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) +static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, + MemOpIdx oi, TCGType data_type) { - TCGReg addr_regl, addr_regh __attribute__((unused)); - TCGReg data_regl, data_regh; - TCGMemOpIdx oi; - MemOp opc; -#if defined(CONFIG_SOFTMMU) - tcg_insn_unit *label_ptr[1]; -#endif - TCGReg base = TCG_REG_TMP0; - - data_regl = *args++; - data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); - addr_regl = *args++; - addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); - oi = *args++; - opc = get_memop(oi); - -#if defined(CONFIG_SOFTMMU) - tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); - add_qemu_ldst_label(s, 0, oi, - (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), - data_regl, data_regh, addr_regl, addr_regh, - s->code_ptr, label_ptr); -#else - if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { - tcg_out_ext32u(s, base, addr_regl); - addr_regl = base; - } - if (guest_base != 0) { - tcg_out_opc_reg(s, OPC_ADD, base, TCG_GUEST_BASE_REG, addr_regl); + TCGLabelQemuLdst *ldst; + TCGReg base; + + ldst = prepare_host_addr(s, &base, addr_reg, oi, false); + tcg_out_qemu_st_direct(s, data_reg, base, get_memop(oi)); + + if (ldst) { + ldst->type = data_type; + ldst->datalo_reg = data_reg; + ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); } - tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); -#endif } static const tcg_insn_unit *tb_ret_addr; +static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) +{ + /* Reuse the zeroing that exists for goto_ptr. */ + if (a0 == 0) { + tcg_out_call_int(s, tcg_code_gen_epilogue, true); + } else { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); + tcg_out_call_int(s, tb_ret_addr, true); + } +} + +static void tcg_out_goto_tb(TCGContext *s, int which) +{ + /* Direct branch will be patched by tb_target_set_jmp_target. */ + set_jmp_insn_offset(s, which); + tcg_out32(s, OPC_JAL); + + /* When branch is out of range, fall through to indirect. */ + tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, + get_jmp_target_addr(s, which)); + tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); + set_jmp_reset_offset(s, which); +} + +void tb_target_set_jmp_target(const TranslationBlock *tb, int n, + uintptr_t jmp_rx, uintptr_t jmp_rw) +{ + uintptr_t addr = tb->jmp_target_addr[n]; + ptrdiff_t offset = addr - jmp_rx; + tcg_insn_unit insn; + + /* Either directly branch, or fall through to indirect branch. */ + if (offset == sextreg(offset, 0, 20)) { + insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset); + } else { + insn = OPC_NOP; + } + qatomic_set((uint32_t *)jmp_rw, insn); + flush_idcache_range(jmp_rx, jmp_rw, 4); +} + static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg args[TCG_MAX_OP_ARGS], const int const_args[TCG_MAX_OP_ARGS]) @@ -1215,25 +1499,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, int c2 = const_args[2]; switch (opc) { - case INDEX_op_exit_tb: - /* Reuse the zeroing that exists for goto_ptr. */ - if (a0 == 0) { - tcg_out_call_int(s, tcg_code_gen_epilogue, true); - } else { - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); - tcg_out_call_int(s, tb_ret_addr, true); - } - break; - - case INDEX_op_goto_tb: - assert(s->tb_jmp_insn_offset == 0); - /* indirect jump method */ - tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, - (uintptr_t)(s->tb_jmp_target_addr + a0)); - tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); - set_jmp_reset_offset(s, a0); - break; - case INDEX_op_goto_ptr: tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); break; @@ -1343,6 +1608,31 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ANDI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_ANDN, a0, a1, a2); + } + break; + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_ORI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_ORN, a0, a1, a2); + } + break; + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_XORI, a0, a1, ~a2); + } else { + tcg_out_opc_reg(s, OPC_XNOR, a0, a1, a2); + } + break; + case INDEX_op_not_i32: case INDEX_op_not_i64: tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); @@ -1435,6 +1725,80 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, } break; + case INDEX_op_rotl_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_RORIW, a0, a1, -a2 & 0x1f); + } else { + tcg_out_opc_reg(s, OPC_ROLW, a0, a1, a2); + } + break; + case INDEX_op_rotl_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_RORI, a0, a1, -a2 & 0x3f); + } else { + tcg_out_opc_reg(s, OPC_ROL, a0, a1, a2); + } + break; + + case INDEX_op_rotr_i32: + if (c2) { + tcg_out_opc_imm(s, OPC_RORIW, a0, a1, a2 & 0x1f); + } else { + tcg_out_opc_reg(s, OPC_RORW, a0, a1, a2); + } + break; + case INDEX_op_rotr_i64: + if (c2) { + tcg_out_opc_imm(s, OPC_RORI, a0, a1, a2 & 0x3f); + } else { + tcg_out_opc_reg(s, OPC_ROR, a0, a1, a2); + } + break; + + case INDEX_op_bswap64_i64: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + break; + case INDEX_op_bswap32_i32: + a2 = 0; + /* fall through */ + case INDEX_op_bswap32_i64: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (a2 & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 32); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 32); + } + break; + case INDEX_op_bswap16_i64: + case INDEX_op_bswap16_i32: + tcg_out_opc_imm(s, OPC_REV8, a0, a1, 0); + if (a2 & TCG_BSWAP_OZ) { + tcg_out_opc_imm(s, OPC_SRLI, a0, a0, 48); + } else { + tcg_out_opc_imm(s, OPC_SRAI, a0, a0, 48); + } + break; + + case INDEX_op_ctpop_i32: + tcg_out_opc_imm(s, OPC_CPOPW, a0, a1, 0); + break; + case INDEX_op_ctpop_i64: + tcg_out_opc_imm(s, OPC_CPOP, a0, a1, 0); + break; + + case INDEX_op_clz_i32: + tcg_out_cltz(s, TCG_TYPE_I32, OPC_CLZW, a0, a1, a2, c2); + break; + case INDEX_op_clz_i64: + tcg_out_cltz(s, TCG_TYPE_I64, OPC_CLZ, a0, a1, a2, c2); + break; + case INDEX_op_ctz_i32: + tcg_out_cltz(s, TCG_TYPE_I32, OPC_CTZW, a0, a1, a2, c2); + break; + case INDEX_op_ctz_i64: + tcg_out_cltz(s, TCG_TYPE_I64, OPC_CTZ, a0, a1, a2, c2); + break; + case INDEX_op_add2_i32: tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], const_args[4], const_args[5], false, true); @@ -1456,60 +1820,38 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_brcond_i64: tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); break; - case INDEX_op_brcond2_i32: - tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); - break; case INDEX_op_setcond_i32: case INDEX_op_setcond_i64: - tcg_out_setcond(s, args[3], a0, a1, a2); - break; - case INDEX_op_setcond2_i32: - tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); - break; - - case INDEX_op_qemu_ld_i32: - tcg_out_qemu_ld(s, args, false); - break; - case INDEX_op_qemu_ld_i64: - tcg_out_qemu_ld(s, args, true); - break; - case INDEX_op_qemu_st_i32: - tcg_out_qemu_st(s, args, false); - break; - case INDEX_op_qemu_st_i64: - tcg_out_qemu_st(s, args, true); + tcg_out_setcond(s, args[3], a0, a1, a2, c2); break; - case INDEX_op_ext8u_i32: - case INDEX_op_ext8u_i64: - tcg_out_ext8u(s, a0, a1); + case INDEX_op_negsetcond_i32: + case INDEX_op_negsetcond_i64: + tcg_out_negsetcond(s, args[3], a0, a1, a2, c2); break; - case INDEX_op_ext16u_i32: - case INDEX_op_ext16u_i64: - tcg_out_ext16u(s, a0, a1); + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: + tcg_out_movcond(s, args[5], a0, a1, a2, c2, + args[3], const_args[3], args[4], const_args[4]); break; - case INDEX_op_ext32u_i64: - case INDEX_op_extu_i32_i64: - tcg_out_ext32u(s, a0, a1); + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); break; - - case INDEX_op_ext8s_i32: - case INDEX_op_ext8s_i64: - tcg_out_ext8s(s, a0, a1); + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: + tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); break; - - case INDEX_op_ext16s_i32: - case INDEX_op_ext16s_i64: - tcg_out_ext16s(s, a0, a1); + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); break; - - case INDEX_op_ext32s_i64: - case INDEX_op_extrl_i64_i32: - case INDEX_op_ext_i32_i64: - tcg_out_ext32s(s, a0, a1); + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: + tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); break; case INDEX_op_extrh_i64_i32: @@ -1533,6 +1875,21 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ case INDEX_op_mov_i64: case INDEX_op_call: /* Always emitted via tcg_out_call. */ + case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ + case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ + case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ + case INDEX_op_ext8s_i64: + case INDEX_op_ext8u_i32: + case INDEX_op_ext8u_i64: + case INDEX_op_ext16s_i32: + case INDEX_op_ext16s_i64: + case INDEX_op_ext16u_i32: + case INDEX_op_ext16u_i64: + case INDEX_op_ext32s_i64: + case INDEX_op_ext32u_i64: + case INDEX_op_ext_i32_i64: + case INDEX_op_extu_i32_i64: + case INDEX_op_extrl_i64_i32: default: g_assert_not_reached(); } @@ -1574,6 +1931,13 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_extrl_i64_i32: case INDEX_op_extrh_i64_i32: case INDEX_op_ext_i32_i64: + case INDEX_op_bswap16_i32: + case INDEX_op_bswap32_i32: + case INDEX_op_bswap16_i64: + case INDEX_op_bswap32_i64: + case INDEX_op_bswap64_i64: + case INDEX_op_ctpop_i32: + case INDEX_op_ctpop_i64: return C_O1_I1(r, r); case INDEX_op_st8_i32: @@ -1593,8 +1957,20 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_and_i64: case INDEX_op_or_i64: case INDEX_op_xor_i64: + case INDEX_op_setcond_i32: + case INDEX_op_setcond_i64: + case INDEX_op_negsetcond_i32: + case INDEX_op_negsetcond_i64: return C_O1_I2(r, r, rI); + case INDEX_op_andc_i32: + case INDEX_op_andc_i64: + case INDEX_op_orc_i32: + case INDEX_op_orc_i64: + case INDEX_op_eqv_i32: + case INDEX_op_eqv_i64: + return C_O1_I2(r, r, rJ); + case INDEX_op_sub_i32: case INDEX_op_sub_i64: return C_O1_I2(r, rZ, rN); @@ -1606,7 +1982,6 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_divu_i32: case INDEX_op_rem_i32: case INDEX_op_remu_i32: - case INDEX_op_setcond_i32: case INDEX_op_mul_i64: case INDEX_op_mulsh_i64: case INDEX_op_muluh_i64: @@ -1614,47 +1989,50 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_divu_i64: case INDEX_op_rem_i64: case INDEX_op_remu_i64: - case INDEX_op_setcond_i64: return C_O1_I2(r, rZ, rZ); case INDEX_op_shl_i32: case INDEX_op_shr_i32: case INDEX_op_sar_i32: + case INDEX_op_rotl_i32: + case INDEX_op_rotr_i32: case INDEX_op_shl_i64: case INDEX_op_shr_i64: case INDEX_op_sar_i64: + case INDEX_op_rotl_i64: + case INDEX_op_rotr_i64: return C_O1_I2(r, r, ri); + case INDEX_op_clz_i32: + case INDEX_op_clz_i64: + case INDEX_op_ctz_i32: + case INDEX_op_ctz_i64: + return C_N1_I2(r, r, rM); + case INDEX_op_brcond_i32: case INDEX_op_brcond_i64: return C_O0_I2(rZ, rZ); + case INDEX_op_movcond_i32: + case INDEX_op_movcond_i64: + return C_O1_I4(r, r, rI, rM, rM); + case INDEX_op_add2_i32: case INDEX_op_add2_i64: case INDEX_op_sub2_i32: case INDEX_op_sub2_i64: return C_O2_I4(r, r, rZ, rZ, rM, rM); - case INDEX_op_brcond2_i32: - return C_O0_I4(rZ, rZ, rZ, rZ); - - case INDEX_op_setcond2_i32: - return C_O1_I4(r, rZ, rZ, rZ, rZ); - - case INDEX_op_qemu_ld_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); - case INDEX_op_qemu_st_i32: - return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS - ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); - case INDEX_op_qemu_ld_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) - : C_O2_I2(r, r, L, L)); - case INDEX_op_qemu_st_i64: - return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) - : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) - : C_O0_I4(LZ, LZ, L, L)); + case INDEX_op_qemu_ld_a32_i32: + case INDEX_op_qemu_ld_a64_i32: + case INDEX_op_qemu_ld_a32_i64: + case INDEX_op_qemu_ld_a64_i64: + return C_O1_I1(r, r); + case INDEX_op_qemu_st_a32_i32: + case INDEX_op_qemu_st_a64_i32: + case INDEX_op_qemu_st_a32_i64: + case INDEX_op_qemu_st_a64_i64: + return C_O0_I2(rZ, r); default: g_assert_not_reached(); @@ -1703,10 +2081,10 @@ static void tcg_target_qemu_prologue(TCGContext *s) TCG_REG_SP, SAVE_OFS + i * REG_SIZE); } -#if !defined(CONFIG_SOFTMMU) - tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); - tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); -#endif + if (!tcg_use_softmmu && guest_base) { + tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); + tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); + } /* Call generated code */ tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); @@ -1727,12 +2105,69 @@ static void tcg_target_qemu_prologue(TCGContext *s) tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); } +static void tcg_out_tb_start(TCGContext *s) +{ + /* nothing to do */ +} + +static volatile sig_atomic_t got_sigill; + +static void sigill_handler(int signo, siginfo_t *si, void *data) +{ + /* Skip the faulty instruction */ + ucontext_t *uc = (ucontext_t *)data; + uc->uc_mcontext.__gregs[REG_PC] += 4; + + got_sigill = 1; +} + +static void tcg_target_detect_isa(void) +{ +#if !defined(have_zba) || !defined(have_zbb) || !defined(have_zicond) + /* + * TODO: It is expected that this will be determinable via + * linux riscv_hwprobe syscall, not yet merged. + * In the meantime, test via sigill. + */ + + struct sigaction sa_old, sa_new; + + memset(&sa_new, 0, sizeof(sa_new)); + sa_new.sa_flags = SA_SIGINFO; + sa_new.sa_sigaction = sigill_handler; + sigaction(SIGILL, &sa_new, &sa_old); + +#ifndef have_zba + /* Probe for Zba: add.uw zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x3b, 0, 0x04, zero, zero, zero" : : : "memory"); + have_zba = !got_sigill; +#endif + +#ifndef have_zbb + /* Probe for Zba: andn zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x33, 7, 0x20, zero, zero, zero" : : : "memory"); + have_zbb = !got_sigill; +#endif + +#ifndef have_zicond + /* Probe for Zicond: czero.eqz zero,zero,zero. */ + got_sigill = 0; + asm volatile(".insn r 0x33, 5, 0x07, zero, zero, zero" : : : "memory"); + have_zicond = !got_sigill; +#endif + + sigaction(SIGILL, &sa_old, NULL); +#endif +} + static void tcg_target_init(TCGContext *s) { + tcg_target_detect_isa(); + tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; - if (TCG_TARGET_REG_BITS == 64) { - tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; - } + tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; tcg_target_call_clobber_regs = -1u; tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); |