blob: dd65f1afcd772f5b7e72c3b50d4b2c68aadcace4 [file] [log] [blame]
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001/*
2 * Optimizations for Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2010 Samsung Electronics.
5 * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 * THE SOFTWARE.
24 */
25
Peter Maydell757e7252016-01-26 18:17:08 +000026#include "qemu/osdep.h"
Philippe Mathieu-Daudédcb32f12020-01-01 12:23:00 +010027#include "tcg/tcg-op.h"
Richard Henderson90163902021-03-18 10:21:45 -060028#include "tcg-internal.h"
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040029
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040030#define CASE_OP_32_64(x) \
31 glue(glue(case INDEX_op_, x), _i32): \
32 glue(glue(case INDEX_op_, x), _i64)
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +040033
Richard Henderson170ba882017-11-22 09:07:11 +010034#define CASE_OP_32_64_VEC(x) \
35 glue(glue(case INDEX_op_, x), _i32): \
36 glue(glue(case INDEX_op_, x), _i64): \
37 glue(glue(case INDEX_op_, x), _vec)
38
Richard Henderson6fcb98e2020-03-30 17:44:30 -070039typedef struct TempOptInfo {
Aurelien Jarnob41059d2015-07-27 12:41:44 +020040 bool is_const;
Richard Henderson63490392017-06-20 13:43:15 -070041 TCGTemp *prev_copy;
42 TCGTemp *next_copy;
Richard Henderson54795542020-09-06 16:21:32 -070043 uint64_t val;
Richard Hendersonb1fde412021-08-23 13:07:49 -070044 uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */
Richard Henderson6fcb98e2020-03-30 17:44:30 -070045} TempOptInfo;
Kirill Batuzov22613af2011-07-07 16:37:13 +040046
Richard Henderson3b3f8472021-08-23 22:06:31 -070047typedef struct OptContext {
Richard Hendersondc849882021-08-24 07:13:45 -070048 TCGContext *tcg;
Richard Hendersond0ed5152021-08-24 07:38:39 -070049 TCGOp *prev_mb;
Richard Henderson3b3f8472021-08-23 22:06:31 -070050 TCGTempSet temps_used;
Richard Henderson137f1f42021-08-24 08:49:25 -070051
52 /* In flight values from optimization. */
53 uint64_t z_mask;
Richard Henderson3b3f8472021-08-23 22:06:31 -070054} OptContext;
55
Richard Henderson6fcb98e2020-03-30 17:44:30 -070056static inline TempOptInfo *ts_info(TCGTemp *ts)
Aurelien Jarnod9c769c2015-07-27 12:41:44 +020057{
Richard Henderson63490392017-06-20 13:43:15 -070058 return ts->state_ptr;
Aurelien Jarnod9c769c2015-07-27 12:41:44 +020059}
60
Richard Henderson6fcb98e2020-03-30 17:44:30 -070061static inline TempOptInfo *arg_info(TCGArg arg)
Aurelien Jarnod9c769c2015-07-27 12:41:44 +020062{
Richard Henderson63490392017-06-20 13:43:15 -070063 return ts_info(arg_temp(arg));
64}
65
66static inline bool ts_is_const(TCGTemp *ts)
67{
68 return ts_info(ts)->is_const;
69}
70
71static inline bool arg_is_const(TCGArg arg)
72{
73 return ts_is_const(arg_temp(arg));
74}
75
76static inline bool ts_is_copy(TCGTemp *ts)
77{
78 return ts_info(ts)->next_copy != ts;
Aurelien Jarnod9c769c2015-07-27 12:41:44 +020079}
80
Aurelien Jarnob41059d2015-07-27 12:41:44 +020081/* Reset TEMP's state, possibly removing the temp for the list of copies. */
Richard Henderson63490392017-06-20 13:43:15 -070082static void reset_ts(TCGTemp *ts)
Kirill Batuzov22613af2011-07-07 16:37:13 +040083{
Richard Henderson6fcb98e2020-03-30 17:44:30 -070084 TempOptInfo *ti = ts_info(ts);
85 TempOptInfo *pi = ts_info(ti->prev_copy);
86 TempOptInfo *ni = ts_info(ti->next_copy);
Richard Henderson63490392017-06-20 13:43:15 -070087
88 ni->prev_copy = ti->prev_copy;
89 pi->next_copy = ti->next_copy;
90 ti->next_copy = ts;
91 ti->prev_copy = ts;
92 ti->is_const = false;
Richard Hendersonb1fde412021-08-23 13:07:49 -070093 ti->z_mask = -1;
Richard Henderson63490392017-06-20 13:43:15 -070094}
95
96static void reset_temp(TCGArg arg)
97{
98 reset_ts(arg_temp(arg));
Kirill Batuzov22613af2011-07-07 16:37:13 +040099}
100
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200101/* Initialize and activate a temporary. */
Richard Henderson3b3f8472021-08-23 22:06:31 -0700102static void init_ts_info(OptContext *ctx, TCGTemp *ts)
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200103{
Richard Henderson63490392017-06-20 13:43:15 -0700104 size_t idx = temp_idx(ts);
Richard Henderson8f17a972020-03-30 19:52:02 -0700105 TempOptInfo *ti;
Richard Henderson63490392017-06-20 13:43:15 -0700106
Richard Henderson3b3f8472021-08-23 22:06:31 -0700107 if (test_bit(idx, ctx->temps_used.l)) {
Richard Henderson8f17a972020-03-30 19:52:02 -0700108 return;
109 }
Richard Henderson3b3f8472021-08-23 22:06:31 -0700110 set_bit(idx, ctx->temps_used.l);
Richard Henderson8f17a972020-03-30 19:52:02 -0700111
112 ti = ts->state_ptr;
113 if (ti == NULL) {
114 ti = tcg_malloc(sizeof(TempOptInfo));
Richard Henderson63490392017-06-20 13:43:15 -0700115 ts->state_ptr = ti;
Richard Henderson8f17a972020-03-30 19:52:02 -0700116 }
117
118 ti->next_copy = ts;
119 ti->prev_copy = ts;
120 if (ts->kind == TEMP_CONST) {
121 ti->is_const = true;
122 ti->val = ts->val;
Richard Hendersonb1fde412021-08-23 13:07:49 -0700123 ti->z_mask = ts->val;
Richard Henderson8f17a972020-03-30 19:52:02 -0700124 if (TCG_TARGET_REG_BITS > 32 && ts->type == TCG_TYPE_I32) {
125 /* High bits of a 32-bit quantity are garbage. */
Richard Hendersonb1fde412021-08-23 13:07:49 -0700126 ti->z_mask |= ~0xffffffffull;
Richard Hendersonc0522132020-03-29 18:55:52 -0700127 }
Richard Henderson8f17a972020-03-30 19:52:02 -0700128 } else {
129 ti->is_const = false;
Richard Hendersonb1fde412021-08-23 13:07:49 -0700130 ti->z_mask = -1;
Aurelien Jarno1208d7d2015-07-27 12:41:44 +0200131 }
132}
133
Richard Henderson63490392017-06-20 13:43:15 -0700134static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts)
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200135{
Richard Henderson4c868ce2020-04-23 09:02:23 -0700136 TCGTemp *i, *g, *l;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200137
Richard Henderson4c868ce2020-04-23 09:02:23 -0700138 /* If this is already readonly, we can't do better. */
139 if (temp_readonly(ts)) {
Richard Henderson63490392017-06-20 13:43:15 -0700140 return ts;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200141 }
142
Richard Henderson4c868ce2020-04-23 09:02:23 -0700143 g = l = NULL;
Richard Henderson63490392017-06-20 13:43:15 -0700144 for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
Richard Henderson4c868ce2020-04-23 09:02:23 -0700145 if (temp_readonly(i)) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200146 return i;
Richard Henderson4c868ce2020-04-23 09:02:23 -0700147 } else if (i->kind > ts->kind) {
148 if (i->kind == TEMP_GLOBAL) {
149 g = i;
150 } else if (i->kind == TEMP_LOCAL) {
151 l = i;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200152 }
153 }
154 }
155
Richard Henderson4c868ce2020-04-23 09:02:23 -0700156 /* If we didn't find a better representation, return the same temp. */
157 return g ? g : l ? l : ts;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200158}
159
Richard Henderson63490392017-06-20 13:43:15 -0700160static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2)
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200161{
Richard Henderson63490392017-06-20 13:43:15 -0700162 TCGTemp *i;
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200163
Richard Henderson63490392017-06-20 13:43:15 -0700164 if (ts1 == ts2) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200165 return true;
166 }
167
Richard Henderson63490392017-06-20 13:43:15 -0700168 if (!ts_is_copy(ts1) || !ts_is_copy(ts2)) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200169 return false;
170 }
171
Richard Henderson63490392017-06-20 13:43:15 -0700172 for (i = ts_info(ts1)->next_copy; i != ts1; i = ts_info(i)->next_copy) {
173 if (i == ts2) {
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +0200174 return true;
175 }
176 }
177
178 return false;
179}
180
Richard Henderson63490392017-06-20 13:43:15 -0700181static bool args_are_copies(TCGArg arg1, TCGArg arg2)
182{
183 return ts_are_copies(arg_temp(arg1), arg_temp(arg2));
184}
185
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700186static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src)
Kirill Batuzov22613af2011-07-07 16:37:13 +0400187{
Richard Henderson63490392017-06-20 13:43:15 -0700188 TCGTemp *dst_ts = arg_temp(dst);
189 TCGTemp *src_ts = arg_temp(src);
Richard Henderson170ba882017-11-22 09:07:11 +0100190 const TCGOpDef *def;
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700191 TempOptInfo *di;
192 TempOptInfo *si;
Richard Hendersonb1fde412021-08-23 13:07:49 -0700193 uint64_t z_mask;
Richard Henderson63490392017-06-20 13:43:15 -0700194 TCGOpcode new_op;
195
196 if (ts_are_copies(dst_ts, src_ts)) {
Richard Hendersondc849882021-08-24 07:13:45 -0700197 tcg_op_remove(ctx->tcg, op);
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700198 return true;
Aurelien Jarno53657182015-06-04 21:53:25 +0200199 }
200
Richard Henderson63490392017-06-20 13:43:15 -0700201 reset_ts(dst_ts);
202 di = ts_info(dst_ts);
203 si = ts_info(src_ts);
Richard Henderson170ba882017-11-22 09:07:11 +0100204 def = &tcg_op_defs[op->opc];
205 if (def->flags & TCG_OPF_VECTOR) {
206 new_op = INDEX_op_mov_vec;
207 } else if (def->flags & TCG_OPF_64BIT) {
208 new_op = INDEX_op_mov_i64;
209 } else {
210 new_op = INDEX_op_mov_i32;
211 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -0700212 op->opc = new_op;
Richard Henderson170ba882017-11-22 09:07:11 +0100213 /* TCGOP_VECL and TCGOP_VECE remain unchanged. */
Richard Henderson63490392017-06-20 13:43:15 -0700214 op->args[0] = dst;
215 op->args[1] = src;
Richard Hendersona62f6f52014-05-22 10:59:12 -0700216
Richard Hendersonb1fde412021-08-23 13:07:49 -0700217 z_mask = si->z_mask;
Richard Henderson24666ba2014-05-22 11:14:10 -0700218 if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) {
219 /* High bits of the destination are now garbage. */
Richard Hendersonb1fde412021-08-23 13:07:49 -0700220 z_mask |= ~0xffffffffull;
Richard Henderson24666ba2014-05-22 11:14:10 -0700221 }
Richard Hendersonb1fde412021-08-23 13:07:49 -0700222 di->z_mask = z_mask;
Richard Henderson24666ba2014-05-22 11:14:10 -0700223
Richard Henderson63490392017-06-20 13:43:15 -0700224 if (src_ts->type == dst_ts->type) {
Richard Henderson6fcb98e2020-03-30 17:44:30 -0700225 TempOptInfo *ni = ts_info(si->next_copy);
Richard Henderson63490392017-06-20 13:43:15 -0700226
227 di->next_copy = si->next_copy;
228 di->prev_copy = src_ts;
229 ni->prev_copy = dst_ts;
230 si->next_copy = dst_ts;
231 di->is_const = si->is_const;
232 di->val = si->val;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -0800233 }
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700234 return true;
Kirill Batuzov22613af2011-07-07 16:37:13 +0400235}
236
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700237static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op,
Richard Hendersondc849882021-08-24 07:13:45 -0700238 TCGArg dst, uint64_t val)
Richard Henderson8fe35e02020-03-30 20:42:43 -0700239{
240 const TCGOpDef *def = &tcg_op_defs[op->opc];
241 TCGType type;
242 TCGTemp *tv;
243
244 if (def->flags & TCG_OPF_VECTOR) {
245 type = TCGOP_VECL(op) + TCG_TYPE_V64;
246 } else if (def->flags & TCG_OPF_64BIT) {
247 type = TCG_TYPE_I64;
248 } else {
249 type = TCG_TYPE_I32;
250 }
251
252 /* Convert movi to mov with constant temp. */
253 tv = tcg_constant_internal(type, val);
Richard Henderson3b3f8472021-08-23 22:06:31 -0700254 init_ts_info(ctx, tv);
Richard Henderson6b99d5b2021-08-24 10:57:56 -0700255 return tcg_opt_gen_mov(ctx, op, dst, temp_arg(tv));
Richard Henderson8fe35e02020-03-30 20:42:43 -0700256}
257
Richard Henderson54795542020-09-06 16:21:32 -0700258static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400259{
Richard Henderson03271522013-08-14 14:35:56 -0700260 uint64_t l64, h64;
261
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400262 switch (op) {
263 CASE_OP_32_64(add):
264 return x + y;
265
266 CASE_OP_32_64(sub):
267 return x - y;
268
269 CASE_OP_32_64(mul):
270 return x * y;
271
Kirill Batuzov9a810902011-07-07 16:37:15 +0400272 CASE_OP_32_64(and):
273 return x & y;
274
275 CASE_OP_32_64(or):
276 return x | y;
277
278 CASE_OP_32_64(xor):
279 return x ^ y;
280
Kirill Batuzov55c09752011-07-07 16:37:16 +0400281 case INDEX_op_shl_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700282 return (uint32_t)x << (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400283
Kirill Batuzov55c09752011-07-07 16:37:16 +0400284 case INDEX_op_shl_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700285 return (uint64_t)x << (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400286
287 case INDEX_op_shr_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700288 return (uint32_t)x >> (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400289
Kirill Batuzov55c09752011-07-07 16:37:16 +0400290 case INDEX_op_shr_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700291 return (uint64_t)x >> (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400292
293 case INDEX_op_sar_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700294 return (int32_t)x >> (y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400295
Kirill Batuzov55c09752011-07-07 16:37:16 +0400296 case INDEX_op_sar_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700297 return (int64_t)x >> (y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400298
299 case INDEX_op_rotr_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700300 return ror32(x, y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400301
Kirill Batuzov55c09752011-07-07 16:37:16 +0400302 case INDEX_op_rotr_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700303 return ror64(x, y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400304
305 case INDEX_op_rotl_i32:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700306 return rol32(x, y & 31);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400307
Kirill Batuzov55c09752011-07-07 16:37:16 +0400308 case INDEX_op_rotl_i64:
Richard Henderson50c5c4d2014-03-18 07:45:39 -0700309 return rol64(x, y & 63);
Kirill Batuzov55c09752011-07-07 16:37:16 +0400310
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700311 CASE_OP_32_64(not):
Kirill Batuzova640f032011-07-07 16:37:17 +0400312 return ~x;
313
Richard Hendersoncb25c802011-08-17 14:11:47 -0700314 CASE_OP_32_64(neg):
315 return -x;
316
317 CASE_OP_32_64(andc):
318 return x & ~y;
319
320 CASE_OP_32_64(orc):
321 return x | ~y;
322
323 CASE_OP_32_64(eqv):
324 return ~(x ^ y);
325
326 CASE_OP_32_64(nand):
327 return ~(x & y);
328
329 CASE_OP_32_64(nor):
330 return ~(x | y);
331
Richard Henderson0e28d002016-11-16 09:23:28 +0100332 case INDEX_op_clz_i32:
333 return (uint32_t)x ? clz32(x) : y;
334
335 case INDEX_op_clz_i64:
336 return x ? clz64(x) : y;
337
338 case INDEX_op_ctz_i32:
339 return (uint32_t)x ? ctz32(x) : y;
340
341 case INDEX_op_ctz_i64:
342 return x ? ctz64(x) : y;
343
Richard Hendersona768e4e2016-11-21 11:13:39 +0100344 case INDEX_op_ctpop_i32:
345 return ctpop32(x);
346
347 case INDEX_op_ctpop_i64:
348 return ctpop64(x);
349
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700350 CASE_OP_32_64(ext8s):
Kirill Batuzova640f032011-07-07 16:37:17 +0400351 return (int8_t)x;
352
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700353 CASE_OP_32_64(ext16s):
Kirill Batuzova640f032011-07-07 16:37:17 +0400354 return (int16_t)x;
355
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700356 CASE_OP_32_64(ext8u):
Kirill Batuzova640f032011-07-07 16:37:17 +0400357 return (uint8_t)x;
358
Richard Henderson25c4d9c2011-08-17 14:11:46 -0700359 CASE_OP_32_64(ext16u):
Kirill Batuzova640f032011-07-07 16:37:17 +0400360 return (uint16_t)x;
361
Richard Henderson64985942018-11-20 08:53:34 +0100362 CASE_OP_32_64(bswap16):
Richard Henderson0b76ff82021-06-13 13:04:00 -0700363 x = bswap16(x);
364 return y & TCG_BSWAP_OS ? (int16_t)x : x;
Richard Henderson64985942018-11-20 08:53:34 +0100365
366 CASE_OP_32_64(bswap32):
Richard Henderson0b76ff82021-06-13 13:04:00 -0700367 x = bswap32(x);
368 return y & TCG_BSWAP_OS ? (int32_t)x : x;
Richard Henderson64985942018-11-20 08:53:34 +0100369
370 case INDEX_op_bswap64_i64:
371 return bswap64(x);
372
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +0200373 case INDEX_op_ext_i32_i64:
Kirill Batuzova640f032011-07-07 16:37:17 +0400374 case INDEX_op_ext32s_i64:
375 return (int32_t)x;
376
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +0200377 case INDEX_op_extu_i32_i64:
Richard Henderson609ad702015-07-24 07:16:00 -0700378 case INDEX_op_extrl_i64_i32:
Kirill Batuzova640f032011-07-07 16:37:17 +0400379 case INDEX_op_ext32u_i64:
380 return (uint32_t)x;
Kirill Batuzova640f032011-07-07 16:37:17 +0400381
Richard Henderson609ad702015-07-24 07:16:00 -0700382 case INDEX_op_extrh_i64_i32:
383 return (uint64_t)x >> 32;
384
Richard Henderson03271522013-08-14 14:35:56 -0700385 case INDEX_op_muluh_i32:
386 return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
387 case INDEX_op_mulsh_i32:
388 return ((int64_t)(int32_t)x * (int32_t)y) >> 32;
389
390 case INDEX_op_muluh_i64:
391 mulu64(&l64, &h64, x, y);
392 return h64;
393 case INDEX_op_mulsh_i64:
394 muls64(&l64, &h64, x, y);
395 return h64;
396
Richard Henderson01547f72013-08-14 15:22:46 -0700397 case INDEX_op_div_i32:
398 /* Avoid crashing on divide by zero, otherwise undefined. */
399 return (int32_t)x / ((int32_t)y ? : 1);
400 case INDEX_op_divu_i32:
401 return (uint32_t)x / ((uint32_t)y ? : 1);
402 case INDEX_op_div_i64:
403 return (int64_t)x / ((int64_t)y ? : 1);
404 case INDEX_op_divu_i64:
405 return (uint64_t)x / ((uint64_t)y ? : 1);
406
407 case INDEX_op_rem_i32:
408 return (int32_t)x % ((int32_t)y ? : 1);
409 case INDEX_op_remu_i32:
410 return (uint32_t)x % ((uint32_t)y ? : 1);
411 case INDEX_op_rem_i64:
412 return (int64_t)x % ((int64_t)y ? : 1);
413 case INDEX_op_remu_i64:
414 return (uint64_t)x % ((uint64_t)y ? : 1);
415
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400416 default:
417 fprintf(stderr,
418 "Unrecognized operation %d in do_constant_folding.\n", op);
419 tcg_abort();
420 }
421}
422
Richard Henderson54795542020-09-06 16:21:32 -0700423static uint64_t do_constant_folding(TCGOpcode op, uint64_t x, uint64_t y)
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400424{
Richard Henderson170ba882017-11-22 09:07:11 +0100425 const TCGOpDef *def = &tcg_op_defs[op];
Richard Henderson54795542020-09-06 16:21:32 -0700426 uint64_t res = do_constant_folding_2(op, x, y);
Richard Henderson170ba882017-11-22 09:07:11 +0100427 if (!(def->flags & TCG_OPF_64BIT)) {
Aurelien Jarno29f3ff82015-07-10 18:03:31 +0200428 res = (int32_t)res;
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400429 }
Kirill Batuzov53108fb2011-07-07 16:37:14 +0400430 return res;
431}
432
Richard Henderson9519da72012-10-02 11:32:26 -0700433static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
434{
435 switch (c) {
436 case TCG_COND_EQ:
437 return x == y;
438 case TCG_COND_NE:
439 return x != y;
440 case TCG_COND_LT:
441 return (int32_t)x < (int32_t)y;
442 case TCG_COND_GE:
443 return (int32_t)x >= (int32_t)y;
444 case TCG_COND_LE:
445 return (int32_t)x <= (int32_t)y;
446 case TCG_COND_GT:
447 return (int32_t)x > (int32_t)y;
448 case TCG_COND_LTU:
449 return x < y;
450 case TCG_COND_GEU:
451 return x >= y;
452 case TCG_COND_LEU:
453 return x <= y;
454 case TCG_COND_GTU:
455 return x > y;
456 default:
457 tcg_abort();
458 }
459}
460
461static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
462{
463 switch (c) {
464 case TCG_COND_EQ:
465 return x == y;
466 case TCG_COND_NE:
467 return x != y;
468 case TCG_COND_LT:
469 return (int64_t)x < (int64_t)y;
470 case TCG_COND_GE:
471 return (int64_t)x >= (int64_t)y;
472 case TCG_COND_LE:
473 return (int64_t)x <= (int64_t)y;
474 case TCG_COND_GT:
475 return (int64_t)x > (int64_t)y;
476 case TCG_COND_LTU:
477 return x < y;
478 case TCG_COND_GEU:
479 return x >= y;
480 case TCG_COND_LEU:
481 return x <= y;
482 case TCG_COND_GTU:
483 return x > y;
484 default:
485 tcg_abort();
486 }
487}
488
489static bool do_constant_folding_cond_eq(TCGCond c)
490{
491 switch (c) {
492 case TCG_COND_GT:
493 case TCG_COND_LTU:
494 case TCG_COND_LT:
495 case TCG_COND_GTU:
496 case TCG_COND_NE:
497 return 0;
498 case TCG_COND_GE:
499 case TCG_COND_GEU:
500 case TCG_COND_LE:
501 case TCG_COND_LEU:
502 case TCG_COND_EQ:
503 return 1;
504 default:
505 tcg_abort();
506 }
507}
508
Richard Henderson8d57bf12021-08-24 08:34:27 -0700509/*
510 * Return -1 if the condition can't be simplified,
511 * and the result of the condition (0 or 1) if it can.
512 */
513static int do_constant_folding_cond(TCGOpcode op, TCGArg x,
514 TCGArg y, TCGCond c)
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200515{
Richard Henderson54795542020-09-06 16:21:32 -0700516 uint64_t xv = arg_info(x)->val;
517 uint64_t yv = arg_info(y)->val;
518
Richard Henderson63490392017-06-20 13:43:15 -0700519 if (arg_is_const(x) && arg_is_const(y)) {
Richard Henderson170ba882017-11-22 09:07:11 +0100520 const TCGOpDef *def = &tcg_op_defs[op];
521 tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR));
522 if (def->flags & TCG_OPF_64BIT) {
Richard Henderson63490392017-06-20 13:43:15 -0700523 return do_constant_folding_cond_64(xv, yv, c);
Richard Henderson170ba882017-11-22 09:07:11 +0100524 } else {
525 return do_constant_folding_cond_32(xv, yv, c);
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200526 }
Richard Henderson63490392017-06-20 13:43:15 -0700527 } else if (args_are_copies(x, y)) {
Richard Henderson9519da72012-10-02 11:32:26 -0700528 return do_constant_folding_cond_eq(c);
Richard Henderson63490392017-06-20 13:43:15 -0700529 } else if (arg_is_const(y) && yv == 0) {
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200530 switch (c) {
531 case TCG_COND_LTU:
532 return 0;
533 case TCG_COND_GEU:
534 return 1;
535 default:
Richard Henderson8d57bf12021-08-24 08:34:27 -0700536 return -1;
Aurelien Jarnob336ceb2012-09-18 19:37:00 +0200537 }
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200538 }
Richard Henderson8d57bf12021-08-24 08:34:27 -0700539 return -1;
Aurelien Jarnof8dd19e2012-09-06 16:47:14 +0200540}
541
Richard Henderson8d57bf12021-08-24 08:34:27 -0700542/*
543 * Return -1 if the condition can't be simplified,
544 * and the result of the condition (0 or 1) if it can.
545 */
546static int do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
Richard Henderson6c4382f2012-10-02 11:32:27 -0700547{
548 TCGArg al = p1[0], ah = p1[1];
549 TCGArg bl = p2[0], bh = p2[1];
550
Richard Henderson63490392017-06-20 13:43:15 -0700551 if (arg_is_const(bl) && arg_is_const(bh)) {
552 tcg_target_ulong blv = arg_info(bl)->val;
553 tcg_target_ulong bhv = arg_info(bh)->val;
554 uint64_t b = deposit64(blv, 32, 32, bhv);
Richard Henderson6c4382f2012-10-02 11:32:27 -0700555
Richard Henderson63490392017-06-20 13:43:15 -0700556 if (arg_is_const(al) && arg_is_const(ah)) {
557 tcg_target_ulong alv = arg_info(al)->val;
558 tcg_target_ulong ahv = arg_info(ah)->val;
559 uint64_t a = deposit64(alv, 32, 32, ahv);
Richard Henderson6c4382f2012-10-02 11:32:27 -0700560 return do_constant_folding_cond_64(a, b, c);
561 }
562 if (b == 0) {
563 switch (c) {
564 case TCG_COND_LTU:
565 return 0;
566 case TCG_COND_GEU:
567 return 1;
568 default:
569 break;
570 }
571 }
572 }
Richard Henderson63490392017-06-20 13:43:15 -0700573 if (args_are_copies(al, bl) && args_are_copies(ah, bh)) {
Richard Henderson6c4382f2012-10-02 11:32:27 -0700574 return do_constant_folding_cond_eq(c);
575 }
Richard Henderson8d57bf12021-08-24 08:34:27 -0700576 return -1;
Richard Henderson6c4382f2012-10-02 11:32:27 -0700577}
578
Richard Henderson24c9ae42012-10-02 11:32:21 -0700579static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
580{
581 TCGArg a1 = *p1, a2 = *p2;
582 int sum = 0;
Richard Henderson63490392017-06-20 13:43:15 -0700583 sum += arg_is_const(a1);
584 sum -= arg_is_const(a2);
Richard Henderson24c9ae42012-10-02 11:32:21 -0700585
586 /* Prefer the constant in second argument, and then the form
587 op a, a, b, which is better handled on non-RISC hosts. */
588 if (sum > 0 || (sum == 0 && dest == a2)) {
589 *p1 = a2;
590 *p2 = a1;
591 return true;
592 }
593 return false;
594}
595
Richard Henderson0bfcb862012-10-02 11:32:23 -0700596static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
597{
598 int sum = 0;
Richard Henderson63490392017-06-20 13:43:15 -0700599 sum += arg_is_const(p1[0]);
600 sum += arg_is_const(p1[1]);
601 sum -= arg_is_const(p2[0]);
602 sum -= arg_is_const(p2[1]);
Richard Henderson0bfcb862012-10-02 11:32:23 -0700603 if (sum > 0) {
604 TCGArg t;
605 t = p1[0], p1[0] = p2[0], p2[0] = t;
606 t = p1[1], p1[1] = p2[1], p2[1] = t;
607 return true;
608 }
609 return false;
610}
611
Richard Hendersone2577ea2021-08-24 08:00:48 -0700612static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args)
613{
614 for (int i = 0; i < nb_args; i++) {
615 TCGTemp *ts = arg_temp(op->args[i]);
616 if (ts) {
617 init_ts_info(ctx, ts);
618 }
619 }
620}
621
Richard Henderson8774dde2021-08-24 08:04:47 -0700622static void copy_propagate(OptContext *ctx, TCGOp *op,
623 int nb_oargs, int nb_iargs)
624{
625 TCGContext *s = ctx->tcg;
626
627 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
628 TCGTemp *ts = arg_temp(op->args[i]);
629 if (ts && ts_is_copy(ts)) {
630 op->args[i] = temp_arg(find_better_copy(s, ts));
631 }
632 }
633}
634
Richard Henderson137f1f42021-08-24 08:49:25 -0700635static void finish_folding(OptContext *ctx, TCGOp *op)
636{
637 const TCGOpDef *def = &tcg_op_defs[op->opc];
638 int i, nb_oargs;
639
640 /*
641 * For an opcode that ends a BB, reset all temp data.
642 * We do no cross-BB optimization.
643 */
644 if (def->flags & TCG_OPF_BB_END) {
645 memset(&ctx->temps_used, 0, sizeof(ctx->temps_used));
646 ctx->prev_mb = NULL;
647 return;
648 }
649
650 nb_oargs = def->nb_oargs;
651 for (i = 0; i < nb_oargs; i++) {
652 reset_temp(op->args[i]);
653 /*
654 * Save the corresponding known-zero bits mask for the
655 * first output argument (only one supported so far).
656 */
657 if (i == 0) {
658 arg_info(op->args[i])->z_mask = ctx->z_mask;
659 }
660 }
661}
662
Richard Henderson2f9f08b2021-08-25 12:03:48 -0700663/*
664 * The fold_* functions return true when processing is complete,
665 * usually by folding the operation to a constant or to a copy,
666 * and calling tcg_opt_gen_{mov,movi}. They may do other things,
667 * like collect information about the value produced, for use in
668 * optimizing a subsequent operation.
669 *
670 * These first fold_* functions are all helpers, used by other
671 * folders for more specific operations.
672 */
673
674static bool fold_const1(OptContext *ctx, TCGOp *op)
675{
676 if (arg_is_const(op->args[1])) {
677 uint64_t t;
678
679 t = arg_info(op->args[1])->val;
680 t = do_constant_folding(op->opc, t, 0);
681 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
682 }
683 return false;
684}
685
686static bool fold_const2(OptContext *ctx, TCGOp *op)
687{
688 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
689 uint64_t t1 = arg_info(op->args[1])->val;
690 uint64_t t2 = arg_info(op->args[2])->val;
691
692 t1 = do_constant_folding(op->opc, t1, t2);
693 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
694 }
695 return false;
696}
697
698/*
699 * These outermost fold_<op> functions are sorted alphabetically.
700 */
701
702static bool fold_add(OptContext *ctx, TCGOp *op)
703{
704 return fold_const2(ctx, op);
705}
706
Richard Hendersone3f7dc22021-08-24 10:30:38 -0700707static bool fold_addsub2_i32(OptContext *ctx, TCGOp *op, bool add)
708{
709 if (arg_is_const(op->args[2]) && arg_is_const(op->args[3]) &&
710 arg_is_const(op->args[4]) && arg_is_const(op->args[5])) {
711 uint32_t al = arg_info(op->args[2])->val;
712 uint32_t ah = arg_info(op->args[3])->val;
713 uint32_t bl = arg_info(op->args[4])->val;
714 uint32_t bh = arg_info(op->args[5])->val;
715 uint64_t a = ((uint64_t)ah << 32) | al;
716 uint64_t b = ((uint64_t)bh << 32) | bl;
717 TCGArg rl, rh;
718 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32);
719
720 if (add) {
721 a += b;
722 } else {
723 a -= b;
724 }
725
726 rl = op->args[0];
727 rh = op->args[1];
728 tcg_opt_gen_movi(ctx, op, rl, (int32_t)a);
729 tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(a >> 32));
730 return true;
731 }
732 return false;
733}
734
735static bool fold_add2_i32(OptContext *ctx, TCGOp *op)
736{
737 return fold_addsub2_i32(ctx, op, true);
738}
739
Richard Henderson2f9f08b2021-08-25 12:03:48 -0700740static bool fold_and(OptContext *ctx, TCGOp *op)
741{
742 return fold_const2(ctx, op);
743}
744
745static bool fold_andc(OptContext *ctx, TCGOp *op)
746{
747 return fold_const2(ctx, op);
748}
749
Richard Henderson079b0802021-08-24 09:30:59 -0700750static bool fold_brcond(OptContext *ctx, TCGOp *op)
751{
752 TCGCond cond = op->args[2];
753 int i = do_constant_folding_cond(op->opc, op->args[0], op->args[1], cond);
754
755 if (i == 0) {
756 tcg_op_remove(ctx->tcg, op);
757 return true;
758 }
759 if (i > 0) {
760 op->opc = INDEX_op_br;
761 op->args[0] = op->args[3];
762 }
763 return false;
764}
765
Richard Henderson764d2ab2021-08-24 09:22:11 -0700766static bool fold_brcond2(OptContext *ctx, TCGOp *op)
767{
768 TCGCond cond = op->args[4];
769 int i = do_constant_folding_cond2(&op->args[0], &op->args[2], cond);
770 TCGArg label = op->args[5];
771 int inv = 0;
772
773 if (i >= 0) {
774 goto do_brcond_const;
775 }
776
777 switch (cond) {
778 case TCG_COND_LT:
779 case TCG_COND_GE:
780 /*
781 * Simplify LT/GE comparisons vs zero to a single compare
782 * vs the high word of the input.
783 */
784 if (arg_is_const(op->args[2]) && arg_info(op->args[2])->val == 0 &&
785 arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0) {
786 goto do_brcond_high;
787 }
788 break;
789
790 case TCG_COND_NE:
791 inv = 1;
792 QEMU_FALLTHROUGH;
793 case TCG_COND_EQ:
794 /*
795 * Simplify EQ/NE comparisons where one of the pairs
796 * can be simplified.
797 */
798 i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[0],
799 op->args[2], cond);
800 switch (i ^ inv) {
801 case 0:
802 goto do_brcond_const;
803 case 1:
804 goto do_brcond_high;
805 }
806
807 i = do_constant_folding_cond(INDEX_op_brcond_i32, op->args[1],
808 op->args[3], cond);
809 switch (i ^ inv) {
810 case 0:
811 goto do_brcond_const;
812 case 1:
813 op->opc = INDEX_op_brcond_i32;
814 op->args[1] = op->args[2];
815 op->args[2] = cond;
816 op->args[3] = label;
817 break;
818 }
819 break;
820
821 default:
822 break;
823
824 do_brcond_high:
825 op->opc = INDEX_op_brcond_i32;
826 op->args[0] = op->args[1];
827 op->args[1] = op->args[3];
828 op->args[2] = cond;
829 op->args[3] = label;
830 break;
831
832 do_brcond_const:
833 if (i == 0) {
834 tcg_op_remove(ctx->tcg, op);
835 return true;
836 }
837 op->opc = INDEX_op_br;
838 op->args[0] = label;
839 break;
840 }
841 return false;
842}
843
Richard Henderson5cf32be2021-08-24 08:17:08 -0700844static bool fold_call(OptContext *ctx, TCGOp *op)
845{
846 TCGContext *s = ctx->tcg;
847 int nb_oargs = TCGOP_CALLO(op);
848 int nb_iargs = TCGOP_CALLI(op);
849 int flags, i;
850
851 init_arguments(ctx, op, nb_oargs + nb_iargs);
852 copy_propagate(ctx, op, nb_oargs, nb_iargs);
853
854 /* If the function reads or writes globals, reset temp data. */
855 flags = tcg_call_flags(op);
856 if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
857 int nb_globals = s->nb_globals;
858
859 for (i = 0; i < nb_globals; i++) {
860 if (test_bit(i, ctx->temps_used.l)) {
861 reset_ts(&ctx->tcg->temps[i]);
862 }
863 }
864 }
865
866 /* Reset temp data for outputs. */
867 for (i = 0; i < nb_oargs; i++) {
868 reset_temp(op->args[i]);
869 }
870
871 /* Stop optimizing MB across calls. */
872 ctx->prev_mb = NULL;
873 return true;
874}
875
Richard Henderson30dd0bf2021-08-24 10:51:34 -0700876static bool fold_count_zeros(OptContext *ctx, TCGOp *op)
877{
878 if (arg_is_const(op->args[1])) {
879 uint64_t t = arg_info(op->args[1])->val;
880
881 if (t != 0) {
882 t = do_constant_folding(op->opc, t, 0);
883 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
884 }
885 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
886 }
887 return false;
888}
889
Richard Henderson2f9f08b2021-08-25 12:03:48 -0700890static bool fold_ctpop(OptContext *ctx, TCGOp *op)
891{
892 return fold_const1(ctx, op);
893}
894
Richard Henderson1b1907b2021-08-24 10:47:04 -0700895static bool fold_deposit(OptContext *ctx, TCGOp *op)
896{
897 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
898 uint64_t t1 = arg_info(op->args[1])->val;
899 uint64_t t2 = arg_info(op->args[2])->val;
900
901 t1 = deposit64(t1, op->args[3], op->args[4], t2);
902 return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
903 }
904 return false;
905}
906
Richard Henderson2f9f08b2021-08-25 12:03:48 -0700907static bool fold_divide(OptContext *ctx, TCGOp *op)
908{
909 return fold_const2(ctx, op);
910}
911
912static bool fold_eqv(OptContext *ctx, TCGOp *op)
913{
914 return fold_const2(ctx, op);
915}
916
Richard Hendersonb6617c82021-08-24 10:44:53 -0700917static bool fold_extract(OptContext *ctx, TCGOp *op)
918{
919 if (arg_is_const(op->args[1])) {
920 uint64_t t;
921
922 t = arg_info(op->args[1])->val;
923 t = extract64(t, op->args[2], op->args[3]);
924 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
925 }
926 return false;
927}
928
Richard Hendersondcd08992021-08-24 10:41:39 -0700929static bool fold_extract2(OptContext *ctx, TCGOp *op)
930{
931 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
932 uint64_t v1 = arg_info(op->args[1])->val;
933 uint64_t v2 = arg_info(op->args[2])->val;
934 int shr = op->args[3];
935
936 if (op->opc == INDEX_op_extract2_i64) {
937 v1 >>= shr;
938 v2 <<= 64 - shr;
939 } else {
940 v1 = (uint32_t)v1 >> shr;
941 v2 = (int32_t)v2 << (32 - shr);
942 }
943 return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2);
944 }
945 return false;
946}
947
Richard Henderson2f9f08b2021-08-25 12:03:48 -0700948static bool fold_exts(OptContext *ctx, TCGOp *op)
949{
950 return fold_const1(ctx, op);
951}
952
953static bool fold_extu(OptContext *ctx, TCGOp *op)
954{
955 return fold_const1(ctx, op);
956}
957
Richard Henderson3eefdf22021-08-25 11:06:43 -0700958static bool fold_mb(OptContext *ctx, TCGOp *op)
959{
960 /* Eliminate duplicate and redundant fence instructions. */
961 if (ctx->prev_mb) {
962 /*
963 * Merge two barriers of the same type into one,
964 * or a weaker barrier into a stronger one,
965 * or two weaker barriers into a stronger one.
966 * mb X; mb Y => mb X|Y
967 * mb; strl => mb; st
968 * ldaq; mb => ld; mb
969 * ldaq; strl => ld; mb; st
970 * Other combinations are also merged into a strong
971 * barrier. This is stricter than specified but for
972 * the purposes of TCG is better than not optimizing.
973 */
974 ctx->prev_mb->args[0] |= op->args[0];
975 tcg_op_remove(ctx->tcg, op);
976 } else {
977 ctx->prev_mb = op;
978 }
979 return true;
980}
981
Richard Henderson0c310a32021-08-24 10:37:24 -0700982static bool fold_movcond(OptContext *ctx, TCGOp *op)
983{
984 TCGOpcode opc = op->opc;
985 TCGCond cond = op->args[5];
986 int i = do_constant_folding_cond(opc, op->args[1], op->args[2], cond);
987
988 if (i >= 0) {
989 return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]);
990 }
991
992 if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) {
993 uint64_t tv = arg_info(op->args[3])->val;
994 uint64_t fv = arg_info(op->args[4])->val;
995
996 opc = (opc == INDEX_op_movcond_i32
997 ? INDEX_op_setcond_i32 : INDEX_op_setcond_i64);
998
999 if (tv == 1 && fv == 0) {
1000 op->opc = opc;
1001 op->args[3] = cond;
1002 } else if (fv == 1 && tv == 0) {
1003 op->opc = opc;
1004 op->args[3] = tcg_invert_cond(cond);
1005 }
1006 }
1007 return false;
1008}
1009
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001010static bool fold_mul(OptContext *ctx, TCGOp *op)
1011{
1012 return fold_const2(ctx, op);
1013}
1014
1015static bool fold_mul_highpart(OptContext *ctx, TCGOp *op)
1016{
1017 return fold_const2(ctx, op);
1018}
1019
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001020static bool fold_mulu2_i32(OptContext *ctx, TCGOp *op)
1021{
1022 if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
1023 uint32_t a = arg_info(op->args[2])->val;
1024 uint32_t b = arg_info(op->args[3])->val;
1025 uint64_t r = (uint64_t)a * b;
1026 TCGArg rl, rh;
1027 TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_mov_i32);
1028
1029 rl = op->args[0];
1030 rh = op->args[1];
1031 tcg_opt_gen_movi(ctx, op, rl, (int32_t)r);
1032 tcg_opt_gen_movi(ctx, op2, rh, (int32_t)(r >> 32));
1033 return true;
1034 }
1035 return false;
1036}
1037
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001038static bool fold_nand(OptContext *ctx, TCGOp *op)
1039{
1040 return fold_const2(ctx, op);
1041}
1042
1043static bool fold_neg(OptContext *ctx, TCGOp *op)
1044{
1045 return fold_const1(ctx, op);
1046}
1047
1048static bool fold_nor(OptContext *ctx, TCGOp *op)
1049{
1050 return fold_const2(ctx, op);
1051}
1052
1053static bool fold_not(OptContext *ctx, TCGOp *op)
1054{
1055 return fold_const1(ctx, op);
1056}
1057
1058static bool fold_or(OptContext *ctx, TCGOp *op)
1059{
1060 return fold_const2(ctx, op);
1061}
1062
1063static bool fold_orc(OptContext *ctx, TCGOp *op)
1064{
1065 return fold_const2(ctx, op);
1066}
1067
Richard Henderson3eefdf22021-08-25 11:06:43 -07001068static bool fold_qemu_ld(OptContext *ctx, TCGOp *op)
1069{
1070 /* Opcodes that touch guest memory stop the mb optimization. */
1071 ctx->prev_mb = NULL;
1072 return false;
1073}
1074
1075static bool fold_qemu_st(OptContext *ctx, TCGOp *op)
1076{
1077 /* Opcodes that touch guest memory stop the mb optimization. */
1078 ctx->prev_mb = NULL;
1079 return false;
1080}
1081
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001082static bool fold_remainder(OptContext *ctx, TCGOp *op)
1083{
1084 return fold_const2(ctx, op);
1085}
1086
Richard Hendersonc63ff552021-08-24 09:35:30 -07001087static bool fold_setcond(OptContext *ctx, TCGOp *op)
1088{
1089 TCGCond cond = op->args[3];
1090 int i = do_constant_folding_cond(op->opc, op->args[1], op->args[2], cond);
1091
1092 if (i >= 0) {
1093 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1094 }
1095 return false;
1096}
1097
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07001098static bool fold_setcond2(OptContext *ctx, TCGOp *op)
1099{
1100 TCGCond cond = op->args[5];
1101 int i = do_constant_folding_cond2(&op->args[1], &op->args[3], cond);
1102 int inv = 0;
1103
1104 if (i >= 0) {
1105 goto do_setcond_const;
1106 }
1107
1108 switch (cond) {
1109 case TCG_COND_LT:
1110 case TCG_COND_GE:
1111 /*
1112 * Simplify LT/GE comparisons vs zero to a single compare
1113 * vs the high word of the input.
1114 */
1115 if (arg_is_const(op->args[3]) && arg_info(op->args[3])->val == 0 &&
1116 arg_is_const(op->args[4]) && arg_info(op->args[4])->val == 0) {
1117 goto do_setcond_high;
1118 }
1119 break;
1120
1121 case TCG_COND_NE:
1122 inv = 1;
1123 QEMU_FALLTHROUGH;
1124 case TCG_COND_EQ:
1125 /*
1126 * Simplify EQ/NE comparisons where one of the pairs
1127 * can be simplified.
1128 */
1129 i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[1],
1130 op->args[3], cond);
1131 switch (i ^ inv) {
1132 case 0:
1133 goto do_setcond_const;
1134 case 1:
1135 goto do_setcond_high;
1136 }
1137
1138 i = do_constant_folding_cond(INDEX_op_setcond_i32, op->args[2],
1139 op->args[4], cond);
1140 switch (i ^ inv) {
1141 case 0:
1142 goto do_setcond_const;
1143 case 1:
1144 op->args[2] = op->args[3];
1145 op->args[3] = cond;
1146 op->opc = INDEX_op_setcond_i32;
1147 break;
1148 }
1149 break;
1150
1151 default:
1152 break;
1153
1154 do_setcond_high:
1155 op->args[1] = op->args[2];
1156 op->args[2] = op->args[4];
1157 op->args[3] = cond;
1158 op->opc = INDEX_op_setcond_i32;
1159 break;
1160 }
1161 return false;
1162
1163 do_setcond_const:
1164 return tcg_opt_gen_movi(ctx, op, op->args[0], i);
1165}
1166
Richard Hendersonb6617c82021-08-24 10:44:53 -07001167static bool fold_sextract(OptContext *ctx, TCGOp *op)
1168{
1169 if (arg_is_const(op->args[1])) {
1170 uint64_t t;
1171
1172 t = arg_info(op->args[1])->val;
1173 t = sextract64(t, op->args[2], op->args[3]);
1174 return tcg_opt_gen_movi(ctx, op, op->args[0], t);
1175 }
1176 return false;
1177}
1178
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001179static bool fold_shift(OptContext *ctx, TCGOp *op)
1180{
1181 return fold_const2(ctx, op);
1182}
1183
1184static bool fold_sub(OptContext *ctx, TCGOp *op)
1185{
1186 return fold_const2(ctx, op);
1187}
1188
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001189static bool fold_sub2_i32(OptContext *ctx, TCGOp *op)
1190{
1191 return fold_addsub2_i32(ctx, op, false);
1192}
1193
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001194static bool fold_xor(OptContext *ctx, TCGOp *op)
1195{
1196 return fold_const2(ctx, op);
1197}
1198
Kirill Batuzov22613af2011-07-07 16:37:13 +04001199/* Propagate constants and copies, fold constant expressions. */
Aurelien Jarno36e60ef2015-06-04 21:53:27 +02001200void tcg_optimize(TCGContext *s)
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001201{
Richard Henderson5cf32be2021-08-24 08:17:08 -07001202 int nb_temps, i;
Richard Hendersond0ed5152021-08-24 07:38:39 -07001203 TCGOp *op, *op_next;
Richard Hendersondc849882021-08-24 07:13:45 -07001204 OptContext ctx = { .tcg = s };
Richard Henderson5d8f5362012-09-21 10:13:38 -07001205
Kirill Batuzov22613af2011-07-07 16:37:13 +04001206 /* Array VALS has an element for each temp.
1207 If this temp holds a constant then its value is kept in VALS' element.
Aurelien Jarnoe590d4e2012-09-11 12:31:21 +02001208 If this temp is a copy of other ones then the other copies are
1209 available through the doubly linked circular list. */
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001210
1211 nb_temps = s->nb_temps;
Richard Henderson8f17a972020-03-30 19:52:02 -07001212 for (i = 0; i < nb_temps; ++i) {
1213 s->temps[i].state_ptr = NULL;
1214 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001215
Richard Henderson15fa08f2017-11-02 15:19:14 +01001216 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
Richard Hendersonb1fde412021-08-23 13:07:49 -07001217 uint64_t z_mask, partmask, affected, tmp;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001218 TCGOpcode opc = op->opc;
Richard Henderson5cf32be2021-08-24 08:17:08 -07001219 const TCGOpDef *def;
Richard Henderson404a1482021-08-24 11:08:21 -07001220 bool done = false;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001221
Richard Henderson5cf32be2021-08-24 08:17:08 -07001222 /* Calls are special. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001223 if (opc == INDEX_op_call) {
Richard Henderson5cf32be2021-08-24 08:17:08 -07001224 fold_call(&ctx, op);
1225 continue;
Richard Hendersoncf066672014-03-22 20:06:52 -07001226 }
Richard Henderson5cf32be2021-08-24 08:17:08 -07001227
1228 def = &tcg_op_defs[opc];
Richard Hendersonec5d4cb2021-08-24 08:20:27 -07001229 init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs);
1230 copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs);
Kirill Batuzov22613af2011-07-07 16:37:13 +04001231
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001232 /* For commutative operations make constant second argument */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001233 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001234 CASE_OP_32_64_VEC(add):
1235 CASE_OP_32_64_VEC(mul):
1236 CASE_OP_32_64_VEC(and):
1237 CASE_OP_32_64_VEC(or):
1238 CASE_OP_32_64_VEC(xor):
Richard Hendersoncb25c802011-08-17 14:11:47 -07001239 CASE_OP_32_64(eqv):
1240 CASE_OP_32_64(nand):
1241 CASE_OP_32_64(nor):
Richard Henderson03271522013-08-14 14:35:56 -07001242 CASE_OP_32_64(muluh):
1243 CASE_OP_32_64(mulsh):
Richard Hendersonacd93702016-12-08 12:28:42 -08001244 swap_commutative(op->args[0], &op->args[1], &op->args[2]);
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001245 break;
Aurelien Jarno65a7cce2012-09-06 16:47:14 +02001246 CASE_OP_32_64(brcond):
Richard Hendersonacd93702016-12-08 12:28:42 -08001247 if (swap_commutative(-1, &op->args[0], &op->args[1])) {
1248 op->args[2] = tcg_swap_cond(op->args[2]);
Aurelien Jarno65a7cce2012-09-06 16:47:14 +02001249 }
1250 break;
1251 CASE_OP_32_64(setcond):
Richard Hendersonacd93702016-12-08 12:28:42 -08001252 if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) {
1253 op->args[3] = tcg_swap_cond(op->args[3]);
Aurelien Jarno65a7cce2012-09-06 16:47:14 +02001254 }
1255 break;
Richard Hendersonfa01a202012-09-21 10:13:37 -07001256 CASE_OP_32_64(movcond):
Richard Hendersonacd93702016-12-08 12:28:42 -08001257 if (swap_commutative(-1, &op->args[1], &op->args[2])) {
1258 op->args[5] = tcg_swap_cond(op->args[5]);
Richard Hendersonfa01a202012-09-21 10:13:37 -07001259 }
Richard Henderson5d8f5362012-09-21 10:13:38 -07001260 /* For movcond, we canonicalize the "false" input reg to match
1261 the destination reg so that the tcg backend can implement
1262 a "move if true" operation. */
Richard Hendersonacd93702016-12-08 12:28:42 -08001263 if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
1264 op->args[5] = tcg_invert_cond(op->args[5]);
Richard Henderson5d8f5362012-09-21 10:13:38 -07001265 }
Richard Henderson1e484e62012-10-02 11:32:22 -07001266 break;
Richard Hendersond7156f72013-02-19 23:51:52 -08001267 CASE_OP_32_64(add2):
Richard Hendersonacd93702016-12-08 12:28:42 -08001268 swap_commutative(op->args[0], &op->args[2], &op->args[4]);
1269 swap_commutative(op->args[1], &op->args[3], &op->args[5]);
Richard Henderson1e484e62012-10-02 11:32:22 -07001270 break;
Richard Hendersond7156f72013-02-19 23:51:52 -08001271 CASE_OP_32_64(mulu2):
Richard Henderson4d3203f2013-02-19 23:51:53 -08001272 CASE_OP_32_64(muls2):
Richard Hendersonacd93702016-12-08 12:28:42 -08001273 swap_commutative(op->args[0], &op->args[2], &op->args[3]);
Richard Henderson14149682012-10-02 11:32:30 -07001274 break;
Richard Henderson0bfcb862012-10-02 11:32:23 -07001275 case INDEX_op_brcond2_i32:
Richard Hendersonacd93702016-12-08 12:28:42 -08001276 if (swap_commutative2(&op->args[0], &op->args[2])) {
1277 op->args[4] = tcg_swap_cond(op->args[4]);
Richard Henderson0bfcb862012-10-02 11:32:23 -07001278 }
1279 break;
1280 case INDEX_op_setcond2_i32:
Richard Hendersonacd93702016-12-08 12:28:42 -08001281 if (swap_commutative2(&op->args[1], &op->args[3])) {
1282 op->args[5] = tcg_swap_cond(op->args[5]);
Richard Henderson0bfcb862012-10-02 11:32:23 -07001283 }
1284 break;
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001285 default:
1286 break;
1287 }
1288
Richard Henderson2d497542013-03-21 09:13:33 -07001289 /* Simplify expressions for "shift/rot r, 0, a => movi r, 0",
1290 and "sub r, 0, a => neg r, a" case. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001291 switch (opc) {
Aurelien Jarno01ee5282012-09-06 16:47:14 +02001292 CASE_OP_32_64(shl):
1293 CASE_OP_32_64(shr):
1294 CASE_OP_32_64(sar):
1295 CASE_OP_32_64(rotl):
1296 CASE_OP_32_64(rotr):
Richard Henderson63490392017-06-20 13:43:15 -07001297 if (arg_is_const(op->args[1])
1298 && arg_info(op->args[1])->val == 0) {
Richard Hendersondc849882021-08-24 07:13:45 -07001299 tcg_opt_gen_movi(&ctx, op, op->args[0], 0);
Aurelien Jarno01ee5282012-09-06 16:47:14 +02001300 continue;
1301 }
1302 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001303 CASE_OP_32_64_VEC(sub):
Richard Henderson2d497542013-03-21 09:13:33 -07001304 {
1305 TCGOpcode neg_op;
1306 bool have_neg;
1307
Richard Henderson63490392017-06-20 13:43:15 -07001308 if (arg_is_const(op->args[2])) {
Richard Henderson2d497542013-03-21 09:13:33 -07001309 /* Proceed with possible constant folding. */
1310 break;
1311 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001312 if (opc == INDEX_op_sub_i32) {
Richard Henderson2d497542013-03-21 09:13:33 -07001313 neg_op = INDEX_op_neg_i32;
1314 have_neg = TCG_TARGET_HAS_neg_i32;
Richard Henderson170ba882017-11-22 09:07:11 +01001315 } else if (opc == INDEX_op_sub_i64) {
Richard Henderson2d497542013-03-21 09:13:33 -07001316 neg_op = INDEX_op_neg_i64;
1317 have_neg = TCG_TARGET_HAS_neg_i64;
Richard Hendersonac383dde2019-04-20 00:27:24 +00001318 } else if (TCG_TARGET_HAS_neg_vec) {
1319 TCGType type = TCGOP_VECL(op) + TCG_TYPE_V64;
1320 unsigned vece = TCGOP_VECE(op);
Richard Henderson170ba882017-11-22 09:07:11 +01001321 neg_op = INDEX_op_neg_vec;
Richard Hendersonac383dde2019-04-20 00:27:24 +00001322 have_neg = tcg_can_emit_vec_op(neg_op, type, vece) > 0;
1323 } else {
1324 break;
Richard Henderson2d497542013-03-21 09:13:33 -07001325 }
1326 if (!have_neg) {
1327 break;
1328 }
Richard Henderson63490392017-06-20 13:43:15 -07001329 if (arg_is_const(op->args[1])
1330 && arg_info(op->args[1])->val == 0) {
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001331 op->opc = neg_op;
Richard Hendersonacd93702016-12-08 12:28:42 -08001332 reset_temp(op->args[0]);
1333 op->args[1] = op->args[2];
Richard Henderson2d497542013-03-21 09:13:33 -07001334 continue;
1335 }
1336 }
1337 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001338 CASE_OP_32_64_VEC(xor):
Richard Hendersone201b562014-01-28 13:15:38 -08001339 CASE_OP_32_64(nand):
Richard Henderson63490392017-06-20 13:43:15 -07001340 if (!arg_is_const(op->args[1])
1341 && arg_is_const(op->args[2])
1342 && arg_info(op->args[2])->val == -1) {
Richard Hendersone201b562014-01-28 13:15:38 -08001343 i = 1;
1344 goto try_not;
1345 }
1346 break;
1347 CASE_OP_32_64(nor):
Richard Henderson63490392017-06-20 13:43:15 -07001348 if (!arg_is_const(op->args[1])
1349 && arg_is_const(op->args[2])
1350 && arg_info(op->args[2])->val == 0) {
Richard Hendersone201b562014-01-28 13:15:38 -08001351 i = 1;
1352 goto try_not;
1353 }
1354 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001355 CASE_OP_32_64_VEC(andc):
Richard Henderson63490392017-06-20 13:43:15 -07001356 if (!arg_is_const(op->args[2])
1357 && arg_is_const(op->args[1])
1358 && arg_info(op->args[1])->val == -1) {
Richard Hendersone201b562014-01-28 13:15:38 -08001359 i = 2;
1360 goto try_not;
1361 }
1362 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001363 CASE_OP_32_64_VEC(orc):
Richard Hendersone201b562014-01-28 13:15:38 -08001364 CASE_OP_32_64(eqv):
Richard Henderson63490392017-06-20 13:43:15 -07001365 if (!arg_is_const(op->args[2])
1366 && arg_is_const(op->args[1])
1367 && arg_info(op->args[1])->val == 0) {
Richard Hendersone201b562014-01-28 13:15:38 -08001368 i = 2;
1369 goto try_not;
1370 }
1371 break;
1372 try_not:
1373 {
1374 TCGOpcode not_op;
1375 bool have_not;
1376
Richard Henderson170ba882017-11-22 09:07:11 +01001377 if (def->flags & TCG_OPF_VECTOR) {
1378 not_op = INDEX_op_not_vec;
1379 have_not = TCG_TARGET_HAS_not_vec;
1380 } else if (def->flags & TCG_OPF_64BIT) {
Richard Hendersone201b562014-01-28 13:15:38 -08001381 not_op = INDEX_op_not_i64;
1382 have_not = TCG_TARGET_HAS_not_i64;
1383 } else {
1384 not_op = INDEX_op_not_i32;
1385 have_not = TCG_TARGET_HAS_not_i32;
1386 }
1387 if (!have_not) {
1388 break;
1389 }
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001390 op->opc = not_op;
Richard Hendersonacd93702016-12-08 12:28:42 -08001391 reset_temp(op->args[0]);
1392 op->args[1] = op->args[i];
Richard Hendersone201b562014-01-28 13:15:38 -08001393 continue;
1394 }
Aurelien Jarno01ee5282012-09-06 16:47:14 +02001395 default:
1396 break;
1397 }
1398
Richard Henderson464a1442014-01-31 07:42:11 -06001399 /* Simplify expression for "op r, a, const => mov r, a" cases */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001400 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001401 CASE_OP_32_64_VEC(add):
1402 CASE_OP_32_64_VEC(sub):
1403 CASE_OP_32_64_VEC(or):
1404 CASE_OP_32_64_VEC(xor):
1405 CASE_OP_32_64_VEC(andc):
Kirill Batuzov55c09752011-07-07 16:37:16 +04001406 CASE_OP_32_64(shl):
1407 CASE_OP_32_64(shr):
1408 CASE_OP_32_64(sar):
Richard Henderson25c4d9c2011-08-17 14:11:46 -07001409 CASE_OP_32_64(rotl):
1410 CASE_OP_32_64(rotr):
Richard Henderson63490392017-06-20 13:43:15 -07001411 if (!arg_is_const(op->args[1])
1412 && arg_is_const(op->args[2])
1413 && arg_info(op->args[2])->val == 0) {
Richard Hendersondc849882021-08-24 07:13:45 -07001414 tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]);
Aurelien Jarno97a79eb2015-06-05 11:19:18 +02001415 continue;
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001416 }
1417 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001418 CASE_OP_32_64_VEC(and):
1419 CASE_OP_32_64_VEC(orc):
Richard Henderson464a1442014-01-31 07:42:11 -06001420 CASE_OP_32_64(eqv):
Richard Henderson63490392017-06-20 13:43:15 -07001421 if (!arg_is_const(op->args[1])
1422 && arg_is_const(op->args[2])
1423 && arg_info(op->args[2])->val == -1) {
Richard Hendersondc849882021-08-24 07:13:45 -07001424 tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]);
Aurelien Jarno97a79eb2015-06-05 11:19:18 +02001425 continue;
Richard Henderson464a1442014-01-31 07:42:11 -06001426 }
1427 break;
Aurelien Jarno56e49432012-09-06 16:47:13 +02001428 default:
1429 break;
1430 }
1431
Aurelien Jarno30312442013-09-03 08:27:38 +02001432 /* Simplify using known-zero bits. Currently only ops with a single
1433 output argument is supported. */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001434 z_mask = -1;
Paolo Bonzini633f6502013-01-11 15:42:53 -08001435 affected = -1;
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001436 switch (opc) {
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001437 CASE_OP_32_64(ext8s):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001438 if ((arg_info(op->args[1])->z_mask & 0x80) != 0) {
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001439 break;
1440 }
Thomas Huthd84568b2020-12-11 16:24:24 +01001441 QEMU_FALLTHROUGH;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001442 CASE_OP_32_64(ext8u):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001443 z_mask = 0xff;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001444 goto and_const;
1445 CASE_OP_32_64(ext16s):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001446 if ((arg_info(op->args[1])->z_mask & 0x8000) != 0) {
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001447 break;
1448 }
Thomas Huthd84568b2020-12-11 16:24:24 +01001449 QEMU_FALLTHROUGH;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001450 CASE_OP_32_64(ext16u):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001451 z_mask = 0xffff;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001452 goto and_const;
1453 case INDEX_op_ext32s_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001454 if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) {
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001455 break;
1456 }
Thomas Huthd84568b2020-12-11 16:24:24 +01001457 QEMU_FALLTHROUGH;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001458 case INDEX_op_ext32u_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001459 z_mask = 0xffffffffU;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001460 goto and_const;
1461
1462 CASE_OP_32_64(and):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001463 z_mask = arg_info(op->args[2])->z_mask;
Richard Henderson63490392017-06-20 13:43:15 -07001464 if (arg_is_const(op->args[2])) {
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001465 and_const:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001466 affected = arg_info(op->args[1])->z_mask & ~z_mask;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001467 }
Richard Hendersonb1fde412021-08-23 13:07:49 -07001468 z_mask = arg_info(op->args[1])->z_mask & z_mask;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001469 break;
1470
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +02001471 case INDEX_op_ext_i32_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001472 if ((arg_info(op->args[1])->z_mask & 0x80000000) != 0) {
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +02001473 break;
1474 }
Thomas Huthd84568b2020-12-11 16:24:24 +01001475 QEMU_FALLTHROUGH;
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +02001476 case INDEX_op_extu_i32_i64:
1477 /* We do not compute affected as it is a size changing op. */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001478 z_mask = (uint32_t)arg_info(op->args[1])->z_mask;
Aurelien Jarno8bcb5c82015-07-27 12:41:45 +02001479 break;
1480
Richard Henderson23ec69ed2014-01-28 12:03:24 -08001481 CASE_OP_32_64(andc):
1482 /* Known-zeros does not imply known-ones. Therefore unless
Richard Hendersonacd93702016-12-08 12:28:42 -08001483 op->args[2] is constant, we can't infer anything from it. */
Richard Henderson63490392017-06-20 13:43:15 -07001484 if (arg_is_const(op->args[2])) {
Richard Hendersonb1fde412021-08-23 13:07:49 -07001485 z_mask = ~arg_info(op->args[2])->z_mask;
Richard Henderson23ec69ed2014-01-28 12:03:24 -08001486 goto and_const;
1487 }
Richard Henderson63490392017-06-20 13:43:15 -07001488 /* But we certainly know nothing outside args[1] may be set. */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001489 z_mask = arg_info(op->args[1])->z_mask;
Richard Henderson23ec69ed2014-01-28 12:03:24 -08001490 break;
1491
Aurelien Jarnoe46b2252013-09-03 08:27:38 +02001492 case INDEX_op_sar_i32:
Richard Henderson63490392017-06-20 13:43:15 -07001493 if (arg_is_const(op->args[2])) {
1494 tmp = arg_info(op->args[2])->val & 31;
Richard Hendersonb1fde412021-08-23 13:07:49 -07001495 z_mask = (int32_t)arg_info(op->args[1])->z_mask >> tmp;
Aurelien Jarnoe46b2252013-09-03 08:27:38 +02001496 }
1497 break;
1498 case INDEX_op_sar_i64:
Richard Henderson63490392017-06-20 13:43:15 -07001499 if (arg_is_const(op->args[2])) {
1500 tmp = arg_info(op->args[2])->val & 63;
Richard Hendersonb1fde412021-08-23 13:07:49 -07001501 z_mask = (int64_t)arg_info(op->args[1])->z_mask >> tmp;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001502 }
1503 break;
1504
Aurelien Jarnoe46b2252013-09-03 08:27:38 +02001505 case INDEX_op_shr_i32:
Richard Henderson63490392017-06-20 13:43:15 -07001506 if (arg_is_const(op->args[2])) {
1507 tmp = arg_info(op->args[2])->val & 31;
Richard Hendersonb1fde412021-08-23 13:07:49 -07001508 z_mask = (uint32_t)arg_info(op->args[1])->z_mask >> tmp;
Aurelien Jarnoe46b2252013-09-03 08:27:38 +02001509 }
1510 break;
1511 case INDEX_op_shr_i64:
Richard Henderson63490392017-06-20 13:43:15 -07001512 if (arg_is_const(op->args[2])) {
1513 tmp = arg_info(op->args[2])->val & 63;
Richard Hendersonb1fde412021-08-23 13:07:49 -07001514 z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> tmp;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001515 }
1516 break;
1517
Richard Henderson609ad702015-07-24 07:16:00 -07001518 case INDEX_op_extrl_i64_i32:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001519 z_mask = (uint32_t)arg_info(op->args[1])->z_mask;
Richard Henderson609ad702015-07-24 07:16:00 -07001520 break;
1521 case INDEX_op_extrh_i64_i32:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001522 z_mask = (uint64_t)arg_info(op->args[1])->z_mask >> 32;
Richard Henderson4bb7a412013-09-09 17:03:24 -07001523 break;
1524
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001525 CASE_OP_32_64(shl):
Richard Henderson63490392017-06-20 13:43:15 -07001526 if (arg_is_const(op->args[2])) {
1527 tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1);
Richard Hendersonb1fde412021-08-23 13:07:49 -07001528 z_mask = arg_info(op->args[1])->z_mask << tmp;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001529 }
1530 break;
1531
1532 CASE_OP_32_64(neg):
1533 /* Set to 1 all bits to the left of the rightmost. */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001534 z_mask = -(arg_info(op->args[1])->z_mask
1535 & -arg_info(op->args[1])->z_mask);
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001536 break;
1537
1538 CASE_OP_32_64(deposit):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001539 z_mask = deposit64(arg_info(op->args[1])->z_mask,
1540 op->args[3], op->args[4],
1541 arg_info(op->args[2])->z_mask);
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001542 break;
1543
Richard Henderson7ec8bab2016-10-14 12:04:32 -05001544 CASE_OP_32_64(extract):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001545 z_mask = extract64(arg_info(op->args[1])->z_mask,
1546 op->args[2], op->args[3]);
Richard Hendersonacd93702016-12-08 12:28:42 -08001547 if (op->args[2] == 0) {
Richard Hendersonb1fde412021-08-23 13:07:49 -07001548 affected = arg_info(op->args[1])->z_mask & ~z_mask;
Richard Henderson7ec8bab2016-10-14 12:04:32 -05001549 }
1550 break;
1551 CASE_OP_32_64(sextract):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001552 z_mask = sextract64(arg_info(op->args[1])->z_mask,
1553 op->args[2], op->args[3]);
1554 if (op->args[2] == 0 && (tcg_target_long)z_mask >= 0) {
1555 affected = arg_info(op->args[1])->z_mask & ~z_mask;
Richard Henderson7ec8bab2016-10-14 12:04:32 -05001556 }
1557 break;
1558
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001559 CASE_OP_32_64(or):
1560 CASE_OP_32_64(xor):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001561 z_mask = arg_info(op->args[1])->z_mask
1562 | arg_info(op->args[2])->z_mask;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001563 break;
1564
Richard Henderson0e28d002016-11-16 09:23:28 +01001565 case INDEX_op_clz_i32:
1566 case INDEX_op_ctz_i32:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001567 z_mask = arg_info(op->args[2])->z_mask | 31;
Richard Henderson0e28d002016-11-16 09:23:28 +01001568 break;
1569
1570 case INDEX_op_clz_i64:
1571 case INDEX_op_ctz_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001572 z_mask = arg_info(op->args[2])->z_mask | 63;
Richard Henderson0e28d002016-11-16 09:23:28 +01001573 break;
1574
Richard Hendersona768e4e2016-11-21 11:13:39 +01001575 case INDEX_op_ctpop_i32:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001576 z_mask = 32 | 31;
Richard Hendersona768e4e2016-11-21 11:13:39 +01001577 break;
1578 case INDEX_op_ctpop_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001579 z_mask = 64 | 63;
Richard Hendersona768e4e2016-11-21 11:13:39 +01001580 break;
1581
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001582 CASE_OP_32_64(setcond):
Richard Hendersona7635512014-04-23 22:18:30 -07001583 case INDEX_op_setcond2_i32:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001584 z_mask = 1;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001585 break;
1586
1587 CASE_OP_32_64(movcond):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001588 z_mask = arg_info(op->args[3])->z_mask
1589 | arg_info(op->args[4])->z_mask;
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001590 break;
1591
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001592 CASE_OP_32_64(ld8u):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001593 z_mask = 0xff;
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001594 break;
1595 CASE_OP_32_64(ld16u):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001596 z_mask = 0xffff;
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001597 break;
1598 case INDEX_op_ld32u_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001599 z_mask = 0xffffffffu;
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001600 break;
1601
1602 CASE_OP_32_64(qemu_ld):
1603 {
Richard Hendersonec5d4cb2021-08-24 08:20:27 -07001604 MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs];
Tony Nguyen14776ab2019-08-24 04:10:58 +10001605 MemOp mop = get_memop(oi);
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001606 if (!(mop & MO_SIGN)) {
Richard Hendersonb1fde412021-08-23 13:07:49 -07001607 z_mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1;
Aurelien Jarnoc8d70272013-09-03 08:27:39 +02001608 }
1609 }
1610 break;
1611
Richard Henderson0b76ff82021-06-13 13:04:00 -07001612 CASE_OP_32_64(bswap16):
Richard Hendersonb1fde412021-08-23 13:07:49 -07001613 z_mask = arg_info(op->args[1])->z_mask;
1614 if (z_mask <= 0xffff) {
Richard Henderson0b76ff82021-06-13 13:04:00 -07001615 op->args[2] |= TCG_BSWAP_IZ;
1616 }
Richard Hendersonb1fde412021-08-23 13:07:49 -07001617 z_mask = bswap16(z_mask);
Richard Henderson0b76ff82021-06-13 13:04:00 -07001618 switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) {
1619 case TCG_BSWAP_OZ:
1620 break;
1621 case TCG_BSWAP_OS:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001622 z_mask = (int16_t)z_mask;
Richard Henderson0b76ff82021-06-13 13:04:00 -07001623 break;
1624 default: /* undefined high bits */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001625 z_mask |= MAKE_64BIT_MASK(16, 48);
Richard Henderson0b76ff82021-06-13 13:04:00 -07001626 break;
1627 }
1628 break;
1629
1630 case INDEX_op_bswap32_i64:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001631 z_mask = arg_info(op->args[1])->z_mask;
1632 if (z_mask <= 0xffffffffu) {
Richard Henderson0b76ff82021-06-13 13:04:00 -07001633 op->args[2] |= TCG_BSWAP_IZ;
1634 }
Richard Hendersonb1fde412021-08-23 13:07:49 -07001635 z_mask = bswap32(z_mask);
Richard Henderson0b76ff82021-06-13 13:04:00 -07001636 switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) {
1637 case TCG_BSWAP_OZ:
1638 break;
1639 case TCG_BSWAP_OS:
Richard Hendersonb1fde412021-08-23 13:07:49 -07001640 z_mask = (int32_t)z_mask;
Richard Henderson0b76ff82021-06-13 13:04:00 -07001641 break;
1642 default: /* undefined high bits */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001643 z_mask |= MAKE_64BIT_MASK(32, 32);
Richard Henderson0b76ff82021-06-13 13:04:00 -07001644 break;
1645 }
1646 break;
1647
Paolo Bonzini3a9d8b12013-01-11 15:42:52 -08001648 default:
1649 break;
1650 }
1651
Richard Hendersonbc8d6882014-06-08 18:24:14 -07001652 /* 32-bit ops generate 32-bit results. For the result is zero test
1653 below, we can ignore high bits, but for further optimizations we
1654 need to record that the high bits contain garbage. */
Richard Hendersonb1fde412021-08-23 13:07:49 -07001655 partmask = z_mask;
Richard Hendersonbc8d6882014-06-08 18:24:14 -07001656 if (!(def->flags & TCG_OPF_64BIT)) {
Richard Hendersonb1fde412021-08-23 13:07:49 -07001657 z_mask |= ~(tcg_target_ulong)0xffffffffu;
Richard Henderson24666ba2014-05-22 11:14:10 -07001658 partmask &= 0xffffffffu;
1659 affected &= 0xffffffffu;
Aurelien Jarnof096dc92013-09-03 08:27:38 +02001660 }
Richard Henderson137f1f42021-08-24 08:49:25 -07001661 ctx.z_mask = z_mask;
Aurelien Jarnof096dc92013-09-03 08:27:38 +02001662
Richard Henderson24666ba2014-05-22 11:14:10 -07001663 if (partmask == 0) {
Richard Hendersondc849882021-08-24 07:13:45 -07001664 tcg_opt_gen_movi(&ctx, op, op->args[0], 0);
Paolo Bonzini633f6502013-01-11 15:42:53 -08001665 continue;
1666 }
1667 if (affected == 0) {
Richard Hendersondc849882021-08-24 07:13:45 -07001668 tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]);
Paolo Bonzini633f6502013-01-11 15:42:53 -08001669 continue;
1670 }
1671
Aurelien Jarno56e49432012-09-06 16:47:13 +02001672 /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001673 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001674 CASE_OP_32_64_VEC(and):
1675 CASE_OP_32_64_VEC(mul):
Richard Henderson03271522013-08-14 14:35:56 -07001676 CASE_OP_32_64(muluh):
1677 CASE_OP_32_64(mulsh):
Richard Henderson63490392017-06-20 13:43:15 -07001678 if (arg_is_const(op->args[2])
1679 && arg_info(op->args[2])->val == 0) {
Richard Hendersondc849882021-08-24 07:13:45 -07001680 tcg_opt_gen_movi(&ctx, op, op->args[0], 0);
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001681 continue;
1682 }
1683 break;
Aurelien Jarno56e49432012-09-06 16:47:13 +02001684 default:
1685 break;
1686 }
1687
1688 /* Simplify expression for "op r, a, a => mov r, a" cases */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001689 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001690 CASE_OP_32_64_VEC(or):
1691 CASE_OP_32_64_VEC(and):
Richard Henderson63490392017-06-20 13:43:15 -07001692 if (args_are_copies(op->args[1], op->args[2])) {
Richard Hendersondc849882021-08-24 07:13:45 -07001693 tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]);
Kirill Batuzov9a810902011-07-07 16:37:15 +04001694 continue;
1695 }
1696 break;
Blue Swirlfe0de7a2011-07-30 19:18:32 +00001697 default:
1698 break;
Kirill Batuzov53108fb2011-07-07 16:37:14 +04001699 }
1700
Aurelien Jarno3c941932012-09-18 19:12:36 +02001701 /* Simplify expression for "op r, a, a => movi r, 0" cases */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001702 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001703 CASE_OP_32_64_VEC(andc):
1704 CASE_OP_32_64_VEC(sub):
1705 CASE_OP_32_64_VEC(xor):
Richard Henderson63490392017-06-20 13:43:15 -07001706 if (args_are_copies(op->args[1], op->args[2])) {
Richard Hendersondc849882021-08-24 07:13:45 -07001707 tcg_opt_gen_movi(&ctx, op, op->args[0], 0);
Aurelien Jarno3c941932012-09-18 19:12:36 +02001708 continue;
1709 }
1710 break;
1711 default:
1712 break;
1713 }
1714
Kirill Batuzov22613af2011-07-07 16:37:13 +04001715 /* Propagate constants through copy operations and do constant
1716 folding. Constants will be substituted to arguments by register
1717 allocator where needed and possible. Also detect copies. */
Richard Hendersonc45cb8b2014-09-19 13:49:15 -07001718 switch (opc) {
Richard Henderson170ba882017-11-22 09:07:11 +01001719 CASE_OP_32_64_VEC(mov):
Richard Henderson404a1482021-08-24 11:08:21 -07001720 done = tcg_opt_gen_mov(&ctx, op, op->args[0], op->args[1]);
1721 break;
Richard Henderson6e14e912012-10-02 11:32:24 -07001722
Richard Henderson170ba882017-11-22 09:07:11 +01001723 case INDEX_op_dup_vec:
1724 if (arg_is_const(op->args[1])) {
1725 tmp = arg_info(op->args[1])->val;
1726 tmp = dup_const(TCGOP_VECE(op), tmp);
Richard Hendersondc849882021-08-24 07:13:45 -07001727 tcg_opt_gen_movi(&ctx, op, op->args[0], tmp);
Richard Hendersonb10f3832021-08-23 22:30:17 -07001728 continue;
Richard Henderson170ba882017-11-22 09:07:11 +01001729 }
Richard Hendersonb10f3832021-08-23 22:30:17 -07001730 break;
Richard Henderson170ba882017-11-22 09:07:11 +01001731
Richard Henderson1dc4fe72020-09-05 17:03:35 -07001732 case INDEX_op_dup2_vec:
1733 assert(TCG_TARGET_REG_BITS == 32);
1734 if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
Richard Hendersondc849882021-08-24 07:13:45 -07001735 tcg_opt_gen_movi(&ctx, op, op->args[0],
Richard Henderson0b4286d2020-09-06 17:33:18 -07001736 deposit64(arg_info(op->args[1])->val, 32, 32,
1737 arg_info(op->args[2])->val));
Richard Hendersonb10f3832021-08-23 22:30:17 -07001738 continue;
Richard Henderson1dc4fe72020-09-05 17:03:35 -07001739 } else if (args_are_copies(op->args[1], op->args[2])) {
1740 op->opc = INDEX_op_dup_vec;
1741 TCGOP_VECE(op) = MO_32;
Richard Henderson1dc4fe72020-09-05 17:03:35 -07001742 }
Richard Hendersonb10f3832021-08-23 22:30:17 -07001743 break;
Richard Henderson1dc4fe72020-09-05 17:03:35 -07001744
Richard Henderson0b76ff82021-06-13 13:04:00 -07001745 CASE_OP_32_64(bswap16):
1746 CASE_OP_32_64(bswap32):
1747 case INDEX_op_bswap64_i64:
1748 if (arg_is_const(op->args[1])) {
1749 tmp = do_constant_folding(opc, arg_info(op->args[1])->val,
1750 op->args[2]);
Richard Hendersondc849882021-08-24 07:13:45 -07001751 tcg_opt_gen_movi(&ctx, op, op->args[0], tmp);
Richard Hendersonb10f3832021-08-23 22:30:17 -07001752 continue;
Richard Henderson0b76ff82021-06-13 13:04:00 -07001753 }
Richard Hendersonb10f3832021-08-23 22:30:17 -07001754 break;
Richard Henderson0b76ff82021-06-13 13:04:00 -07001755
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001756 default:
1757 break;
1758
1759 /* ---------------------------------------------------------- */
1760 /* Sorted alphabetically by opcode as much as possible. */
1761
1762 CASE_OP_32_64_VEC(add):
1763 done = fold_add(&ctx, op);
1764 break;
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001765 case INDEX_op_add2_i32:
1766 done = fold_add2_i32(&ctx, op);
1767 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001768 CASE_OP_32_64_VEC(and):
1769 done = fold_and(&ctx, op);
1770 break;
1771 CASE_OP_32_64_VEC(andc):
1772 done = fold_andc(&ctx, op);
1773 break;
Richard Henderson079b0802021-08-24 09:30:59 -07001774 CASE_OP_32_64(brcond):
1775 done = fold_brcond(&ctx, op);
1776 break;
Richard Henderson764d2ab2021-08-24 09:22:11 -07001777 case INDEX_op_brcond2_i32:
1778 done = fold_brcond2(&ctx, op);
1779 break;
Richard Henderson30dd0bf2021-08-24 10:51:34 -07001780 CASE_OP_32_64(clz):
1781 CASE_OP_32_64(ctz):
1782 done = fold_count_zeros(&ctx, op);
1783 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001784 CASE_OP_32_64(ctpop):
1785 done = fold_ctpop(&ctx, op);
1786 break;
Richard Henderson1b1907b2021-08-24 10:47:04 -07001787 CASE_OP_32_64(deposit):
1788 done = fold_deposit(&ctx, op);
1789 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001790 CASE_OP_32_64(div):
1791 CASE_OP_32_64(divu):
1792 done = fold_divide(&ctx, op);
1793 break;
1794 CASE_OP_32_64(eqv):
1795 done = fold_eqv(&ctx, op);
1796 break;
Richard Hendersonb6617c82021-08-24 10:44:53 -07001797 CASE_OP_32_64(extract):
1798 done = fold_extract(&ctx, op);
1799 break;
Richard Hendersondcd08992021-08-24 10:41:39 -07001800 CASE_OP_32_64(extract2):
1801 done = fold_extract2(&ctx, op);
1802 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001803 CASE_OP_32_64(ext8s):
1804 CASE_OP_32_64(ext16s):
1805 case INDEX_op_ext32s_i64:
1806 case INDEX_op_ext_i32_i64:
1807 done = fold_exts(&ctx, op);
1808 break;
1809 CASE_OP_32_64(ext8u):
1810 CASE_OP_32_64(ext16u):
1811 case INDEX_op_ext32u_i64:
1812 case INDEX_op_extu_i32_i64:
1813 case INDEX_op_extrl_i64_i32:
1814 case INDEX_op_extrh_i64_i32:
1815 done = fold_extu(&ctx, op);
1816 break;
Richard Henderson3eefdf22021-08-25 11:06:43 -07001817 case INDEX_op_mb:
1818 done = fold_mb(&ctx, op);
1819 break;
Richard Henderson0c310a32021-08-24 10:37:24 -07001820 CASE_OP_32_64(movcond):
1821 done = fold_movcond(&ctx, op);
1822 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001823 CASE_OP_32_64(mul):
1824 done = fold_mul(&ctx, op);
1825 break;
1826 CASE_OP_32_64(mulsh):
1827 CASE_OP_32_64(muluh):
1828 done = fold_mul_highpart(&ctx, op);
1829 break;
Richard Henderson6b8ac0d2021-08-24 10:24:12 -07001830 case INDEX_op_mulu2_i32:
1831 done = fold_mulu2_i32(&ctx, op);
1832 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001833 CASE_OP_32_64(nand):
1834 done = fold_nand(&ctx, op);
1835 break;
1836 CASE_OP_32_64(neg):
1837 done = fold_neg(&ctx, op);
1838 break;
1839 CASE_OP_32_64(nor):
1840 done = fold_nor(&ctx, op);
1841 break;
1842 CASE_OP_32_64_VEC(not):
1843 done = fold_not(&ctx, op);
1844 break;
1845 CASE_OP_32_64_VEC(or):
1846 done = fold_or(&ctx, op);
1847 break;
1848 CASE_OP_32_64_VEC(orc):
1849 done = fold_orc(&ctx, op);
1850 break;
Richard Henderson3eefdf22021-08-25 11:06:43 -07001851 case INDEX_op_qemu_ld_i32:
1852 case INDEX_op_qemu_ld_i64:
1853 done = fold_qemu_ld(&ctx, op);
1854 break;
1855 case INDEX_op_qemu_st_i32:
1856 case INDEX_op_qemu_st8_i32:
1857 case INDEX_op_qemu_st_i64:
1858 done = fold_qemu_st(&ctx, op);
1859 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001860 CASE_OP_32_64(rem):
1861 CASE_OP_32_64(remu):
1862 done = fold_remainder(&ctx, op);
1863 break;
1864 CASE_OP_32_64(rotl):
1865 CASE_OP_32_64(rotr):
1866 CASE_OP_32_64(sar):
1867 CASE_OP_32_64(shl):
1868 CASE_OP_32_64(shr):
1869 done = fold_shift(&ctx, op);
1870 break;
Richard Hendersonc63ff552021-08-24 09:35:30 -07001871 CASE_OP_32_64(setcond):
1872 done = fold_setcond(&ctx, op);
1873 break;
Richard Hendersonbc47b1a2021-08-24 09:09:35 -07001874 case INDEX_op_setcond2_i32:
1875 done = fold_setcond2(&ctx, op);
1876 break;
Richard Hendersonb6617c82021-08-24 10:44:53 -07001877 CASE_OP_32_64(sextract):
1878 done = fold_sextract(&ctx, op);
1879 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001880 CASE_OP_32_64_VEC(sub):
1881 done = fold_sub(&ctx, op);
1882 break;
Richard Hendersone3f7dc22021-08-24 10:30:38 -07001883 case INDEX_op_sub2_i32:
1884 done = fold_sub2_i32(&ctx, op);
1885 break;
Richard Henderson2f9f08b2021-08-25 12:03:48 -07001886 CASE_OP_32_64_VEC(xor):
1887 done = fold_xor(&ctx, op);
Richard Hendersonb10f3832021-08-23 22:30:17 -07001888 break;
1889 }
1890
Richard Henderson404a1482021-08-24 11:08:21 -07001891 if (!done) {
1892 finish_folding(&ctx, op);
1893 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001894 }
Kirill Batuzov8f2e8c02011-07-07 16:37:12 +04001895}