21#include "internal/variable.h"
30#include "insns_info.inc"
33#define NOT_COMPILED_STACK_SIZE -1
34#define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE)
37struct inlined_call_context {
47struct compile_status {
49 int *stack_size_for_pos;
61 struct rb_mjit_compile_info *compile_info;
62 bool merge_ivar_guards_p;
64 size_t max_ivar_index;
67 struct inlined_call_context inline_context;
73struct compile_branch {
74 unsigned int stack_size;
78struct case_dispatch_var {
80 unsigned int base_pos;
96captured_cc_entries(
const struct compile_status *status)
98 VM_ASSERT(status->cc_entries_index != -1);
106 return vm_cc_cme(cc) !=
NULL;
114 return has_valid_method_type(cc) &&
127 && vm_call_iseq_optimizable_p(ci, cc);
145 struct case_dispatch_var *var = (
struct case_dispatch_var *)arg;
148 if (var->last_value != value) {
150 var->last_value = value;
151 fprintf(var->f,
" case %d:\n",
offset);
152 fprintf(var->f,
" goto label_%d;\n", var->base_pos +
offset);
153 fprintf(var->f,
" break;\n");
162#ifdef MJIT_COMMENT_ID
169 e = RSTRING_END(
name);
173 case '*':
case '/':
if (prev != (c ^ (
'/' ^
'*')))
break;
174 case '\\':
case '"': fputc(
'\\',
f);
184 unsigned int pos,
struct compile_status *status);
194 const unsigned int pos,
struct compile_status *status,
struct compile_branch *b)
196 unsigned int next_pos = pos + insn_len(insn);
199 #include "mjit_compile.inc"
204 if (!b->finish_p && next_pos < body->iseq_size && ALREADY_COMPILED_P(status, next_pos)) {
205 fprintf(
f,
"goto label_%d;\n", next_pos);
208 if ((
unsigned int)status->stack_size_for_pos[next_pos] != b->stack_size) {
210 fprintf(stderr,
"MJIT warning: JIT stack assumption is not the same between branches (%d != %u)\n",
211 status->stack_size_for_pos[next_pos], b->stack_size);
212 status->success =
false;
223 unsigned int pos,
struct compile_status *status)
226 struct compile_branch branch;
228 branch.stack_size = stack_size;
229 branch.finish_p =
false;
231 while (pos < body->iseq_size && !ALREADY_COMPILED_P(status, pos) && !branch.finish_p) {
232#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
237 status->stack_size_for_pos[pos] = (
int)branch.stack_size;
239 fprintf(
f,
"\nlabel_%d: /* %s */\n", pos, insn_name(insn));
240 pos = compile_insn(
f, body, insn, body->
iseq_encoded + (pos+1), pos, status, &branch);
241 if (status->success && branch.stack_size > body->
stack_max) {
243 fprintf(stderr,
"MJIT warning: JIT stack size (%d) exceeded its max size (%d)\n", branch.stack_size, body->
stack_max);
244 status->success =
false;
246 if (!status->success)
255 fprintf(
f,
"\ncancel:\n");
256 fprintf(
f,
" RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
257 fprintf(
f,
" rb_mjit_recompile_inlining(original_iseq);\n");
260 fprintf(
f,
" const VALUE *current_pc = reg_cfp->pc;\n");
261 fprintf(
f,
" VALUE *current_sp = reg_cfp->sp;\n");
262 fprintf(
f,
" reg_cfp->pc = orig_pc;\n");
263 fprintf(
f,
" reg_cfp->sp = orig_sp;\n\n");
266 fprintf(
f,
" struct rb_calling_info calling;\n");
267 fprintf(
f,
" calling.block_handler = VM_BLOCK_HANDLER_NONE;\n");
268 fprintf(
f,
" calling.argc = %d;\n", inline_context->orig_argc);
269 fprintf(
f,
" calling.recv = reg_cfp->self;\n");
270 fprintf(
f,
" reg_cfp->self = orig_self;\n");
271 fprintf(
f,
" vm_call_iseq_setup_normal(ec, reg_cfp, &calling, (const rb_callable_method_entry_t *)0x%"PRIxVALUE", 0, %d, %d);\n\n",
272 inline_context->me, inline_context->param_size, inline_context->local_size);
275 fprintf(
f,
" reg_cfp = ec->cfp;\n");
276 fprintf(
f,
" reg_cfp->pc = current_pc;\n");
277 fprintf(
f,
" reg_cfp->sp = current_sp;\n");
278 for (
unsigned int i = 0; i < body->
stack_max; i++) {
279 fprintf(
f,
" *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
283 fprintf(
f,
" return vm_exec(ec, false);\n");
290 if (status->inlined_iseqs ==
NULL) {
291 compile_inlined_cancel_handler(
f, body, &status->inline_context);
295 fprintf(
f,
"\nsend_cancel:\n");
296 fprintf(
f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_send_inline);\n");
297 fprintf(
f,
" rb_mjit_recompile_send(original_iseq);\n");
298 fprintf(
f,
" goto cancel;\n");
300 fprintf(
f,
"\nivar_cancel:\n");
301 fprintf(
f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_ivar_inline);\n");
302 fprintf(
f,
" rb_mjit_recompile_ivar(original_iseq);\n");
303 fprintf(
f,
" goto cancel;\n");
305 fprintf(
f,
"\nexivar_cancel:\n");
306 fprintf(
f,
" RB_DEBUG_COUNTER_INC(mjit_cancel_exivar_inline);\n");
307 fprintf(
f,
" rb_mjit_recompile_exivar(original_iseq);\n");
308 fprintf(
f,
" goto cancel;\n");
310 fprintf(
f,
"\nconst_cancel:\n");
311 fprintf(
f,
" rb_mjit_recompile_const(original_iseq);\n");
312 fprintf(
f,
" goto cancel;\n");
314 fprintf(
f,
"\ncancel:\n");
315 fprintf(
f,
" RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
316 if (status->local_stack_p) {
317 for (
unsigned int i = 0; i < body->
stack_max; i++) {
318 fprintf(
f,
" *(vm_base_ptr(reg_cfp) + %d) = stack[%d];\n", i, i);
321 fprintf(
f,
" return Qundef;\n");
333 if (is_entries ==
NULL)
339mjit_compile_body(
FILE *
f,
const rb_iseq_t *iseq,
struct compile_status *status)
342 status->success =
true;
345 if (status->local_stack_p) {
346 fprintf(
f,
" VALUE stack[%d];\n", body->
stack_max);
349 fprintf(
f,
" VALUE *stack = reg_cfp->sp;\n");
351 if (status->inlined_iseqs !=
NULL)
352 fprintf(
f,
" static const rb_iseq_t *original_iseq = (const rb_iseq_t *)0x%"PRIxVALUE";\n", (
VALUE)iseq);
353 fprintf(
f,
" static const VALUE *const original_body_iseq = (VALUE *)0x%"PRIxVALUE";\n",
355 fprintf(
f,
" VALUE cfp_self = reg_cfp->self;\n");
356 fprintf(
f,
"#define GET_SELF() cfp_self\n");
359 if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
360 fprintf(
f,
" if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
362 fprintf(
f,
"%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index);
365 fprintf(
f,
"ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())");
367 fprintf(
f,
"))) {\n");
368 fprintf(
f,
" goto ivar_cancel;\n");
377 fprintf(
f,
" switch (reg_cfp->pc - reg_cfp->iseq->body->iseq_encoded) {\n");
380 fprintf(
f,
" case %"PRIdVALUE":\n", pc_offset);
381 fprintf(
f,
" goto label_%"PRIdVALUE";\n", pc_offset);
386 compile_insns(
f, body, 0, 0, status);
387 compile_cancel_handler(
f, body, status);
388 fprintf(
f,
"#undef GET_SELF");
389 return status->success;
407 unsigned int pos = 0;
409#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
419 if (insn == BIN(invokebuiltin) || insn == BIN(opt_invokebuiltin_delegate) || insn == BIN(opt_invokebuiltin_delegate_leave)) {
424 else if (insn != BIN(leave) && insn_may_depend_on_sp_or_pc(insn, body->
iseq_encoded + (pos + 1)))
429 case BIN(getlocal_WC_0):
430 case BIN(getlocal_WC_1):
432 case BIN(setlocal_WC_0):
433 case BIN(setlocal_WC_1):
434 case BIN(getblockparam):
435 case BIN(getblockparamproxy):
436 case BIN(setblockparam):
439 pos += insn_len(insn);
449 if (has_valid_method_type(cc) &&
452 fastpath_applied_iseq_p(ci, cc, iseq = def_iseq_ptr(vm_cc_cme(cc)->
def)) &&
454 inlinable_iseq_p(iseq->
body)) {
463 mjit_capture_is_entries(body, status->is_entries);
466 unsigned int pos = 0;
467 status->max_ivar_index = 0;
468 status->ivar_serial = 0;
471#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
476 if (insn == BIN(getinstancevariable) || insn == BIN(setinstancevariable)) {
479 if (ic_copy->entry) {
482 if (status->max_ivar_index < ic_copy->entry->index) {
483 status->max_ivar_index = ic_copy->
entry->
index;
486 if (status->ivar_serial == 0) {
487 status->ivar_serial = ic_copy->entry->class_serial;
489 else if (status->ivar_serial != ic_copy->entry->class_serial) {
491 status->merge_ivar_guards_p =
false;
496 pos += insn_len(insn);
498 status->merge_ivar_guards_p = status->ivar_serial > 0 && num_ivars >= 2;
502#define INIT_COMPILE_STATUS(status, body, compile_root_p) do { \
503 status = (struct compile_status){ \
504 .stack_size_for_pos = (int *)alloca(sizeof(int) * body->iseq_size), \
505 .inlined_iseqs = compile_root_p ? \
506 alloca(sizeof(const struct rb_iseq_constant_body *) * body->iseq_size) : NULL, \
507 .is_entries = (body->is_size > 0) ? \
508 alloca(sizeof(union iseq_inline_storage_entry) * body->is_size) : NULL, \
509 .cc_entries_index = (body->ci_size > 0) ? \
510 mjit_capture_cc_entries(status.compiled_iseq, body) : -1, \
511 .compiled_id = status.compiled_id, \
512 .compiled_iseq = status.compiled_iseq, \
513 .compile_info = compile_root_p ? \
514 rb_mjit_iseq_compile_info(body) : alloca(sizeof(struct rb_mjit_compile_info)) \
516 memset(status.stack_size_for_pos, NOT_COMPILED_STACK_SIZE, sizeof(int) * body->iseq_size); \
517 if (compile_root_p) \
518 memset((void *)status.inlined_iseqs, 0, sizeof(const struct rb_iseq_constant_body *) * body->iseq_size); \
520 memset(status.compile_info, 0, sizeof(struct rb_mjit_compile_info)); \
525precompile_inlinable_iseqs(
FILE *
f,
const rb_iseq_t *iseq,
struct compile_status *status)
528 unsigned int pos = 0;
530#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
535 if (insn == BIN(opt_send_without_block)) {
538 const struct rb_callcache *cc = captured_cc_entries(status)[call_data_index(cd, body)];
543 status->inlined_iseqs[pos] = child_iseq->
body;
546 fprintf(stderr,
"JIT inline: %s@%s:%d => %s@%s:%d\n",
552 struct compile_status child_status = { .compiled_iseq = status->compiled_iseq, .compiled_id = status->compiled_id };
553 INIT_COMPILE_STATUS(child_status, child_iseq->
body,
false);
554 child_status.inline_context = (
struct inlined_call_context){
555 .orig_argc = vm_ci_argc(ci),
556 .me = (
VALUE)vm_cc_cme(cc),
560 if (child_iseq->
body->
ci_size > 0 && child_status.cc_entries_index == -1) {
563 init_ivar_compile_status(child_iseq->
body, &child_status);
565 fprintf(
f,
"ALWAYS_INLINE(static VALUE _mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq));\n", status->compiled_id, pos);
566 fprintf(
f,
"static inline VALUE\n_mjit%d_inlined_%d(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE orig_self, const rb_iseq_t *original_iseq)\n{\n", status->compiled_id, pos);
567 fprintf(
f,
" const VALUE *orig_pc = reg_cfp->pc;\n");
568 fprintf(
f,
" VALUE *orig_sp = reg_cfp->sp;\n");
569 bool success = mjit_compile_body(
f, child_iseq, &child_status);
570 fprintf(
f,
"\n} /* end of _mjit%d_inlined_%d */\n\n", status->compiled_id, pos);
576 pos += insn_len(insn);
583mjit_compile(
FILE *
f,
const rb_iseq_t *iseq,
const char *funcname,
int id)
585 struct compile_status status = { .compiled_iseq = iseq->
body, .compiled_id =
id };
586 INIT_COMPILE_STATUS(status, iseq->
body,
true);
587 if (iseq->
body->
ci_size > 0 && status.cc_entries_index == -1) {
590 init_ivar_compile_status(iseq->
body, &status);
592 if (!status.compile_info->disable_send_cache && !status.compile_info->disable_inlining) {
593 if (!precompile_inlinable_iseqs(
f, iseq, &status))
598 fprintf(
f,
"__declspec(dllexport)\n");
600 fprintf(
f,
"VALUE\n%s(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)\n{\n", funcname);
601 bool success = mjit_compile_body(
f, iseq, &status);
602 fprintf(
f,
"\n} // end of %s\n", funcname);
Internal header for the compiler.
int rb_vm_insn_addr2insn(const void *)
#define RSTRING_PTR(string)
VALUE rb_cNilClass
NilClass class.
VALUE rb_cFalseClass
FalseClass class.
VALUE rb_cTrueClass
TrueClass class.
Thin wrapper to ruby/config.h.
Internal header for Hash.
Internal header for Object.
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
VALUE rb_iseq_path(const rb_iseq_t *iseq)
@ VM_METHOD_TYPE_ISEQ
Ruby method.
@ VM_METHOD_TYPE_CFUNC
C method.
int mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq)
struct mjit_options mjit_opts
const struct rb_callcache ** mjit_iseq_cc_entries(const struct rb_iseq_constant_body *const body)
bool rb_mjit_compiling_iseq_p(const rb_iseq_t *iseq)
#define ROBJECT_EMBED_LEN_MAX
#define PRI_SERIALT_PREFIX
unsigned LONG_LONG rb_serial_t
struct rb_iv_index_tbl_entry * entry
const struct rb_callinfo * ci
unsigned int local_table_size
struct rb_iseq_constant_body::@188::@190 flags
union iseq_inline_storage_entry * is_entries
rb_iseq_location_t location
struct rb_iseq_constant_body::@188 param
parameter information
struct rb_call_data * call_data
struct rb_iseq_constant_body * body
struct rb_call_data * CALL_DATA
struct iseq_inline_iv_cache_entry * IVC
bool rb_vm_opt_cfunc_p(CALL_CACHE cc, int insn)
bool rb_simple_iseq_p(const rb_iseq_t *iseq)
if((ID)(DISPID) nameid !=nameid)
int def(FILE *source, FILE *dest, int level)