Ruby: yjit.c Source File

Ruby 3.2.5p208 (2024-07-26 revision 31d0f1a2e7dbfb60731d1f05b868e1d578cda493)

yjit.c
1// This part of YJIT helps interfacing with the rest of CRuby and with the OS.
2// Sometimes our FFI binding generation tool gives undesirable outputs when it
3// sees C features that Rust doesn't support well. We mitigate that by binding
4// functions which have simple parameter types. The boilerplate C functions for
5// that purpose are in this file.
6// Similarly, we wrap OS facilities we need in simple functions to help with
7// FFI and to avoid the need to use external crates.io Rust libraries.
8
9#include "internal.h"
10#include "internal/sanitizers.h"
11#include "internal/string.h"
12#include "internal/hash.h"
13#include "internal/variable.h"
14#include "internal/compile.h"
15#include "internal/class.h"
16#include "internal/fixnum.h"
17#include "gc.h"
18#include "vm_core.h"
19#include "vm_callinfo.h"
20#include "builtin.h"
21#include "insns.inc"
22#include "insns_info.inc"
23#include "vm_sync.h"
24#include "yjit.h"
25#include "vm_insnhelper.h"
26#include "probes.h"
27#include "probes_helper.h"
28#include "iseq.h"
29#include "ruby/debug.h"
30#include "internal/cont.h"
31
32// For mmapp(), sysconf()
33#ifndef _WIN32
34#include <unistd.h>
35#include <sys/mman.h>
36#endif
37
38#include <errno.h>
39
40// We need size_t to have a known size to simplify code generation and FFI.
41// TODO(alan): check this in configure.ac to fail fast on 32 bit platforms.
42STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX);
43// I don't know any C implementation that has uint64_t and puts padding bits
44// into size_t but the standard seems to allow it.
45STATIC_ASSERT(size_t_no_padding_bits, sizeof(size_t) == sizeof(uint64_t));
46
47// This build config impacts the pointer tagging scheme and we only want to
48// support one scheme for simplicity.
49STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM);
50
51// NOTE: We can trust that uint8_t has no "padding bits" since the C spec
52// guarantees it. Wording about padding bits is more explicit in C11 compared
53// to C99. See C11 7.20.1.1p2. All this is to say we have _some_ standards backing to
54// use a Rust `*mut u8` to represent a C `uint8_t *`.
55//
56// If we don't want to trust that we can interpreter the C standard correctly, we
57// could outsource that work to the Rust standard library by sticking to fundamental
58// types in C such as int, long, etc. and use `std::os::raw::c_long` and friends on
59// the Rust side.
60//
61// What's up with the long prefix? Even though we build with `-fvisibility=hidden`
62// we are sometimes a static library where the option doesn't prevent name collision.
63// The "_yjit_" part is for trying to be informative. We might want different
64// suffixes for symbols meant for Rust and symbols meant for broader CRuby.
65
66bool
67rb_yjit_mark_writable(void *mem_block, uint32_t mem_size)
68{
69 return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
70}
71
72void
73rb_yjit_mark_executable(void *mem_block, uint32_t mem_size)
74{
75 // Do not call mprotect when mem_size is zero. Some platforms may return
76 // an error for it. https://github.com/Shopify/ruby/issues/450
77 if (mem_size == 0) {
78 return;
79 }
80 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
81 rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
82 mem_block, (unsigned long)mem_size, strerror(errno));
83 }
84}
85
86// Free the specified memory block.
87bool
88rb_yjit_mark_unused(void *mem_block, uint32_t mem_size)
89{
90 // On Linux, you need to use madvise MADV_DONTNEED to free memory.
91 // We might not need to call this on macOS, but it's not really documented.
92 // We generally prefer to do the same thing on both to ease testing too.
93 madvise(mem_block, mem_size, MADV_DONTNEED);
94
95 // On macOS, mprotect PROT_NONE seems to reduce RSS.
96 // We also call this on Linux to avoid executing unused pages.
97 return mprotect(mem_block, mem_size, PROT_NONE) == 0;
98}
99
100// `start` is inclusive and `end` is exclusive.
101void
102rb_yjit_icache_invalidate(void *start, void *end)
103{
104 // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
105 // but required on ARM before running freshly written code.
106 // On Darwin it's the same as calling sys_icache_invalidate().
107#ifdef __GNUC__
108 __builtin___clear_cache(start, end);
109#elif defined(__aarch64__)
110#error No instruction cache clear available with this compiler on Aarch64!
111#endif
112}
113
114# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
115
116// For a given raw_sample (frame), set the hash with the caller's
117// name, file, and line number. Return the hash with collected frame_info.
118static void
119rb_yjit_add_frame(VALUE hash, VALUE frame)
120{
121 VALUE frame_id = PTR2NUM(frame);
122
123 if (RTEST(rb_hash_aref(hash, frame_id))) {
124 return;
125 }
126 else {
127 VALUE frame_info = rb_hash_new();
128 // Full label for the frame
129 VALUE name = rb_profile_frame_full_label(frame);
130 // Absolute path of the frame from rb_iseq_realpath
131 VALUE file = rb_profile_frame_absolute_path(frame);
132 // Line number of the frame
133 VALUE line = rb_profile_frame_first_lineno(frame);
134
135 // If absolute path isn't available use the rb_iseq_path
136 if (NIL_P(file)) {
137 file = rb_profile_frame_path(frame);
138 }
139
140 rb_hash_aset(frame_info, ID2SYM(rb_intern("name")), name);
141 rb_hash_aset(frame_info, ID2SYM(rb_intern("file")), file);
142 rb_hash_aset(frame_info, ID2SYM(rb_intern("samples")), INT2NUM(0));
143 rb_hash_aset(frame_info, ID2SYM(rb_intern("total_samples")), INT2NUM(0));
144 rb_hash_aset(frame_info, ID2SYM(rb_intern("edges")), rb_hash_new());
145 rb_hash_aset(frame_info, ID2SYM(rb_intern("lines")), rb_hash_new());
146
147 if (line != INT2FIX(0)) {
148 rb_hash_aset(frame_info, ID2SYM(rb_intern("line")), line);
149 }
150
151 rb_hash_aset(hash, frame_id, frame_info);
152 }
153}
154
155// Parses the YjitExitLocations raw_samples and line_samples collected by
156// rb_yjit_record_exit_stack and turns them into 3 hashes (raw, lines, and frames) to
157// be used by RubyVM::YJIT.exit_locations. yjit_raw_samples represents the raw frames information
158// (without name, file, and line), and yjit_line_samples represents the line information
159// of the iseq caller.
160 VALUE
161rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int samples_len)
162{
163 VALUE result = rb_hash_new();
164 VALUE raw_samples = rb_ary_new_capa(samples_len);
165 VALUE line_samples = rb_ary_new_capa(samples_len);
166 VALUE frames = rb_hash_new();
167 int idx = 0;
168
169 // While the index is less than samples_len, parse yjit_raw_samples and
170 // yjit_line_samples, then add casted values to raw_samples and line_samples array.
171 while (idx < samples_len) {
172 int num = (int)yjit_raw_samples[idx];
173 int line_num = (int)yjit_line_samples[idx];
174 idx++;
175
176 rb_ary_push(raw_samples, SIZET2NUM(num));
177 rb_ary_push(line_samples, INT2NUM(line_num));
178
179 // Loop through the length of samples_len and add data to the
180 // frames hash. Also push the current value onto the raw_samples
181 // and line_samples array respectively.
182 for (int o = 0; o < num; o++) {
183 rb_yjit_add_frame(frames, yjit_raw_samples[idx]);
184 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
185 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
186 idx++;
187 }
188
189 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
190 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
191 idx++;
192
193 rb_ary_push(raw_samples, SIZET2NUM(yjit_raw_samples[idx]));
194 rb_ary_push(line_samples, INT2NUM(yjit_line_samples[idx]));
195 idx++;
196 }
197
198 // Set add the raw_samples, line_samples, and frames to the results
199 // hash.
200 rb_hash_aset(result, ID2SYM(rb_intern("raw")), raw_samples);
201 rb_hash_aset(result, ID2SYM(rb_intern("lines")), line_samples);
202 rb_hash_aset(result, ID2SYM(rb_intern("frames")), frames);
203
204 return result;
205}
206
207uint32_t
208rb_yjit_get_page_size(void)
209{
210#if defined(_SC_PAGESIZE)
211 long page_size = sysconf(_SC_PAGESIZE);
212 if (page_size <= 0) rb_bug("yjit: failed to get page size");
213
214 // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected.
215 // Though our design sort of assume we have fine grained control over memory protection
216 // which require small page sizes.
217 if (page_size > 0x40000000l) rb_bug("yjit page size too large");
218
219 return (uint32_t)page_size;
220#else
221#error "YJIT supports POSIX only for now"
222#endif
223}
224
225#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
226// Align the current write position to a multiple of bytes
227static uint8_t *
228align_ptr(uint8_t *ptr, uint32_t multiple)
229{
230 // Compute the pointer modulo the given alignment boundary
231 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
232
233 // If the pointer is already aligned, stop
234 if (rem == 0)
235 return ptr;
236
237 // Pad the pointer by the necessary amount to align it
238 uint32_t pad = multiple - rem;
239
240 return ptr + pad;
241}
242#endif
243
244// Address space reservation. Memory pages are mapped on an as needed basis.
245// See the Rust mm module for details.
246uint8_t *
247rb_yjit_reserve_addr_space(uint32_t mem_size)
248{
249#ifndef _WIN32
250 uint8_t *mem_block;
251
252 // On Linux
253 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
254 uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
255 uint8_t *const cfunc_sample_addr = (void *)&rb_yjit_reserve_addr_space;
256 uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX;
257 // Align the requested address to page size
258 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
259
260 // Probe for addresses close to this function using MAP_FIXED_NOREPLACE
261 // to improve odds of being in range for 32-bit relative call instructions.
262 do {
263 mem_block = mmap(
264 req_addr,
265 mem_size,
266 PROT_NONE,
267 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
268 -1,
269 0
270 );
271
272 // If we succeeded, stop
273 if (mem_block != MAP_FAILED) {
274 break;
275 }
276
277 // +4MB
278 req_addr += 4 * 1024 * 1024;
279 } while (req_addr < probe_region_end);
280
281 // On MacOS and other platforms
282 #else
283 // Try to map a chunk of memory as executable
284 mem_block = mmap(
285 (void *)rb_yjit_reserve_addr_space,
286 mem_size,
287 PROT_NONE,
288 MAP_PRIVATE | MAP_ANONYMOUS,
289 -1,
290 0
291 );
292 #endif
293
294 // Fallback
295 if (mem_block == MAP_FAILED) {
296 // Try again without the address hint (e.g., valgrind)
297 mem_block = mmap(
298 NULL,
299 mem_size,
300 PROT_NONE,
301 MAP_PRIVATE | MAP_ANONYMOUS,
302 -1,
303 0
304 );
305 }
306
307 // Check that the memory mapping was successful
308 if (mem_block == MAP_FAILED) {
309 perror("ruby: yjit: mmap:");
310 if(errno == ENOMEM) {
311 // No crash report if it's only insufficient memory
312 exit(EXIT_FAILURE);
313 }
314 rb_bug("mmap failed");
315 }
316
317 return mem_block;
318#else
319 // Windows not supported for now
320 return NULL;
321#endif
322}
323
324// Is anyone listening for :c_call and :c_return event currently?
325bool
326rb_c_method_tracing_currently_enabled(rb_execution_context_t *ec)
327{
328 rb_event_flag_t tracing_events;
329 if (rb_multi_ractor_p()) {
330 tracing_events = ruby_vm_event_enabled_global_flags;
331 }
332 else {
333 // At the time of writing, events are never removed from
334 // ruby_vm_event_enabled_global_flags so always checking using it would
335 // mean we don't compile even after tracing is disabled.
336 tracing_events = rb_ec_ractor_hooks(ec)->events;
337 }
338
339 return tracing_events & (RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN);
340}
341
342// The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event
343// like the interpreter. When tracing for c_return is enabled, we patch the code after
344// the C method return to call into this to fire the event.
345void
346rb_full_cfunc_return(rb_execution_context_t *ec, VALUE return_value)
347{
348 rb_control_frame_t *cfp = ec->cfp;
349 RUBY_ASSERT_ALWAYS(cfp == GET_EC()->cfp);
350 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp);
351
352 RUBY_ASSERT_ALWAYS(RUBYVM_CFUNC_FRAME_P(cfp));
353 RUBY_ASSERT_ALWAYS(me->def->type == VM_METHOD_TYPE_CFUNC);
354
355 // CHECK_CFP_CONSISTENCY("full_cfunc_return"); TODO revive this
356
357 // Pop the C func's frame and fire the c_return TracePoint event
358 // Note that this is the same order as vm_call_cfunc_with_frame().
359 rb_vm_pop_frame(ec);
360 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
361 // Note, this deviates from the interpreter in that users need to enable
362 // a c_return TracePoint for this DTrace hook to work. A reasonable change
363 // since the Ruby return event works this way as well.
364 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
365
366 // Push return value into the caller's stack. We know that it's a frame that
367 // uses cfp->sp because we are patching a call done with gen_send_cfunc().
368 ec->cfp->sp[0] = return_value;
369 ec->cfp->sp++;
370}
371
372unsigned int
373rb_iseq_encoded_size(const rb_iseq_t *iseq)
374{
375 return iseq->body->iseq_size;
376}
377
378// TODO(alan): consider using an opaque pointer for the payload rather than a void pointer
379void *
380rb_iseq_get_yjit_payload(const rb_iseq_t *iseq)
381{
382 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
383 if (iseq->body) {
384 return iseq->body->yjit_payload;
385 }
386 else {
387 // Body is NULL when constructing the iseq.
388 return NULL;
389 }
390}
391
392void
393rb_iseq_set_yjit_payload(const rb_iseq_t *iseq, void *payload)
394{
395 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
396 RUBY_ASSERT_ALWAYS(iseq->body);
397 RUBY_ASSERT_ALWAYS(NULL == iseq->body->yjit_payload);
398 iseq->body->yjit_payload = payload;
399}
400
401void
402rb_iseq_reset_jit_func(const rb_iseq_t *iseq)
403{
404 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
405 iseq->body->jit_func = NULL;
406 // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint,
407 // we'd like to re-compile ISEQs that haven't been converted to trace_* insns.
408 iseq->body->total_calls = 0;
409}
410
411// Get the PC for a given index in an iseq
412 VALUE *
413rb_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx)
414{
415 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq));
416 RUBY_ASSERT_ALWAYS(insn_idx < iseq->body->iseq_size);
417 VALUE *encoded = iseq->body->iseq_encoded;
418 VALUE *pc = &encoded[insn_idx];
419 return pc;
420}
421
422// Get the opcode given a program counter. Can return trace opcode variants.
423int
424rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc)
425{
426 // YJIT should only use iseqs after AST to bytecode compilation
427 RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED));
428
429 const VALUE at_pc = *pc;
430 return rb_vm_insn_addr2opcode((const void *)at_pc);
431}
432
433// used by jit_rb_str_bytesize in codegen.rs
434 VALUE
435rb_str_bytesize(VALUE str)
436{
437 return LONG2NUM(RSTRING_LEN(str));
438}
439
440unsigned long
441rb_RSTRING_LEN(VALUE str)
442{
443 return RSTRING_LEN(str);
444}
445
446char *
447rb_RSTRING_PTR(VALUE str)
448{
449 return RSTRING_PTR(str);
450}
451
452 rb_proc_t *
453rb_yjit_get_proc_ptr(VALUE procv)
454{
455 rb_proc_t *proc;
456 GetProcPtr(procv, proc);
457 return proc;
458}
459
460// This is defined only as a named struct inside rb_iseq_constant_body.
461// By giving it a separate typedef, we make it nameable by rust-bindgen.
462// Bindgen's temp/anon name isn't guaranteed stable.
463typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct;
464
465const char *
466rb_insn_name(VALUE insn)
467{
468 return insn_name(insn);
469}
470
471// Query the instruction length in bytes for YARV opcode insn
472int
473rb_insn_len(VALUE insn)
474{
475 return insn_len(insn);
476}
477
478unsigned int
479rb_vm_ci_argc(const struct rb_callinfo *ci)
480{
481 return vm_ci_argc(ci);
482}
483
484 ID
485rb_vm_ci_mid(const struct rb_callinfo *ci)
486{
487 return vm_ci_mid(ci);
488}
489
490unsigned int
491rb_vm_ci_flag(const struct rb_callinfo *ci)
492{
493 return vm_ci_flag(ci);
494}
495
496const struct rb_callinfo_kwarg *
497rb_vm_ci_kwarg(const struct rb_callinfo *ci)
498{
499 return vm_ci_kwarg(ci);
500}
501
502int
503rb_get_cikw_keyword_len(const struct rb_callinfo_kwarg *cikw)
504{
505 return cikw->keyword_len;
506}
507
508 VALUE
509rb_get_cikw_keywords_idx(const struct rb_callinfo_kwarg *cikw, int idx)
510{
511 return cikw->keywords[idx];
512}
513
514rb_method_visibility_t
515rb_METHOD_ENTRY_VISI(const rb_callable_method_entry_t *me)
516{
517 return METHOD_ENTRY_VISI(me);
518}
519
520rb_method_type_t
521rb_get_cme_def_type(const rb_callable_method_entry_t *cme)
522{
523 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
524 return VM_METHOD_TYPE_UNDEF;
525 }
526 else {
527 return cme->def->type;
528 }
529}
530
531 ID
532rb_get_cme_def_body_attr_id(const rb_callable_method_entry_t *cme)
533{
534 return cme->def->body.attr.id;
535}
536
537 ID rb_get_symbol_id(VALUE namep);
538
539enum method_optimized_type
540rb_get_cme_def_body_optimized_type(const rb_callable_method_entry_t *cme)
541{
542 return cme->def->body.optimized.type;
543}
544
545unsigned int
546rb_get_cme_def_body_optimized_index(const rb_callable_method_entry_t *cme)
547{
548 return cme->def->body.optimized.index;
549}
550
551 rb_method_cfunc_t *
552rb_get_cme_def_body_cfunc(const rb_callable_method_entry_t *cme)
553{
554 return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
555}
556
557uintptr_t
558rb_get_def_method_serial(const rb_method_definition_t *def)
559{
560 return def->method_serial;
561}
562
563 ID
564rb_get_def_original_id(const rb_method_definition_t *def)
565{
566 return def->original_id;
567}
568
569int
570rb_get_mct_argc(const rb_method_cfunc_t *mct)
571{
572 return mct->argc;
573}
574
575void *
576rb_get_mct_func(const rb_method_cfunc_t *mct)
577{
578 return (void*)mct->func; // this field is defined as type VALUE (*func)(ANYARGS)
579}
580
581const rb_iseq_t *
582rb_get_def_iseq_ptr(rb_method_definition_t *def)
583{
584 return def_iseq_ptr(def);
585}
586
587 VALUE
588rb_get_def_bmethod_proc(rb_method_definition_t *def)
589{
590 RUBY_ASSERT(def->type == VM_METHOD_TYPE_BMETHOD);
591 return def->body.bmethod.proc;
592}
593
594const rb_iseq_t *
595rb_get_iseq_body_local_iseq(const rb_iseq_t *iseq)
596{
597 return iseq->body->local_iseq;
598}
599
600const rb_iseq_t *
601rb_get_iseq_body_parent_iseq(const rb_iseq_t *iseq)
602{
603 return iseq->body->parent_iseq;
604}
605
606unsigned int
607rb_get_iseq_body_local_table_size(const rb_iseq_t *iseq)
608{
609 return iseq->body->local_table_size;
610}
611
612 VALUE *
613rb_get_iseq_body_iseq_encoded(const rb_iseq_t *iseq)
614{
615 return iseq->body->iseq_encoded;
616}
617
618bool
619rb_get_iseq_body_builtin_inline_p(const rb_iseq_t *iseq)
620{
621 return iseq->body->builtin_inline_p;
622}
623
624unsigned
625rb_get_iseq_body_stack_max(const rb_iseq_t *iseq)
626{
627 return iseq->body->stack_max;
628}
629
630bool
631rb_get_iseq_flags_has_lead(const rb_iseq_t *iseq)
632{
633 return iseq->body->param.flags.has_lead;
634}
635
636bool
637rb_get_iseq_flags_has_opt(const rb_iseq_t *iseq)
638{
639 return iseq->body->param.flags.has_opt;
640}
641
642bool
643rb_get_iseq_flags_has_kw(const rb_iseq_t *iseq)
644{
645 return iseq->body->param.flags.has_kw;
646}
647
648bool
649rb_get_iseq_flags_has_post(const rb_iseq_t *iseq)
650{
651 return iseq->body->param.flags.has_post;
652}
653
654bool
655rb_get_iseq_flags_has_kwrest(const rb_iseq_t *iseq)
656{
657 return iseq->body->param.flags.has_kwrest;
658}
659
660bool
661rb_get_iseq_flags_has_rest(const rb_iseq_t *iseq)
662{
663 return iseq->body->param.flags.has_rest;
664}
665
666bool
667rb_get_iseq_flags_ruby2_keywords(const rb_iseq_t *iseq)
668{
669 return iseq->body->param.flags.ruby2_keywords;
670}
671
672bool
673rb_get_iseq_flags_has_block(const rb_iseq_t *iseq)
674{
675 return iseq->body->param.flags.has_block;
676}
677
678bool
679rb_get_iseq_flags_ambiguous_param0(const rb_iseq_t *iseq)
680{
681 return iseq->body->param.flags.ambiguous_param0;
682}
683
684bool
685rb_get_iseq_flags_accepts_no_kwarg(const rb_iseq_t *iseq)
686{
687 return iseq->body->param.flags.accepts_no_kwarg;
688}
689
690const rb_seq_param_keyword_struct *
691rb_get_iseq_body_param_keyword(const rb_iseq_t *iseq)
692{
693 return iseq->body->param.keyword;
694}
695
696unsigned
697rb_get_iseq_body_param_size(const rb_iseq_t *iseq)
698{
699 return iseq->body->param.size;
700}
701
702int
703rb_get_iseq_body_param_lead_num(const rb_iseq_t *iseq)
704{
705 return iseq->body->param.lead_num;
706}
707
708int
709rb_get_iseq_body_param_opt_num(const rb_iseq_t *iseq)
710{
711 return iseq->body->param.opt_num;
712}
713
714const VALUE *
715rb_get_iseq_body_param_opt_table(const rb_iseq_t *iseq)
716{
717 return iseq->body->param.opt_table;
718}
719
720 VALUE
721rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv, int kw_splat, VALUE block_handler)
722{
723 rb_proc_t *proc;
724 GetProcPtr(recv, proc);
725 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
726}
727
728
729// If true, the iseq is leaf and it can be replaced by a single C call.
730bool
731rb_leaf_invokebuiltin_iseq_p(const rb_iseq_t *iseq)
732{
733 unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
734 unsigned int leave_len = insn_len(BIN(leave));
735
736 return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
737 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
738 rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
739 iseq->body->builtin_inline_p
740 );
741}
742
743// Return an rb_builtin_function if the iseq contains only that leaf builtin function.
744const struct rb_builtin_function *
745rb_leaf_builtin_function(const rb_iseq_t *iseq)
746{
747 if (!rb_leaf_invokebuiltin_iseq_p(iseq))
748 return NULL;
749 return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
750}
751
752 VALUE
753rb_yjit_str_simple_append(VALUE str1, VALUE str2)
754{
755 return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2));
756}
757
758struct rb_control_frame_struct *
759rb_get_ec_cfp(const rb_execution_context_t *ec)
760{
761 return ec->cfp;
762}
763
764 VALUE *
765rb_get_cfp_pc(struct rb_control_frame_struct *cfp)
766{
767 return (VALUE*)cfp->pc;
768}
769
770 VALUE *
771rb_get_cfp_sp(struct rb_control_frame_struct *cfp)
772{
773 return cfp->sp;
774}
775
776void
777rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc)
778{
779 cfp->pc = pc;
780}
781
782void
783rb_set_cfp_sp(struct rb_control_frame_struct *cfp, VALUE *sp)
784{
785 cfp->sp = sp;
786}
787
788 rb_iseq_t *
789rb_cfp_get_iseq(struct rb_control_frame_struct *cfp)
790{
791 // TODO(alan) could assert frame type here to make sure that it's a ruby frame with an iseq.
792 return (rb_iseq_t*)cfp->iseq;
793}
794
795 VALUE
796rb_get_cfp_self(struct rb_control_frame_struct *cfp)
797{
798 return cfp->self;
799}
800
801 VALUE *
802rb_get_cfp_ep(struct rb_control_frame_struct *cfp)
803{
804 return (VALUE*)cfp->ep;
805}
806
807const VALUE *
808rb_get_cfp_ep_level(struct rb_control_frame_struct *cfp, uint32_t lv)
809{
810 uint32_t i;
811 const VALUE *ep = (VALUE*)cfp->ep;
812 for (i = 0; i < lv; i++) {
813 ep = VM_ENV_PREV_EP(ep);
814 }
815 return ep;
816}
817
818 VALUE
819rb_yarv_class_of(VALUE obj)
820{
821 return rb_class_of(obj);
822}
823
824// YJIT needs this function to never allocate and never raise
825 VALUE
826rb_yarv_str_eql_internal(VALUE str1, VALUE str2)
827{
828 // We wrap this since it's static inline
829 return rb_str_eql_internal(str1, str2);
830}
831
832// YJIT needs this function to never allocate and never raise
833 VALUE
834rb_yarv_ary_entry_internal(VALUE ary, long offset)
835{
836 return rb_ary_entry_internal(ary, offset);
837}
838
839 VALUE
840rb_yarv_fix_mod_fix(VALUE recv, VALUE obj)
841{
842 return rb_fix_mod_fix(recv, obj);
843}
844
845// Print the Ruby source location of some ISEQ for debugging purposes
846void
847rb_yjit_dump_iseq_loc(const rb_iseq_t *iseq, uint32_t insn_idx)
848{
849 char *ptr;
850 long len;
851 VALUE path = rb_iseq_path(iseq);
852 RSTRING_GETMEM(path, ptr, len);
853 fprintf(stderr, "%s %.*s:%u\n", __func__, (int)len, ptr, rb_iseq_line_no(iseq, insn_idx));
854}
855
856// The FL_TEST() macro
857 VALUE
858rb_FL_TEST(VALUE obj, VALUE flags)
859{
860 return RB_FL_TEST(obj, flags);
861}
862
863// The FL_TEST_RAW() macro, normally an internal implementation detail
864 VALUE
865rb_FL_TEST_RAW(VALUE obj, VALUE flags)
866{
867 return FL_TEST_RAW(obj, flags);
868}
869
870// The RB_TYPE_P macro
871bool
872rb_RB_TYPE_P(VALUE obj, enum ruby_value_type t)
873{
874 return RB_TYPE_P(obj, t);
875}
876
877long
878rb_RSTRUCT_LEN(VALUE st)
879{
880 return RSTRUCT_LEN(st);
881}
882
883// There are RSTRUCT_SETs in ruby/internal/core/rstruct.h and internal/struct.h
884// with different types (int vs long) for k. Here we use the one from ruby/internal/core/rstruct.h,
885// which takes an int.
886void
887rb_RSTRUCT_SET(VALUE st, int k, VALUE v)
888{
889 RSTRUCT_SET(st, k, v);
890}
891
892const struct rb_callinfo *
893rb_get_call_data_ci(const struct rb_call_data *cd)
894{
895 return cd->ci;
896}
897
898bool
899rb_BASIC_OP_UNREDEFINED_P(enum ruby_basic_operators bop, uint32_t klass)
900{
901 return BASIC_OP_UNREDEFINED_P(bop, klass);
902}
903
904 VALUE
905rb_RCLASS_ORIGIN(VALUE c)
906{
907 return RCLASS_ORIGIN(c);
908}
909
910// Return the string encoding index
911int
912rb_ENCODING_GET(VALUE obj)
913{
914 return RB_ENCODING_GET(obj);
915}
916
917bool
918rb_yjit_multi_ractor_p(void)
919{
920 return rb_multi_ractor_p();
921}
922
923// For debug builds
924void
925rb_assert_iseq_handle(VALUE handle)
926{
927 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
928 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_iseq));
929}
930
931int
932rb_IMEMO_TYPE_P(VALUE imemo, enum imemo_type imemo_type)
933{
934 return IMEMO_TYPE_P(imemo, imemo_type);
935}
936
937void
938rb_assert_cme_handle(VALUE handle)
939{
940 RUBY_ASSERT_ALWAYS(rb_objspace_markable_object_p(handle));
941 RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_ment));
942}
943
944// Used for passing a callback and other data over rb_objspace_each_objects
945 struct iseq_callback_data {
946 rb_iseq_callback callback;
947 void *data;
948};
949
950// Heap-walking callback for rb_yjit_for_each_iseq().
951static int
952for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data)
953{
954 const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data;
955 VALUE v = (VALUE)vstart;
956 for (; v != (VALUE)vend; v += stride) {
957 void *ptr = asan_poisoned_object_p(v);
958 asan_unpoison_object(v, false);
959
960 if (rb_obj_is_iseq(v)) {
961 rb_iseq_t *iseq = (rb_iseq_t *)v;
962 callback_data->callback(iseq, callback_data->data);
963 }
964
965 asan_poison_object_if(ptr, v);
966 }
967 return 0;
968}
969
970// Iterate through the whole GC heap and invoke a callback for each iseq.
971// Used for global code invalidation.
972void
973rb_yjit_for_each_iseq(rb_iseq_callback callback, void *data)
974{
975 struct iseq_callback_data callback_data = { .callback = callback, .data = data };
976 rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data);
977}
978
979// For running write barriers from Rust. Required when we add a new edge in the
980// object graph from `old` to `young`.
981void
982rb_yjit_obj_written(VALUE old, VALUE young, const char *file, int line)
983{
984 rb_obj_written(old, Qundef, young, file, line);
985}
986
987// Acquire the VM lock and then signal all other Ruby threads (ractors) to
988// contend for the VM lock, putting them to sleep. YJIT uses this to evict
989// threads running inside generated code so among other things, it can
990// safely change memory protection of regions housing generated code.
991void
992rb_yjit_vm_lock_then_barrier(unsigned int *recursive_lock_level, const char *file, int line)
993{
994 rb_vm_lock_enter(recursive_lock_level, file, line);
995 rb_vm_barrier();
996}
997
998// Release the VM lock. The lock level must point to the same integer used to
999// acquire the lock.
1000void
1001rb_yjit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line)
1002{
1003 rb_vm_lock_leave(recursive_lock_level, file, line);
1004}
1005
1006// Pointer to a YJIT entry point (machine code generated by YJIT)
1007typedef VALUE (*yjit_func_t)(rb_execution_context_t *, rb_control_frame_t *);
1008
1009bool
1010rb_yjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec)
1011{
1012 bool success = true;
1013 RB_VM_LOCK_ENTER();
1014 rb_vm_barrier();
1015
1016 // Compile a block version starting at the first instruction
1017 uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec); // defined in Rust
1018 uint8_t *code_ptr = rb_yjit_iseq_gen_entry_point(iseq, ec);
1019
1020 if (code_ptr) {
1021 iseq->body->jit_func = (yjit_func_t)code_ptr;
1022 }
1023 else {
1024 iseq->body->jit_func = 0;
1025 success = false;
1026 }
1027
1028 RB_VM_LOCK_LEAVE();
1029 return success;
1030}
1031
1032// GC root for interacting with the GC
1033 struct yjit_root_struct {
1034 bool unused; // empty structs are not legal in C99
1035};
1036
1037static void
1038yjit_root_free(void *ptr)
1039{
1040 // Do nothing. The root lives as long as the process.
1041}
1042
1043static size_t
1044yjit_root_memsize(const void *ptr)
1045{
1046 // Count off-gc-heap allocation size of the dependency table
1047 return 0; // TODO: more accurate accounting
1048}
1049
1050// GC callback during compaction
1051static void
1052yjit_root_update_references(void *ptr)
1053{
1054 // Do nothing since we use rb_gc_mark(), which pins.
1055}
1056
1057void rb_yjit_root_mark(void *ptr); // in Rust
1058
1059// Custom type for interacting with the GC
1060// TODO: make this write barrier protected
1061static const rb_data_type_t yjit_root_type = {
1062 "yjit_root",
1063 {rb_yjit_root_mark, yjit_root_free, yjit_root_memsize, yjit_root_update_references},
1064 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1065};
1066
1067// For dealing with refinements
1068void
1069rb_yjit_invalidate_all_method_lookup_assumptions(void)
1070{
1071 // It looks like Module#using actually doesn't need to invalidate all the
1072 // method caches, so we do nothing here for now.
1073}
1074
1075// Number of object shapes, which might be useful for investigating YJIT exit reasons.
1076static VALUE
1077object_shape_count(rb_execution_context_t *ec, VALUE self)
1078{
1079 // next_shape_id starts from 0, so it's the same as the count
1080 return ULONG2NUM((unsigned long)GET_VM()->next_shape_id);
1081}
1082
1083// Primitives used by yjit.rb
1084 VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self);
1085 VALUE rb_yjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self);
1086 VALUE rb_yjit_get_stats(rb_execution_context_t *ec, VALUE self);
1087 VALUE rb_yjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self);
1088 VALUE rb_yjit_disasm_iseq(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1089 VALUE rb_yjit_insns_compiled(rb_execution_context_t *ec, VALUE self, VALUE iseq);
1090 VALUE rb_yjit_code_gc(rb_execution_context_t *ec, VALUE self);
1091 VALUE rb_yjit_simulate_oom_bang(rb_execution_context_t *ec, VALUE self);
1092 VALUE rb_yjit_get_exit_locations(rb_execution_context_t *ec, VALUE self);
1093
1094// Preprocessed yjit.rb generated during build
1095#include "yjit.rbinc"
1096
1097// Can raise RuntimeError
1098void
1099rb_yjit_init(void)
1100{
1101 // Call the Rust initialization code
1102 void rb_yjit_init_rust(void);
1103 rb_yjit_init_rust();
1104
1105 // Initialize the GC hooks. Do this second as some code depend on Rust initialization.
1106 struct yjit_root_struct *root;
1107 VALUE yjit_root = TypedData_Make_Struct(0, struct yjit_root_struct, &yjit_root_type, root);
1108 rb_gc_register_mark_object(yjit_root);
1109}
RUBY_ASSERT
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:177
RUBY_ASSERT_ALWAYS
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition assert.h:167
debug.h
rb_profile_frame_full_label
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
Definition vm_backtrace.c:1808
rb_profile_frame_absolute_path
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
Definition vm_backtrace.c:1688
rb_profile_frame_path
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
Definition vm_backtrace.c:1656
rb_profile_frame_first_lineno
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
Definition vm_backtrace.c:1717
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:39
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:40
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:103
RB_FL_TEST
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
Definition fl_type.h:527
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
ULONG2NUM
#define ULONG2NUM
Old name of RB_ULONG2NUM.
Definition long.h:60
SIZET2NUM
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
FL_TEST_RAW
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:140
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
INT2NUM
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
rb_bug
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition error.c:794
rb_class_of
static VALUE rb_class_of(VALUE obj)
Object to class mapping function.
Definition globals.h:172
RB_ENCODING_GET
static int RB_ENCODING_GET(VALUE obj)
Just another name of rb_enc_get_index.
Definition encoding.h:211
rb_str_cat
VALUE rb_str_cat(VALUE dst, const char *src, long srclen)
Destructively appends the passed contents to the string.
Definition string.c:3177
rb_intern
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
Definition symbol.c:796
RSTRING_GETMEM
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:574
RSTRING_LEN
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
Definition rstring.h:484
RSTRING_PTR
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
Definition rstring.h:498
TypedData_Make_Struct
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition rtypeddata.h:489
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
USE_FLONUM
#define USE_FLONUM
Definition special_consts.h:45
iseq_callback_data
Definition yjit.c:945
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:496
rb_callable_method_entry_struct
Definition method.h:62
rb_callinfo_kwarg
Definition vm_callinfo.h:46
rb_callinfo
Definition vm_callinfo.h:62
rb_control_frame_struct
Definition vm_core.h:818
rb_data_type_struct
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:190
rb_execution_context_struct
Definition vm_core.h:899
rb_iseq_constant_body::param
struct rb_iseq_constant_body::@131 param
parameter information
rb_iseq_struct
Definition vm_core.h:526
rb_method_cfunc_struct
Definition method.h:139
rb_method_definition_struct
Definition method.h:179
rb_proc_t
Definition vm_core.h:1137
yjit_root_struct
Definition yjit.c:1033
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:375
ruby_value_type
ruby_value_type
C-level type of an object.
Definition value_type.h:112