Ruby: mjit.c Source File

Ruby 3.2.5p208 (2024-07-26 revision 31d0f1a2e7dbfb60731d1f05b868e1d578cda493)

mjit.c
1/**********************************************************************
2
3 mjit.c - MRI method JIT compiler functions
4
5 Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>.
6 Copyright (C) 2017 Takashi Kokubun <k0kubun@ruby-lang.org>.
7
8**********************************************************************/
9
10/* We utilize widely used C compilers (GCC and LLVM Clang) to
11 implement MJIT. We feed them a C code generated from ISEQ. The
12 industrial C compilers are slower than regular JIT engines.
13 Generated code performance of the used C compilers has a higher
14 priority over the compilation speed.
15
16 So our major goal is to minimize the ISEQ compilation time when we
17 use widely optimization level (-O2). It is achieved by
18
19 o Using a precompiled version of the header
20 o Keeping all files in `/tmp`. On modern Linux `/tmp` is a file
21 system in memory. So it is pretty fast
22 o Implementing MJIT as a multi-threaded code because we want to
23 compile ISEQs in parallel with iseq execution to speed up Ruby
24 code execution. MJIT has one thread (*worker*) to do
25 parallel compilations:
26 o It prepares a precompiled code of the minimized header.
27 It starts at the MRI execution start
28 o It generates PIC object files of ISEQs
29 o It takes one JIT unit from a priority queue unless it is empty.
30 o It translates the JIT unit ISEQ into C-code using the precompiled
31 header, calls CC and load PIC code when it is ready
32 o Currently MJIT put ISEQ in the queue when ISEQ is called
33 o MJIT can reorder ISEQs in the queue if some ISEQ has been called
34 many times and its compilation did not start yet
35 o MRI reuses the machine code if it already exists for ISEQ
36 o The machine code we generate can stop and switch to the ISEQ
37 interpretation if some condition is not satisfied as the machine
38 code can be speculative or some exception raises
39 o Speculative machine code can be canceled.
40
41 Here is a diagram showing the MJIT organization:
42
43 _______
44 |header |
45 |_______|
46 | MRI building
47 --------------|----------------------------------------
48 | MRI execution
49 |
50 _____________|_____
51 | | |
52 | ___V__ | CC ____________________
53 | | |----------->| precompiled header |
54 | | | | |____________________|
55 | | | | |
56 | | MJIT | | |
57 | | | | |
58 | | | | ____V___ CC __________
59 | |______|----------->| C code |--->| .so file |
60 | | |________| |__________|
61 | | |
62 | | |
63 | MRI machine code |<-----------------------------
64 |___________________| loading
65
66*/
67
68#include "ruby/internal/config.h" // defines USE_MJIT
69
70#if USE_MJIT
71
72#include "constant.h"
73#include "id_table.h"
74#include "internal.h"
75#include "internal/class.h"
76#include "internal/cmdlineopt.h"
77#include "internal/cont.h"
78#include "internal/file.h"
79#include "internal/hash.h"
80#include "internal/process.h"
81#include "internal/warnings.h"
82#include "vm_sync.h"
83#include "ractor_core.h"
84
85#ifdef __sun
86#define __EXTENSIONS__ 1
87#endif
88
89#include "vm_core.h"
90#include "vm_callinfo.h"
91#include "mjit.h"
92#include "mjit_c.h"
93#include "gc.h"
94#include "ruby_assert.h"
95#include "ruby/debug.h"
96#include "ruby/thread.h"
97#include "ruby/version.h"
98#include "builtin.h"
99#include "insns.inc"
100#include "insns_info.inc"
101#include "internal/compile.h"
102
103#include <sys/wait.h>
104#include <sys/time.h>
105#include <dlfcn.h>
106#include <errno.h>
107#ifdef HAVE_FCNTL_H
108#include <fcntl.h>
109#endif
110#ifdef HAVE_SYS_PARAM_H
111# include <sys/param.h>
112#endif
113#include "dln.h"
114
115#include "ruby/util.h"
116#undef strdup // ruby_strdup may trigger GC
117
118#ifndef MAXPATHLEN
119# define MAXPATHLEN 1024
120#endif
121
122// Atomically set function pointer if possible.
123#define MJIT_ATOMIC_SET(var, val) (void)ATOMIC_PTR_EXCHANGE(var, val)
124
125#define MJIT_TMP_PREFIX "_ruby_mjit_"
126
127extern void rb_native_mutex_lock(rb_nativethread_lock_t *lock);
128extern void rb_native_mutex_unlock(rb_nativethread_lock_t *lock);
129extern void rb_native_mutex_initialize(rb_nativethread_lock_t *lock);
130extern void rb_native_mutex_destroy(rb_nativethread_lock_t *lock);
131
132// process.c
133extern void mjit_add_waiting_pid(rb_vm_t *vm, rb_pid_t pid);
134
135// A copy of MJIT portion of MRI options since MJIT initialization. We
136// need them as MJIT threads still can work when the most MRI data were
137// freed.
138struct mjit_options mjit_opts;
139
140// true if MJIT is enabled.
141bool mjit_enabled = false;
142// true if JIT-ed code should be called. When `ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS`
143// and `mjit_call_p == false`, any JIT-ed code execution is cancelled as soon as possible.
144bool mjit_call_p = false;
145// A flag to communicate that mjit_call_p should be disabled while it's temporarily false.
146bool mjit_cancel_p = false;
147// There's an ISEQ in unit_queue whose total_calls reached 2 * call_threshold.
148// If this is true, check_unit_queue will start compiling ISEQs in unit_queue.
149static bool mjit_compile_p = false;
150// The actual number of units in active_units
151static int active_units_length = 0;
152// The actual number of units in compact_units
153static int compact_units_length = 0;
154
155// Priority queue of iseqs waiting for JIT compilation.
156// This variable is a pointer to head unit of the queue.
157static struct rb_mjit_unit_list unit_queue = { CCAN_LIST_HEAD_INIT(unit_queue.head) };
158// List of units which are successfully compiled.
159static struct rb_mjit_unit_list active_units = { CCAN_LIST_HEAD_INIT(active_units.head) };
160// List of compacted so files which will be cleaned up by `free_list()` in `mjit_finish()`.
161static struct rb_mjit_unit_list compact_units = { CCAN_LIST_HEAD_INIT(compact_units.head) };
162// List of units before recompilation and just waiting for dlclose().
163static struct rb_mjit_unit_list stale_units = { CCAN_LIST_HEAD_INIT(stale_units.head) };
164// The number of so far processed ISEQs, used to generate unique id.
165static int current_unit_num;
166// A mutex for conitionals and critical sections.
167static rb_nativethread_lock_t mjit_engine_mutex;
168// Set to true to stop worker.
169static bool stop_worker_p;
170// Set to true if worker is stopped.
171static bool worker_stopped = true;
172
173// Path of "/tmp", which is different on Windows or macOS. See: system_default_tmpdir()
174static char *tmp_dir;
175
176// Used C compiler path.
177static const char *cc_path;
178// Used C compiler flags.
179static const char **cc_common_args;
180// Used C compiler flags added by --mjit-debug=...
181static char **cc_added_args;
182// Name of the precompiled header file.
183static char *pch_file;
184// The process id which should delete the pch_file on mjit_finish.
185static rb_pid_t pch_owner_pid;
186// Status of the precompiled header creation. The status is
187// shared by the workers and the pch thread.
188static enum {PCH_NOT_READY, PCH_FAILED, PCH_SUCCESS} pch_status;
189
190// The start timestamp of current compilation
191static double current_cc_ms = 0.0; // TODO: make this part of unit?
192// Currently compiling MJIT unit
193static struct rb_mjit_unit *current_cc_unit = NULL;
194// PID of currently running C compiler process. 0 if nothing is running.
195static pid_t current_cc_pid = 0; // TODO: make this part of unit?
196
197// Name of the header file.
198static char *header_file;
199
200#include "mjit_config.h"
201
202#if defined(__GNUC__) && \
203 (!defined(__clang__) || \
204 (defined(__clang__) && (defined(__FreeBSD__) || defined(__GLIBC__))))
205# define GCC_PIC_FLAGS "-Wfatal-errors", "-fPIC", "-shared", "-w", "-pipe",
206# define MJIT_CFLAGS_PIPE 1
207#else
208# define GCC_PIC_FLAGS /* empty */
209# define MJIT_CFLAGS_PIPE 0
210#endif
211
212// Use `-nodefaultlibs -nostdlib` for GCC where possible, which does not work on cygwin, AIX, and OpenBSD.
213// This seems to improve MJIT performance on GCC.
214#if defined __GNUC__ && !defined __clang__ && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__)
215# define GCC_NOSTDLIB_FLAGS "-nodefaultlibs", "-nostdlib",
216#else
217# define GCC_NOSTDLIB_FLAGS // empty
218#endif
219
220static const char *const CC_COMMON_ARGS[] = {
221 MJIT_CC_COMMON MJIT_CFLAGS GCC_PIC_FLAGS
222 NULL
223};
224
225static const char *const CC_DEBUG_ARGS[] = {MJIT_DEBUGFLAGS NULL};
226static const char *const CC_OPTIMIZE_ARGS[] = {MJIT_OPTFLAGS NULL};
227
228static const char *const CC_LDSHARED_ARGS[] = {MJIT_LDSHARED MJIT_CFLAGS GCC_PIC_FLAGS NULL};
229static const char *const CC_DLDFLAGS_ARGS[] = {MJIT_DLDFLAGS NULL};
230// `CC_LINKER_ARGS` are linker flags which must be passed to `-c` as well.
231static const char *const CC_LINKER_ARGS[] = {
232#if defined __GNUC__ && !defined __clang__ && !defined(__OpenBSD__)
233 "-nostartfiles",
234#endif
235 GCC_NOSTDLIB_FLAGS NULL
236};
237
238static const char *const CC_LIBS[] = {
239#if defined(__CYGWIN__)
240 MJIT_LIBS // mswin, cygwin
241#endif
242#if defined __GNUC__ && !defined __clang__
243 "-lgcc", // cygwin, and GCC platforms using `-nodefaultlibs -nostdlib`
244#endif
245#if defined __ANDROID__
246 "-lm", // to avoid 'cannot locate symbol "modf" referenced by .../_ruby_mjit_XXX.so"'
247#endif
248 NULL
249};
250
251#define CC_CODEFLAG_ARGS (mjit_opts.debug ? CC_DEBUG_ARGS : CC_OPTIMIZE_ARGS)
252
253// Print the arguments according to FORMAT to stderr only if MJIT
254// verbose option value is more or equal to LEVEL.
255PRINTF_ARGS(static void, 2, 3)
256verbose(int level, const char *format, ...)
257{
258 if (mjit_opts.verbose >= level) {
259 va_list args;
260 size_t len = strlen(format);
261 char *full_format = alloca(sizeof(char) * (len + 2));
262
263 // Creating `format + '\n'` to atomically print format and '\n'.
264 memcpy(full_format, format, len);
265 full_format[len] = '\n';
266 full_format[len+1] = '\0';
267
268 va_start(args, format);
269 vfprintf(stderr, full_format, args);
270 va_end(args);
271 }
272}
273
274PRINTF_ARGS(static void, 1, 2)
275mjit_warning(const char *format, ...)
276{
277 if (mjit_opts.warnings || mjit_opts.verbose) {
278 va_list args;
279
280 fprintf(stderr, "MJIT warning: ");
281 va_start(args, format);
282 vfprintf(stderr, format, args);
283 va_end(args);
284 fprintf(stderr, "\n");
285 }
286}
287
288// Add unit node to the tail of doubly linked `list`. It should be not in
289// the list before.
290static void
291add_to_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
292{
293 ccan_list_add_tail(&list->head, &unit->unode);
294 list->length++;
295}
296
297static void
298remove_from_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
299{
300 ccan_list_del(&unit->unode);
301 list->length--;
302}
303
304static void
305remove_file(const char *filename)
306{
307 if (remove(filename)) {
308 mjit_warning("failed to remove \"%s\": %s", filename, strerror(errno));
309 }
310}
311
312// This is called in the following situations:
313// 1) On dequeue or `unload_units()`, associated ISeq is already GCed.
314// 2) The unit is not called often and unloaded by `unload_units()`.
315// 3) Freeing lists on `mjit_finish()`.
316//
317// `jit_func` value does not matter for 1 and 3 since the unit won't be used anymore.
318// For the situation 2, this sets the ISeq's JIT state to MJIT_FUNC_FAILED
319// to prevent the situation that the same methods are continuously compiled.
320static void
321free_unit(struct rb_mjit_unit *unit)
322{
323 if (unit->iseq) { // ISeq is not GCed
324 ISEQ_BODY(unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED;
325 ISEQ_BODY(unit->iseq)->mjit_unit = NULL;
326 }
327 if (unit->cc_entries) {
328 void *entries = (void *)unit->cc_entries;
329 free(entries);
330 }
331 if (unit->handle && dlclose(unit->handle)) { // handle is NULL if it's in queue
332 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
333 }
334 xfree(unit);
335}
336
337// Start a critical section. Use message `msg` to print debug info at `level`.
338static inline void
339CRITICAL_SECTION_START(int level, const char *msg)
340{
341 verbose(level, "Locking %s", msg);
342 rb_native_mutex_lock(&mjit_engine_mutex);
343 verbose(level, "Locked %s", msg);
344}
345
346// Finish the current critical section. Use message `msg` to print
347// debug info at `level`.
348static inline void
349CRITICAL_SECTION_FINISH(int level, const char *msg)
350{
351 verbose(level, "Unlocked %s", msg);
352 rb_native_mutex_unlock(&mjit_engine_mutex);
353}
354
355static pid_t mjit_pid = 0;
356
357static int
358sprint_uniq_filename(char *str, size_t size, unsigned long id, const char *prefix, const char *suffix)
359{
360 return snprintf(str, size, "%s/%sp%"PRI_PIDT_PREFIX"uu%lu%s", tmp_dir, prefix, mjit_pid, id, suffix);
361}
362
363// Return time in milliseconds as a double.
364#ifdef __APPLE__
365double ruby_real_ms_time(void);
366# define real_ms_time() ruby_real_ms_time()
367#else
368static double
369real_ms_time(void)
370{
371# ifdef HAVE_CLOCK_GETTIME
372 struct timespec tv;
373# ifdef CLOCK_MONOTONIC
374 const clockid_t c = CLOCK_MONOTONIC;
375# else
376 const clockid_t c = CLOCK_REALTIME;
377# endif
378
379 clock_gettime(c, &tv);
380 return tv.tv_nsec / 1000000.0 + tv.tv_sec * 1000.0;
381# else
382 struct timeval tv;
383
384 gettimeofday(&tv, NULL);
385 return tv.tv_usec / 1000.0 + tv.tv_sec * 1000.0;
386# endif
387}
388#endif
389
390// Return the best unit from list. The best is the first
391// high priority unit or the unit whose iseq has the biggest number
392// of calls so far.
393static struct rb_mjit_unit *
394get_from_list(struct rb_mjit_unit_list *list)
395{
396 // Find iseq with max total_calls
397 struct rb_mjit_unit *unit = NULL, *next, *best = NULL;
398 ccan_list_for_each_safe(&list->head, unit, next, unode) {
399 if (unit->iseq == NULL) { // ISeq is GCed.
400 remove_from_list(unit, list);
401 free_unit(unit);
402 continue;
403 }
404
405 if (best == NULL || ISEQ_BODY(best->iseq)->total_calls < ISEQ_BODY(unit->iseq)->total_calls) {
406 best = unit;
407 }
408 }
409
410 if (best) {
411 remove_from_list(best, list);
412 }
413 return best;
414}
415
416// Return length of NULL-terminated array `args` excluding the NULL marker.
417static size_t
418args_len(char *const *args)
419{
420 size_t i;
421
422 for (i = 0; (args[i]) != NULL;i++)
423 ;
424 return i;
425}
426
427// Concatenate `num` passed NULL-terminated arrays of strings, put the
428// result (with NULL end marker) into the heap, and return the result.
429static char **
430form_args(int num, ...)
431{
432 va_list argp;
433 size_t len, n;
434 int i;
435 char **args, **res, **tmp;
436
437 va_start(argp, num);
438 res = NULL;
439 for (i = len = 0; i < num; i++) {
440 args = va_arg(argp, char **);
441 n = args_len(args);
442 if ((tmp = (char **)realloc(res, sizeof(char *) * (len + n + 1))) == NULL) {
443 free(res);
444 res = NULL;
445 break;
446 }
447 res = tmp;
448 MEMCPY(res + len, args, char *, n + 1);
449 len += n;
450 }
451 va_end(argp);
452 return res;
453}
454
455COMPILER_WARNING_PUSH
456#if __has_warning("-Wdeprecated-declarations") || RBIMPL_COMPILER_IS(GCC)
457COMPILER_WARNING_IGNORED(-Wdeprecated-declarations)
458#endif
459// Start an OS process of absolute executable path with arguments `argv`.
460// Return PID of the process.
461static pid_t
462start_process(const char *abspath, char *const *argv)
463{
464 // Not calling non-async-signal-safe functions between vfork
465 // and execv for safety
466 int dev_null = rb_cloexec_open(ruby_null_device, O_WRONLY, 0);
467 if (dev_null < 0) {
468 verbose(1, "MJIT: Failed to open a null device: %s", strerror(errno));
469 return -1;
470 }
471 if (mjit_opts.verbose >= 2) {
472 const char *arg;
473 fprintf(stderr, "Starting process: %s", abspath);
474 for (int i = 0; (arg = argv[i]) != NULL; i++)
475 fprintf(stderr, " %s", arg);
476 fprintf(stderr, "\n");
477 }
478
479 pid_t pid;
480 if ((pid = vfork()) == 0) { /* TODO: reuse some function in process.c */
481 umask(0077);
482 if (mjit_opts.verbose == 0) {
483 // CC can be started in a thread using a file which has been
484 // already removed while MJIT is finishing. Discard the
485 // messages about missing files.
486 dup2(dev_null, STDERR_FILENO);
487 dup2(dev_null, STDOUT_FILENO);
488 }
489 (void)close(dev_null);
490 pid = execv(abspath, argv); // Pid will be negative on an error
491 // Even if we successfully found CC to compile PCH we still can
492 // fail with loading the CC in very rare cases for some reasons.
493 // Stop the forked process in this case.
494 verbose(1, "MJIT: Error in execv: %s", abspath);
495 _exit(1);
496 }
497 (void)close(dev_null);
498 return pid;
499}
500COMPILER_WARNING_POP
501
502// Execute an OS process of executable PATH with arguments ARGV.
503// Return -1 or -2 if failed to execute, otherwise exit code of the process.
504// TODO: Use a similar function in process.c
505static int
506exec_process(const char *path, char *const argv[])
507{
508 int stat, exit_code = -2;
509 pid_t pid = start_process(path, argv);
510 for (;pid > 0;) {
511 pid_t r = waitpid(pid, &stat, 0);
512 if (r == -1) {
513 if (errno == EINTR) continue;
514 fprintf(stderr, "[%"PRI_PIDT_PREFIX"d] waitpid(%lu): %s (SIGCHLD=%d,%u)\n",
515 getpid(), (unsigned long)pid, strerror(errno),
516 RUBY_SIGCHLD, SIGCHLD_LOSSY);
517 break;
518 }
519 else if (r == pid) {
520 if (WIFEXITED(stat)) {
521 exit_code = WEXITSTATUS(stat);
522 break;
523 }
524 else if (WIFSIGNALED(stat)) {
525 exit_code = -1;
526 break;
527 }
528 }
529 }
530 return exit_code;
531}
532
533static void
534remove_so_file(const char *so_file, struct rb_mjit_unit *unit)
535{
536 remove_file(so_file);
537}
538
539// Print _mjitX, but make a human-readable funcname when --mjit-debug is used
540static void
541sprint_funcname(char *funcname, size_t funcname_size, const struct rb_mjit_unit *unit)
542{
543 const rb_iseq_t *iseq = unit->iseq;
544 if (iseq == NULL || (!mjit_opts.debug && !mjit_opts.debug_flags)) {
545 snprintf(funcname, funcname_size, "_mjit%d", unit->id);
546 return;
547 }
548
549 // Generate a short path
550 const char *path = RSTRING_PTR(rb_iseq_path(iseq));
551 const char *lib = "/lib/";
552 const char *version = "/" STRINGIZE(RUBY_API_VERSION_MAJOR) "." STRINGIZE(RUBY_API_VERSION_MINOR) "." STRINGIZE(RUBY_API_VERSION_TEENY) "/";
553 while (strstr(path, lib)) // skip "/lib/"
554 path = strstr(path, lib) + strlen(lib);
555 while (strstr(path, version)) // skip "/x.y.z/"
556 path = strstr(path, version) + strlen(version);
557
558 // Annotate all-normalized method names
559 const char *method = RSTRING_PTR(ISEQ_BODY(iseq)->location.label);
560 if (!strcmp(method, "[]")) method = "AREF";
561 if (!strcmp(method, "[]=")) method = "ASET";
562
563 // Print and normalize
564 snprintf(funcname, funcname_size, "_mjit%d_%s_%s", unit->id, path, method);
565 for (size_t i = 0; i < strlen(funcname); i++) {
566 char c = funcname[i];
567 if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_')) {
568 funcname[i] = '_';
569 }
570 }
571}
572
573static const int c_file_access_mode =
574#ifdef O_BINARY
575 O_BINARY|
576#endif
577 O_WRONLY|O_EXCL|O_CREAT;
578
579#define append_str2(p, str, len) ((char *)memcpy((p), str, (len))+(len))
580#define append_str(p, str) append_str2(p, str, sizeof(str)-1)
581#define append_lit(p, str) append_str2(p, str, rb_strlen_lit(str))
582
583// The function producing the pre-compiled header.
584static void
585make_pch(void)
586{
587 const char *rest_args[] = {
588# ifdef __clang__
589 "-emit-pch",
590 "-c",
591# endif
592 // -nodefaultlibs is a linker flag, but it may affect cc1 behavior on Gentoo, which should NOT be changed on pch:
593 // https://gitweb.gentoo.org/proj/gcc-patches.git/tree/7.3.0/gentoo/13_all_default-ssp-fix.patch
594 GCC_NOSTDLIB_FLAGS
595 "-o", pch_file, header_file,
596 NULL,
597 };
598
599 verbose(2, "Creating precompiled header");
600 char **args = form_args(4, cc_common_args, CC_CODEFLAG_ARGS, cc_added_args, rest_args);
601 if (args == NULL) {
602 mjit_warning("making precompiled header failed on forming args");
603 pch_status = PCH_FAILED;
604 return;
605 }
606
607 int exit_code = exec_process(cc_path, args);
608 free(args);
609
610 if (exit_code == 0) {
611 pch_status = PCH_SUCCESS;
612 }
613 else {
614 mjit_warning("Making precompiled header failed on compilation. Stopping MJIT worker...");
615 pch_status = PCH_FAILED;
616 }
617}
618
619static int
620c_compile(const char *c_file, const char *so_file)
621{
622 const char *so_args[] = {
623 "-o", so_file,
624# ifdef __clang__
625 "-include-pch", pch_file,
626# endif
627 c_file, NULL
628 };
629
630# if defined(__MACH__)
631 extern VALUE rb_libruby_selfpath;
632 const char *loader_args[] = {"-bundle_loader", StringValuePtr(rb_libruby_selfpath), NULL};
633# else
634 const char *loader_args[] = {NULL};
635# endif
636
637 char **args = form_args(8, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS, cc_added_args,
638 so_args, loader_args, CC_LIBS, CC_DLDFLAGS_ARGS, CC_LINKER_ARGS);
639 if (args == NULL) return 1;
640
641 int exit_code = exec_process(cc_path, args);
642 if (!mjit_opts.save_temps)
643 remove_file(c_file);
644
645 free(args);
646 return exit_code;
647}
648
649static int
650c_compile_unit(struct rb_mjit_unit *unit)
651{
652 static const char c_ext[] = ".c";
653 static const char so_ext[] = DLEXT;
654 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
655
656 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
657 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
658
659 return c_compile(c_file, so_file);
660}
661
662static void compile_prelude(FILE *f);
663
664static bool
665mjit_batch(struct rb_mjit_unit *unit)
666{
667 VM_ASSERT(unit->type == MJIT_UNIT_BATCH);
668 static const char c_ext[] = ".c";
669 static const char so_ext[] = DLEXT;
670 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
671
672 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
673 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
674
675 FILE *f;
676 int fd = rb_cloexec_open(c_file, c_file_access_mode, 0600);
677 if (fd < 0 || (f = fdopen(fd, "w")) == NULL) {
678 int e = errno;
679 if (fd >= 0) (void)close(fd);
680 verbose(1, "Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
681 return false;
682 }
683
684 compile_prelude(f);
685
686 bool success = true;
687 struct rb_mjit_unit *child_unit = 0;
688 ccan_list_for_each(&unit->units.head, child_unit, unode) {
689 if (!success) continue;
690 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
691
692 char funcname[MAXPATHLEN];
693 sprint_funcname(funcname, sizeof(funcname), child_unit);
694
695 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
696 const char *sep = "@";
697 const char *iseq_label = RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
698 const char *iseq_path = RSTRING_PTR(rb_iseq_path(child_unit->iseq));
699 if (!iseq_label) iseq_label = sep = "";
700 fprintf(f, "\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
701 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
702 }
703
704 fclose(f);
705 return success;
706}
707
708// Compile all cached .c files and build a single .so file. Reload all JIT func from it.
709// This improves the code locality for better performance in terms of iTLB and iCache.
710static bool
711mjit_compact(struct rb_mjit_unit *unit)
712{
713 VM_ASSERT(unit->type == MJIT_UNIT_COMPACT);
714 static const char c_ext[] = ".c";
715 static const char so_ext[] = DLEXT;
716 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
717
718 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
719 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
720
721 FILE *f;
722 int fd = rb_cloexec_open(c_file, c_file_access_mode, 0600);
723 if (fd < 0 || (f = fdopen(fd, "w")) == NULL) {
724 int e = errno;
725 if (fd >= 0) (void)close(fd);
726 verbose(1, "Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
727 return false;
728 }
729
730 compile_prelude(f);
731
732 bool success = true;
733 compact_units_length = 0;
734 struct rb_mjit_unit *batch_unit = 0, *child_unit = 0;
735 ccan_list_for_each(&active_units.head, batch_unit, unode) {
736 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
737 if (!success) continue;
738 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
739
740 char funcname[MAXPATHLEN];
741 sprint_funcname(funcname, sizeof(funcname), child_unit);
742
743 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
744 const char *sep = "@";
745 const char *iseq_label = RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
746 const char *iseq_path = RSTRING_PTR(rb_iseq_path(child_unit->iseq));
747 if (!iseq_label) iseq_label = sep = "";
748 fprintf(f, "\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
749 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
750 compact_units_length++;
751 }
752 }
753
754 fclose(f);
755 return success;
756}
757
758static void
759load_batch_funcs_from_so(struct rb_mjit_unit *unit, char *c_file, char *so_file)
760{
761 double end_time = real_ms_time();
762
763 void *handle = dlopen(so_file, RTLD_NOW);
764 if (handle == NULL) {
765 mjit_warning("failure in loading code from batched '%s': %s", so_file, dlerror());
766 xfree(unit);
767 return;
768 }
769 unit->handle = handle;
770
771 // lazily dlclose handle on `mjit_finish()`.
772 add_to_list(unit, &active_units);
773 active_units_length += unit->units.length;
774
775 if (!mjit_opts.save_temps)
776 remove_so_file(so_file, unit);
777
778 struct rb_mjit_unit *child_unit = 0;
779 ccan_list_for_each(&unit->units.head, child_unit, unode) {
780 char funcname[MAXPATHLEN];
781 sprint_funcname(funcname, sizeof(funcname), child_unit);
782
783 void *func;
784 if ((func = dlsym(handle, funcname)) == NULL) {
785 mjit_warning("skipping to load '%s' from '%s': %s", funcname, so_file, dlerror());
786 continue;
787 }
788
789 if (child_unit->iseq) { // Check whether GCed or not
790 // Usage of jit_code might be not in a critical section.
791 const rb_iseq_t *iseq = child_unit->iseq;
792 MJIT_ATOMIC_SET(ISEQ_BODY(iseq)->jit_func, (jit_func_t)func);
793
794 verbose(1, "JIT success: %s@%s:%d",
795 RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
796 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
797 }
798 else {
799 verbose(1, "JIT skip: A compiled method has been GCed");
800 }
801 }
802 verbose(1, "JIT batch (%.1fms): Batched %d methods %s -> %s", end_time - current_cc_ms, unit->units.length, c_file, so_file);
803}
804
805static void
806load_compact_funcs_from_so(struct rb_mjit_unit *unit, char *c_file, char *so_file)
807{
808 double end_time = real_ms_time();
809
810 void *handle = dlopen(so_file, RTLD_NOW);
811 if (handle == NULL) {
812 mjit_warning("failure in loading code from compacted '%s': %s", so_file, dlerror());
813 xfree(unit);
814 return;
815 }
816 unit->handle = handle;
817
818 // lazily dlclose handle on `mjit_finish()`.
819 add_to_list(unit, &compact_units);
820
821 if (!mjit_opts.save_temps)
822 remove_so_file(so_file, unit);
823
824 struct rb_mjit_unit *batch_unit = 0, *child_unit = 0;
825 ccan_list_for_each(&active_units.head, batch_unit, unode) {
826 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
827 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
828
829 char funcname[MAXPATHLEN];
830 sprint_funcname(funcname, sizeof(funcname), child_unit);
831
832 void *func;
833 if ((func = dlsym(handle, funcname)) == NULL) {
834 mjit_warning("skipping to reload '%s' from '%s': %s", funcname, so_file, dlerror());
835 continue;
836 }
837
838 if (child_unit->iseq) { // Check whether GCed or not
839 // Usage of jit_code might be not in a critical section.
840 MJIT_ATOMIC_SET(ISEQ_BODY(child_unit->iseq)->jit_func, (jit_func_t)func);
841 }
842 }
843 }
844 verbose(1, "JIT compaction (%.1fms): Compacted %d methods %s -> %s", end_time - current_cc_ms, active_units_length, c_file, so_file);
845}
846
847#ifndef __clang__
848static const char *
849header_name_end(const char *s)
850{
851 const char *e = s + strlen(s);
852# ifdef __GNUC__ // don't chomp .pch for mswin
853 static const char suffix[] = ".gch";
854
855 // chomp .gch suffix
856 if (e > s+sizeof(suffix)-1 && strcmp(e-sizeof(suffix)+1, suffix) == 0) {
857 e -= sizeof(suffix)-1;
858 }
859# endif
860 return e;
861}
862#endif
863
864// Print platform-specific prerequisites in generated code.
865static void
866compile_prelude(FILE *f)
867{
868#ifndef __clang__ // -include-pch is used for Clang
869 const char *s = pch_file;
870 const char *e = header_name_end(s);
871
872 fprintf(f, "#include \"");
873 // print pch_file except .gch for gcc, but keep .pch for mswin
874 for (; s < e; s++) {
875 switch (*s) {
876 case '\\': case '"':
877 fputc('\\', f);
878 }
879 fputc(*s, f);
880 }
881 fprintf(f, "\"\n");
882#endif
883}
884
885static pid_t
886start_c_compile_unit(struct rb_mjit_unit *unit)
887{
888 extern pid_t rb_mjit_fork();
889 pid_t pid = rb_mjit_fork();
890 if (pid == 0) {
891 int exit_code = c_compile_unit(unit);
892 exit(exit_code);
893 }
894 else {
895 return pid;
896 }
897}
898
899// Capture cc entries of `captured_iseq` and append them to `compiled_iseq->mjit_unit->cc_entries`.
900// This is needed when `captured_iseq` is inlined by `compiled_iseq` and GC needs to mark inlined cc.
901//
902// Index to refer to `compiled_iseq->mjit_unit->cc_entries` is returned instead of the address
903// because old addresses may be invalidated by `realloc` later. -1 is returned on failure.
904//
905// This assumes that it's safe to reference cc without acquiring GVL.
906int
907mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq)
908{
909 VM_ASSERT(compiled_iseq != NULL);
910 VM_ASSERT(compiled_iseq->mjit_unit != NULL);
911 VM_ASSERT(captured_iseq != NULL);
912
913 struct rb_mjit_unit *unit = compiled_iseq->mjit_unit;
914 unsigned int new_entries_size = unit->cc_entries_size + captured_iseq->ci_size;
915 VM_ASSERT(captured_iseq->ci_size > 0);
916
917 // Allocate new cc_entries and append them to unit->cc_entries
918 const struct rb_callcache **cc_entries;
919 int cc_entries_index = unit->cc_entries_size;
920 if (unit->cc_entries_size == 0) {
921 VM_ASSERT(unit->cc_entries == NULL);
922 unit->cc_entries = cc_entries = malloc(sizeof(struct rb_callcache *) * new_entries_size);
923 if (cc_entries == NULL) return -1;
924 }
925 else {
926 void *cc_ptr = (void *)unit->cc_entries; // get rid of bogus warning by VC
927 cc_entries = realloc(cc_ptr, sizeof(struct rb_callcache *) * new_entries_size);
928 if (cc_entries == NULL) return -1;
929 unit->cc_entries = cc_entries;
930 cc_entries += cc_entries_index;
931 }
932 unit->cc_entries_size = new_entries_size;
933
934 // Capture cc to cc_enties
935 for (unsigned int i = 0; i < captured_iseq->ci_size; i++) {
936 cc_entries[i] = captured_iseq->call_data[i].cc;
937 }
938
939 return cc_entries_index;
940}
941
942static void mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info);
943
944// Return an unique file name in /tmp with PREFIX and SUFFIX and
945// number ID. Use getpid if ID == 0. The return file name exists
946// until the next function call.
947static char *
948get_uniq_filename(unsigned long id, const char *prefix, const char *suffix)
949{
950 char buff[70], *str = buff;
951 int size = sprint_uniq_filename(buff, sizeof(buff), id, prefix, suffix);
952 str = 0;
953 ++size;
954 str = xmalloc(size);
955 if (size <= (int)sizeof(buff)) {
956 memcpy(str, buff, size);
957 }
958 else {
959 sprint_uniq_filename(str, size, id, prefix, suffix);
960 }
961 return str;
962}
963
964// Prohibit calling JIT-ed code and let existing JIT-ed frames exit before the next insn.
965void
966mjit_cancel_all(const char *reason)
967{
968 if (!mjit_enabled)
969 return;
970
971 mjit_call_p = false;
972 mjit_cancel_p = true;
973 if (mjit_opts.warnings || mjit_opts.verbose) {
974 fprintf(stderr, "JIT cancel: Disabled JIT-ed code because %s\n", reason);
975 }
976}
977
978// Deal with ISeq movement from compactor
979void
980mjit_update_references(const rb_iseq_t *iseq)
981{
982 if (!mjit_enabled)
983 return;
984
985 CRITICAL_SECTION_START(4, "mjit_update_references");
986 if (ISEQ_BODY(iseq)->mjit_unit) {
987 ISEQ_BODY(iseq)->mjit_unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)ISEQ_BODY(iseq)->mjit_unit->iseq);
988 // We need to invalidate JIT-ed code for the ISeq because it embeds pointer addresses.
989 // To efficiently do that, we use the same thing as TracePoint and thus everything is cancelled for now.
990 // See mjit.h and tool/ruby_vm/views/_mjit_compile_insn.erb for how `mjit_call_p` is used.
991 mjit_cancel_all("GC.compact is used"); // TODO: instead of cancelling all, invalidate only this one and recompile it with some threshold.
992 }
993
994 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
995 // `ISEQ_BODY(iseq)->mjit_unit` anymore (because new one replaces that). So we need to check them too.
996 // TODO: we should be able to reduce the number of units checked here.
997 struct rb_mjit_unit *unit = NULL;
998 ccan_list_for_each(&stale_units.head, unit, unode) {
999 if (unit->iseq == iseq) {
1000 unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)unit->iseq);
1001 }
1002 }
1003 CRITICAL_SECTION_FINISH(4, "mjit_update_references");
1004}
1005
1006// Iseqs can be garbage collected. This function should call when it
1007// happens. It removes iseq from the unit.
1008void
1009mjit_free_iseq(const rb_iseq_t *iseq)
1010{
1011 if (!mjit_enabled)
1012 return;
1013
1014 if (ISEQ_BODY(iseq)->mjit_unit) {
1015 // mjit_unit is not freed here because it may be referred by multiple
1016 // lists of units. `get_from_list` and `mjit_finish` do the job.
1017 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL;
1018 }
1019 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
1020 // `ISEQ_BODY(iseq)->mjit_unit` anymore (because new one replaces that). So we need to check them too.
1021 // TODO: we should be able to reduce the number of units checked here.
1022 struct rb_mjit_unit *unit = NULL;
1023 ccan_list_for_each(&stale_units.head, unit, unode) {
1024 if (unit->iseq == iseq) {
1025 unit->iseq = NULL;
1026 }
1027 }
1028}
1029
1030// Free unit list. This should be called only when worker is finished
1031// because node of unit_queue and one of active_units may have the same unit
1032// during proceeding unit.
1033static void
1034free_list(struct rb_mjit_unit_list *list, bool close_handle_p)
1035{
1036 struct rb_mjit_unit *unit = 0, *next;
1037
1038 ccan_list_for_each_safe(&list->head, unit, next, unode) {
1039 ccan_list_del(&unit->unode);
1040 if (!close_handle_p) unit->handle = NULL; /* Skip dlclose in free_unit() */
1041
1042 if (list == &stale_units) { // `free_unit(unit)` crashes after GC.compact on `stale_units`
1043 /*
1044 * TODO: REVERT THIS BRANCH
1045 * Debug the crash on stale_units w/ GC.compact and just use `free_unit(unit)`!!
1046 */
1047 if (unit->handle && dlclose(unit->handle)) {
1048 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
1049 }
1050 xfree(unit);
1051 }
1052 else {
1053 free_unit(unit);
1054 }
1055 }
1056 list->length = 0;
1057}
1058
1059static struct rb_mjit_unit*
1060create_unit(enum rb_mjit_unit_type type)
1061{
1062 struct rb_mjit_unit *unit = ZALLOC_N(struct rb_mjit_unit, 1);
1063 unit->id = current_unit_num++;
1064 unit->type = type;
1065 if (type == MJIT_UNIT_BATCH) {
1066 ccan_list_head_init(&unit->units.head);
1067 }
1068 return unit;
1069}
1070
1071static struct rb_mjit_unit*
1072create_iseq_unit(const rb_iseq_t *iseq)
1073{
1074 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_ISEQ);
1075 unit->iseq = (rb_iseq_t *)iseq;
1076 ISEQ_BODY(iseq)->mjit_unit = unit;
1077 return unit;
1078}
1079
1080static void mjit_wait(struct rb_mjit_unit *unit);
1081
1082// Check the unit queue and start mjit_compile if nothing is in progress.
1083static void
1084check_unit_queue(void)
1085{
1086 if (mjit_opts.custom) return; // Custom RubyVM::MJIT.compile is in use
1087 if (worker_stopped) return;
1088 if (current_cc_pid != 0) return; // still compiling
1089
1090 // TODO: resurrect unload_units
1091 if (active_units_length >= mjit_opts.max_cache_size) return; // wait until unload_units makes a progress
1092
1093 // No ISEQ in unit_queue has enough calls to trigger JIT
1094 if (!mjit_compile_p) return;
1095 mjit_compile_p = false;
1096
1097 // Compile all ISEQs in unit_queue together
1098 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_BATCH);
1099 struct rb_mjit_unit *child_unit = NULL;
1100 VM_ASSERT(unit_queue.length > 0);
1101 while ((child_unit = get_from_list(&unit_queue)) != NULL && (active_units_length + unit->units.length) < mjit_opts.max_cache_size) {
1102 add_to_list(child_unit, &unit->units);
1103 ISEQ_BODY(child_unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_COMPILING;
1104 }
1105
1106 // Run the MJIT compiler synchronously
1107 current_cc_ms = real_ms_time();
1108 current_cc_unit = unit;
1109 bool success = mjit_batch(unit);
1110 if (!success) {
1111 mjit_notify_waitpid(1);
1112 return;
1113 }
1114
1115 // Run the C compiler asynchronously (unless --mjit-wait)
1116 if (mjit_opts.wait) {
1117 int exit_code = c_compile_unit(unit);
1118 mjit_notify_waitpid(exit_code);
1119 }
1120 else {
1121 current_cc_pid = start_c_compile_unit(unit);
1122 if (current_cc_pid == -1) { // JIT failure
1123 mjit_notify_waitpid(1);
1124 }
1125 }
1126}
1127
1128// Check if it should compact all JIT code and start it as needed
1129static void
1130check_compaction(void)
1131{
1132 // Allow only `max_cache_size / 100` times (default: 100) of compaction.
1133 // Note: GC of compacted code has not been implemented yet.
1134 int max_compact_size = mjit_opts.max_cache_size / 100;
1135 if (max_compact_size < 10) max_compact_size = 10;
1136
1137 // Run JIT compaction only when it's going to add 10%+ units.
1138 int throttle_threshold = active_units_length / 10;
1139
1140 if (compact_units.length < max_compact_size
1141 && active_units_length - compact_units_length > throttle_threshold
1142 && ((!mjit_opts.wait && unit_queue.length == 0 && active_units.length > 1)
1143 || (active_units_length == mjit_opts.max_cache_size))) {
1144 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_COMPACT);
1145
1146 // Run the MJIT compiler synchronously
1147 current_cc_ms = real_ms_time();
1148 current_cc_unit = unit;
1149 bool success = mjit_compact(unit);
1150 if (!success) {
1151 mjit_notify_waitpid(1);
1152 return;
1153 }
1154
1155 // Run the C compiler asynchronously (unless --mjit-wait)
1156 if (mjit_opts.wait) {
1157 int exit_code = c_compile_unit(unit);
1158 mjit_notify_waitpid(exit_code);
1159 }
1160 else {
1161 current_cc_pid = start_c_compile_unit(unit);
1162 if (current_cc_pid == -1) { // JIT failure
1163 mjit_notify_waitpid(1);
1164 }
1165 }
1166 }
1167}
1168
1169// Check the current CC process if any, and start a next C compiler process as needed.
1170void
1171mjit_notify_waitpid(int exit_code)
1172{
1173 VM_ASSERT(mjit_opts.wait || current_cc_pid != 0);
1174 current_cc_pid = 0;
1175
1176 // Delete .c file
1177 char c_file[MAXPATHLEN];
1178 sprint_uniq_filename(c_file, (int)sizeof(c_file), current_cc_unit->id, MJIT_TMP_PREFIX, ".c");
1179
1180 // Check the result
1181 if (exit_code != 0) {
1182 verbose(2, "Failed to generate so");
1183 // TODO: set MJIT_FUNC_FAILED to unit->units
1184 // TODO: free list of unit->units
1185 free_unit(current_cc_unit);
1186 current_cc_unit = NULL;
1187 return;
1188 }
1189
1190 // Load .so file
1191 char so_file[MAXPATHLEN];
1192 sprint_uniq_filename(so_file, (int)sizeof(so_file), current_cc_unit->id, MJIT_TMP_PREFIX, DLEXT);
1193 switch (current_cc_unit->type) {
1194 case MJIT_UNIT_ISEQ:
1195 rb_bug("unreachable: current_cc_unit->type must not be MJIT_UNIT_ISEQ");
1196 case MJIT_UNIT_BATCH:
1197 load_batch_funcs_from_so(current_cc_unit, c_file, so_file);
1198 current_cc_unit = NULL;
1199
1200 // Run compaction if it should
1201 if (!stop_worker_p) {
1202 check_compaction();
1203 }
1204 break;
1205 case MJIT_UNIT_COMPACT:
1206 load_compact_funcs_from_so(current_cc_unit, c_file, so_file);
1207 current_cc_unit = NULL;
1208 break;
1209 }
1210
1211 // Skip further compilation if mjit_finish is trying to stop it
1212 if (!stop_worker_p) {
1213 // Start the next one as needed
1214 check_unit_queue();
1215 }
1216}
1217
1218// Return true if given ISeq body should be compiled by MJIT
1219static inline int
1220mjit_target_iseq_p(const rb_iseq_t *iseq)
1221{
1222 struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
1223 return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK)
1224 && !body->builtin_inline_p
1225 && strcmp("<internal:mjit>", RSTRING_PTR(rb_iseq_path(iseq))) != 0;
1226}
1227
1228// RubyVM::MJIT
1229static VALUE rb_mMJIT = 0;
1230// RubyVM::MJIT::C
1231static VALUE rb_mMJITC = 0;
1232// RubyVM::MJIT::Compiler
1233static VALUE rb_cMJITCompiler = 0;
1234// RubyVM::MJIT::CPointer::Struct_rb_iseq_t
1235static VALUE rb_cMJITIseqPtr = 0;
1236// RubyVM::MJIT::CPointer::Struct_IC
1237static VALUE rb_cMJITICPtr = 0;
1238// RubyVM::MJIT::Compiler
1239static VALUE rb_mMJITHooks = 0;
1240
1241#define WITH_MJIT_DISABLED(stmt) do { \
1242 bool original_call_p = mjit_call_p; \
1243 mjit_call_p = false; \
1244 stmt; \
1245 mjit_call_p = original_call_p; \
1246 if (mjit_cancel_p) mjit_call_p = false; \
1247} while (0);
1248
1249// Hook MJIT when BOP is redefined.
1250MJIT_FUNC_EXPORTED void
1251rb_mjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop)
1252{
1253 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1254 WITH_MJIT_DISABLED({
1255 rb_funcall(rb_mMJITHooks, rb_intern("on_bop_redefined"), 2, INT2NUM(redefined_flag), INT2NUM((int)bop));
1256 });
1257}
1258
1259// Hook MJIT when CME is invalidated.
1260MJIT_FUNC_EXPORTED void
1261rb_mjit_cme_invalidate(rb_callable_method_entry_t *cme)
1262{
1263 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1264 WITH_MJIT_DISABLED({
1265 VALUE cme_klass = rb_funcall(rb_mMJITC, rb_intern("rb_callable_method_entry_struct"), 0);
1266 VALUE cme_ptr = rb_funcall(cme_klass, rb_intern("new"), 1, SIZET2NUM((size_t)cme));
1267 rb_funcall(rb_mMJITHooks, rb_intern("on_cme_invalidate"), 1, cme_ptr);
1268 });
1269}
1270
1271// Hook MJIT when Ractor is spawned.
1272void
1273rb_mjit_before_ractor_spawn(void)
1274{
1275 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1276 WITH_MJIT_DISABLED({
1277 rb_funcall(rb_mMJITHooks, rb_intern("on_ractor_spawn"), 0);
1278 });
1279}
1280
1281static void
1282mjit_constant_state_changed(void *data)
1283{
1284 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1285 ID id = (ID)data;
1286 WITH_MJIT_DISABLED({
1287 rb_funcall(rb_mMJITHooks, rb_intern("on_constant_state_changed"), 1, ID2SYM(id));
1288 });
1289}
1290
1291// Hook MJIT when constant state is changed.
1292MJIT_FUNC_EXPORTED void
1293rb_mjit_constant_state_changed(ID id)
1294{
1295 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1296 // Asynchronously hook the Ruby code since this is hooked during a "Ruby critical section".
1297 extern int rb_workqueue_register(unsigned flags, rb_postponed_job_func_t func, void *data);
1298 rb_workqueue_register(0, mjit_constant_state_changed, (void *)id);
1299}
1300
1301// Hook MJIT when constant IC is updated.
1302MJIT_FUNC_EXPORTED void
1303rb_mjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx)
1304{
1305 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1306 WITH_MJIT_DISABLED({
1307 VALUE iseq_ptr = rb_funcall(rb_cMJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
1308 VALUE ic_ptr = rb_funcall(rb_cMJITICPtr, rb_intern("new"), 1, SIZET2NUM((size_t)ic));
1309 rb_funcall(rb_mMJITHooks, rb_intern("on_constant_ic_update"), 3, iseq_ptr, ic_ptr, UINT2NUM(insn_idx));
1310 });
1311}
1312
1313// Hook MJIT when TracePoint is enabled.
1314MJIT_FUNC_EXPORTED void
1315rb_mjit_tracing_invalidate_all(rb_event_flag_t new_iseq_events)
1316{
1317 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1318 WITH_MJIT_DISABLED({
1319 rb_funcall(rb_mMJITHooks, rb_intern("on_tracing_invalidate_all"), 1, UINT2NUM(new_iseq_events));
1320 });
1321}
1322
1323// [experimental] Call custom RubyVM::MJIT.compile if defined
1324static void
1325mjit_hook_custom_compile(const rb_iseq_t *iseq)
1326{
1327 WITH_MJIT_DISABLED({
1328 VALUE iseq_class = rb_funcall(rb_mMJITC, rb_intern("rb_iseq_t"), 0);
1329 VALUE iseq_ptr = rb_funcall(iseq_class, rb_intern("new"), 1, ULONG2NUM((size_t)iseq));
1330 VALUE jit_func = rb_funcall(rb_mMJIT, rb_intern("compile"), 1, iseq_ptr);
1331 ISEQ_BODY(iseq)->jit_func = (jit_func_t)NUM2ULONG(jit_func);
1332 });
1333}
1334
1335static void
1336mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info)
1337{
1338 if (!mjit_enabled) return;
1339 if (mjit_opts.custom) { // Hook custom RubyVM::MJIT.compile if defined
1340 mjit_hook_custom_compile(iseq);
1341 return;
1342 }
1343 if (pch_status != PCH_SUCCESS || !rb_ractor_main_p()) // TODO: Support non-main Ractors
1344 return;
1345 if (!mjit_target_iseq_p(iseq)) {
1346 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED; // skip mjit_wait
1347 return;
1348 }
1349
1350 // For batching multiple ISEQs, we only enqueue ISEQs when total_calls reaches call_threshold,
1351 // and compile all enqueued ISEQs when any ISEQ reaches call_threshold * 2.
1352 bool recompile_p = !MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func);
1353 if (!ISEQ_BODY(iseq)->mjit_unit || recompile_p) { // call_threshold, or recompile
1354 // Discard an old unit with recompile_p
1355 if (recompile_p) {
1356 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL; // Ignore this from compaction
1357 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_NOT_COMPILED;
1358 active_units_length--;
1359 }
1360
1361 // Create a new unit and enqueue it
1362 struct rb_mjit_unit *unit = create_iseq_unit(iseq);
1363 if (recompile_p) {
1364 VM_ASSERT(compile_info != NULL);
1365 unit->compile_info = *compile_info;
1366 }
1367 add_to_list(unit, &unit_queue);
1368 ISEQ_BODY(iseq)->total_calls = 0; // come here again :)
1369 }
1370 else { // call_threshold * 2
1371 VM_ASSERT(compile_info == NULL);
1372 mjit_compile_p = true; // compile all ISEQs in unit_queue
1373 }
1374}
1375
1376// Add ISEQ to be JITed in parallel with the current thread.
1377// Unload some JIT codes if there are too many of them.
1378void
1379rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq)
1380{
1381 mjit_add_iseq_to_process(iseq, NULL);
1382 check_unit_queue();
1383}
1384
1385// For this timeout seconds, mjit_finish will wait for JIT compilation finish.
1386#define MJIT_WAIT_TIMEOUT_SECONDS 5
1387
1388static void
1389mjit_wait(struct rb_mjit_unit *unit)
1390{
1391 pid_t initial_pid = current_cc_pid;
1392 if (initial_pid == 0) {
1393 mjit_warning("initial_pid was 0 on mjit_wait");
1394 return;
1395 }
1396 if (pch_status == PCH_FAILED) return;
1397
1398 int tries = 0;
1399 struct timeval tv = { .tv_sec = 0, .tv_usec = 1000 };
1400 while (current_cc_pid == initial_pid) {
1401 tries++;
1402 if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS) {
1403 if (unit->type == MJIT_UNIT_ISEQ) {
1404 unit->iseq->body->jit_func = (jit_func_t)MJIT_FUNC_FAILED; // C compiler was too slow. Give up.
1405 }
1406 mjit_warning("timed out to wait for JIT finish");
1407 break;
1408 }
1409
1410 rb_thread_wait_for(tv);
1411 }
1412}
1413
1414struct rb_mjit_compile_info*
1415rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body)
1416{
1417 VM_ASSERT(body->mjit_unit != NULL);
1418 return &body->mjit_unit->compile_info;
1419}
1420
1421static void
1422mjit_recompile(const rb_iseq_t *iseq)
1423{
1424 if (MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func))
1425 return;
1426
1427 verbose(1, "JIT recompile: %s@%s:%d", RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
1428 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
1429 VM_ASSERT(ISEQ_BODY(iseq)->mjit_unit != NULL);
1430
1431 mjit_add_iseq_to_process(iseq, &ISEQ_BODY(iseq)->mjit_unit->compile_info);
1432 check_unit_queue();
1433}
1434
1435// Recompile iseq, disabling send optimization
1436void
1437rb_mjit_recompile_send(const rb_iseq_t *iseq)
1438{
1439 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_send_cache = true;
1440 mjit_recompile(iseq);
1441}
1442
1443// Recompile iseq, disabling ivar optimization
1444void
1445rb_mjit_recompile_ivar(const rb_iseq_t *iseq)
1446{
1447 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_ivar_cache = true;
1448 mjit_recompile(iseq);
1449}
1450
1451// Recompile iseq, disabling exivar optimization
1452void
1453rb_mjit_recompile_exivar(const rb_iseq_t *iseq)
1454{
1455 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_exivar_cache = true;
1456 mjit_recompile(iseq);
1457}
1458
1459// Recompile iseq, disabling method inlining
1460void
1461rb_mjit_recompile_inlining(const rb_iseq_t *iseq)
1462{
1463 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_inlining = true;
1464 mjit_recompile(iseq);
1465}
1466
1467// Recompile iseq, disabling getconstant inlining
1468void
1469rb_mjit_recompile_const(const rb_iseq_t *iseq)
1470{
1471 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_const_cache = true;
1472 mjit_recompile(iseq);
1473}
1474
1475extern VALUE ruby_archlibdir_path, ruby_prefix_path;
1476
1477// Initialize header_file, pch_file, libruby_pathflag. Return true on success.
1478static bool
1479init_header_filename(void)
1480{
1481 int fd;
1482#ifdef LOAD_RELATIVE
1483 // Root path of the running ruby process. Equal to RbConfig::TOPDIR.
1484 VALUE basedir_val;
1485#endif
1486 const char *basedir = "";
1487 size_t baselen = 0;
1488 char *p;
1489
1490#ifdef LOAD_RELATIVE
1491 basedir_val = ruby_prefix_path;
1492 basedir = StringValuePtr(basedir_val);
1493 baselen = RSTRING_LEN(basedir_val);
1494#else
1495 if (getenv("MJIT_SEARCH_BUILD_DIR")) {
1496 // This path is not intended to be used on production, but using build directory's
1497 // header file here because people want to run `make test-all` without running
1498 // `make install`. Don't use $MJIT_SEARCH_BUILD_DIR except for test-all.
1499
1500 struct stat st;
1501 const char *hdr = dlsym(RTLD_DEFAULT, "MJIT_HEADER");
1502 if (!hdr) {
1503 verbose(1, "No MJIT_HEADER");
1504 }
1505 else if (hdr[0] != '/') {
1506 verbose(1, "Non-absolute header file path: %s", hdr);
1507 }
1508 else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) {
1509 verbose(1, "Non-file header file path: %s", hdr);
1510 }
1511 else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
1512 !rb_path_check(hdr)) {
1513 verbose(1, "Unsafe header file: uid=%ld mode=%#o %s",
1514 (long)st.st_uid, (unsigned)st.st_mode, hdr);
1515 return FALSE;
1516 }
1517 else {
1518 // Do not pass PRELOADENV to child processes, on
1519 // multi-arch environment
1520 verbose(3, "PRELOADENV("PRELOADENV")=%s", getenv(PRELOADENV));
1521 // assume no other PRELOADENV in test-all
1522 unsetenv(PRELOADENV);
1523 verbose(3, "MJIT_HEADER: %s", hdr);
1524 header_file = ruby_strdup(hdr);
1525 if (!header_file) return false;
1526 }
1527 }
1528 else
1529#endif
1530 {
1531 // A name of the header file included in any C file generated by MJIT for iseqs.
1532 static const char header_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_MIN_HEADER_NAME;
1533 const size_t header_name_len = sizeof(header_name) - 1;
1534
1535 header_file = xmalloc(baselen + header_name_len + 1);
1536 p = append_str2(header_file, basedir, baselen);
1537 p = append_str2(p, header_name, header_name_len + 1);
1538
1539 if ((fd = rb_cloexec_open(header_file, O_RDONLY, 0)) < 0) {
1540 verbose(1, "Cannot access header file: %s", header_file);
1541 xfree(header_file);
1542 header_file = NULL;
1543 return false;
1544 }
1545 (void)close(fd);
1546 }
1547
1548 pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX "h", ".h.gch");
1549
1550 return true;
1551}
1552
1553static char *
1554system_default_tmpdir(void)
1555{
1556 // c.f. ext/etc/etc.c:etc_systmpdir()
1557#if defined _CS_DARWIN_USER_TEMP_DIR
1558 char path[MAXPATHLEN];
1559 size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path, sizeof(path));
1560 if (len > 0) {
1561 char *tmpdir = xmalloc(len);
1562 if (len > sizeof(path)) {
1563 confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
1564 }
1565 else {
1566 memcpy(tmpdir, path, len);
1567 }
1568 return tmpdir;
1569 }
1570#endif
1571 return 0;
1572}
1573
1574static int
1575check_tmpdir(const char *dir)
1576{
1577 struct stat st;
1578
1579 if (!dir) return FALSE;
1580 if (stat(dir, &st)) return FALSE;
1581#ifndef S_ISDIR
1582# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
1583#endif
1584 if (!S_ISDIR(st.st_mode)) return FALSE;
1585#ifndef S_IWOTH
1586# define S_IWOTH 002
1587#endif
1588 if (st.st_mode & S_IWOTH) {
1589#ifdef S_ISVTX
1590 if (!(st.st_mode & S_ISVTX)) return FALSE;
1591#else
1592 return FALSE;
1593#endif
1594 }
1595 if (access(dir, W_OK)) return FALSE;
1596 return TRUE;
1597}
1598
1599static char *
1600system_tmpdir(void)
1601{
1602 char *tmpdir;
1603# define RETURN_ENV(name) \
1604 if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
1605 RETURN_ENV("TMPDIR");
1606 RETURN_ENV("TMP");
1607 tmpdir = system_default_tmpdir();
1608 if (check_tmpdir(tmpdir)) return tmpdir;
1609 return ruby_strdup("/tmp");
1610# undef RETURN_ENV
1611}
1612
1613// Minimum value for JIT cache size.
1614#define MIN_CACHE_SIZE 10
1615// Default permitted number of units with a JIT code kept in memory.
1616#define DEFAULT_MAX_CACHE_SIZE 100
1617// A default threshold used to add iseq to JIT.
1618#define DEFAULT_CALL_THRESHOLD 10000
1619
1620// Start MJIT worker. Return TRUE if worker is successfully started.
1621static bool
1622start_worker(void)
1623{
1624 stop_worker_p = false;
1625 worker_stopped = false;
1626 return true;
1627}
1628
1629// There's no strndup on Windows
1630static char*
1631ruby_strndup(const char *str, size_t n)
1632{
1633 char *ret = xmalloc(n + 1);
1634 memcpy(ret, str, n);
1635 ret[n] = '\0';
1636 return ret;
1637}
1638
1639// Convert "foo bar" to {"foo", "bar", NULL} array. Caller is responsible for
1640// freeing a returned buffer and its elements.
1641static char **
1642split_flags(const char *flags)
1643{
1644 char *buf[MAXPATHLEN];
1645 int i = 0;
1646 char *next;
1647 for (; flags != NULL; flags = next) {
1648 next = strchr(flags, ' ');
1649 if (next == NULL) {
1650 if (strlen(flags) > 0)
1651 buf[i++] = strdup(flags);
1652 }
1653 else {
1654 if (next > flags)
1655 buf[i++] = ruby_strndup(flags, next - flags);
1656 next++; // skip space
1657 }
1658 }
1659
1660 char **ret = xmalloc(sizeof(char *) * (i + 1));
1661 memcpy(ret, buf, sizeof(char *) * i);
1662 ret[i] = NULL;
1663 return ret;
1664}
1665
1666#define opt_match_noarg(s, l, name) \
1667 opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --mjit-" name " is ignored"), 1) : 1)
1668#define opt_match_arg(s, l, name) \
1669 opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--mjit-" name " needs an argument"), 0))
1670
1671void
1672mjit_setup_options(const char *s, struct mjit_options *mjit_opt)
1673{
1674 const size_t l = strlen(s);
1675 if (l == 0) {
1676 return;
1677 }
1678 else if (opt_match_noarg(s, l, "warnings")) {
1679 mjit_opt->warnings = true;
1680 }
1681 else if (opt_match(s, l, "debug")) {
1682 if (*s)
1683 mjit_opt->debug_flags = strdup(s + 1);
1684 else
1685 mjit_opt->debug = true;
1686 }
1687 else if (opt_match_noarg(s, l, "wait")) {
1688 mjit_opt->wait = true;
1689 }
1690 else if (opt_match_noarg(s, l, "save-temps")) {
1691 mjit_opt->save_temps = true;
1692 }
1693 else if (opt_match(s, l, "verbose")) {
1694 mjit_opt->verbose = *s ? atoi(s + 1) : 1;
1695 }
1696 else if (opt_match_arg(s, l, "max-cache")) {
1697 mjit_opt->max_cache_size = atoi(s + 1);
1698 }
1699 else if (opt_match_arg(s, l, "call-threshold")) {
1700 mjit_opt->call_threshold = atoi(s + 1);
1701 }
1702 // --mjit=pause is an undocumented feature for experiments
1703 else if (opt_match_noarg(s, l, "pause")) {
1704 mjit_opt->pause = true;
1705 }
1706 else {
1707 rb_raise(rb_eRuntimeError,
1708 "invalid MJIT option `%s' (--help will show valid MJIT options)", s);
1709 }
1710}
1711
1712#define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc)
1713const struct ruby_opt_message mjit_option_messages[] = {
1714 M("--mjit-warnings", "", "Enable printing JIT warnings"),
1715 M("--mjit-debug", "", "Enable JIT debugging (very slow), or add cflags if specified"),
1716 M("--mjit-wait", "", "Wait until JIT compilation finishes every time (for testing)"),
1717 M("--mjit-save-temps", "", "Save JIT temporary files in $TMP or /tmp (for testing)"),
1718 M("--mjit-verbose=num", "", "Print JIT logs of level num or less to stderr (default: 0)"),
1719 M("--mjit-max-cache=num", "", "Max number of methods to be JIT-ed in a cache (default: "
1720 STRINGIZE(DEFAULT_MAX_CACHE_SIZE) ")"),
1721 M("--mjit-call-threshold=num", "", "Number of calls to trigger JIT (for testing, default: "
1722 STRINGIZE(DEFAULT_CALL_THRESHOLD) ")"),
1723 {0}
1724};
1725#undef M
1726
1727// Initialize MJIT. Start a thread creating the precompiled header and
1728// processing ISeqs. The function should be called first for using MJIT.
1729// If everything is successful, MJIT_INIT_P will be TRUE.
1730void
1731mjit_init(const struct mjit_options *opts)
1732{
1733 VM_ASSERT(mjit_enabled);
1734 mjit_opts = *opts;
1735
1736 // MJIT doesn't support miniruby, but it might reach here by MJIT_FORCE_ENABLE.
1737 rb_mMJIT = rb_const_get(rb_cRubyVM, rb_intern("MJIT"));
1738 if (!rb_const_defined(rb_mMJIT, rb_intern("Compiler"))) {
1739 verbose(1, "Disabling MJIT because RubyVM::MJIT::Compiler is not defined");
1740 mjit_enabled = false;
1741 return;
1742 }
1743 rb_mMJITC = rb_const_get(rb_mMJIT, rb_intern("C"));
1744 rb_cMJITCompiler = rb_funcall(rb_const_get(rb_mMJIT, rb_intern("Compiler")), rb_intern("new"), 0);
1745 rb_cMJITIseqPtr = rb_funcall(rb_mMJITC, rb_intern("rb_iseq_t"), 0);
1746 rb_cMJITICPtr = rb_funcall(rb_mMJITC, rb_intern("IC"), 0);
1747 rb_funcall(rb_cMJITICPtr, rb_intern("new"), 1, SIZET2NUM(0)); // Trigger no-op constant events before enabling hooks
1748 rb_mMJITHooks = rb_const_get(rb_mMJIT, rb_intern("Hooks"));
1749
1750 mjit_call_p = true;
1751 mjit_pid = getpid();
1752
1753 // Normalize options
1754 if (mjit_opts.call_threshold == 0)
1755 mjit_opts.call_threshold = DEFAULT_CALL_THRESHOLD;
1756 if (mjit_opts.call_threshold % 2 == 1) {
1757 mjit_opts.call_threshold += 1;
1758 mjit_warning("--mjit-call-threshold must be an even number. Using %d instead.", mjit_opts.call_threshold);
1759 }
1760 mjit_opts.call_threshold /= 2; // Half for enqueue, half for trigger
1761 if (mjit_opts.max_cache_size <= 0)
1762 mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
1763 if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
1764 mjit_opts.max_cache_size = MIN_CACHE_SIZE;
1765
1766 // Initialize variables for compilation
1767 pch_status = PCH_NOT_READY;
1768 cc_path = CC_COMMON_ARGS[0];
1769 verbose(2, "MJIT: CC defaults to %s", cc_path);
1770 cc_common_args = xmalloc(sizeof(CC_COMMON_ARGS));
1771 memcpy((void *)cc_common_args, CC_COMMON_ARGS, sizeof(CC_COMMON_ARGS));
1772 cc_added_args = split_flags(opts->debug_flags);
1773 xfree(opts->debug_flags);
1774#if MJIT_CFLAGS_PIPE
1775 // Filter out `-save-temps`. It's a C compiler flag used by update-deps and not compatible with `-pipe`.
1776 for (size_t i = 0, j = 0; i < sizeof(CC_COMMON_ARGS) / sizeof(char *); i++) {
1777 if (CC_COMMON_ARGS[i] && strncmp("-save-temps", CC_COMMON_ARGS[i], strlen("-save-temps")) == 0)
1778 continue; // Skip `-save-temps`
1779 cc_common_args[j] = CC_COMMON_ARGS[i];
1780 j++;
1781 }
1782#endif
1783
1784 tmp_dir = system_tmpdir();
1785 verbose(2, "MJIT: tmp_dir is %s", tmp_dir);
1786
1787 if (!init_header_filename()) {
1788 mjit_enabled = false;
1789 verbose(1, "Failure in MJIT header file name initialization\n");
1790 return;
1791 }
1792 pch_owner_pid = getpid();
1793
1794 // Initialize mutex
1795 rb_native_mutex_initialize(&mjit_engine_mutex);
1796
1797 // If --mjit=pause is given, lazily start MJIT when RubyVM::MJIT.resume is called.
1798 // You can use it to control MJIT warmup, or to customize the JIT implementation.
1799 if (!mjit_opts.pause) {
1800 // TODO: Consider running C compiler asynchronously
1801 make_pch();
1802
1803 // Enable MJIT compilation
1804 start_worker();
1805 }
1806}
1807
1808static void
1809stop_worker(void)
1810{
1811 stop_worker_p = true;
1812 if (current_cc_unit != NULL) {
1813 mjit_wait(current_cc_unit);
1814 }
1815 worker_stopped = true;
1816}
1817
1818// Stop JIT-compiling methods but compiled code is kept available.
1819 VALUE
1820mjit_pause(bool wait_p)
1821{
1822 if (!mjit_enabled) {
1823 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
1824 }
1825 if (worker_stopped) {
1826 return Qfalse;
1827 }
1828
1829 // Flush all queued units with no option or `wait: true`
1830 if (wait_p) {
1831 while (current_cc_unit != NULL) {
1832 mjit_wait(current_cc_unit);
1833 }
1834 }
1835
1836 stop_worker();
1837 return Qtrue;
1838}
1839
1840// Restart JIT-compiling methods after mjit_pause.
1841 VALUE
1842mjit_resume(void)
1843{
1844 if (!mjit_enabled) {
1845 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
1846 }
1847 if (!worker_stopped) {
1848 return Qfalse;
1849 }
1850
1851 // Lazily prepare PCH when --mjit=pause is given
1852 if (pch_status == PCH_NOT_READY) {
1853 if (rb_respond_to(rb_mMJIT, rb_intern("compile"))) {
1854 // [experimental] defining RubyVM::MJIT.compile allows you to replace JIT
1855 mjit_opts.custom = true;
1856 pch_status = PCH_SUCCESS;
1857 }
1858 else {
1859 // Lazy MJIT boot
1860 make_pch();
1861 }
1862 }
1863
1864 if (!start_worker()) {
1865 rb_raise(rb_eRuntimeError, "Failed to resume MJIT worker");
1866 }
1867 return Qtrue;
1868}
1869
1870// This is called after fork initiated by Ruby's method to launch MJIT worker thread
1871// for child Ruby process.
1872//
1873// In multi-process Ruby applications, child Ruby processes do most of the jobs.
1874// Thus we want child Ruby processes to enqueue ISeqs to MJIT worker's queue and
1875// call the JIT-ed code.
1876//
1877// But unfortunately current MJIT-generated code is process-specific. After the fork,
1878// JIT-ed code created by parent Ruby process cannot be used in child Ruby process
1879// because the code could rely on inline cache values (ivar's IC, send's CC) which
1880// may vary between processes after fork or embed some process-specific addresses.
1881//
1882// So child Ruby process can't request parent process to JIT an ISeq and use the code.
1883// Instead of that, MJIT worker thread is created for all child Ruby processes, even
1884// while child processes would end up with compiling the same ISeqs.
1885void
1886mjit_child_after_fork(void)
1887{
1888 if (!mjit_enabled)
1889 return;
1890
1891 /* MJIT worker thread is not inherited on fork. Start it for this child process. */
1892 start_worker();
1893}
1894
1895// Finish the threads processing units and creating PCH, finalize
1896// and free MJIT data. It should be called last during MJIT
1897// life.
1898//
1899// If close_handle_p is true, it calls dlclose() for JIT-ed code. So it should be false
1900// if the code can still be on stack. ...But it means to leak JIT-ed handle forever (FIXME).
1901void
1902mjit_finish(bool close_handle_p)
1903{
1904 if (!mjit_enabled)
1905 return;
1906
1907 // Stop worker
1908 verbose(2, "Stopping worker thread");
1909 stop_worker();
1910
1911 rb_native_mutex_destroy(&mjit_engine_mutex);
1912
1913 if (!mjit_opts.save_temps && getpid() == pch_owner_pid && pch_status == PCH_SUCCESS && !mjit_opts.custom)
1914 remove_file(pch_file);
1915
1916 xfree(header_file); header_file = NULL;
1917 xfree((void *)cc_common_args); cc_common_args = NULL;
1918 for (char **flag = cc_added_args; *flag != NULL; flag++)
1919 xfree(*flag);
1920 xfree((void *)cc_added_args); cc_added_args = NULL;
1921 xfree(tmp_dir); tmp_dir = NULL;
1922 xfree(pch_file); pch_file = NULL;
1923
1924 mjit_call_p = false;
1925 free_list(&unit_queue, close_handle_p);
1926 free_list(&active_units, close_handle_p);
1927 free_list(&compact_units, close_handle_p);
1928 free_list(&stale_units, close_handle_p);
1929
1930 mjit_enabled = false;
1931 verbose(1, "Successful MJIT finish");
1932}
1933
1934// Called by rb_vm_mark().
1935//
1936// Mark active_units so that we do not GC ISeq which may still be
1937// referenced by mjit_recompile() or mjit_compact().
1938void
1939mjit_mark(void)
1940{
1941 if (!mjit_enabled)
1942 return;
1943 RUBY_MARK_ENTER("mjit");
1944
1945 // Mark objects used by the MJIT compiler
1946 rb_gc_mark(rb_cMJITCompiler);
1947 rb_gc_mark(rb_cMJITIseqPtr);
1948 rb_gc_mark(rb_cMJITICPtr);
1949 rb_gc_mark(rb_mMJITHooks);
1950
1951 // Mark JIT-compiled ISEQs
1952 struct rb_mjit_unit *unit = NULL;
1953 ccan_list_for_each(&active_units.head, unit, unode) {
1954 rb_gc_mark((VALUE)unit->iseq);
1955 }
1956
1957 RUBY_MARK_LEAVE("mjit");
1958}
1959
1960// Called by rb_iseq_mark() to mark cc_entries captured for MJIT
1961void
1962mjit_mark_cc_entries(const struct rb_iseq_constant_body *const body)
1963{
1964 const struct rb_callcache **cc_entries;
1965 if (body->mjit_unit && (cc_entries = body->mjit_unit->cc_entries) != NULL) {
1966 // It must be `body->mjit_unit->cc_entries_size` instead of `body->ci_size` to mark children's cc_entries
1967 for (unsigned int i = 0; i < body->mjit_unit->cc_entries_size; i++) {
1968 const struct rb_callcache *cc = cc_entries[i];
1969 if (cc != NULL && vm_cc_markable(cc)) {
1970 // Pin `cc` and `cc->cme` against GC.compact as their addresses may be written in JIT-ed code.
1971 rb_gc_mark((VALUE)cc);
1972 rb_gc_mark((VALUE)vm_cc_cme(cc));
1973 }
1974 }
1975 }
1976}
1977
1978// Compile ISeq to C code in `f`. It returns true if it succeeds to compile.
1979bool
1980mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id)
1981{
1982 bool original_call_p = mjit_call_p;
1983 mjit_call_p = false; // Avoid impacting JIT metrics by itself
1984
1985 VALUE iseq_ptr = rb_funcall(rb_cMJITIseqPtr, rb_intern("new"), 1, ULONG2NUM((size_t)iseq));
1986 VALUE src = rb_funcall(rb_cMJITCompiler, rb_intern("compile"), 3,
1987 iseq_ptr, rb_str_new_cstr(funcname), INT2NUM(id));
1988 if (!NIL_P(src)) {
1989 fprintf(f, "%s", RSTRING_PTR(src));
1990 }
1991
1992 mjit_call_p = original_call_p;
1993 return !NIL_P(src);
1994}
1995
1996#include "mjit.rbinc"
1997
1998#endif // USE_MJIT
debug.h
rb_postponed_job_func_t
void(* rb_postponed_job_func_t)(void *arg)
Type of postponed jobs.
Definition debug.h:608
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:103
NUM2ULONG
#define NUM2ULONG
Old name of RB_NUM2ULONG.
Definition long.h:52
xfree
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
ULONG2NUM
#define ULONG2NUM
Old name of RB_ULONG2NUM.
Definition long.h:60
SIZET2NUM
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
xmalloc
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
ZALLOC_N
#define ZALLOC_N
Old name of RB_ZALLOC_N.
Definition memory.h:395
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
INT2NUM
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
UINT2NUM
#define UINT2NUM
Old name of RB_UINT2NUM.
Definition int.h:46
rb_raise
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
Definition error.c:3150
rb_bug
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition error.c:794
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1089
rb_funcall
VALUE rb_funcall(VALUE recv, ID mid, int n,...)
Calls a method.
Definition vm_eval.c:1102
rb_cloexec_open
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
Definition io.c:310
rb_str_new_cstr
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1514
rb_thread_wait_for
void rb_thread_wait_for(struct timeval time)
Identical to rb_thread_sleep(), except it takes struct timeval instead.
Definition thread.c:1425
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:2896
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3204
rb_respond_to
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
Definition vm_method.c:2805
rb_intern
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
Definition symbol.c:796
thread.h
util.h
ruby_strdup
char * ruby_strdup(const char *str)
This is our own version of strdup(3) that uses ruby_xmalloc() instead of system malloc (benefits our ...
Definition util.c:538
strdup
#define strdup(s)
Just another name of ruby_strdup.
Definition util.h:176
version.h
RUBY_API_VERSION_TEENY
#define RUBY_API_VERSION_TEENY
Teeny version.
Definition version.h:76
RUBY_API_VERSION_MAJOR
#define RUBY_API_VERSION_MAJOR
Major version.
Definition version.h:64
RUBY_API_VERSION_MINOR
#define RUBY_API_VERSION_MINOR
Minor version.
Definition version.h:70
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:366
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
PRI_PIDT_PREFIX
#define PRI_PIDT_PREFIX
A rb_sprintf() format prefix to be used for a pid_t parameter.
Definition pid_t.h:38
StringValuePtr
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
Definition rstring.h:82
RSTRING_LEN
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
Definition rstring.h:484
RSTRING_PTR
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
Definition rstring.h:498
__sFILE
Definition vsnprintf.c:160
iseq_inline_constant_cache
Definition vm_core.h:259
rb_callable_method_entry_struct
Definition method.h:62
rb_callcache
Definition vm_callinfo.h:275
rb_iseq_constant_body
Definition vm_core.h:371
rb_iseq_struct
Definition vm_core.h:526
rb_mjit_unit_list
Definition mjit_c.h:18
rb_mjit_unit
Definition mjit_c.h:34
rb_vm_struct
Definition vm_core.h:615
ruby_opt_message
Definition cmdlineopt.h:44
timespec
Definition missing.h:74
timeval
Definition missing.h:63
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_mutex_initialize
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_mutex_destroy
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40