Ruby 3.0.5p211 (2022-11-24 revision ba5cf0f7c52d4d35cc6a173c89eda98ceffa2dcf)
mjit.c
Go to the documentation of this file.
1/**********************************************************************
2
3 mjit.c - MRI method JIT compiler functions for Ruby's main thread
4
5 Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>.
6
7**********************************************************************/
8
9// Functions in this file are never executed on MJIT worker thread.
10// So you can safely use Ruby methods and GC in this file.
11
12// To share variables privately, include mjit_worker.c instead of linking.
13
14#include "ruby/internal/config.h" // defines USE_MJIT
15
16#if USE_MJIT
17
18#include "constant.h"
19#include "id_table.h"
20#include "internal.h"
21#include "internal/class.h"
22#include "internal/cont.h"
23#include "internal/file.h"
24#include "internal/hash.h"
25#include "internal/warnings.h"
26
27#include "mjit_worker.c"
28
29extern int rb_thread_create_mjit_thread(void (*worker_func)(void));
30
31// Return an unique file name in /tmp with PREFIX and SUFFIX and
32// number ID. Use getpid if ID == 0. The return file name exists
33// until the next function call.
34static char *
35get_uniq_filename(unsigned long id, const char *prefix, const char *suffix)
36{
37 char buff[70], *str = buff;
38 int size = sprint_uniq_filename(buff, sizeof(buff), id, prefix, suffix);
39 str = 0;
40 ++size;
41 str = xmalloc(size);
42 if (size <= (int)sizeof(buff)) {
43 memcpy(str, buff, size);
44 }
45 else {
46 sprint_uniq_filename(str, size, id, prefix, suffix);
47 }
48 return str;
49}
50
51// Wait until workers don't compile any iseq. It is called at the
52// start of GC.
53void
54mjit_gc_start_hook(void)
55{
56 if (!mjit_enabled)
57 return;
58 CRITICAL_SECTION_START(4, "mjit_gc_start_hook");
59 while (in_jit) {
60 verbose(4, "Waiting wakeup from a worker for GC");
61 rb_native_cond_wait(&mjit_client_wakeup, &mjit_engine_mutex);
62 verbose(4, "Getting wakeup from a worker for GC");
63 }
64 in_gc++;
65 CRITICAL_SECTION_FINISH(4, "mjit_gc_start_hook");
66}
67
68// Send a signal to workers to continue iseq compilations. It is
69// called at the end of GC.
70void
71mjit_gc_exit_hook(void)
72{
73 if (!mjit_enabled)
74 return;
75 CRITICAL_SECTION_START(4, "mjit_gc_exit_hook");
76 in_gc--;
77 RUBY_ASSERT_ALWAYS(in_gc >= 0);
78 if (!in_gc) {
79 verbose(4, "Sending wakeup signal to workers after GC");
80 rb_native_cond_broadcast(&mjit_gc_wakeup);
81 }
82 CRITICAL_SECTION_FINISH(4, "mjit_gc_exit_hook");
83}
84
85// Deal with ISeq movement from compactor
86void
87mjit_update_references(const rb_iseq_t *iseq)
88{
89 if (!mjit_enabled)
90 return;
91
92 CRITICAL_SECTION_START(4, "mjit_update_references");
93 if (iseq->body->jit_unit) {
94 iseq->body->jit_unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)iseq->body->jit_unit->iseq);
95 // We need to invalidate JIT-ed code for the ISeq because it embeds pointer addresses.
96 // To efficiently do that, we use the same thing as TracePoint and thus everything is cancelled for now.
97 // See mjit.h and tool/ruby_vm/views/_mjit_compile_insn.erb for how `mjit_call_p` is used.
98 mjit_call_p = false; // TODO: instead of cancelling all, invalidate only this one and recompile it with some threshold.
99 }
100
101 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
102 // `iseq->body->jit_unit` anymore (because new one replaces that). So we need to check them too.
103 // TODO: we should be able to reduce the number of units checked here.
104 struct rb_mjit_unit *unit = NULL;
105 list_for_each(&stale_units.head, unit, unode) {
106 if (unit->iseq == iseq) {
107 unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)unit->iseq);
108 }
109 }
110 CRITICAL_SECTION_FINISH(4, "mjit_update_references");
111}
112
113// Iseqs can be garbage collected. This function should call when it
114// happens. It removes iseq from the unit.
115void
116mjit_free_iseq(const rb_iseq_t *iseq)
117{
118 if (!mjit_enabled)
119 return;
120
121 CRITICAL_SECTION_START(4, "mjit_free_iseq");
122 RUBY_ASSERT_ALWAYS(in_gc);
123 RUBY_ASSERT_ALWAYS(!in_jit);
124 if (iseq->body->jit_unit) {
125 // jit_unit is not freed here because it may be referred by multiple
126 // lists of units. `get_from_list` and `mjit_finish` do the job.
127 iseq->body->jit_unit->iseq = NULL;
128 }
129 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
130 // `iseq->body->jit_unit` anymore (because new one replaces that). So we need to check them too.
131 // TODO: we should be able to reduce the number of units checked here.
132 struct rb_mjit_unit *unit = NULL;
133 list_for_each(&stale_units.head, unit, unode) {
134 if (unit->iseq == iseq) {
135 unit->iseq = NULL;
136 }
137 }
138 CRITICAL_SECTION_FINISH(4, "mjit_free_iseq");
139}
140
141// Free unit list. This should be called only when worker is finished
142// because node of unit_queue and one of active_units may have the same unit
143// during proceeding unit.
144static void
145free_list(struct rb_mjit_unit_list *list, bool close_handle_p)
146{
147 struct rb_mjit_unit *unit = 0, *next;
148
149 list_for_each_safe(&list->head, unit, next, unode) {
150 list_del(&unit->unode);
151 if (!close_handle_p) unit->handle = NULL; /* Skip dlclose in free_unit() */
152
153 if (list == &stale_units) { // `free_unit(unit)` crashes after GC.compact on `stale_units`
154 /*
155 * TODO: REVERT THIS BRANCH
156 * Debug the crash on stale_units w/ GC.compact and just use `free_unit(unit)`!!
157 */
158 if (unit->handle && dlclose(unit->handle)) {
159 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
160 }
161 clean_temp_files(unit);
162 free(unit);
163 }
164 else {
165 free_unit(unit);
166 }
167 }
168 list->length = 0;
169}
170
171// Register a new continuation with execution context `ec`. Return MJIT info about
172// the continuation.
173struct mjit_cont *
174mjit_cont_new(rb_execution_context_t *ec)
175{
176 struct mjit_cont *cont;
177
178 // We need to use calloc instead of something like ZALLOC to avoid triggering GC here.
179 // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup,
180 // the thread is still being prepared and marking it causes SEGV.
181 cont = calloc(1, sizeof(struct mjit_cont));
182 if (cont == NULL)
183 rb_memerror();
184 cont->ec = ec;
185
186 CRITICAL_SECTION_START(3, "in mjit_cont_new");
187 if (first_cont == NULL) {
188 cont->next = cont->prev = NULL;
189 }
190 else {
191 cont->prev = NULL;
192 cont->next = first_cont;
193 first_cont->prev = cont;
194 }
195 first_cont = cont;
196 CRITICAL_SECTION_FINISH(3, "in mjit_cont_new");
197
198 return cont;
199}
200
201// Unregister continuation `cont`.
202void
203mjit_cont_free(struct mjit_cont *cont)
204{
205 CRITICAL_SECTION_START(3, "in mjit_cont_new");
206 if (cont == first_cont) {
207 first_cont = cont->next;
208 if (first_cont != NULL)
209 first_cont->prev = NULL;
210 }
211 else {
212 cont->prev->next = cont->next;
213 if (cont->next != NULL)
214 cont->next->prev = cont->prev;
215 }
216 CRITICAL_SECTION_FINISH(3, "in mjit_cont_new");
217
218 free(cont);
219}
220
221// Finish work with continuation info.
222static void
223finish_conts(void)
224{
225 struct mjit_cont *cont, *next;
226
227 for (cont = first_cont; cont != NULL; cont = next) {
228 next = cont->next;
229 xfree(cont);
230 }
231}
232
233// Create unit for `iseq`. This function may be called from an MJIT worker.
234static void
235create_unit(const rb_iseq_t *iseq)
236{
237 struct rb_mjit_unit *unit;
238
239 unit = calloc(1, sizeof(struct rb_mjit_unit));
240 if (unit == NULL)
241 return;
242
243 unit->id = current_unit_num++;
244 unit->iseq = (rb_iseq_t *)iseq;
245 iseq->body->jit_unit = unit;
246}
247
248// This is called from an MJIT worker when worker_p is true.
249static void
250mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info, bool worker_p)
251{
252 if (!mjit_enabled || pch_status == PCH_FAILED)
253 return;
254
255 RB_DEBUG_COUNTER_INC(mjit_add_iseq_to_process);
256 iseq->body->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC;
257 create_unit(iseq);
258 if (iseq->body->jit_unit == NULL)
259 // Failure in creating the unit.
260 return;
261 if (compile_info != NULL)
262 iseq->body->jit_unit->compile_info = *compile_info;
263
264 if (!worker_p) {
265 CRITICAL_SECTION_START(3, "in add_iseq_to_process");
266 }
267 add_to_list(iseq->body->jit_unit, &unit_queue);
268 if (active_units.length >= mjit_opts.max_cache_size) {
269 unload_requests++;
270 }
271 if (!worker_p) {
272 verbose(3, "Sending wakeup signal to workers in mjit_add_iseq_to_process");
273 rb_native_cond_broadcast(&mjit_worker_wakeup);
274 CRITICAL_SECTION_FINISH(3, "in add_iseq_to_process");
275 }
276}
277
278// Add ISEQ to be JITed in parallel with the current thread.
279// Unload some JIT codes if there are too many of them.
280void
281rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq)
282{
283 mjit_add_iseq_to_process(iseq, NULL, false);
284}
285
286// For this timeout seconds, --jit-wait will wait for JIT compilation finish.
287#define MJIT_WAIT_TIMEOUT_SECONDS 60
288
289static void
290mjit_wait(struct rb_iseq_constant_body *body)
291{
292 struct timeval tv;
293 int tries = 0;
294 tv.tv_sec = 0;
295 tv.tv_usec = 1000;
296 while (body->jit_func == (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC) {
297 tries++;
298 if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS || pch_status == PCH_FAILED) {
299 CRITICAL_SECTION_START(3, "in rb_mjit_wait_call to set jit_func");
300 body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC; // JIT worker seems dead. Give up.
301 CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call to set jit_func");
302 mjit_warning("timed out to wait for JIT finish");
303 break;
304 }
305
306 CRITICAL_SECTION_START(3, "in rb_mjit_wait_call for a client wakeup");
307 rb_native_cond_broadcast(&mjit_worker_wakeup);
308 CRITICAL_SECTION_FINISH(3, "in rb_mjit_wait_call for a client wakeup");
310 }
311}
312
313// Wait for JIT compilation finish for --jit-wait, and call the function pointer
314// if the compiled result is not NOT_COMPILED_JIT_ISEQ_FUNC.
315VALUE
316rb_mjit_wait_call(rb_execution_context_t *ec, struct rb_iseq_constant_body *body)
317{
318 mjit_wait(body);
319 if ((uintptr_t)body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC) {
320 return Qundef;
321 }
322 return body->jit_func(ec, ec->cfp);
323}
324
325struct rb_mjit_compile_info*
326rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body)
327{
328 assert(body->jit_unit != NULL);
329 return &body->jit_unit->compile_info;
330}
331
332static void
333mjit_recompile(const rb_iseq_t *iseq)
334{
335 if ((uintptr_t)iseq->body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC)
336 return;
337
338 verbose(1, "JIT recompile: %s@%s:%d", RSTRING_PTR(iseq->body->location.label),
340 assert(iseq->body->jit_unit != NULL);
341
342 if (UNLIKELY(mjit_opts.wait)) {
343 remove_from_list(iseq->body->jit_unit, &active_units);
344 add_to_list(iseq->body->jit_unit, &stale_units);
345 mjit_add_iseq_to_process(iseq, &iseq->body->jit_unit->compile_info, false);
346 mjit_wait(iseq->body);
347 }
348 else {
349 // Lazily move active_units to stale_units to avoid race conditions around active_units with compaction.
350 // Also, it's lazily moved to unit_queue as well because otherwise it won't be added to stale_units properly.
351 // It's good to avoid a race condition between mjit_add_iseq_to_process and mjit_compile around jit_unit as well.
352 CRITICAL_SECTION_START(3, "in rb_mjit_recompile_iseq");
353 iseq->body->jit_unit->stale_p = true;
354 iseq->body->jit_func = (mjit_func_t)NOT_ADDED_JIT_ISEQ_FUNC;
355 pending_stale_p = true;
356 CRITICAL_SECTION_FINISH(3, "in rb_mjit_recompile_iseq");
357 }
358}
359
360// Recompile iseq, disabling send optimization
361void
362rb_mjit_recompile_send(const rb_iseq_t *iseq)
363{
364 rb_mjit_iseq_compile_info(iseq->body)->disable_send_cache = true;
365 mjit_recompile(iseq);
366}
367
368// Recompile iseq, disabling ivar optimization
369void
370rb_mjit_recompile_ivar(const rb_iseq_t *iseq)
371{
372 rb_mjit_iseq_compile_info(iseq->body)->disable_ivar_cache = true;
373 mjit_recompile(iseq);
374}
375
376// Recompile iseq, disabling exivar optimization
377void
378rb_mjit_recompile_exivar(const rb_iseq_t *iseq)
379{
380 rb_mjit_iseq_compile_info(iseq->body)->disable_exivar_cache = true;
381 mjit_recompile(iseq);
382}
383
384// Recompile iseq, disabling method inlining
385void
386rb_mjit_recompile_inlining(const rb_iseq_t *iseq)
387{
388 rb_mjit_iseq_compile_info(iseq->body)->disable_inlining = true;
389 mjit_recompile(iseq);
390}
391
392// Recompile iseq, disabling getconstant inlining
393void
394rb_mjit_recompile_const(const rb_iseq_t *iseq)
395{
396 rb_mjit_iseq_compile_info(iseq->body)->disable_const_cache = true;
397 mjit_recompile(iseq);
398}
399
401
402// Initialize header_file, pch_file, libruby_pathflag. Return true on success.
403static bool
404init_header_filename(void)
405{
406 int fd;
407#ifdef LOAD_RELATIVE
408 // Root path of the running ruby process. Equal to RbConfig::TOPDIR.
409 VALUE basedir_val;
410#endif
411 const char *basedir = "";
412 size_t baselen = 0;
413 char *p;
414#ifdef _WIN32
415 static const char libpathflag[] =
416# ifdef _MSC_VER
417 "-LIBPATH:"
418# else
419 "-L"
420# endif
421 ;
422 const size_t libpathflag_len = sizeof(libpathflag) - 1;
423#endif
424
425#ifdef LOAD_RELATIVE
426 basedir_val = ruby_prefix_path;
427 basedir = StringValuePtr(basedir_val);
428 baselen = RSTRING_LEN(basedir_val);
429#else
430 if (getenv("MJIT_SEARCH_BUILD_DIR")) {
431 // This path is not intended to be used on production, but using build directory's
432 // header file here because people want to run `make test-all` without running
433 // `make install`. Don't use $MJIT_SEARCH_BUILD_DIR except for test-all.
434
435 struct stat st;
436 const char *hdr = dlsym(RTLD_DEFAULT, "MJIT_HEADER");
437 if (!hdr) {
438 verbose(1, "No MJIT_HEADER");
439 }
440 else if (hdr[0] != '/') {
441 verbose(1, "Non-absolute header file path: %s", hdr);
442 }
443 else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) {
444 verbose(1, "Non-file header file path: %s", hdr);
445 }
446 else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
447 !rb_path_check(hdr)) {
448 verbose(1, "Unsafe header file: uid=%ld mode=%#o %s",
449 (long)st.st_uid, (unsigned)st.st_mode, hdr);
450 return FALSE;
451 }
452 else {
453 // Do not pass PRELOADENV to child processes, on
454 // multi-arch environment
455 verbose(3, "PRELOADENV("PRELOADENV")=%s", getenv(PRELOADENV));
456 // assume no other PRELOADENV in test-all
457 unsetenv(PRELOADENV);
458 verbose(3, "MJIT_HEADER: %s", hdr);
459 header_file = ruby_strdup(hdr);
460 if (!header_file) return false;
461 }
462 }
463 else
464#endif
465#ifndef _MSC_VER
466 {
467 // A name of the header file included in any C file generated by MJIT for iseqs.
468 static const char header_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_MIN_HEADER_NAME;
469 const size_t header_name_len = sizeof(header_name) - 1;
470
471 header_file = xmalloc(baselen + header_name_len + 1);
472 p = append_str2(header_file, basedir, baselen);
473 p = append_str2(p, header_name, header_name_len + 1);
474
475 if ((fd = rb_cloexec_open(header_file, O_RDONLY, 0)) < 0) {
476 verbose(1, "Cannot access header file: %s", header_file);
477 xfree(header_file);
478 header_file = NULL;
479 return false;
480 }
481 (void)close(fd);
482 }
483
484 pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX "h", ".h.gch");
485#else
486 {
487 static const char pch_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_PRECOMPILED_HEADER_NAME;
488 const size_t pch_name_len = sizeof(pch_name) - 1;
489
490 pch_file = xmalloc(baselen + pch_name_len + 1);
491 p = append_str2(pch_file, basedir, baselen);
492 p = append_str2(p, pch_name, pch_name_len + 1);
493 if ((fd = rb_cloexec_open(pch_file, O_RDONLY, 0)) < 0) {
494 verbose(1, "Cannot access precompiled header file: %s", pch_file);
495 xfree(pch_file);
496 pch_file = NULL;
497 return false;
498 }
499 (void)close(fd);
500 }
501#endif
502
503#ifdef _WIN32
504 basedir_val = ruby_archlibdir_path;
505 basedir = StringValuePtr(basedir_val);
506 baselen = RSTRING_LEN(basedir_val);
507 libruby_pathflag = p = xmalloc(libpathflag_len + baselen + 1);
508 p = append_str(p, libpathflag);
509 p = append_str2(p, basedir, baselen);
510 *p = '\0';
511#endif
512
513 return true;
514}
515
517valid_class_serials_add_i(ID key, VALUE v, void *unused)
518{
520 VALUE value = ce->value;
521
523 if (RB_TYPE_P(value, T_MODULE) || RB_TYPE_P(value, T_CLASS)) {
524 mjit_add_class_serial(RCLASS_SERIAL(value));
525 }
526 return ID_TABLE_CONTINUE;
527}
528
529#ifdef _WIN32
530UINT rb_w32_system_tmpdir(WCHAR *path, UINT len);
531#endif
532
533static char *
534system_default_tmpdir(void)
535{
536 // c.f. ext/etc/etc.c:etc_systmpdir()
537#ifdef _WIN32
538 WCHAR tmppath[_MAX_PATH];
539 UINT len = rb_w32_system_tmpdir(tmppath, numberof(tmppath));
540 if (len) {
541 int blen = WideCharToMultiByte(CP_UTF8, 0, tmppath, len, NULL, 0, NULL, NULL);
542 char *tmpdir = xmalloc(blen + 1);
543 WideCharToMultiByte(CP_UTF8, 0, tmppath, len, tmpdir, blen, NULL, NULL);
544 tmpdir[blen] = '\0';
545 return tmpdir;
546 }
547#elif defined _CS_DARWIN_USER_TEMP_DIR
548 char path[MAXPATHLEN];
549 size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path, sizeof(path));
550 if (len > 0) {
551 char *tmpdir = xmalloc(len);
552 if (len > sizeof(path)) {
553 confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
554 }
555 else {
556 memcpy(tmpdir, path, len);
557 }
558 return tmpdir;
559 }
560#endif
561 return 0;
562}
563
564static int
565check_tmpdir(const char *dir)
566{
567 struct stat st;
568
569 if (!dir) return FALSE;
570 if (stat(dir, &st)) return FALSE;
571#ifndef S_ISDIR
572# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
573#endif
574 if (!S_ISDIR(st.st_mode)) return FALSE;
575#ifndef _WIN32
576# ifndef S_IWOTH
577# define S_IWOTH 002
578# endif
579 if (st.st_mode & S_IWOTH) {
580# ifdef S_ISVTX
581 if (!(st.st_mode & S_ISVTX)) return FALSE;
582# else
583 return FALSE;
584# endif
585 }
586 if (access(dir, W_OK)) return FALSE;
587#endif
588 return TRUE;
589}
590
591static char *
592system_tmpdir(void)
593{
594 char *tmpdir;
595# define RETURN_ENV(name) \
596 if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
597 RETURN_ENV("TMPDIR");
598 RETURN_ENV("TMP");
599 tmpdir = system_default_tmpdir();
600 if (check_tmpdir(tmpdir)) return tmpdir;
601 return ruby_strdup("/tmp");
602# undef RETURN_ENV
603}
604
605// Minimum value for JIT cache size.
606#define MIN_CACHE_SIZE 10
607// Default permitted number of units with a JIT code kept in memory.
608#define DEFAULT_MAX_CACHE_SIZE 100
609// A default threshold used to add iseq to JIT.
610#define DEFAULT_MIN_CALLS_TO_ADD 10000
611
612// Start MJIT worker. Return TRUE if worker is successfully started.
613static bool
614start_worker(void)
615{
616 stop_worker_p = false;
617 worker_stopped = false;
618
619 if (!rb_thread_create_mjit_thread(mjit_worker)) {
620 mjit_enabled = false;
621 rb_native_mutex_destroy(&mjit_engine_mutex);
622 rb_native_cond_destroy(&mjit_pch_wakeup);
623 rb_native_cond_destroy(&mjit_client_wakeup);
624 rb_native_cond_destroy(&mjit_worker_wakeup);
625 rb_native_cond_destroy(&mjit_gc_wakeup);
626 verbose(1, "Failure in MJIT thread initialization\n");
627 return false;
628 }
629 return true;
630}
631
632// There's no strndup on Windows
633static char*
634ruby_strndup(const char *str, size_t n)
635{
636 char *ret = xmalloc(n + 1);
637 memcpy(ret, str, n);
638 ret[n] = '\0';
639 return ret;
640}
641
642// Convert "foo bar" to {"foo", "bar", NULL} array. Caller is responsible for
643// freeing a returned buffer and its elements.
644static char **
645split_flags(const char *flags)
646{
647 char *buf[MAXPATHLEN];
648 int i = 0;
649 char *next;
650 for (; flags != NULL; flags = next) {
651 next = strchr(flags, ' ');
652 if (next == NULL) {
653 if (strlen(flags) > 0)
654 buf[i++] = strdup(flags);
655 }
656 else {
657 if (next > flags)
658 buf[i++] = ruby_strndup(flags, next - flags);
659 next++; // skip space
660 }
661 }
662
663 char **ret = xmalloc(sizeof(char *) * (i + 1));
664 memcpy(ret, buf, sizeof(char *) * i);
665 ret[i] = NULL;
666 return ret;
667}
668
669// Initialize MJIT. Start a thread creating the precompiled header and
670// processing ISeqs. The function should be called first for using MJIT.
671// If everything is successful, MJIT_INIT_P will be TRUE.
672void
673mjit_init(const struct mjit_options *opts)
674{
675 mjit_opts = *opts;
676 mjit_enabled = true;
677 mjit_call_p = true;
678
679 // Normalize options
680 if (mjit_opts.min_calls == 0)
681 mjit_opts.min_calls = DEFAULT_MIN_CALLS_TO_ADD;
682 if (mjit_opts.max_cache_size <= 0)
683 mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
684 if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
685 mjit_opts.max_cache_size = MIN_CACHE_SIZE;
686
687 // Initialize variables for compilation
688#ifdef _MSC_VER
689 pch_status = PCH_SUCCESS; // has prebuilt precompiled header
690#else
691 pch_status = PCH_NOT_READY;
692#endif
693 cc_path = CC_COMMON_ARGS[0];
694 verbose(2, "MJIT: CC defaults to %s", cc_path);
695 cc_common_args = xmalloc(sizeof(CC_COMMON_ARGS));
696 memcpy((void *)cc_common_args, CC_COMMON_ARGS, sizeof(CC_COMMON_ARGS));
697 cc_added_args = split_flags(opts->debug_flags);
698 xfree(opts->debug_flags);
699#if MJIT_CFLAGS_PIPE
700 // eliminate a flag incompatible with `-pipe`
701 for (size_t i = 0, j = 0; i < sizeof(CC_COMMON_ARGS) / sizeof(char *); i++) {
702 if (CC_COMMON_ARGS[i] && strncmp("-save-temps", CC_COMMON_ARGS[i], strlen("-save-temps")) == 0)
703 continue; // skip -save-temps flag
704 cc_common_args[j] = CC_COMMON_ARGS[i];
705 j++;
706 }
707#endif
708
709 tmp_dir = system_tmpdir();
710 verbose(2, "MJIT: tmp_dir is %s", tmp_dir);
711
712 if (!init_header_filename()) {
713 mjit_enabled = false;
714 verbose(1, "Failure in MJIT header file name initialization\n");
715 return;
716 }
717 pch_owner_pid = getpid();
718
719 // Initialize mutex
720 rb_native_mutex_initialize(&mjit_engine_mutex);
721 rb_native_cond_initialize(&mjit_pch_wakeup);
722 rb_native_cond_initialize(&mjit_client_wakeup);
723 rb_native_cond_initialize(&mjit_worker_wakeup);
724 rb_native_cond_initialize(&mjit_gc_wakeup);
725
726 // Make sure the saved_ec of the initial thread's root_fiber is scanned by mark_ec_units.
727 //
728 // rb_threadptr_root_fiber_setup for the initial thread is called before mjit_init,
729 // meaning mjit_cont_new is skipped for the root_fiber. Therefore we need to call
730 // rb_fiber_init_mjit_cont again with mjit_enabled=true to set the root_fiber's mjit_cont.
731 rb_fiber_init_mjit_cont(GET_EC()->fiber_ptr);
732
733 // Initialize class_serials cache for compilation
734 valid_class_serials = rb_hash_new();
735 rb_obj_hide(valid_class_serials);
736 rb_gc_register_mark_object(valid_class_serials);
737 mjit_add_class_serial(RCLASS_SERIAL(rb_cObject));
738 mjit_add_class_serial(RCLASS_SERIAL(CLASS_OF(rb_vm_top_self())));
740 rb_id_table_foreach(RCLASS_CONST_TBL(rb_cObject), valid_class_serials_add_i, NULL);
741 }
742
743 // Initialize worker thread
744 start_worker();
745}
746
747static void
748stop_worker(void)
749{
750 rb_execution_context_t *ec = GET_EC();
751
752 while (!worker_stopped) {
753 verbose(3, "Sending cancel signal to worker");
754 CRITICAL_SECTION_START(3, "in stop_worker");
755 stop_worker_p = true; // Setting this inside loop because RUBY_VM_CHECK_INTS may make this false.
756 rb_native_cond_broadcast(&mjit_worker_wakeup);
757 CRITICAL_SECTION_FINISH(3, "in stop_worker");
759 }
760}
761
762// Stop JIT-compiling methods but compiled code is kept available.
763VALUE
764mjit_pause(bool wait_p)
765{
766 if (!mjit_enabled) {
767 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
768 }
769 if (worker_stopped) {
770 return Qfalse;
771 }
772
773 // Flush all queued units with no option or `wait: true`
774 if (wait_p) {
775 struct timeval tv;
776 tv.tv_sec = 0;
777 tv.tv_usec = 1000;
778
779 while (unit_queue.length > 0 && active_units.length < mjit_opts.max_cache_size) { // inverse of condition that waits for mjit_worker_wakeup
780 CRITICAL_SECTION_START(3, "in mjit_pause for a worker wakeup");
781 rb_native_cond_broadcast(&mjit_worker_wakeup);
782 CRITICAL_SECTION_FINISH(3, "in mjit_pause for a worker wakeup");
784 }
785 }
786
787 stop_worker();
788 return Qtrue;
789}
790
791// Restart JIT-compiling methods after mjit_pause.
792VALUE
793mjit_resume(void)
794{
795 if (!mjit_enabled) {
796 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
797 }
798 if (!worker_stopped) {
799 return Qfalse;
800 }
801
802 if (!start_worker()) {
803 rb_raise(rb_eRuntimeError, "Failed to resume MJIT worker");
804 }
805 return Qtrue;
806}
807
808// Skip calling `clean_temp_files` for units which currently exist in the list.
809static void
810skip_cleaning_object_files(struct rb_mjit_unit_list *list)
811{
812 struct rb_mjit_unit *unit = NULL, *next;
813
814 // No mutex for list, assuming MJIT worker does not exist yet since it's immediately after fork.
815 list_for_each_safe(&list->head, unit, next, unode) {
816#if defined(_WIN32) // mswin doesn't reach here either. This is for MinGW.
817 if (unit->so_file) unit->so_file = NULL;
818#endif
819 }
820}
821
822// This is called after fork initiated by Ruby's method to launch MJIT worker thread
823// for child Ruby process.
824//
825// In multi-process Ruby applications, child Ruby processes do most of the jobs.
826// Thus we want child Ruby processes to enqueue ISeqs to MJIT worker's queue and
827// call the JIT-ed code.
828//
829// But unfortunately current MJIT-generated code is process-specific. After the fork,
830// JIT-ed code created by parent Ruby process cannot be used in child Ruby process
831// because the code could rely on inline cache values (ivar's IC, send's CC) which
832// may vary between processes after fork or embed some process-specific addresses.
833//
834// So child Ruby process can't request parent process to JIT an ISeq and use the code.
835// Instead of that, MJIT worker thread is created for all child Ruby processes, even
836// while child processes would end up with compiling the same ISeqs.
837void
838mjit_child_after_fork(void)
839{
840 if (!mjit_enabled)
841 return;
842
843 /* Let parent process delete the already-compiled object files.
844 This must be done before starting MJIT worker on child process. */
845 skip_cleaning_object_files(&active_units);
846
847 /* MJIT worker thread is not inherited on fork. Start it for this child process. */
848 start_worker();
849}
850
851// Edit 0 to 1 to enable this feature for investigating hot methods
852#define MJIT_COUNTER 0
853#if MJIT_COUNTER
854static void
855mjit_dump_total_calls(void)
856{
857 struct rb_mjit_unit *unit;
858 fprintf(stderr, "[MJIT_COUNTER] total_calls of active_units:\n");
859 list_for_each(&active_units.head, unit, unode) {
860 const rb_iseq_t *iseq = unit->iseq;
861 fprintf(stderr, "%8ld: %s@%s:%d\n", iseq->body->total_calls, RSTRING_PTR(iseq->body->location.label),
863 }
864}
865#endif
866
867// Finish the threads processing units and creating PCH, finalize
868// and free MJIT data. It should be called last during MJIT
869// life.
870//
871// If close_handle_p is true, it calls dlclose() for JIT-ed code. So it should be false
872// if the code can still be on stack. ...But it means to leak JIT-ed handle forever (FIXME).
873void
874mjit_finish(bool close_handle_p)
875{
876 if (!mjit_enabled)
877 return;
878
879 // Wait for pch finish
880 verbose(2, "Stopping worker thread");
881 CRITICAL_SECTION_START(3, "in mjit_finish to wakeup from pch");
882 // As our threads are detached, we could just cancel them. But it
883 // is a bad idea because OS processes (C compiler) started by
884 // threads can produce temp files. And even if the temp files are
885 // removed, the used C compiler still complaint about their
886 // absence. So wait for a clean finish of the threads.
887 while (pch_status == PCH_NOT_READY) {
888 verbose(3, "Waiting wakeup from make_pch");
889 rb_native_cond_wait(&mjit_pch_wakeup, &mjit_engine_mutex);
890 }
891 CRITICAL_SECTION_FINISH(3, "in mjit_finish to wakeup from pch");
892
893 // Stop worker
894 stop_worker();
895
896 rb_native_mutex_destroy(&mjit_engine_mutex);
897 rb_native_cond_destroy(&mjit_pch_wakeup);
898 rb_native_cond_destroy(&mjit_client_wakeup);
899 rb_native_cond_destroy(&mjit_worker_wakeup);
900 rb_native_cond_destroy(&mjit_gc_wakeup);
901
902#if MJIT_COUNTER
903 mjit_dump_total_calls();
904#endif
905
906#ifndef _MSC_VER // mswin has prebuilt precompiled header
907 if (!mjit_opts.save_temps && getpid() == pch_owner_pid)
908 remove_file(pch_file);
909
910 xfree(header_file); header_file = NULL;
911#endif
912 xfree((void *)cc_common_args); cc_common_args = NULL;
913 for (char **flag = cc_added_args; *flag != NULL; flag++)
914 xfree(*flag);
915 xfree((void *)cc_added_args); cc_added_args = NULL;
916 xfree(tmp_dir); tmp_dir = NULL;
917 xfree(pch_file); pch_file = NULL;
918
919 mjit_call_p = false;
920 free_list(&unit_queue, close_handle_p);
921 free_list(&active_units, close_handle_p);
922 free_list(&compact_units, close_handle_p);
923 free_list(&stale_units, close_handle_p);
924 finish_conts();
925
926 mjit_enabled = false;
927 verbose(1, "Successful MJIT finish");
928}
929
930// Called by rb_vm_mark().
931//
932// Mark an ISeq being compiled to prevent its CCs from being GC-ed, which
933// an MJIT worker may concurrently see.
934//
935// Also mark active_units so that we do not GC ISeq which may still be
936// referred to by mjit_recompile() or compact_all_jit_code().
937void
938mjit_mark(void)
939{
940 if (!mjit_enabled)
941 return;
942 RUBY_MARK_ENTER("mjit");
943
944 // We need to release a lock when calling rb_gc_mark to avoid doubly acquiring
945 // a lock by by mjit_gc_start_hook inside rb_gc_mark.
946 //
947 // Because an MJIT worker may modify active_units anytime, we need to convert
948 // the linked list to an array to safely loop its ISeqs without keeping a lock.
949 CRITICAL_SECTION_START(4, "mjit_mark");
950 int length = 0;
951 if (compiling_iseqs != NULL) {
952 while (compiling_iseqs[length]) length++;
953 }
954 length += active_units.length;
955 const rb_iseq_t **iseqs = ALLOCA_N(const rb_iseq_t *, length);
956
957 struct rb_mjit_unit *unit = NULL;
958 int i = 0;
959 if (compiling_iseqs != NULL) {
960 while (compiling_iseqs[i]) {
961 iseqs[i] = compiling_iseqs[i];
962 i++;
963 }
964 }
965 list_for_each(&active_units.head, unit, unode) {
966 iseqs[i] = unit->iseq;
967 i++;
968 }
969 assert(i == length);
970 CRITICAL_SECTION_FINISH(4, "mjit_mark");
971
972 for (i = 0; i < length; i++) {
973 if (iseqs[i] == NULL) // ISeq is GC-ed
974 continue;
975 rb_gc_mark((VALUE)iseqs[i]);
976 }
977
978 RUBY_MARK_LEAVE("mjit");
979}
980
981// Called by rb_iseq_mark() to mark cc_entries captured for MJIT
982void
983mjit_mark_cc_entries(const struct rb_iseq_constant_body *const body)
984{
985 const struct rb_callcache **cc_entries;
986 if (body->jit_unit && (cc_entries = body->jit_unit->cc_entries) != NULL) {
987 // It must be `body->jit_unit->cc_entries_size` instead of `body->ci_size` to mark children's cc_entries
988 for (unsigned int i = 0; i < body->jit_unit->cc_entries_size; i++) {
989 const struct rb_callcache *cc = cc_entries[i];
990 if (cc != NULL && vm_cc_markable(cc)) {
991 // Pin `cc` and `cc->cme` against GC.compact as their addresses may be written in JIT-ed code.
992 rb_gc_mark((VALUE)cc);
993 rb_gc_mark((VALUE)vm_cc_cme(cc));
994 }
995 }
996 }
997}
998
999// A hook to update valid_class_serials.
1000void
1001mjit_add_class_serial(rb_serial_t class_serial)
1002{
1003 if (!mjit_enabled)
1004 return;
1005
1006 // Do not wrap CRITICAL_SECTION here. This function is only called in main thread
1007 // and guarded by GVL, and `rb_hash_aset` may cause GC and deadlock in it.
1008 rb_hash_aset(valid_class_serials, LONG2FIX(class_serial), Qtrue);
1009}
1010
1011// A hook to update valid_class_serials.
1012void
1013mjit_remove_class_serial(rb_serial_t class_serial)
1014{
1015 if (!mjit_enabled)
1016 return;
1017
1018 CRITICAL_SECTION_START(3, "in mjit_remove_class_serial");
1019 rb_hash_delete_entry(valid_class_serials, LONG2FIX(class_serial));
1020 CRITICAL_SECTION_FINISH(3, "in mjit_remove_class_serial");
1021}
1022
1023#endif
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
Definition: assert.h:167
void rb_fiber_init_mjit_cont(struct rb_fiber_struct *fiber)
Definition: cont.c:1165
#define RB_DEBUG_COUNTER_INC(type)
char * strchr(char *, char)
#define S_ISDIR(m)
Definition: dir.c:1913
#define assert(x)
Definition: dlmalloc.c:1176
#define MAXPATHLEN
Definition: dln.c:69
#define free(x)
Definition: dln.c:52
#define S_ISREG(m)
uint8_t len
Definition: escape.c:17
char str[HTML_ESCAPE_MAX_LEN+1]
Definition: escape.c:18
#define numberof(array)
Definition: etc.c:649
#define RSTRING_LEN(string)
Definition: fbuffer.h:22
#define RSTRING_PTR(string)
Definition: fbuffer.h:19
#define UNLIKELY(x)
Definition: ffi_common.h:126
#define memcpy(d, s, n)
Definition: ffi_common.h:55
int rb_path_check(const char *path)
Definition: file.c:6250
void rb_memerror(void)
Definition: gc.c:10309
VALUE rb_gc_location(VALUE value)
Definition: gc.c:9003
void rb_gc_mark(VALUE ptr)
Definition: gc.c:6112
void rb_gc_register_mark_object(VALUE obj)
Inform the garbage collector that object is a live Ruby object that should not be moved.
Definition: gc.c:8022
#define RUBY_MARK_LEAVE(msg)
Definition: gc.h:65
#define RUBY_MARK_ENTER(msg)
Definition: gc.h:64
#define CLASS_OF
Definition: globals.h:153
void rb_raise(VALUE exc, const char *fmt,...)
Definition: error.c:2917
VALUE rb_eRuntimeError
Definition: error.c:1055
VALUE rb_cObject
Object class.
Definition: object.c:49
VALUE rb_obj_hide(VALUE obj)
Make the object invisible from Ruby code.
Definition: object.c:92
unsigned char suffix[65536]
Definition: gun.c:164
unsigned short prefix[65536]
Definition: gun.c:163
#define RTLD_DEFAULT
Definition: handle.c:291
VALUE rb_hash_delete_entry(VALUE hash, VALUE key)
Definition: hash.c:2309
VALUE rb_hash_aset(VALUE hash, VALUE key, VALUE val)
Definition: hash.c:2901
VALUE rb_hash_new(void)
Definition: hash.c:1538
void rb_id_table_foreach(struct rb_id_table *tbl, rb_id_table_foreach_func_t *func, void *data)
Definition: id_table.c:292
rb_id_table_iterator_result
Definition: id_table.h:10
@ ID_TABLE_CONTINUE
Definition: id_table.h:11
Thin wrapper to ruby/config.h.
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Definition: io.c:307
int rb_is_const_id(ID)
Definition: symbol.c:1004
void rb_thread_wait_for(struct timeval)
Definition: thread.c:1562
char * ruby_strdup(const char *)
Definition: util.c:531
#define strdup(s)
Definition: util.h:39
#define FIX2INT
Definition: int.h:41
Internal header for Class.
#define RCLASS_SERIAL(c)
Definition: class.h:90
#define RCLASS_CONST_TBL(c)
Definition: class.h:78
Internal header for Fiber.
Internal header for File.
Internal header for Hash.
VALUE rb_vm_top_self(void)
Definition: vm.c:3752
voidpf void uLong size
Definition: ioapi.h:138
voidpf void * buf
Definition: ioapi.h:138
VALUE rb_iseq_path(const rb_iseq_t *iseq)
Definition: iseq.c:1087
#define LONG2FIX
Definition: long.h:49
#define ALLOCA_N(type, n)
Definition: memory.h:112
#define W_OK
Definition: file.h:16
#define mjit_enabled
Definition: mjit.h:210
#define MJIT_HEADER_INSTALL_DIR
Definition: mjit_config.h:7
#define MJIT_MIN_HEADER_NAME
Definition: mjit_config.h:9
#define PRELOADENV
Definition: mjit_config.h:17
void mjit_worker(void)
Definition: mjit_worker.c:1418
#define append_str(p, str)
Definition: mjit_worker.c:788
#define append_str2(p, str, len)
Definition: mjit_worker.c:787
struct mjit_options mjit_opts
Definition: mjit_worker.c:199
bool mjit_call_p
Definition: mjit_worker.c:205
#define MJIT_TMP_PREFIX
Definition: mjit_worker.c:142
@ PCH_SUCCESS
Definition: mjit_worker.c:263
@ PCH_FAILED
Definition: mjit_worker.c:263
@ PCH_NOT_READY
Definition: mjit_worker.c:263
#define TRUE
Definition: nkf.h:175
#define FALSE
Definition: nkf.h:174
#define NULL
Definition: regenc.h:69
#define StringValuePtr(v)
Definition: rstring.h:51
VALUE ruby_prefix_path
Definition: ruby.c:609
VALUE ruby_archlibdir_path
Definition: ruby.c:609
unsigned LONG_LONG rb_serial_t
Definition: serial.h:19
#define Qundef
#define Qtrue
#define Qfalse
#define calloc
Definition: st.c:171
size_t strlen(const char *)
rb_execution_context_t * ec
Definition: mjit_worker.c:1337
struct mjit_cont * next
Definition: mjit_worker.c:1338
struct mjit_cont * prev
Definition: mjit_worker.c:1338
Definition: constant.h:33
VALUE value
Definition: constant.h:36
rb_control_frame_t * cfp
Definition: vm_core.h:858
rb_iseq_location_t location
Definition: vm_core.h:393
struct rb_iseq_constant_body * body
Definition: vm_core.h:448
struct list_head head
Definition: mjit_worker.c:178
void * handle
Definition: mjit_worker.c:159
rb_iseq_t * iseq
Definition: mjit_worker.c:160
struct rb_mjit_compile_info compile_info
Definition: mjit_worker.c:170
struct list_node unode
Definition: mjit_worker.c:155
time_t tv_sec
Definition: missing.h:52
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
unsigned long VALUE
Definition: value.h:38
unsigned long ID
Definition: value.h:39
#define T_MODULE
Definition: value_type.h:69
#define T_CLASS
Definition: value_type.h:57
#define RUBY_VM_CHECK_INTS(ec)
Definition: vm_core.h:1921
Internal header to suppres / mandate warnings.
#define getenv(name)
Definition: win32.c:80
UINT rb_w32_system_tmpdir(WCHAR *path, UINT len)
Definition: win32.c:522
#define stat
Definition: win32.h:195
rb_uid_t getuid(void)
Definition: win32.c:2821
#define S_IWOTH
Definition: win32.h:386
unsigned int uintptr_t
Definition: win32.h:106
#define access(path, mode)
Definition: win32.h:205
#define xfree
Definition: xmalloc.h:49
#define xmalloc
Definition: xmalloc.h:44