12#include "internal/gc.h"
15#include "internal/static_assert.h"
17#include "internal/variable.h"
30#ifndef TRANSIENT_HEAP_CHECK_MODE
31#define TRANSIENT_HEAP_CHECK_MODE 0
33#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
40#define TRANSIENT_HEAP_DEBUG 0
46#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
48#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
55#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
58#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
61#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 )
62#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32)
63#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 )
64#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
65#define TRANSIENT_HEAP_USABLE_SIZE (TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header))
67#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
68#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
70#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
71#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
73enum transient_heap_status {
75 transient_heap_marking,
76 transient_heap_escaping
79struct transient_heap_block {
80 struct transient_heap_block_header {
82 int16_t last_marked_index;
84 struct transient_heap_block *next_block;
86 char buff[TRANSIENT_HEAP_USABLE_SIZE];
89struct transient_heap {
90 struct transient_heap_block *using_blocks;
91 struct transient_heap_block *marked_blocks;
92 struct transient_heap_block *free_blocks;
94 int total_marked_objects;
96 enum transient_heap_status status;
98 VALUE *promoted_objects;
99 int promoted_objects_size;
100 int promoted_objects_index;
102 struct transient_heap_block *arena;
106struct transient_alloc_header {
109 int16_t next_marked_index;
114static struct transient_heap global_transient_heap;
116static void transient_heap_promote_add(
struct transient_heap* theap,
VALUE obj);
117static const void *transient_heap_ptr(
VALUE obj,
int error);
118static int transient_header_managed_ptr_p(
struct transient_heap* theap,
const void *
ptr);
120#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
123transient_heap_block_dump(
struct transient_heap* theap,
struct transient_heap_block *block)
127 while (i<block->info.index) {
128 void *
ptr = &block->buff[i];
129 struct transient_alloc_header *header =
ptr;
130 fprintf(stderr,
"%4d %8d %p size:%4d next:%4d %s\n", n, i,
ptr, header->size, header->next_marked_index,
rb_obj_info(header->obj));
137transient_heap_blocks_dump(
struct transient_heap* theap,
struct transient_heap_block *block,
const char *type_str)
140 fprintf(stderr,
"- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
141 type_str, (
void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (
void *)block->info.next_block);
143 transient_heap_block_dump(theap, block);
144 block = block->info.next_block;
149transient_heap_dump(
struct transient_heap* theap)
151 fprintf(stderr,
"transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
152 transient_heap_blocks_dump(theap, theap->using_blocks,
"using_blocks");
153 transient_heap_blocks_dump(theap, theap->marked_blocks,
"marked_blocks");
154 transient_heap_blocks_dump(theap, theap->free_blocks,
"free_blocks");
159rb_transient_heap_dump(
void)
161 transient_heap_dump(&global_transient_heap);
164#if TRANSIENT_HEAP_CHECK_MODE >= 2
167transient_heap_ptr_check(
struct transient_heap *theap,
VALUE obj)
170 const void *
ptr = transient_heap_ptr(obj,
FALSE);
171 TH_ASSERT(
ptr ==
NULL || transient_header_managed_ptr_p(theap,
ptr));
177transient_heap_block_verify(
struct transient_heap *theap,
struct transient_heap_block *block)
180 struct transient_alloc_header *header;
182 while (i<block->info.index) {
183 header = (
void *)&block->buff[i];
184 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
185 transient_heap_ptr_check(theap, header->obj);
189 TH_ASSERT(block->info.objects == n);
195transient_heap_blocks_verify(
struct transient_heap *theap,
struct transient_heap_block *blocks,
int *block_num_ptr)
198 struct transient_heap_block *block = blocks;
200 n += transient_heap_block_verify(theap, block);
202 block = block->info.next_block;
210transient_heap_verify(
struct transient_heap *theap)
212#if TRANSIENT_HEAP_CHECK_MODE >= 2
213 int n=0, block_num=0;
215 n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
216 n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
218 TH_ASSERT(n == theap->total_objects);
219 TH_ASSERT(n >= theap->total_marked_objects);
220 TH_ASSERT(block_num == theap->total_blocks);
228 transient_heap_verify(&global_transient_heap);
231static struct transient_heap*
232transient_heap_get(
void)
234 struct transient_heap* theap = &global_transient_heap;
235 transient_heap_verify(theap);
240reset_block(
struct transient_heap_block *block)
242 __msan_allocated_memory(block,
sizeof block);
243 block->info.index = 0;
244 block->info.objects = 0;
245 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
246 block->info.next_block =
NULL;
247 __asan_poison_memory_region(&block->buff,
sizeof block->buff);
251connect_to_free_blocks(
struct transient_heap *theap,
struct transient_heap_block *block)
253 block->info.next_block = theap->free_blocks;
254 theap->free_blocks = block;
258connect_to_using_blocks(
struct transient_heap *theap,
struct transient_heap_block *block)
260 block->info.next_block = theap->using_blocks;
261 theap->using_blocks = block;
266connect_to_marked_blocks(
struct transient_heap *theap,
struct transient_heap_block *block)
268 block->info.next_block = theap->marked_blocks;
269 theap->marked_blocks = block;
274append_to_marked_blocks(
struct transient_heap *theap,
struct transient_heap_block *append_blocks)
276 if (theap->marked_blocks) {
277 struct transient_heap_block *block = theap->marked_blocks, *last_block =
NULL;
280 block = block->info.next_block;
283 TH_ASSERT(last_block->info.next_block ==
NULL);
284 last_block->info.next_block = append_blocks;
287 theap->marked_blocks = append_blocks;
291static struct transient_heap_block *
292transient_heap_block_alloc(
struct transient_heap* theap)
294 struct transient_heap_block *block;
295#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
296 block = mmap(
NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
297 MAP_PRIVATE | MAP_ANONYMOUS,
299 if (block == MAP_FAILED)
rb_bug(
"transient_heap_block_alloc: err:%d\n", errno);
301 if (theap->arena ==
NULL) {
302 theap->arena =
rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
303 if (theap->arena ==
NULL) {
304 rb_bug(
"transient_heap_block_alloc: failed\n");
308 TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
309 block = &theap->arena[theap->arena_index++];
310 TH_ASSERT(((
intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
314 TH_ASSERT(((
intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
315 if (0) fprintf(stderr,
"transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (
void *)block);
320static struct transient_heap_block *
321transient_heap_allocatable_block(
struct transient_heap* theap)
323 struct transient_heap_block *block;
325#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
326 block = transient_heap_block_alloc(theap);
327 theap->total_blocks++;
330 block = theap->free_blocks;
332 theap->free_blocks = block->info.next_block;
333 block->info.next_block =
NULL;
334 theap->total_blocks++;
341static struct transient_alloc_header *
342transient_heap_allocatable_header(
struct transient_heap* theap,
size_t size)
344 struct transient_heap_block *block = theap->using_blocks;
347 TH_ASSERT(block->info.index <= (int16_t)TRANSIENT_HEAP_USABLE_SIZE);
349 if (TRANSIENT_HEAP_USABLE_SIZE - block->info.index >=
size) {
350 struct transient_alloc_header *header = (
void *)&block->buff[block->info.index];
351 block->info.index +=
size;
352 block->info.objects++;
356 block = transient_heap_allocatable_block(theap);
357 if (block) connect_to_using_blocks(theap, block);
371 struct transient_heap* theap = transient_heap_get();
372 size_t size =
ROUND_UP(req_size +
sizeof(
struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
374 TH_ASSERT(RB_TYPE_P(obj,
T_ARRAY) ||
379 if (
size > TRANSIENT_HEAP_ALLOC_MAX) {
380 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_alloc: [too big: %ld] %s\n", (
long)
size,
rb_obj_info(obj));
383#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
384 else if (RB_OBJ_PROMOTED_RAW(obj)) {
385 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_alloc: [promoted object] %s\n",
rb_obj_info(obj));
390 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_alloc: [hidden object] %s\n",
rb_obj_info(obj));
395 struct transient_alloc_header *header = transient_heap_allocatable_header(theap,
size);
401 asan_unpoison_memory_region(header,
sizeof *header,
true);
404 header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
405 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
409 asan_poison_memory_region(header,
sizeof *header);
412 theap->total_objects++;
414#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
415 if (RB_OBJ_PROMOTED_RAW(obj)) {
416 transient_heap_promote_add(theap, obj);
419 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (
void *)header,
ptr, (
int)
size,
rb_obj_info(obj));
424 asan_unpoison_memory_region(
ptr,
size -
sizeof *header,
true);
428 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_alloc: [no enough space: %ld] %s\n", (
long)
size,
rb_obj_info(obj));
438Init_TransientHeap(
void)
441 struct transient_heap* theap = transient_heap_get();
443#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
446 TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
447 block_num = TRANSIENT_HEAP_BLOCK_NUM;
449 for (i=0; i<block_num; i++) {
450 connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
452 theap->using_blocks = transient_heap_allocatable_block(theap);
454 theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
455 theap->promoted_objects_index = 0;
457 theap->promoted_objects =
malloc(
sizeof(
VALUE) * theap->promoted_objects_size);
460 sizeof(
VALUE) <=
SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE);
461 if (theap->promoted_objects ==
NULL)
rb_bug(
"Init_TransientHeap: malloc failed.");
464static struct transient_heap_block *
465blocks_alloc_header_to_block(
struct transient_heap *theap,
struct transient_heap_block *blocks,
struct transient_alloc_header *header)
467 struct transient_heap_block *block = blocks;
470 if (block->buff <= (
char *)header && (
char *)header < block->buff + TRANSIENT_HEAP_USABLE_SIZE) {
473 block = block->info.next_block;
479static struct transient_heap_block *
480alloc_header_to_block_verbose(
struct transient_heap *theap,
struct transient_alloc_header *header)
482 struct transient_heap_block *block;
484 if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) !=
NULL) {
485 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"alloc_header_to_block: found in marked_blocks\n");
488 else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) !=
NULL) {
489 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"alloc_header_to_block: found in using_blocks\n");
497static struct transient_alloc_header *
498ptr_to_alloc_header(
const void *
ptr)
500 struct transient_alloc_header *header = (
void *)
ptr;
506transient_header_managed_ptr_p(
struct transient_heap* theap,
const void *
ptr)
508 if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(
ptr))) {
518rb_transient_heap_managed_ptr_p(
const void *
ptr)
520 return transient_header_managed_ptr_p(transient_heap_get(),
ptr);
523static struct transient_heap_block *
524alloc_header_to_block(
struct transient_heap *theap,
struct transient_alloc_header *header)
526 struct transient_heap_block *block;
527#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
528 block = alloc_header_to_block_verbose(theap, header);
530 transient_heap_dump(theap);
531 rb_bug(
"alloc_header_to_block: not found in mark_blocks (%p)\n", header);
534 block = (
void *)((
intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
535 TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
545 struct transient_alloc_header *header = ptr_to_alloc_header(
ptr);
546 asan_unpoison_memory_region(header,
sizeof *header,
false);
547 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC)
rb_bug(
"rb_transient_heap_mark: wrong header, %s (%p)",
rb_obj_info(obj),
ptr);
548 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_mark: %s (%p)\n",
rb_obj_info(obj),
ptr);
550#if TRANSIENT_HEAP_CHECK_MODE > 0
552 struct transient_heap* theap = transient_heap_get();
553 TH_ASSERT(theap->status == transient_heap_marking);
554 TH_ASSERT(transient_header_managed_ptr_p(theap,
ptr));
556 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
557 transient_heap_dump(theap);
558 rb_bug(
"rb_transient_heap_mark: magic is broken");
560 else if (header->obj != obj) {
562 rb_bug(
"rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
568 if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
573 struct transient_heap* theap = transient_heap_get();
574 struct transient_heap_block *block = alloc_header_to_block(theap, header);
575 __asan_unpoison_memory_region(&block->info,
sizeof block->info);
576 header->next_marked_index = block->info.last_marked_index;
577 block->info.last_marked_index = (
int)((
char *)header - block->buff);
578 theap->total_marked_objects++;
580 transient_heap_verify(theap);
592 if (RARRAY_TRANSIENT_P(obj)) {
598 if (ROBJ_TRANSIENT_P(obj)) {
599 ptr = ROBJECT_IVPTR(obj);
603 if (RSTRUCT_TRANSIENT_P(obj)) {
604 ptr = rb_struct_const_heap_ptr(obj);
608 if (RHASH_TRANSIENT_P(obj)) {
609 TH_ASSERT(RHASH_AR_TABLE_P(obj));
626transient_heap_promote_add(
struct transient_heap* theap,
VALUE obj)
628 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
"rb_transient_heap_promote: %s\n",
rb_obj_info(obj));
630 if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
633 for (i=0; i<theap->promoted_objects_index; i++) {
634 if (theap->promoted_objects[i] == obj)
return;
638 if (theap->promoted_objects_size <= theap->promoted_objects_index) {
639 theap->promoted_objects_size *= 2;
640 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
643 theap->promoted_objects =
NULL;
646 theap->promoted_objects =
realloc(theap->promoted_objects, theap->promoted_objects_size *
sizeof(
VALUE));
648 if (theap->promoted_objects ==
NULL)
rb_bug(
"rb_transient_heap_promote: realloc failed");
650 theap->promoted_objects[theap->promoted_objects_index++] = obj;
658 if (transient_heap_ptr(obj,
FALSE)) {
659 struct transient_heap* theap = transient_heap_get();
660 transient_heap_promote_add(theap, obj);
667static struct transient_alloc_header *
668alloc_header(
struct transient_heap_block* block,
int index)
670 return (
void *)&block->buff[index];
674transient_heap_reset(
void)
678 struct transient_heap* theap = transient_heap_get();
679 struct transient_heap_block* block;
681 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"!! transient_heap_reset\n");
683 block = theap->marked_blocks;
685 struct transient_heap_block *next_block = block->info.next_block;
686 theap->total_objects -= block->info.objects;
687#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
688 if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
689 rb_bug(
"madvise err:%d", errno);
691 if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
692 rb_bug(
"mprotect err:%d", errno);
696 connect_to_free_blocks(theap, block);
698 theap->total_blocks--;
702 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"!! transient_heap_reset block_num:%d\n", theap->total_blocks);
704 theap->marked_blocks =
NULL;
705 theap->total_marked_objects = 0;
709transient_heap_block_evacuate(
struct transient_heap* theap,
struct transient_heap_block* block)
711 int marked_index = block->info.last_marked_index;
712 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
714 while (marked_index >= 0) {
715 struct transient_alloc_header *header = alloc_header(block, marked_index);
716 asan_unpoison_memory_region(header,
sizeof *header,
true);
717 VALUE obj = header->obj;
718 TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
719 if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC)
rb_bug(
"transient_heap_block_evacuate: wrong header %p %s\n", (
void *)header,
rb_obj_info(obj));
721 if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr,
" * transient_heap_block_evacuate %p %s\n", (
void *)header,
rb_obj_info(obj));
744 marked_index = header->next_marked_index;
745 asan_poison_memory_region(header,
sizeof *header);
749#if USE_RUBY_DEBUG_LOG
751transient_heap_status_cstr(
enum transient_heap_status status)
754 case transient_heap_none:
return "none";
755 case transient_heap_marking:
return "marking";
756 case transient_heap_escaping:
return "escaping";
763transient_heap_update_status(
struct transient_heap* theap,
enum transient_heap_status status)
766 transient_heap_status_cstr(theap->status),
767 transient_heap_status_cstr(status));
769 TH_ASSERT(theap->status != status);
770 theap->status = status;
774transient_heap_evacuate(
void *dmy)
776 struct transient_heap* theap = transient_heap_get();
778 if (theap->total_marked_objects == 0)
return;
780 if (theap->status == transient_heap_marking) {
781 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"!! transient_heap_evacuate: skip while transient_heap_marking\n");
786 struct transient_heap_block* block;
790 if (TRANSIENT_HEAP_DEBUG >= 1) {
792 fprintf(stderr,
"!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
793 if (TRANSIENT_HEAP_DEBUG >= 4) {
794 for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr,
"%4d %s\n", i,
rb_obj_info(theap->promoted_objects[i]));
797 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
799 TH_ASSERT(theap->status == transient_heap_none);
800 transient_heap_update_status(theap, transient_heap_escaping);
803 block = theap->marked_blocks;
805 transient_heap_block_evacuate(theap, block);
806 block = block->info.next_block;
811 block = theap->using_blocks;
813 transient_heap_block_evacuate(theap, block);
814 block = block->info.next_block;
818 transient_heap_reset();
820 if (TRANSIENT_HEAP_DEBUG > 0) {
821 fprintf(stderr,
"!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
824 transient_heap_verify(theap);
825 transient_heap_update_status(theap, transient_heap_none);
835 transient_heap_evacuate(
NULL);
839clear_marked_index(
struct transient_heap_block* block)
841 int marked_index = block->info.last_marked_index;
843 while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
844 struct transient_alloc_header *header = alloc_header(block, marked_index);
847 asan_unpoison_memory_region(header,
sizeof *header,
false);
848 TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
849 if (0) fprintf(stderr,
"clear_marked_index - block:%p mark_index:%d\n", (
void *)block, marked_index);
851 marked_index = header->next_marked_index;
852 header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
855 block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
859blocks_clear_marked_index(
struct transient_heap_block* block)
862 clear_marked_index(block);
863 block = block->info.next_block;
868transient_heap_block_update_refs(
struct transient_heap* theap,
struct transient_heap_block* block)
870 int marked_index = block->info.last_marked_index;
872 while (marked_index >= 0) {
873 struct transient_alloc_header *header = alloc_header(block, marked_index);
875 asan_unpoison_memory_region(header,
sizeof *header,
false);
879 marked_index = header->next_marked_index;
880 asan_poison_memory_region(header,
sizeof *header);
885transient_heap_blocks_update_refs(
struct transient_heap* theap,
struct transient_heap_block *block,
const char *type_str)
888 transient_heap_block_update_refs(theap, block);
889 block = block->info.next_block;
898 struct transient_heap* theap = transient_heap_get();
901 transient_heap_blocks_update_refs(theap, theap->using_blocks,
"using_blocks");
902 transient_heap_blocks_update_refs(theap, theap->marked_blocks,
"marked_blocks");
904 for (i=0; i<theap->promoted_objects_index; i++) {
905 VALUE obj = theap->promoted_objects[i];
916 struct transient_heap* theap = transient_heap_get();
918 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"!! rb_transient_heap_start_marking objects:%d blocks:%d promoted:%d full_marking:%d\n",
919 theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
920 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
922 blocks_clear_marked_index(theap->marked_blocks);
923 blocks_clear_marked_index(theap->using_blocks);
925 if (theap->using_blocks) {
926 if (theap->using_blocks->info.objects > 0) {
927 append_to_marked_blocks(theap, theap->using_blocks);
928 theap->using_blocks =
NULL;
931 append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
932 theap->using_blocks->info.next_block =
NULL;
936 if (theap->using_blocks ==
NULL) {
937 theap->using_blocks = transient_heap_allocatable_block(theap);
940 TH_ASSERT(theap->status == transient_heap_none);
941 transient_heap_update_status(theap, transient_heap_marking);
942 theap->total_marked_objects = 0;
945 theap->promoted_objects_index = 0;
949 for (i=0; i<theap->promoted_objects_index; i++) {
950 VALUE obj = theap->promoted_objects[i];
951 const void *
ptr = transient_heap_ptr(obj,
TRUE);
958 transient_heap_verify(theap);
967 struct transient_heap* theap = transient_heap_get();
970 theap->total_objects,
971 theap->total_marked_objects);
972 if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
974 TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
976 TH_ASSERT(theap->status == transient_heap_marking);
977 transient_heap_update_status(theap, transient_heap_none);
979 if (theap->total_marked_objects > 0) {
980 if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr,
"-> rb_transient_heap_finish_marking register escape func.\n");
984 transient_heap_reset();
987 transient_heap_verify(theap);
#define UNREACHABLE_RETURN
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
#define RB_DEBUG_COUNTER_INC(type)
VALUE rb_gc_location(VALUE value)
void * rb_aligned_malloc(size_t alignment, size_t size)
const char * rb_obj_info(VALUE obj)
VALUE rb_gc_disable_no_rest(void)
void rb_bug(const char *fmt,...)
Internal header for Hash.
Internal header for Struct.
typedef int(ZCALLBACK *close_file_func) OF((voidpf opaque
#define RARRAY_EMBED_FLAG
Internal header for ASAN / MSAN / etc.
#define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(x)
#define rb_transient_heap_finish_marking()
#define rb_obj_transient_heap_evacuate(x, y)
#define rb_transient_heap_promote(obj)
#define rb_transient_heap_verify()
#define rb_ary_transient_heap_evacuate(x, y)
#define rb_hash_transient_heap_evacuate(x, y)
#define rb_struct_transient_heap_evacuate(x, y)
#define rb_transient_heap_evacuate()
#define rb_transient_heap_alloc(o, s)
#define rb_transient_heap_mark(obj, ptr)
#define rb_transient_heap_update_references()
#define rb_transient_heap_start_marking(full_marking)
void error(const char *msg)
rb_ractor_t * ruby_single_main_ractor
#define RUBY_DEBUG_LOG(fmt,...)
#define ASSERT_vm_locking()