30#if defined (__APPLE__)
31#define AARCH64_STACK_ALIGN 1
33#define AARCH64_STACK_ALIGN 16
39#define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
56#if defined (__clang__) && defined (__APPLE__)
58sys_icache_invalidate (
void *start,
size_t len);
62ffi_clear_cache (
void *start,
void *end)
64#if defined (__clang__) && defined (__APPLE__)
65 sys_icache_invalidate (start, (
char *)end - (
char *)start);
66#elif defined (__GNUC__)
67 __builtin___clear_cache (start, end);
69#error "Missing builtin to flush instruction cache"
76 return &context->
x[n];
82#if defined __AARCH64EB__
83 return &context->
v[n].
d[1].
s[1];
85 return &context->
v[n].
d[0].
s[0];
92#if defined __AARCH64EB__
93 return &context->
v[n].
d[1];
95 return &context->
v[n].
d[0];
102 return &context->
v[n];
109get_basic_type_addr (
unsigned short type,
struct call_context *context,
115 return get_s_addr (context, n);
116 case FFI_TYPE_DOUBLE:
117 return get_d_addr (context, n);
118#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
120 return get_v_addr (context, n);
124 case FFI_TYPE_UINT16:
125 case FFI_TYPE_SINT16:
126 case FFI_TYPE_UINT32:
127 case FFI_TYPE_SINT32:
129 case FFI_TYPE_POINTER:
130 case FFI_TYPE_UINT64:
131 case FFI_TYPE_SINT64:
132 return get_x_addr (context, n);
144get_basic_type_alignment (
unsigned short type)
149#if defined (__APPLE__)
152 case FFI_TYPE_DOUBLE:
154#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
156 return sizeof (
long double);
160#if defined (__APPLE__)
161 return sizeof (
UINT8);
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165#if defined (__APPLE__)
168 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171#if defined (__APPLE__)
174 case FFI_TYPE_POINTER:
175 case FFI_TYPE_UINT64:
176 case FFI_TYPE_SINT64:
188get_basic_type_size (
unsigned short type)
194 case FFI_TYPE_DOUBLE:
196#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
198 return sizeof (
long double);
201 return sizeof (
UINT8);
203 return sizeof (
SINT8);
204 case FFI_TYPE_UINT16:
206 case FFI_TYPE_SINT16:
208 case FFI_TYPE_UINT32:
211 case FFI_TYPE_SINT32:
213 case FFI_TYPE_POINTER:
214 case FFI_TYPE_UINT64:
216 case FFI_TYPE_SINT64:
239is_floating_type (
unsigned short type)
241 return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE
248get_homogeneous_type (ffi_type *ty)
250 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
253 unsigned short candidate_type
254 = get_homogeneous_type (ty->elements[0]);
255 for (i =1; ty->elements[i]; i++)
257 unsigned short iteration_type = 0;
261 if (ty->elements[i]->type == FFI_TYPE_STRUCT
262 && ty->elements[i]->elements)
264 iteration_type = get_homogeneous_type (ty->elements[i]);
268 iteration_type = ty->elements[i]->type;
272 if (candidate_type != iteration_type)
273 return FFI_TYPE_STRUCT;
275 return candidate_type;
290element_count (ffi_type *ty)
292 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
296 for (n = 0; ty->elements[n]; n++)
298 if (ty->elements[n]->type == FFI_TYPE_STRUCT
299 && ty->elements[n]->elements)
300 elems += element_count (ty->elements[n]);
319 if (ty->type == FFI_TYPE_STRUCT
321 && is_floating_type (get_homogeneous_type (ty)))
323 unsigned n = element_count (ty);
324 return n >= 1 && n <= 4;
342is_register_candidate (ffi_type *ty)
348 case FFI_TYPE_DOUBLE:
349#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
353 case FFI_TYPE_UINT16:
354 case FFI_TYPE_UINT32:
355 case FFI_TYPE_UINT64:
356 case FFI_TYPE_POINTER:
358 case FFI_TYPE_SINT16:
359 case FFI_TYPE_SINT32:
361 case FFI_TYPE_SINT64:
364 case FFI_TYPE_STRUCT:
369 else if (ty->size > 16)
397is_v_register_candidate (ffi_type *ty)
399 return is_floating_type (ty->type)
400 || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
415#if defined (__APPLE__)
416 unsigned allocating_variadic;
428#if defined (__APPLE__)
429 state->allocating_variadic = 0;
455 return get_x_addr (context, (
state->ngrn)++);
462 return get_s_addr (context, (
state->nsrn)++);
469 return get_d_addr (context, (
state->nsrn)++);
476 return get_v_addr (context, (
state->nsrn)++);
481allocate_to_stack (
struct arg_state *
state,
void *stack,
size_t alignment,
490#if defined (__APPLE__)
491 if (
state->allocating_variadic)
497 allocation = stack +
state->nsaa;
504copy_basic_type (
void *dest,
void *source,
unsigned short type)
511 *(
float *) dest = *(
float *) source;
513 case FFI_TYPE_DOUBLE:
514 *(
double *) dest = *(
double *) source;
516#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
518 *(
long double *) dest = *(
long double *) source;
527 case FFI_TYPE_UINT16:
530 case FFI_TYPE_SINT16:
533 case FFI_TYPE_UINT32:
537 case FFI_TYPE_SINT32:
540 case FFI_TYPE_POINTER:
541 case FFI_TYPE_UINT64:
544 case FFI_TYPE_SINT64:
556copy_hfa_to_reg_or_stack (
void *memory,
559 unsigned char *stack,
562 unsigned elems = element_count (ty);
563 if (available_v (
state) < elems)
569 memcpy (allocate_to_stack (
state, stack, ty->alignment, ty->size),
576 unsigned short type = get_homogeneous_type (ty);
577 for (i = 0; i < elems; i++)
579 void *reg = allocate_to_v (context,
state);
580 copy_basic_type (reg, memory, type);
581 memory += get_basic_type_size (type);
591allocate_to_register_or_stack (
struct call_context *context,
592 unsigned char *stack,
596 size_t alignment = get_basic_type_alignment (type);
597 size_t size = alignment;
605 case FFI_TYPE_DOUBLE:
607 return allocate_to_d (context,
state);
610#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
613 return allocate_to_v (context,
state);
619 case FFI_TYPE_UINT16:
620 case FFI_TYPE_SINT16:
621 case FFI_TYPE_UINT32:
622 case FFI_TYPE_SINT32:
624 case FFI_TYPE_POINTER:
625 case FFI_TYPE_UINT64:
626 case FFI_TYPE_SINT64:
628 return allocate_to_x (context,
state);
635 return allocate_to_stack (
state, stack, alignment,
size);
643 unsigned char *stack,
649 allocate_to_register_or_stack (context, stack,
state, type),
658aarch64_prep_args (
struct call_context *context,
unsigned char *stack,
666 for (i = 0; i < ecif->
cif->nargs; i++)
668 ffi_type *ty = ecif->
cif->arg_types[i];
678 case FFI_TYPE_DOUBLE:
679#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
684 case FFI_TYPE_UINT16:
685 case FFI_TYPE_SINT16:
686 case FFI_TYPE_UINT32:
688 case FFI_TYPE_SINT32:
689 case FFI_TYPE_POINTER:
690 case FFI_TYPE_UINT64:
691 case FFI_TYPE_SINT64:
692 copy_to_register_or_stack (context, stack, &
state,
693 ecif->
avalue[i], ty->type);
696 case FFI_TYPE_STRUCT:
699 copy_hfa_to_reg_or_stack (ecif->
avalue[i], ty, context,
702 else if (ty->size > 16)
708 copy_to_register_or_stack (context, stack, &
state,
709 &(ecif->
avalue[i]), FFI_TYPE_POINTER);
711 else if (available_x (&
state) >= (ty->size + 7) / 8)
718 for (j = 0; j < (ty->size + 7) / 8; j++)
733 memcpy (allocate_to_stack (&
state, stack, ty->alignment,
734 ty->size), ecif->
avalue + i, ty->size);
743#if defined (__APPLE__)
744 if (i + 1 == ecif->
cif->aarch64_nfixedargs)
749 state.allocating_variadic = 1;
754 return ecif->
cif->aarch64_flags;
768 cif->aarch64_flags = 0;
770 if (is_v_register_candidate (cif->rtype))
777 for (i = 0; i < cif->nargs; i++)
778 if (is_v_register_candidate (cif->arg_types[i]))
785#if defined (__APPLE__)
786 cif->aarch64_nfixedargs = 0;
792#if defined (__APPLE__)
796 unsigned int nfixedargs,
797 unsigned int ntotalargs)
803 cif->aarch64_nfixedargs = nfixedargs;
813ffi_call (ffi_cif *cif,
void (*fn)(
void),
void *rvalue,
void **avalue)
832 stack_bytes =
ALIGN(cif->bytes, 16);
834 memset (&context, 0,
sizeof (context));
835 if (is_register_candidate (cif->rtype))
837 ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
838 switch (cif->rtype->type)
842 case FFI_TYPE_DOUBLE:
843#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
848 case FFI_TYPE_UINT16:
849 case FFI_TYPE_SINT16:
850 case FFI_TYPE_UINT32:
851 case FFI_TYPE_SINT32:
852 case FFI_TYPE_POINTER:
853 case FFI_TYPE_UINT64:
855 case FFI_TYPE_SINT64:
857 void *addr = get_basic_type_addr (cif->rtype->type,
859 copy_basic_type (rvalue, addr, cif->rtype->type);
863 case FFI_TYPE_STRUCT:
864 if (is_hfa (cif->rtype))
867 unsigned short type = get_homogeneous_type (cif->rtype);
868 unsigned elems = element_count (cif->rtype);
869 for (j = 0; j < elems; j++)
871 void *reg = get_basic_type_addr (type, &context, j);
872 copy_basic_type (rvalue, reg, type);
873 rvalue += get_basic_type_size (type);
879 memcpy (rvalue, get_x_addr (&context, 0),
size);
894 memcpy (get_x_addr (&context, 8), &rvalue,
sizeof (
UINT64));
907static unsigned char trampoline [] =
908{ 0x70, 0x00, 0x00, 0x58,
909 0x91, 0x00, 0x00, 0x10,
910 0x00, 0x02, 0x1f, 0xd6
915#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \
916 ({unsigned char *__tramp = (unsigned char*)(TRAMP); \
917 UINT64 __fun = (UINT64)(FUN); \
918 UINT64 __ctx = (UINT64)(CTX); \
919 UINT64 __flags = (UINT64)(FLAGS); \
920 memcpy (__tramp, trampoline, sizeof (trampoline)); \
921 memcpy (__tramp + 12, &__fun, sizeof (__fun)); \
922 memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \
923 memcpy (__tramp + 28, &__flags, sizeof (__flags)); \
924 ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \
930 void (*fun)(ffi_cif*,
void*,
void**,
void*),
941 closure->user_data = user_data;
967 ffi_cif *cif = closure->cif;
968 void **avalue = (
void**)
alloca (cif->nargs * sizeof (
void*));
975 for (i = 0; i < cif->nargs; i++)
977 ffi_type *ty = cif->arg_types[i];
987 case FFI_TYPE_UINT16:
988 case FFI_TYPE_SINT16:
989 case FFI_TYPE_UINT32:
990 case FFI_TYPE_SINT32:
992 case FFI_TYPE_POINTER:
993 case FFI_TYPE_UINT64:
994 case FFI_TYPE_SINT64:
996 case FFI_TYPE_DOUBLE:
997#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
999 avalue[i] = allocate_to_register_or_stack (context, stack,
1004 case FFI_TYPE_STRUCT:
1007 unsigned n = element_count (ty);
1008 if (available_v (&
state) < n)
1011 avalue[i] = allocate_to_stack (&
state, stack, ty->alignment,
1016 switch (get_homogeneous_type (ty))
1018 case FFI_TYPE_FLOAT:
1032 for (j = 0; j < element_count (ty); j++)
1034 allocate_to_s (context, &
state),
1039 case FFI_TYPE_DOUBLE:
1053 for (j = 0; j < element_count (ty); j++)
1055 allocate_to_d (context, &
state),
1060#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1063 allocate_to_v (context, &
state),
1074 else if (ty->size > 16)
1079 allocate_to_register_or_stack (context, stack,
1080 &
state, FFI_TYPE_POINTER),
1081 sizeof (avalue[i]));
1083 else if (available_x (&
state) >= (ty->size + 7) / 8)
1085 avalue[i] = get_x_addr (context,
state.ngrn);
1086 state.ngrn += (ty->size + 7) / 8;
1092 avalue[i] = allocate_to_stack (&
state, stack, ty->alignment,
1107 if (is_register_candidate (cif->rtype))
1115 rvalue =
alloca (cif->rtype->size);
1116 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1120 switch (cif->rtype->type)
1125 case FFI_TYPE_UINT8:
1126 case FFI_TYPE_UINT16:
1127 case FFI_TYPE_UINT32:
1128 case FFI_TYPE_POINTER:
1129 case FFI_TYPE_UINT64:
1130 case FFI_TYPE_SINT8:
1131 case FFI_TYPE_SINT16:
1133 case FFI_TYPE_SINT32:
1134 case FFI_TYPE_SINT64:
1135 case FFI_TYPE_FLOAT:
1136 case FFI_TYPE_DOUBLE:
1137#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1141 void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
1142 copy_basic_type (addr, rvalue, cif->rtype->type);
1145 case FFI_TYPE_STRUCT:
1146 if (is_hfa (cif->rtype))
1149 unsigned short type = get_homogeneous_type (cif->rtype);
1150 unsigned elems = element_count (cif->rtype);
1151 for (j = 0; j < elems; j++)
1153 void *reg = get_basic_type_addr (type, context, j);
1154 copy_basic_type (reg, rvalue, type);
1155 rvalue += get_basic_type_size (type);
1158 else if ((cif->rtype->size + 7) / 8 <
N_X_ARG_REG)
1161 memcpy (get_x_addr (context, 0), rvalue,
size);
1175 memcpy (&rvalue, get_x_addr (context, 8),
sizeof (
UINT64));
1176 (closure->fun) (cif, rvalue, avalue, closure->user_data);
void FFI_HIDDEN ffi_closure_SYSV_inner(ffi_closure *closure, struct call_context *context, void *stack)
ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void(*fun)(ffi_cif *, void *, void **, void *), void *user_data, void *codeloc)
ffi_status ffi_prep_cif_machdep(ffi_cif *cif)
void ffi_call(ffi_cif *cif, void(*fn)(void), void *rvalue, void **avalue)
#define AARCH64_STACK_ALIGN
#define FFI_INIT_TRAMPOLINE(TRAMP, FUN, CTX, FLAGS)
#define AARCH64_FFI_WITH_V
void ffi_call_SYSV(unsigned(*)(struct call_context *context, unsigned char *, extended_cif *), struct call_context *context, extended_cif *, size_t, void(*fn)(void))
#define FFI_TYPE_LONGDOUBLE
ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs)
void ffi_closure_SYSV(void)
VALUE type(ANYARGS)
ANYARGS-ed function type.
struct call_context::@22 v[AARCH64_N_VREG]