Hello everybody,
I'm writing a scripting virtual machine from which I want to directly call C/C++ functions.
To do this I have a function: void x64_InvokeStatic(NativeTypes* args, NativeTypes retType, char** stack, void* func);
- func -> Function pointer.
- stack -> A pointer to the VM's stack.
- retType -> The return type.
- args -> The argument types.
This function is written in x64 assembly.
NativeTypes is defined as:
enum class NativeTypes : char
{
NT_Void = 1,
NT_Int8 = 2,
NT_UInt8 = 2,
NT_Int16 = 3,
NT_UInt16 = 3,
NT_Int32 = 4,
NT_UInt32 = 4,
NT_Object = 1,
NT_Int64 = 5,
NT_UInt64 = 5
};The args array is 0-terminated (example, a single int32 value: NativeTypes args[2] = { NativeTypes::NT_Int32, (NativeTypes)0}; )
The values of the NativeTypes values group the size and type (eg. NT_Int32 and NT_UInt32 have the same size and do not require seperated handling).
Working on x64 Windows, I know the arguments are passed:
rcx - first argument
rdx - second argument
r8 - third argument
r9 - fourth argument
stack - rest -> all 64 bit values.
All integer values are returned in rax.
My problem is with the values pushed on the (x64) stack, they are not passed correctly.
I suspect this has to do with my own setup of the (x64) stack but after several days of trying things I still haven't found a solution.
Does anyone have an idea about what I'm doing wrong (and possibly how to fix it)?
Thanks a lot in advance.
My x64 assembly:
.code
; Help function to get values from the VM's stack, puts them in rax.
get_arg proc
; snipped away, does not modify the (x64) stack.
get_arg endp
; void x64_InvokeStatic(NativeTypes* args, NativeTypes retType, char** stack, void* func);
x64_InvokeStatic proc
; Reserve stack space
push rbp
sub rsp, 72
; Store values
mov [rsp+8], rsp
mov [rsp+16], r15
mov [rsp+24], r14
mov [rsp+32], r13
mov [rsp+40], r12
mov [rsp+48], r9
mov [rsp+56], r8
mov [rsp+64], rdx
mov [rsp+72], rcx
mov rbp, rsp
; Load argument array.
xor r12, r12
mov r12, [rsp+72]
;;;;;;;;;;;;;;;;;;;
; First argument ;
;;;;;;;;;;;;;;;;;;;
mov r13b, [r12]
cmp r13b, 0
je call_func
; Load VM stack
mov r14, [rsp+56]
mov r15, [r14]
call get_arg
mov rcx, rax
;;;;;;;;;;;;;;;;;;;;
; Second argument ;
;;;;;;;;;;;;;;;;;;;;
second_argument:
add r12, 1
mov r13b, [r12]
cmp r13b, 0
je call_func
call get_arg
mov rdx, rax
;;;;;;;;;;;;;;;;;;;
; Third argument ;
;;;;;;;;;;;;;;;;;;;
third_argument:
add r12, 1
mov r13b, [r12]
cmp r13b, 0
je call_func
call get_arg
mov r8, rax
;;;;;;;;;;;;;;;;;;;;
; Fourth argument ;
;;;;;;;;;;;;;;;;;;;;
fourth_argument:
add r12, 1
mov r13b, [r12]
cmp r13b, 0
je call_func
call get_arg
mov r9, rax
;;;;;;;;;;;;;;;;;;;;
; Stack arguments ;
;;;;;;;;;;;;;;;;;;;;
push_stack:
add r12, 1
mov r13b, [r12]
cmp r13b, 0
je call_func
call get_arg
push rax
jmp push_stack
; Call function
call_func:
mov rax, [rbp+48]
call rax
; Discard all pushed variables.
mov r8, rbp
mov rsp, r8
; Load return type, if void -> done.
mov r12, [rsp+64]
cmp r12, 1
je done
; Load stackpointerpointer en stackpointer.
mov r13, [rsp+56]
mov r14, [r13]
; Determine what type to return.
cmp r12, 2
je ret_int8
cmp r12, 3
je ret_int16
cmp r12, 4
je ret_int32
cmp r12, 5
je ret_int64
; No known returntype, return for now.
jmp done
; Process various return type sizes.
; - Snipped, does not modify the x64 stack -
; done, restore stack and then return.
done:
mov r15, [rsp+16]
mov r14, [rsp+24]
mov r13, [rsp+32]
mov r12, [rsp+40]
add rsp, 72
pop rbp
ret
x64_InvokeStatic endp
end
