wasm_runtime.c 148 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379
  1. /*
  2. * Copyright (C) 2019 Intel Corporation. All rights reserved.
  3. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  4. */
  5. #include "wasm_runtime.h"
  6. #include "wasm.h"
  7. #include "wasm_loader.h"
  8. #include "wasm_interp.h"
  9. #include "bh_common.h"
  10. #include "bh_log.h"
  11. #include "mem_alloc.h"
  12. #include "../common/wasm_runtime_common.h"
  13. #include "../common/wasm_memory.h"
  14. #if WASM_ENABLE_GC != 0
  15. #include "../common/gc/gc_object.h"
  16. #endif
  17. #if WASM_ENABLE_SHARED_MEMORY != 0
  18. #include "../common/wasm_shared_memory.h"
  19. #endif
  20. #if WASM_ENABLE_THREAD_MGR != 0
  21. #include "../libraries/thread-mgr/thread_manager.h"
  22. #endif
  23. #if WASM_ENABLE_DEBUG_INTERP != 0
  24. #include "../libraries/debug-engine/debug_engine.h"
  25. #endif
  26. #if WASM_ENABLE_FAST_JIT != 0
  27. #include "../fast-jit/jit_compiler.h"
  28. #endif
  29. #if WASM_ENABLE_JIT != 0
  30. #include "../aot/aot_runtime.h"
  31. #endif
  32. static void
  33. set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
  34. {
  35. if (error_buf != NULL) {
  36. snprintf(error_buf, error_buf_size,
  37. "WASM module instantiate failed: %s", string);
  38. }
  39. }
  40. static void
  41. set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...)
  42. {
  43. va_list args;
  44. char buf[128];
  45. if (error_buf != NULL) {
  46. va_start(args, format);
  47. vsnprintf(buf, sizeof(buf), format, args);
  48. va_end(args);
  49. snprintf(error_buf, error_buf_size,
  50. "WASM module instantiate failed: %s", buf);
  51. }
  52. }
  53. WASMModule *
  54. wasm_load(uint8 *buf, uint32 size,
  55. #if WASM_ENABLE_MULTI_MODULE != 0
  56. bool main_module,
  57. #endif
  58. char *error_buf, uint32 error_buf_size)
  59. {
  60. return wasm_loader_load(buf, size,
  61. #if WASM_ENABLE_MULTI_MODULE != 0
  62. main_module,
  63. #endif
  64. error_buf, error_buf_size);
  65. }
  66. WASMModule *
  67. wasm_load_from_sections(WASMSection *section_list, char *error_buf,
  68. uint32 error_buf_size)
  69. {
  70. return wasm_loader_load_from_sections(section_list, error_buf,
  71. error_buf_size);
  72. }
  73. void
  74. wasm_unload(WASMModule *module)
  75. {
  76. wasm_loader_unload(module);
  77. }
  78. static void *
  79. runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
  80. {
  81. void *mem;
  82. if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
  83. set_error_buf(error_buf, error_buf_size, "allocate memory failed");
  84. return NULL;
  85. }
  86. memset(mem, 0, (uint32)size);
  87. return mem;
  88. }
  89. #if WASM_ENABLE_MULTI_MODULE != 0
  90. static WASMModuleInstance *
  91. get_sub_module_inst(const WASMModuleInstance *parent_module_inst,
  92. const WASMModule *sub_module)
  93. {
  94. bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list;
  95. WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list);
  96. while (node && sub_module != node->module_inst->module) {
  97. node = bh_list_elem_next(node);
  98. }
  99. return node ? node->module_inst : NULL;
  100. }
  101. #endif
  102. /**
  103. * Destroy memory instances.
  104. */
  105. static void
  106. memories_deinstantiate(WASMModuleInstance *module_inst,
  107. WASMMemoryInstance **memories, uint32 count)
  108. {
  109. uint32 i;
  110. if (memories) {
  111. for (i = 0; i < count; i++) {
  112. if (memories[i]) {
  113. #if WASM_ENABLE_MULTI_MODULE != 0
  114. WASMModule *module = module_inst->module;
  115. if (i < module->import_memory_count
  116. && module->import_memories[i].u.memory.import_module) {
  117. continue;
  118. }
  119. #endif
  120. #if WASM_ENABLE_SHARED_MEMORY != 0
  121. if (shared_memory_is_shared(memories[i])) {
  122. uint32 ref_count = shared_memory_dec_reference(memories[i]);
  123. /* if the reference count is not zero,
  124. don't free the memory */
  125. if (ref_count > 0)
  126. continue;
  127. }
  128. #endif
  129. if (memories[i]->heap_handle) {
  130. mem_allocator_destroy(memories[i]->heap_handle);
  131. wasm_runtime_free(memories[i]->heap_handle);
  132. memories[i]->heap_handle = NULL;
  133. }
  134. if (memories[i]->memory_data) {
  135. wasm_deallocate_linear_memory(memories[i]);
  136. }
  137. }
  138. }
  139. wasm_runtime_free(memories);
  140. }
  141. (void)module_inst;
  142. }
  143. static WASMMemoryInstance *
  144. memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
  145. WASMMemoryInstance *memory, uint32 memory_idx,
  146. uint32 num_bytes_per_page, uint32 init_page_count,
  147. uint32 max_page_count, uint32 heap_size, uint32 flags,
  148. char *error_buf, uint32 error_buf_size)
  149. {
  150. WASMModule *module = module_inst->module;
  151. uint64 memory_data_size, max_memory_data_size;
  152. uint32 heap_offset = num_bytes_per_page * init_page_count;
  153. uint32 inc_page_count, aux_heap_base, global_idx;
  154. uint32 bytes_of_last_page, bytes_to_page_end;
  155. uint8 *global_addr;
  156. bool is_shared_memory = false;
  157. #if WASM_ENABLE_SHARED_MEMORY != 0
  158. is_shared_memory = flags & 0x02 ? true : false;
  159. /* shared memory */
  160. if (is_shared_memory && parent != NULL) {
  161. bh_assert(parent->memory_count > memory_idx);
  162. memory = parent->memories[memory_idx];
  163. shared_memory_inc_reference(memory);
  164. return memory;
  165. }
  166. #else
  167. (void)parent;
  168. (void)memory_idx;
  169. (void)flags;
  170. #endif /* end of WASM_ENABLE_SHARED_MEMORY */
  171. if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1
  172. && module_inst->module->free_function != (uint32)-1) {
  173. /* Disable app heap, use malloc/free function exported
  174. by wasm app to allocate/free memory instead */
  175. heap_size = 0;
  176. }
  177. if (init_page_count == max_page_count && init_page_count == 1) {
  178. /* If only one page and at most one page, we just append
  179. the app heap to the end of linear memory, enlarge the
  180. num_bytes_per_page, and don't change the page count */
  181. heap_offset = num_bytes_per_page;
  182. num_bytes_per_page += heap_size;
  183. if (num_bytes_per_page < heap_size) {
  184. set_error_buf(error_buf, error_buf_size,
  185. "failed to insert app heap into linear memory, "
  186. "try using `--heap-size=0` option");
  187. return NULL;
  188. }
  189. }
  190. else if (heap_size > 0) {
  191. if (init_page_count == max_page_count && init_page_count == 0) {
  192. /* If the memory data size is always 0, we resize it to
  193. one page for app heap */
  194. num_bytes_per_page = heap_size;
  195. heap_offset = 0;
  196. inc_page_count = 1;
  197. }
  198. else if (module->aux_heap_base_global_index != (uint32)-1
  199. && module->aux_heap_base
  200. < num_bytes_per_page * init_page_count) {
  201. /* Insert app heap before __heap_base */
  202. aux_heap_base = module->aux_heap_base;
  203. bytes_of_last_page = aux_heap_base % num_bytes_per_page;
  204. if (bytes_of_last_page == 0)
  205. bytes_of_last_page = num_bytes_per_page;
  206. bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
  207. inc_page_count =
  208. (heap_size - bytes_to_page_end + num_bytes_per_page - 1)
  209. / num_bytes_per_page;
  210. heap_offset = aux_heap_base;
  211. aux_heap_base += heap_size;
  212. bytes_of_last_page = aux_heap_base % num_bytes_per_page;
  213. if (bytes_of_last_page == 0)
  214. bytes_of_last_page = num_bytes_per_page;
  215. bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
  216. if (bytes_to_page_end < 1 * BH_KB) {
  217. aux_heap_base += 1 * BH_KB;
  218. inc_page_count++;
  219. }
  220. /* Adjust __heap_base global value */
  221. global_idx = module->aux_heap_base_global_index;
  222. bh_assert(module_inst->e->globals
  223. && global_idx < module_inst->e->global_count);
  224. global_addr = module_inst->global_data
  225. + module_inst->e->globals[global_idx].data_offset;
  226. *(uint32 *)global_addr = aux_heap_base;
  227. LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base);
  228. }
  229. else {
  230. /* Insert app heap before new page */
  231. inc_page_count =
  232. (heap_size + num_bytes_per_page - 1) / num_bytes_per_page;
  233. heap_offset = num_bytes_per_page * init_page_count;
  234. heap_size = num_bytes_per_page * inc_page_count;
  235. if (heap_size > 0)
  236. heap_size -= 1 * BH_KB;
  237. }
  238. init_page_count += inc_page_count;
  239. max_page_count += inc_page_count;
  240. if (init_page_count > DEFAULT_MAX_PAGES) {
  241. set_error_buf(error_buf, error_buf_size,
  242. "failed to insert app heap into linear memory, "
  243. "try using `--heap-size=0` option");
  244. return NULL;
  245. }
  246. else if (init_page_count == DEFAULT_MAX_PAGES) {
  247. num_bytes_per_page = UINT32_MAX;
  248. init_page_count = max_page_count = 1;
  249. }
  250. if (max_page_count > DEFAULT_MAX_PAGES)
  251. max_page_count = DEFAULT_MAX_PAGES;
  252. }
  253. else { /* heap_size == 0 */
  254. if (init_page_count == DEFAULT_MAX_PAGES) {
  255. num_bytes_per_page = UINT32_MAX;
  256. init_page_count = max_page_count = 1;
  257. }
  258. }
  259. LOG_VERBOSE("Memory instantiate:");
  260. LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u",
  261. num_bytes_per_page, init_page_count, max_page_count);
  262. LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size);
  263. max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
  264. bh_assert(max_memory_data_size <= 4 * (uint64)BH_GB);
  265. (void)max_memory_data_size;
  266. bh_assert(memory != NULL);
  267. if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory,
  268. num_bytes_per_page, init_page_count,
  269. max_page_count, &memory_data_size)
  270. != BHT_OK) {
  271. set_error_buf(error_buf, error_buf_size,
  272. "allocate linear memory failed");
  273. return NULL;
  274. }
  275. memory->module_type = Wasm_Module_Bytecode;
  276. memory->num_bytes_per_page = num_bytes_per_page;
  277. memory->cur_page_count = init_page_count;
  278. memory->max_page_count = max_page_count;
  279. memory->memory_data_size = (uint32)memory_data_size;
  280. memory->heap_data = memory->memory_data + heap_offset;
  281. memory->heap_data_end = memory->heap_data + heap_size;
  282. memory->memory_data_end = memory->memory_data + (uint32)memory_data_size;
  283. /* Initialize heap */
  284. if (heap_size > 0) {
  285. uint32 heap_struct_size = mem_allocator_get_heap_struct_size();
  286. if (!(memory->heap_handle = runtime_malloc(
  287. (uint64)heap_struct_size, error_buf, error_buf_size))) {
  288. goto fail1;
  289. }
  290. if (!mem_allocator_create_with_struct_and_pool(
  291. memory->heap_handle, heap_struct_size, memory->heap_data,
  292. heap_size)) {
  293. set_error_buf(error_buf, error_buf_size, "init app heap failed");
  294. goto fail2;
  295. }
  296. }
  297. if (memory_data_size > 0) {
  298. wasm_runtime_set_mem_bound_check_bytes(memory, memory_data_size);
  299. }
  300. #if WASM_ENABLE_SHARED_MEMORY != 0
  301. if (is_shared_memory) {
  302. memory->is_shared_memory = 1;
  303. memory->ref_count = 1;
  304. }
  305. #endif
  306. LOG_VERBOSE("Memory instantiate success.");
  307. return memory;
  308. fail2:
  309. if (heap_size > 0)
  310. wasm_runtime_free(memory->heap_handle);
  311. fail1:
  312. if (memory->memory_data)
  313. wasm_deallocate_linear_memory(memory);
  314. return NULL;
  315. }
  316. /**
  317. * Instantiate memories in a module.
  318. */
  319. static WASMMemoryInstance **
  320. memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  321. WASMModuleInstance *parent, uint32 heap_size,
  322. char *error_buf, uint32 error_buf_size)
  323. {
  324. WASMImport *import;
  325. uint32 mem_index = 0, i,
  326. memory_count = module->import_memory_count + module->memory_count;
  327. uint64 total_size;
  328. WASMMemoryInstance **memories, *memory;
  329. total_size = sizeof(WASMMemoryInstance *) * (uint64)memory_count;
  330. if (!(memories = runtime_malloc(total_size, error_buf, error_buf_size))) {
  331. return NULL;
  332. }
  333. memory = module_inst->global_table_data.memory_instances;
  334. /* instantiate memories from import section */
  335. import = module->import_memories;
  336. for (i = 0; i < module->import_memory_count; i++, import++, memory++) {
  337. uint32 num_bytes_per_page = import->u.memory.num_bytes_per_page;
  338. uint32 init_page_count = import->u.memory.init_page_count;
  339. uint32 max_page_count = import->u.memory.max_page_count;
  340. uint32 flags = import->u.memory.flags;
  341. uint32 actual_heap_size = heap_size;
  342. #if WASM_ENABLE_MULTI_MODULE != 0
  343. if (import->u.memory.import_module != NULL) {
  344. WASMModuleInstance *module_inst_linked;
  345. if (!(module_inst_linked = get_sub_module_inst(
  346. module_inst, import->u.memory.import_module))) {
  347. set_error_buf(error_buf, error_buf_size, "unknown memory");
  348. memories_deinstantiate(module_inst, memories, memory_count);
  349. return NULL;
  350. }
  351. if (!(memories[mem_index++] = wasm_lookup_memory(
  352. module_inst_linked, import->u.memory.field_name))) {
  353. set_error_buf(error_buf, error_buf_size, "unknown memory");
  354. memories_deinstantiate(module_inst, memories, memory_count);
  355. return NULL;
  356. }
  357. }
  358. else
  359. #endif
  360. {
  361. if (!(memories[mem_index] = memory_instantiate(
  362. module_inst, parent, memory, mem_index,
  363. num_bytes_per_page, init_page_count, max_page_count,
  364. actual_heap_size, flags, error_buf, error_buf_size))) {
  365. memories_deinstantiate(module_inst, memories, memory_count);
  366. return NULL;
  367. }
  368. mem_index++;
  369. }
  370. }
  371. /* instantiate memories from memory section */
  372. for (i = 0; i < module->memory_count; i++, memory++) {
  373. if (!(memories[mem_index] = memory_instantiate(
  374. module_inst, parent, memory, mem_index,
  375. module->memories[i].num_bytes_per_page,
  376. module->memories[i].init_page_count,
  377. module->memories[i].max_page_count, heap_size,
  378. module->memories[i].flags, error_buf, error_buf_size))) {
  379. memories_deinstantiate(module_inst, memories, memory_count);
  380. return NULL;
  381. }
  382. mem_index++;
  383. }
  384. bh_assert(mem_index == memory_count);
  385. (void)module_inst;
  386. return memories;
  387. }
  388. /**
  389. * Destroy table instances.
  390. */
  391. static void
  392. tables_deinstantiate(WASMModuleInstance *module_inst)
  393. {
  394. if (module_inst->tables) {
  395. wasm_runtime_free(module_inst->tables);
  396. }
  397. #if WASM_ENABLE_MULTI_MODULE != 0
  398. if (module_inst->e->table_insts_linked) {
  399. wasm_runtime_free(module_inst->e->table_insts_linked);
  400. }
  401. #endif
  402. }
  403. /**
  404. * Instantiate tables in a module.
  405. */
  406. static WASMTableInstance **
  407. tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  408. WASMTableInstance *first_table, char *error_buf,
  409. uint32 error_buf_size)
  410. {
  411. WASMImport *import;
  412. uint32 table_index = 0, i;
  413. uint32 table_count = module->import_table_count + module->table_count;
  414. uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count;
  415. WASMTableInstance **tables, *table = first_table;
  416. #if WASM_ENABLE_MULTI_MODULE != 0
  417. uint64 total_size_of_tables_linked =
  418. (uint64)sizeof(WASMTableInstance *) * module->import_table_count;
  419. WASMTableInstance **table_linked = NULL;
  420. #endif
  421. if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) {
  422. return NULL;
  423. }
  424. #if WASM_ENABLE_MULTI_MODULE != 0
  425. if (module->import_table_count > 0
  426. && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc(
  427. total_size_of_tables_linked, error_buf, error_buf_size))) {
  428. goto fail;
  429. }
  430. #endif
  431. /* instantiate tables from import section */
  432. import = module->import_tables;
  433. for (i = 0; i < module->import_table_count; i++, import++) {
  434. uint32 max_size_fixed = 0;
  435. #if WASM_ENABLE_MULTI_MODULE != 0
  436. WASMTableInstance *table_inst_linked = NULL;
  437. WASMModuleInstance *module_inst_linked = NULL;
  438. if (import->u.table.import_module) {
  439. if (!(module_inst_linked = get_sub_module_inst(
  440. module_inst, import->u.table.import_module))) {
  441. set_error_buf(error_buf, error_buf_size, "unknown table");
  442. goto fail;
  443. }
  444. if (!(table_inst_linked = wasm_lookup_table(
  445. module_inst_linked, import->u.table.field_name))) {
  446. set_error_buf(error_buf, error_buf_size, "unknown table");
  447. goto fail;
  448. }
  449. total_size = offsetof(WASMTableInstance, elems);
  450. }
  451. else
  452. #endif
  453. {
  454. /* in order to save memory, alloc resource as few as possible */
  455. max_size_fixed = import->u.table.possible_grow
  456. ? import->u.table.max_size
  457. : import->u.table.init_size;
  458. /* it is a built-in table, every module has its own */
  459. total_size = offsetof(WASMTableInstance, elems);
  460. /* store function indexes for non-gc, object pointers for gc */
  461. total_size += (uint64)sizeof(table_elem_type_t) * max_size_fixed;
  462. }
  463. tables[table_index++] = table;
  464. #if WASM_ENABLE_GC == 0
  465. /* Set all elements to -1 to mark them as uninitialized elements */
  466. memset(table, -1, (uint32)total_size);
  467. #else
  468. /* For GC, all elements have already been set to NULL_REF (0) as
  469. uninitialized elements */
  470. #endif
  471. #if WASM_ENABLE_MULTI_MODULE != 0
  472. *table_linked = table_inst_linked;
  473. if (table_inst_linked != NULL) {
  474. #if WASM_ENABLE_GC != 0
  475. table->elem_type = table_inst_linked->elem_type;
  476. table->elem_ref_type = table_inst_linked->elem_ref_type;
  477. #endif
  478. table->cur_size = table_inst_linked->cur_size;
  479. table->max_size = table_inst_linked->max_size;
  480. }
  481. else
  482. #endif
  483. {
  484. #if WASM_ENABLE_GC != 0
  485. table->elem_type = import->u.table.elem_type;
  486. table->elem_ref_type.elem_ref_type = import->u.table.elem_ref_type;
  487. #endif
  488. table->cur_size = import->u.table.init_size;
  489. table->max_size = max_size_fixed;
  490. }
  491. table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size);
  492. #if WASM_ENABLE_MULTI_MODULE != 0
  493. table_linked++;
  494. #endif
  495. }
  496. /* instantiate tables from table section */
  497. for (i = 0; i < module->table_count; i++) {
  498. uint32 max_size_fixed = 0;
  499. total_size = offsetof(WASMTableInstance, elems);
  500. #if WASM_ENABLE_MULTI_MODULE != 0
  501. /* in case, a module which imports this table will grow it */
  502. max_size_fixed = module->tables[i].max_size;
  503. #else
  504. max_size_fixed = module->tables[i].possible_grow
  505. ? module->tables[i].max_size
  506. : module->tables[i].init_size;
  507. #endif
  508. #if WASM_ENABLE_GC == 0
  509. /* Store function indexes */
  510. total_size += sizeof(uintptr_t) * (uint64)max_size_fixed;
  511. #else
  512. /* Store object pointers */
  513. total_size += sizeof(uintptr_t) * (uint64)max_size_fixed;
  514. #endif
  515. tables[table_index++] = table;
  516. #if WASM_ENABLE_GC == 0
  517. /* Set all elements to -1 to mark them as uninitialized elements */
  518. memset(table, -1, (uint32)total_size);
  519. #else
  520. /* For GC, all elements have already been set to NULL_REF (0) as
  521. uninitialized elements */
  522. #endif
  523. #if WASM_ENABLE_GC != 0
  524. table->elem_type = module->tables[i].elem_type;
  525. table->elem_ref_type.elem_ref_type = module->tables[i].elem_ref_type;
  526. #endif
  527. table->cur_size = module->tables[i].init_size;
  528. table->max_size = max_size_fixed;
  529. table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size);
  530. }
  531. bh_assert(table_index == table_count);
  532. (void)module_inst;
  533. return tables;
  534. #if WASM_ENABLE_MULTI_MODULE != 0
  535. fail:
  536. wasm_runtime_free(tables);
  537. return NULL;
  538. #endif
  539. }
  540. /**
  541. * Destroy function instances.
  542. */
  543. static void
  544. functions_deinstantiate(WASMFunctionInstance *functions)
  545. {
  546. if (functions) {
  547. wasm_runtime_free(functions);
  548. }
  549. }
  550. /**
  551. * Instantiate functions in a module.
  552. */
  553. static WASMFunctionInstance *
  554. functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  555. char *error_buf, uint32 error_buf_size)
  556. {
  557. WASMImport *import;
  558. uint32 i,
  559. function_count = module->import_function_count + module->function_count;
  560. uint64 total_size = sizeof(WASMFunctionInstance) * (uint64)function_count;
  561. WASMFunctionInstance *functions, *function;
  562. if (!(functions = runtime_malloc(total_size, error_buf, error_buf_size))) {
  563. return NULL;
  564. }
  565. total_size = sizeof(void *) * (uint64)module->import_function_count;
  566. if (total_size > 0
  567. && !(module_inst->import_func_ptrs =
  568. runtime_malloc(total_size, error_buf, error_buf_size))) {
  569. wasm_runtime_free(functions);
  570. return NULL;
  571. }
  572. /* instantiate functions from import section */
  573. function = functions;
  574. import = module->import_functions;
  575. for (i = 0; i < module->import_function_count; i++, import++) {
  576. function->is_import_func = true;
  577. #if WASM_ENABLE_MULTI_MODULE != 0
  578. if (import->u.function.import_module) {
  579. function->import_module_inst = get_sub_module_inst(
  580. module_inst, import->u.function.import_module);
  581. if (function->import_module_inst) {
  582. function->import_func_inst =
  583. wasm_lookup_function(function->import_module_inst,
  584. import->u.function.field_name, NULL);
  585. }
  586. }
  587. #endif /* WASM_ENABLE_MULTI_MODULE */
  588. function->u.func_import = &import->u.function;
  589. function->param_cell_num = import->u.function.func_type->param_cell_num;
  590. function->ret_cell_num = import->u.function.func_type->ret_cell_num;
  591. function->param_count =
  592. (uint16)function->u.func_import->func_type->param_count;
  593. function->param_types = function->u.func_import->func_type->types;
  594. function->local_cell_num = 0;
  595. function->local_count = 0;
  596. function->local_types = NULL;
  597. /* Copy the function pointer to current instance */
  598. module_inst->import_func_ptrs[i] =
  599. function->u.func_import->func_ptr_linked;
  600. function++;
  601. }
  602. /* instantiate functions from function section */
  603. for (i = 0; i < module->function_count; i++) {
  604. function->is_import_func = false;
  605. function->u.func = module->functions[i];
  606. function->param_cell_num = function->u.func->param_cell_num;
  607. function->ret_cell_num = function->u.func->ret_cell_num;
  608. function->local_cell_num = function->u.func->local_cell_num;
  609. function->param_count =
  610. (uint16)function->u.func->func_type->param_count;
  611. function->local_count = (uint16)function->u.func->local_count;
  612. function->param_types = function->u.func->func_type->types;
  613. function->local_types = function->u.func->local_types;
  614. function->local_offsets = function->u.func->local_offsets;
  615. #if WASM_ENABLE_FAST_INTERP != 0
  616. function->const_cell_num = function->u.func->const_cell_num;
  617. #endif
  618. function++;
  619. }
  620. bh_assert((uint32)(function - functions) == function_count);
  621. #if WASM_ENABLE_FAST_JIT != 0
  622. module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
  623. #endif
  624. return functions;
  625. }
  626. #if WASM_ENABLE_TAGS != 0
  627. /**
  628. * Destroy tags instances.
  629. */
  630. static void
  631. tags_deinstantiate(WASMTagInstance *tags, void **import_tag_ptrs)
  632. {
  633. if (tags) {
  634. wasm_runtime_free(tags);
  635. }
  636. if (import_tag_ptrs) {
  637. wasm_runtime_free(import_tag_ptrs);
  638. }
  639. }
  640. /**
  641. * Instantiate tags in a module.
  642. */
  643. static WASMTagInstance *
  644. tags_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  645. char *error_buf, uint32 error_buf_size)
  646. {
  647. WASMImport *import;
  648. uint32 i, tag_count = module->import_tag_count + module->tag_count;
  649. uint64 total_size = sizeof(WASMTagInstance) * (uint64)tag_count;
  650. WASMTagInstance *tags, *tag;
  651. if (!(tags = runtime_malloc(total_size, error_buf, error_buf_size))) {
  652. return NULL;
  653. }
  654. total_size = sizeof(void *) * (uint64)module->import_tag_count;
  655. if (total_size > 0
  656. && !(module_inst->e->import_tag_ptrs =
  657. runtime_malloc(total_size, error_buf, error_buf_size))) {
  658. wasm_runtime_free(tags);
  659. return NULL;
  660. }
  661. /* instantiate tags from import section */
  662. tag = tags;
  663. import = module->import_tags;
  664. for (i = 0; i < module->import_tag_count; i++, import++) {
  665. tag->is_import_tag = true;
  666. tag->u.tag_import = &import->u.tag;
  667. tag->type = import->u.tag.type;
  668. tag->attribute = import->u.tag.attribute;
  669. #if WASM_ENABLE_MULTI_MODULE != 0
  670. if (import->u.tag.import_module) {
  671. if (!(tag->import_module_inst = get_sub_module_inst(
  672. module_inst, import->u.tag.import_module))) {
  673. set_error_buf(error_buf, error_buf_size, "unknown tag");
  674. goto fail;
  675. }
  676. if (!(tag->import_tag_inst =
  677. wasm_lookup_tag(tag->import_module_inst,
  678. import->u.tag.field_name, NULL))) {
  679. set_error_buf(error_buf, error_buf_size, "unknown tag");
  680. goto fail;
  681. }
  682. /* Copy the imported tag to current instance */
  683. module_inst->e->import_tag_ptrs[i] =
  684. tag->u.tag_import->import_tag_linked;
  685. }
  686. #endif
  687. tag++;
  688. }
  689. /* instantiate tags from tag section */
  690. for (i = 0; i < module->tag_count; i++) {
  691. tag->is_import_tag = false;
  692. tag->type = module->tags[i]->type;
  693. tag->u.tag = module->tags[i];
  694. #if WASM_ENABLE_FAST_INTERP != 0
  695. /* tag->const_cell_num = function->u.func->const_cell_num; */
  696. #endif
  697. tag++;
  698. }
  699. bh_assert((uint32)(tag - tags) == tag_count);
  700. return tags;
  701. #if WASM_ENABLE_MULTI_MODULE != 0
  702. fail:
  703. tags_deinstantiate(tags, module_inst->e->import_tag_ptrs);
  704. /* clean up */
  705. module_inst->e->import_tag_ptrs = NULL;
  706. return NULL;
  707. #endif
  708. }
  709. #endif /* end of WASM_ENABLE_TAGS != 0 */
  710. /**
  711. * Destroy global instances.
  712. */
  713. static void
  714. globals_deinstantiate(WASMGlobalInstance *globals)
  715. {
  716. if (globals)
  717. wasm_runtime_free(globals);
  718. }
  719. static bool
  720. check_global_init_expr(const WASMModule *module, uint32 global_index,
  721. char *error_buf, uint32 error_buf_size)
  722. {
  723. if (global_index >= module->import_global_count + module->global_count) {
  724. set_error_buf_v(error_buf, error_buf_size, "unknown global %d",
  725. global_index);
  726. return false;
  727. }
  728. #if WASM_ENABLE_GC == 0
  729. /**
  730. * Currently, constant expressions occurring as initializers of
  731. * globals are further constrained in that contained global.get
  732. * instructions are only allowed to refer to imported globals.
  733. *
  734. * And initializer expression cannot reference a mutable global.
  735. */
  736. if (global_index >= module->import_global_count
  737. || (module->import_globals + global_index)->u.global.is_mutable) {
  738. set_error_buf(error_buf, error_buf_size,
  739. "constant expression required");
  740. return false;
  741. }
  742. #endif
  743. return true;
  744. }
  745. /**
  746. * Instantiate globals in a module.
  747. */
  748. static WASMGlobalInstance *
  749. globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst,
  750. char *error_buf, uint32 error_buf_size)
  751. {
  752. WASMImport *import;
  753. uint32 global_data_offset = 0;
  754. uint32 i, global_count = module->import_global_count + module->global_count;
  755. uint64 total_size = sizeof(WASMGlobalInstance) * (uint64)global_count;
  756. WASMGlobalInstance *globals, *global;
  757. if (!(globals = runtime_malloc(total_size, error_buf, error_buf_size))) {
  758. return NULL;
  759. }
  760. /* instantiate globals from import section */
  761. global = globals;
  762. import = module->import_globals;
  763. for (i = 0; i < module->import_global_count; i++, import++) {
  764. WASMGlobalImport *global_import = &import->u.global;
  765. global->type = global_import->type;
  766. global->is_mutable = global_import->is_mutable;
  767. #if WASM_ENABLE_GC != 0
  768. global->ref_type = global_import->ref_type;
  769. #endif
  770. #if WASM_ENABLE_MULTI_MODULE != 0
  771. if (global_import->import_module) {
  772. if (!(global->import_module_inst = get_sub_module_inst(
  773. module_inst, global_import->import_module))) {
  774. set_error_buf(error_buf, error_buf_size, "unknown global");
  775. goto fail;
  776. }
  777. if (!(global->import_global_inst = wasm_lookup_global(
  778. global->import_module_inst, global_import->field_name))) {
  779. set_error_buf(error_buf, error_buf_size, "unknown global");
  780. goto fail;
  781. }
  782. /* The linked global instance has been initialized, we
  783. just need to copy the value. */
  784. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  785. &(global_import->import_global_linked->init_expr),
  786. sizeof(WASMValue));
  787. }
  788. else
  789. #endif
  790. {
  791. /* native globals share their initial_values in one module */
  792. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  793. &(global_import->global_data_linked),
  794. sizeof(WASMValue));
  795. }
  796. #if WASM_ENABLE_FAST_JIT != 0
  797. bh_assert(global_data_offset == global_import->data_offset);
  798. #endif
  799. global->data_offset = global_data_offset;
  800. global_data_offset += wasm_value_type_size(global->type);
  801. global++;
  802. }
  803. /* instantiate globals from global section */
  804. for (i = 0; i < module->global_count; i++) {
  805. InitializerExpression *init_expr = &(module->globals[i].init_expr);
  806. uint8 flag = init_expr->init_expr_type;
  807. global->type = module->globals[i].type;
  808. global->is_mutable = module->globals[i].is_mutable;
  809. #if WASM_ENABLE_FAST_JIT != 0
  810. bh_assert(global_data_offset == module->globals[i].data_offset);
  811. #endif
  812. global->data_offset = global_data_offset;
  813. global_data_offset += wasm_value_type_size(global->type);
  814. #if WASM_ENABLE_GC != 0
  815. global->ref_type = module->globals[i].ref_type;
  816. #endif
  817. switch (flag) {
  818. case INIT_EXPR_TYPE_GET_GLOBAL:
  819. {
  820. if (!check_global_init_expr(module, init_expr->u.global_index,
  821. error_buf, error_buf_size)) {
  822. goto fail;
  823. }
  824. bh_memcpy_s(
  825. &(global->initial_value), sizeof(WASMValue),
  826. &(globals[init_expr->u.global_index].initial_value),
  827. sizeof(globals[init_expr->u.global_index].initial_value));
  828. break;
  829. }
  830. #if WASM_ENABLE_GC != 0
  831. case INIT_EXPR_TYPE_STRUCT_NEW:
  832. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  833. {
  834. WASMRttType *rtt_type;
  835. WASMStructObjectRef struct_obj;
  836. WASMStructType *struct_type;
  837. WASMStructNewInitValues *init_values = NULL;
  838. uint32 type_idx;
  839. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  840. init_values = (WASMStructNewInitValues *)init_expr->u.data;
  841. type_idx = init_values->type_idx;
  842. }
  843. else {
  844. type_idx = init_expr->u.type_index;
  845. }
  846. struct_type = (WASMStructType *)module->types[type_idx];
  847. if (!(rtt_type = wasm_rtt_type_new(
  848. (WASMType *)struct_type, type_idx, module->rtt_types,
  849. module->type_count, &module->rtt_type_lock))) {
  850. set_error_buf(error_buf, error_buf_size,
  851. "create rtt object failed");
  852. goto fail;
  853. }
  854. if (!(struct_obj = wasm_struct_obj_new_internal(
  855. module_inst->e->common.gc_heap_handle, rtt_type))) {
  856. set_error_buf(error_buf, error_buf_size,
  857. "create struct object failed");
  858. goto fail;
  859. }
  860. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  861. uint32 field_idx;
  862. bh_assert(init_values->count == struct_type->field_count);
  863. for (field_idx = 0; field_idx < init_values->count;
  864. field_idx++) {
  865. wasm_struct_obj_set_field(
  866. struct_obj, field_idx,
  867. &init_values->fields[field_idx]);
  868. }
  869. }
  870. global->initial_value.gc_obj = (void *)struct_obj;
  871. break;
  872. }
  873. case INIT_EXPR_TYPE_ARRAY_NEW:
  874. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  875. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  876. {
  877. WASMRttType *rtt_type;
  878. WASMArrayObjectRef array_obj;
  879. WASMArrayType *array_type;
  880. WASMArrayNewInitValues *init_values = NULL;
  881. WASMValue *arr_init_val = NULL, empty_val = { 0 };
  882. uint32 type_idx, len;
  883. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) {
  884. type_idx = init_expr->u.array_new_default.type_index;
  885. len = init_expr->u.array_new_default.length;
  886. arr_init_val = &empty_val;
  887. }
  888. else {
  889. init_values = (WASMArrayNewInitValues *)init_expr->u.data;
  890. type_idx = init_values->type_idx;
  891. len = init_values->length;
  892. if (flag == INIT_EXPR_TYPE_ARRAY_NEW) {
  893. arr_init_val = init_values->elem_data;
  894. }
  895. }
  896. array_type = (WASMArrayType *)module->types[type_idx];
  897. if (!(rtt_type = wasm_rtt_type_new(
  898. (WASMType *)array_type, type_idx, module->rtt_types,
  899. module->type_count, &module->rtt_type_lock))) {
  900. set_error_buf(error_buf, error_buf_size,
  901. "create rtt object failed");
  902. goto fail;
  903. }
  904. if (!(array_obj = wasm_array_obj_new_internal(
  905. module_inst->e->common.gc_heap_handle, rtt_type, len,
  906. arr_init_val))) {
  907. set_error_buf(error_buf, error_buf_size,
  908. "create array object failed");
  909. goto fail;
  910. }
  911. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  912. uint32 elem_idx;
  913. bh_assert(init_values);
  914. for (elem_idx = 0; elem_idx < len; elem_idx++) {
  915. wasm_array_obj_set_elem(
  916. array_obj, elem_idx,
  917. &init_values->elem_data[elem_idx]);
  918. }
  919. }
  920. global->initial_value.gc_obj = (void *)array_obj;
  921. break;
  922. }
  923. case INIT_EXPR_TYPE_I31_NEW:
  924. {
  925. global->initial_value.gc_obj =
  926. (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32);
  927. break;
  928. }
  929. #endif /* end of WASM_ENABLE_GC != 0 */
  930. default:
  931. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  932. &(init_expr->u), sizeof(init_expr->u));
  933. break;
  934. }
  935. global++;
  936. }
  937. bh_assert((uint32)(global - globals) == global_count);
  938. bh_assert(global_data_offset == module->global_data_size);
  939. (void)module_inst;
  940. return globals;
  941. fail:
  942. wasm_runtime_free(globals);
  943. return NULL;
  944. }
  945. /**
  946. * Return export function count in module export section.
  947. */
  948. static uint32
  949. get_export_count(const WASMModule *module, uint8 kind)
  950. {
  951. WASMExport *export = module->exports;
  952. uint32 count = 0, i;
  953. for (i = 0; i < module->export_count; i++, export ++)
  954. if (export->kind == kind)
  955. count++;
  956. return count;
  957. }
  958. /**
  959. * Destroy export function instances.
  960. */
  961. static void
  962. export_functions_deinstantiate(WASMExportFuncInstance *functions)
  963. {
  964. if (functions)
  965. wasm_runtime_free(functions);
  966. }
  967. /**
  968. * Instantiate export functions in a module.
  969. */
  970. static WASMExportFuncInstance *
  971. export_functions_instantiate(const WASMModule *module,
  972. WASMModuleInstance *module_inst,
  973. uint32 export_func_count, char *error_buf,
  974. uint32 error_buf_size)
  975. {
  976. WASMExportFuncInstance *export_funcs, *export_func;
  977. WASMExport *export = module->exports;
  978. uint32 i;
  979. uint64 total_size =
  980. sizeof(WASMExportFuncInstance) * (uint64)export_func_count;
  981. if (!(export_func = export_funcs =
  982. runtime_malloc(total_size, error_buf, error_buf_size))) {
  983. return NULL;
  984. }
  985. for (i = 0; i < module->export_count; i++, export ++)
  986. if (export->kind == EXPORT_KIND_FUNC) {
  987. export_func->name = export->name;
  988. export_func->function = &module_inst->e->functions[export->index];
  989. export_func++;
  990. }
  991. bh_assert((uint32)(export_func - export_funcs) == export_func_count);
  992. return export_funcs;
  993. }
  994. #if WASM_ENABLE_TAGS != 0
  995. /**
  996. * Destroy export function instances.
  997. */
  998. static void
  999. export_tags_deinstantiate(WASMExportTagInstance *tags)
  1000. {
  1001. if (tags)
  1002. wasm_runtime_free(tags);
  1003. }
  1004. /**
  1005. * Instantiate export functions in a module.
  1006. */
  1007. static WASMExportTagInstance *
  1008. export_tags_instantiate(const WASMModule *module,
  1009. WASMModuleInstance *module_inst,
  1010. uint32 export_tag_count, char *error_buf,
  1011. uint32 error_buf_size)
  1012. {
  1013. WASMExportTagInstance *export_tags, *export_tag;
  1014. WASMExport *export = module->exports;
  1015. uint32 i;
  1016. uint64 total_size =
  1017. sizeof(WASMExportTagInstance) * (uint64)export_tag_count;
  1018. if (!(export_tag = export_tags =
  1019. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1020. return NULL;
  1021. }
  1022. for (i = 0; i < module->export_count; i++, export ++)
  1023. if (export->kind == EXPORT_KIND_TAG) {
  1024. export_tag->name = export->name;
  1025. bh_assert(module_inst->e->tags);
  1026. export_tag->tag = &module_inst->e->tags[export->index];
  1027. export_tag++;
  1028. }
  1029. bh_assert((uint32)(export_tag - export_tags) == export_tag_count);
  1030. return export_tags;
  1031. }
  1032. #endif /* end of WASM_ENABLE_TAGS != 0 */
  1033. #if WASM_ENABLE_MULTI_MODULE != 0
  1034. static void
  1035. export_globals_deinstantiate(WASMExportGlobInstance *globals)
  1036. {
  1037. if (globals)
  1038. wasm_runtime_free(globals);
  1039. }
  1040. static WASMExportGlobInstance *
  1041. export_globals_instantiate(const WASMModule *module,
  1042. WASMModuleInstance *module_inst,
  1043. uint32 export_glob_count, char *error_buf,
  1044. uint32 error_buf_size)
  1045. {
  1046. WASMExportGlobInstance *export_globals, *export_global;
  1047. WASMExport *export = module->exports;
  1048. uint32 i;
  1049. uint64 total_size =
  1050. sizeof(WASMExportGlobInstance) * (uint64)export_glob_count;
  1051. if (!(export_global = export_globals =
  1052. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1053. return NULL;
  1054. }
  1055. for (i = 0; i < module->export_count; i++, export ++)
  1056. if (export->kind == EXPORT_KIND_GLOBAL) {
  1057. export_global->name = export->name;
  1058. export_global->global = &module_inst->e->globals[export->index];
  1059. export_global++;
  1060. }
  1061. bh_assert((uint32)(export_global - export_globals) == export_glob_count);
  1062. return export_globals;
  1063. }
  1064. #endif
  1065. static WASMFunctionInstance *
  1066. lookup_post_instantiate_func(WASMModuleInstance *module_inst,
  1067. const char *func_name)
  1068. {
  1069. WASMFunctionInstance *func;
  1070. WASMFuncType *func_type;
  1071. if (!(func = wasm_lookup_function(module_inst, func_name, NULL)))
  1072. /* Not found */
  1073. return NULL;
  1074. func_type = func->u.func->func_type;
  1075. if (!(func_type->param_count == 0 && func_type->result_count == 0))
  1076. /* Not a valid function type, ignore it */
  1077. return NULL;
  1078. return func;
  1079. }
  1080. static bool
  1081. execute_post_instantiate_functions(WASMModuleInstance *module_inst,
  1082. bool is_sub_inst, WASMExecEnv *exec_env_main)
  1083. {
  1084. WASMFunctionInstance *start_func = module_inst->e->start_function;
  1085. WASMFunctionInstance *initialize_func = NULL;
  1086. WASMFunctionInstance *post_inst_func = NULL;
  1087. WASMFunctionInstance *call_ctors_func = NULL;
  1088. #if WASM_ENABLE_LIBC_WASI != 0
  1089. WASMModule *module = module_inst->module;
  1090. #endif
  1091. WASMModuleInstanceCommon *module_inst_main = NULL;
  1092. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1093. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1094. #endif
  1095. WASMExecEnv *exec_env = NULL, *exec_env_created = NULL;
  1096. bool ret = false;
  1097. #if WASM_ENABLE_LIBC_WASI != 0
  1098. /*
  1099. * WASI reactor instances may assume that _initialize will be called by
  1100. * the environment at most once, and that none of their other exports
  1101. * are accessed before that call.
  1102. */
  1103. if (!is_sub_inst && module->import_wasi_api) {
  1104. initialize_func =
  1105. lookup_post_instantiate_func(module_inst, "_initialize");
  1106. }
  1107. #endif
  1108. /* Execute possible "__post_instantiate" function if wasm app is
  1109. compiled by emsdk's early version */
  1110. if (!is_sub_inst) {
  1111. post_inst_func =
  1112. lookup_post_instantiate_func(module_inst, "__post_instantiate");
  1113. }
  1114. #if WASM_ENABLE_BULK_MEMORY != 0
  1115. /* Only execute the memory init function for main instance since
  1116. the data segments will be dropped once initialized */
  1117. if (!is_sub_inst
  1118. #if WASM_ENABLE_LIBC_WASI != 0
  1119. && !module->import_wasi_api
  1120. #endif
  1121. ) {
  1122. call_ctors_func =
  1123. lookup_post_instantiate_func(module_inst, "__wasm_call_ctors");
  1124. }
  1125. #endif
  1126. if (!start_func && !initialize_func && !post_inst_func
  1127. && !call_ctors_func) {
  1128. /* No post instantiation functions to call */
  1129. return true;
  1130. }
  1131. if (is_sub_inst) {
  1132. bh_assert(exec_env_main);
  1133. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1134. bh_assert(exec_env_tls == exec_env_main);
  1135. (void)exec_env_tls;
  1136. #endif
  1137. exec_env = exec_env_main;
  1138. /* Temporarily replace parent exec_env's module inst to current
  1139. module inst to avoid checking failure when calling the
  1140. wasm functions, and ensure that the exec_env's module inst
  1141. is the correct one. */
  1142. module_inst_main = exec_env_main->module_inst;
  1143. wasm_exec_env_set_module_inst(exec_env,
  1144. (WASMModuleInstanceCommon *)module_inst);
  1145. }
  1146. else {
  1147. /* Try using the existing exec_env */
  1148. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1149. exec_env = exec_env_tls;
  1150. #endif
  1151. #if WASM_ENABLE_THREAD_MGR != 0
  1152. if (!exec_env)
  1153. exec_env = wasm_clusters_search_exec_env(
  1154. (WASMModuleInstanceCommon *)module_inst);
  1155. #endif
  1156. if (!exec_env) {
  1157. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1158. (WASMModuleInstanceCommon *)module_inst,
  1159. module_inst->default_wasm_stack_size))) {
  1160. wasm_set_exception(module_inst, "allocate memory failed");
  1161. return false;
  1162. }
  1163. }
  1164. else {
  1165. /* Temporarily replace exec_env's module inst with current
  1166. module inst to ensure that the exec_env's module inst
  1167. is the correct one. */
  1168. module_inst_main = exec_env->module_inst;
  1169. wasm_exec_env_set_module_inst(
  1170. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1171. }
  1172. }
  1173. /* Execute start function for both main insance and sub instance */
  1174. if (start_func && !wasm_call_function(exec_env, start_func, 0, NULL)) {
  1175. goto fail;
  1176. }
  1177. #if WASM_ENABLE_LIBC_WASI != 0
  1178. if (initialize_func
  1179. && !wasm_call_function(exec_env, initialize_func, 0, NULL)) {
  1180. goto fail;
  1181. }
  1182. #else
  1183. (void)initialize_func;
  1184. #endif
  1185. if (post_inst_func
  1186. && !wasm_call_function(exec_env, post_inst_func, 0, NULL)) {
  1187. goto fail;
  1188. }
  1189. if (call_ctors_func
  1190. && !wasm_call_function(exec_env, call_ctors_func, 0, NULL)) {
  1191. goto fail;
  1192. }
  1193. ret = true;
  1194. fail:
  1195. if (is_sub_inst) {
  1196. /* Restore the parent exec_env's module inst */
  1197. wasm_exec_env_restore_module_inst(exec_env_main, module_inst_main);
  1198. }
  1199. else {
  1200. if (module_inst_main)
  1201. /* Restore the existing exec_env's module inst */
  1202. wasm_exec_env_restore_module_inst(exec_env, module_inst_main);
  1203. if (exec_env_created)
  1204. wasm_exec_env_destroy(exec_env_created);
  1205. }
  1206. return ret;
  1207. }
  1208. static bool
  1209. execute_malloc_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
  1210. WASMFunctionInstance *malloc_func,
  1211. WASMFunctionInstance *retain_func, uint32 size,
  1212. uint32 *p_result)
  1213. {
  1214. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1215. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1216. #endif
  1217. WASMExecEnv *exec_env_created = NULL;
  1218. WASMModuleInstanceCommon *module_inst_old = NULL;
  1219. uint32 argv[2], argc;
  1220. bool ret;
  1221. argv[0] = size;
  1222. argc = 1;
  1223. /* if __retain is exported, then this module is compiled by
  1224. assemblyscript, the memory should be managed by as's runtime,
  1225. in this case we need to call the retain function after malloc
  1226. the memory */
  1227. if (retain_func) {
  1228. /* the malloc functino from assemblyscript is:
  1229. function __new(size: usize, id: u32)
  1230. id = 0 means this is an ArrayBuffer object */
  1231. argv[1] = 0;
  1232. argc = 2;
  1233. }
  1234. if (exec_env) {
  1235. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1236. if (exec_env_tls) {
  1237. bh_assert(exec_env_tls == exec_env);
  1238. }
  1239. #endif
  1240. bh_assert(exec_env->module_inst
  1241. == (WASMModuleInstanceCommon *)module_inst);
  1242. }
  1243. else {
  1244. /* Try using the existing exec_env */
  1245. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1246. exec_env = exec_env_tls;
  1247. #endif
  1248. #if WASM_ENABLE_THREAD_MGR != 0
  1249. if (!exec_env)
  1250. exec_env = wasm_clusters_search_exec_env(
  1251. (WASMModuleInstanceCommon *)module_inst);
  1252. #endif
  1253. if (!exec_env) {
  1254. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1255. (WASMModuleInstanceCommon *)module_inst,
  1256. module_inst->default_wasm_stack_size))) {
  1257. wasm_set_exception(module_inst, "allocate memory failed");
  1258. return false;
  1259. }
  1260. }
  1261. else {
  1262. /* Temporarily replace exec_env's module inst with current
  1263. module inst to ensure that the exec_env's module inst
  1264. is the correct one. */
  1265. module_inst_old = exec_env->module_inst;
  1266. wasm_exec_env_set_module_inst(
  1267. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1268. }
  1269. }
  1270. ret = wasm_call_function(exec_env, malloc_func, argc, argv);
  1271. if (retain_func && ret)
  1272. ret = wasm_call_function(exec_env, retain_func, 1, argv);
  1273. if (module_inst_old)
  1274. /* Restore the existing exec_env's module inst */
  1275. wasm_exec_env_restore_module_inst(exec_env, module_inst_old);
  1276. if (exec_env_created)
  1277. wasm_exec_env_destroy(exec_env_created);
  1278. if (ret)
  1279. *p_result = argv[0];
  1280. return ret;
  1281. }
  1282. static bool
  1283. execute_free_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
  1284. WASMFunctionInstance *free_func, uint32 offset)
  1285. {
  1286. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1287. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1288. #endif
  1289. WASMExecEnv *exec_env_created = NULL;
  1290. WASMModuleInstanceCommon *module_inst_old = NULL;
  1291. uint32 argv[2];
  1292. bool ret;
  1293. argv[0] = offset;
  1294. if (exec_env) {
  1295. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1296. if (exec_env_tls) {
  1297. bh_assert(exec_env_tls == exec_env);
  1298. }
  1299. #endif
  1300. bh_assert(exec_env->module_inst
  1301. == (WASMModuleInstanceCommon *)module_inst);
  1302. }
  1303. else {
  1304. /* Try using the existing exec_env */
  1305. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1306. exec_env = exec_env_tls;
  1307. #endif
  1308. #if WASM_ENABLE_THREAD_MGR != 0
  1309. if (!exec_env)
  1310. exec_env = wasm_clusters_search_exec_env(
  1311. (WASMModuleInstanceCommon *)module_inst);
  1312. #endif
  1313. if (!exec_env) {
  1314. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1315. (WASMModuleInstanceCommon *)module_inst,
  1316. module_inst->default_wasm_stack_size))) {
  1317. wasm_set_exception(module_inst, "allocate memory failed");
  1318. return false;
  1319. }
  1320. }
  1321. else {
  1322. /* Temporarily replace exec_env's module inst with current
  1323. module inst to ensure that the exec_env's module inst
  1324. is the correct one. */
  1325. module_inst_old = exec_env->module_inst;
  1326. wasm_exec_env_set_module_inst(
  1327. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1328. }
  1329. }
  1330. ret = wasm_call_function(exec_env, free_func, 1, argv);
  1331. if (module_inst_old)
  1332. /* Restore the existing exec_env's module inst */
  1333. wasm_exec_env_restore_module_inst(exec_env, module_inst_old);
  1334. if (exec_env_created)
  1335. wasm_exec_env_destroy(exec_env_created);
  1336. return ret;
  1337. }
  1338. static bool
  1339. check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf,
  1340. uint32 error_buf_size)
  1341. {
  1342. WASMModule *module = module_inst->module;
  1343. uint32 i;
  1344. for (i = 0; i < module->import_function_count; i++) {
  1345. WASMFunctionImport *func =
  1346. &((module->import_functions + i)->u.function);
  1347. if (!func->func_ptr_linked
  1348. #if WASM_ENABLE_MULTI_MODULE != 0
  1349. && !func->import_func_linked
  1350. #endif
  1351. ) {
  1352. #if WASM_ENABLE_WAMR_COMPILER == 0
  1353. LOG_WARNING("warning: failed to link import function (%s, %s)",
  1354. func->module_name, func->field_name);
  1355. /* will throw exception only if calling */
  1356. #else
  1357. /* do nothing to avoid confused message */
  1358. #endif /* WASM_ENABLE_WAMR_COMPILER == 0 */
  1359. }
  1360. }
  1361. for (i = 0; i < module->import_global_count; i++) {
  1362. WASMGlobalImport *global = &((module->import_globals + i)->u.global);
  1363. if (!global->is_linked) {
  1364. #if WASM_ENABLE_SPEC_TEST != 0
  1365. set_error_buf(error_buf, error_buf_size,
  1366. "unknown import or incompatible import type");
  1367. return false;
  1368. #else
  1369. #if WASM_ENABLE_WAMR_COMPILER == 0
  1370. set_error_buf_v(error_buf, error_buf_size,
  1371. "failed to link import global (%s, %s)",
  1372. global->module_name, global->field_name);
  1373. return false;
  1374. #else
  1375. /* do nothing to avoid confused message */
  1376. #endif /* WASM_ENABLE_WAMR_COMPILER == 0 */
  1377. #endif /* WASM_ENABLE_SPEC_TEST != 0 */
  1378. }
  1379. }
  1380. return true;
  1381. }
  1382. #if WASM_ENABLE_JIT != 0
  1383. static bool
  1384. init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module,
  1385. char *error_buf, uint32 error_buf_size)
  1386. {
  1387. uint32 i;
  1388. void **func_ptrs;
  1389. uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count;
  1390. /* Allocate memory */
  1391. if (!(func_ptrs = module_inst->func_ptrs =
  1392. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1393. return false;
  1394. }
  1395. /* Set import function pointers */
  1396. for (i = 0; i < module->import_function_count; i++, func_ptrs++) {
  1397. WASMFunctionImport *import_func =
  1398. &module->import_functions[i].u.function;
  1399. /* TODO: handle multi module */
  1400. *func_ptrs = import_func->func_ptr_linked;
  1401. }
  1402. /* The defined function pointers will be set in
  1403. wasm_runtime_set_running_mode, no need to set them here */
  1404. return true;
  1405. }
  1406. #endif /* end of WASM_ENABLE_JIT != 0 */
  1407. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  1408. static uint32
  1409. get_smallest_type_idx(WASMModule *module, WASMFuncType *func_type)
  1410. {
  1411. uint32 i;
  1412. for (i = 0; i < module->type_count; i++) {
  1413. if (func_type == (WASMFuncType *)module->types[i])
  1414. return i;
  1415. }
  1416. bh_assert(0);
  1417. return -1;
  1418. }
  1419. static bool
  1420. init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf,
  1421. uint32 error_buf_size)
  1422. {
  1423. uint32 i;
  1424. uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count;
  1425. /* Allocate memory */
  1426. if (!(module_inst->func_type_indexes =
  1427. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1428. return false;
  1429. }
  1430. for (i = 0; i < module_inst->e->function_count; i++) {
  1431. WASMFunctionInstance *func_inst = module_inst->e->functions + i;
  1432. WASMFuncType *func_type = func_inst->is_import_func
  1433. ? func_inst->u.func_import->func_type
  1434. : func_inst->u.func->func_type;
  1435. module_inst->func_type_indexes[i] =
  1436. get_smallest_type_idx(module_inst->module, func_type);
  1437. }
  1438. return true;
  1439. }
  1440. #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
  1441. #if WASM_ENABLE_GC != 0
  1442. void *
  1443. wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
  1444. bool throw_exce, char *error_buf, uint32 error_buf_size)
  1445. {
  1446. WASMModule *module = module_inst->module;
  1447. WASMRttTypeRef rtt_type;
  1448. WASMFuncObjectRef func_obj;
  1449. WASMFunctionInstance *func_inst;
  1450. WASMFuncType *func_type;
  1451. uint32 type_idx;
  1452. if (throw_exce) {
  1453. error_buf = module_inst->cur_exception;
  1454. error_buf_size = sizeof(module_inst->cur_exception);
  1455. }
  1456. if (func_idx >= module_inst->e->function_count) {
  1457. set_error_buf_v(error_buf, error_buf_size, "unknown function %d",
  1458. func_idx);
  1459. return NULL;
  1460. }
  1461. func_inst = &module_inst->e->functions[func_idx];
  1462. func_type = func_inst->is_import_func ? func_inst->u.func_import->func_type
  1463. : func_inst->u.func->func_type;
  1464. type_idx = func_inst->is_import_func ? func_inst->u.func_import->type_idx
  1465. : func_inst->u.func->type_idx;
  1466. if (!(rtt_type = wasm_rtt_type_new((WASMType *)func_type, type_idx,
  1467. module->rtt_types, module->type_count,
  1468. &module->rtt_type_lock))) {
  1469. set_error_buf(error_buf, error_buf_size, "create rtt object failed");
  1470. return NULL;
  1471. }
  1472. if (!(func_obj = wasm_func_obj_new_internal(
  1473. module_inst->e->common.gc_heap_handle, rtt_type, func_idx))) {
  1474. set_error_buf(error_buf, error_buf_size, "create func object failed");
  1475. return NULL;
  1476. }
  1477. return func_obj;
  1478. }
  1479. static bool
  1480. wasm_global_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap)
  1481. {
  1482. WASMGlobalInstance *global = module_inst->e->globals;
  1483. WASMGlobalInstance *global_end = global + module_inst->e->global_count;
  1484. uint8 *global_data = module_inst->global_data;
  1485. WASMObjectRef gc_obj;
  1486. while (global < global_end) {
  1487. if (wasm_is_type_reftype(global->type)) {
  1488. gc_obj = GET_REF_FROM_ADDR(
  1489. (uint32 *)(global_data + global->data_offset));
  1490. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1491. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1492. return false;
  1493. }
  1494. }
  1495. global++;
  1496. }
  1497. return true;
  1498. }
  1499. static bool
  1500. wasm_table_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap)
  1501. {
  1502. WASMTableInstance **tables = module_inst->tables, *table;
  1503. uint32 table_count = module_inst->table_count, i, j;
  1504. WASMObjectRef gc_obj, *table_elems;
  1505. for (i = 0; i < table_count; i++) {
  1506. table = tables[i];
  1507. table_elems = (WASMObjectRef *)table->elems;
  1508. for (j = 0; j < table->cur_size; j++) {
  1509. gc_obj = table_elems[j];
  1510. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1511. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1512. return false;
  1513. }
  1514. }
  1515. }
  1516. return true;
  1517. }
  1518. static bool
  1519. local_object_refs_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap)
  1520. {
  1521. WASMLocalObjectRef *r;
  1522. WASMObjectRef gc_obj;
  1523. for (r = exec_env->cur_local_object_ref; r; r = r->prev) {
  1524. gc_obj = r->val;
  1525. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1526. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1527. return false;
  1528. }
  1529. }
  1530. return true;
  1531. }
  1532. bool
  1533. wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap)
  1534. {
  1535. WASMModuleInstance *module_inst =
  1536. (WASMModuleInstance *)exec_env->module_inst;
  1537. bool ret;
  1538. ret = wasm_global_traverse_gc_rootset(module_inst, heap);
  1539. if (!ret)
  1540. return ret;
  1541. ret = wasm_table_traverse_gc_rootset(module_inst, heap);
  1542. if (!ret)
  1543. return ret;
  1544. ret = local_object_refs_traverse_gc_rootset(exec_env, heap);
  1545. if (!ret)
  1546. return ret;
  1547. return wasm_interp_traverse_gc_rootset(exec_env, heap);
  1548. }
  1549. #endif /* end of WASM_ENABLE_GC != 0 */
  1550. static bool
  1551. set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
  1552. bool first_time_set)
  1553. {
  1554. WASMModule *module = module_inst->module;
  1555. if (running_mode == Mode_Default) {
  1556. #if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0
  1557. running_mode = Mode_Interp;
  1558. #elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0
  1559. running_mode = Mode_Fast_JIT;
  1560. #elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0
  1561. running_mode = Mode_LLVM_JIT;
  1562. #else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */
  1563. #if WASM_ENABLE_LAZY_JIT == 0
  1564. running_mode = Mode_LLVM_JIT;
  1565. #else
  1566. running_mode = Mode_Multi_Tier_JIT;
  1567. #endif
  1568. #endif
  1569. }
  1570. if (!wasm_runtime_is_running_mode_supported(running_mode))
  1571. return false;
  1572. #if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  1573. && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
  1574. (void)first_time_set;
  1575. module_inst->e->running_mode = running_mode;
  1576. if (running_mode == Mode_Interp) {
  1577. /* Do nothing for Mode_Interp */
  1578. }
  1579. else if (running_mode == Mode_Fast_JIT) {
  1580. /* Do nothing for Mode_Fast_JIT since
  1581. module_inst->fast_jit_func_ptrs is same as
  1582. module->fast_jit_func_ptrs */
  1583. }
  1584. #if WASM_ENABLE_JIT != 0
  1585. else if (running_mode == Mode_LLVM_JIT) {
  1586. /* Set defined function pointers */
  1587. bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
  1588. sizeof(void *) * module->function_count, module->func_ptrs,
  1589. sizeof(void *) * module->function_count);
  1590. }
  1591. #endif
  1592. else {
  1593. bh_assert(0);
  1594. }
  1595. #else /* Possible multi-tier JIT */
  1596. os_mutex_lock(&module->instance_list_lock);
  1597. module_inst->e->running_mode = running_mode;
  1598. if (running_mode == Mode_Interp) {
  1599. /* Do nothing for Mode_Interp */
  1600. }
  1601. #if WASM_ENABLE_FAST_JIT != 0
  1602. else if (running_mode == Mode_Fast_JIT) {
  1603. JitGlobals *jit_globals = jit_compiler_get_jit_globals();
  1604. uint32 i;
  1605. /* Allocate memory for fast_jit_func_ptrs if needed */
  1606. if (!module_inst->fast_jit_func_ptrs
  1607. || module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) {
  1608. uint64 total_size = (uint64)sizeof(void *) * module->function_count;
  1609. if (!(module_inst->fast_jit_func_ptrs =
  1610. runtime_malloc(total_size, NULL, 0))) {
  1611. os_mutex_unlock(&module->instance_list_lock);
  1612. return false;
  1613. }
  1614. }
  1615. for (i = 0; i < module->function_count; i++) {
  1616. if (module->functions[i]->fast_jit_jitted_code) {
  1617. /* current fast jit function has been compiled */
  1618. module_inst->fast_jit_func_ptrs[i] =
  1619. module->functions[i]->fast_jit_jitted_code;
  1620. }
  1621. else {
  1622. module_inst->fast_jit_func_ptrs[i] =
  1623. jit_globals->compile_fast_jit_and_then_call;
  1624. }
  1625. }
  1626. }
  1627. #endif
  1628. #if WASM_ENABLE_JIT != 0
  1629. else if (running_mode == Mode_LLVM_JIT) {
  1630. void **llvm_jit_func_ptrs;
  1631. uint32 i;
  1632. /* Notify backend threads to start llvm jit compilation */
  1633. module->enable_llvm_jit_compilation = true;
  1634. /* Wait until llvm jit finishes initialization */
  1635. os_mutex_lock(&module->tierup_wait_lock);
  1636. while (!module->llvm_jit_inited) {
  1637. os_cond_reltimedwait(&module->tierup_wait_cond,
  1638. &module->tierup_wait_lock, 10000);
  1639. if (module->orcjit_stop_compiling) {
  1640. /* init_llvm_jit_functions_stage2 failed */
  1641. os_mutex_unlock(&module->tierup_wait_lock);
  1642. os_mutex_unlock(&module->instance_list_lock);
  1643. return false;
  1644. }
  1645. }
  1646. os_mutex_unlock(&module->tierup_wait_lock);
  1647. llvm_jit_func_ptrs =
  1648. module_inst->func_ptrs + module->import_function_count;
  1649. for (i = 0; i < module->function_count; i++) {
  1650. llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr;
  1651. }
  1652. }
  1653. #endif
  1654. else if (running_mode == Mode_Multi_Tier_JIT) {
  1655. /* Notify backend threads to start llvm jit compilation */
  1656. module->enable_llvm_jit_compilation = true;
  1657. /* Free fast_jit_func_ptrs if it is allocated before */
  1658. if (module_inst->fast_jit_func_ptrs
  1659. && module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) {
  1660. wasm_runtime_free(module_inst->fast_jit_func_ptrs);
  1661. }
  1662. module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
  1663. /* Copy all llvm jit func ptrs from the module */
  1664. bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
  1665. sizeof(void *) * module->function_count, module->func_ptrs,
  1666. sizeof(void *) * module->function_count);
  1667. }
  1668. else {
  1669. bh_assert(0);
  1670. }
  1671. /* Add module instance into module's instance list if not added */
  1672. if (first_time_set) {
  1673. bool found = false;
  1674. WASMModuleInstance *node = module->instance_list;
  1675. while (node) {
  1676. if (node == module_inst) {
  1677. found = true;
  1678. break;
  1679. }
  1680. node = node->e->next;
  1681. }
  1682. if (!found) {
  1683. module_inst->e->next = module->instance_list;
  1684. module->instance_list = module_inst;
  1685. }
  1686. }
  1687. os_mutex_unlock(&module->instance_list_lock);
  1688. #endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  1689. && WASM_ENABLE_LAZY_JIT != 0) */
  1690. (void)module;
  1691. return true;
  1692. }
  1693. bool
  1694. wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode)
  1695. {
  1696. return set_running_mode(module_inst, running_mode, false);
  1697. }
  1698. /**
  1699. * Instantiate module
  1700. */
  1701. WASMModuleInstance *
  1702. wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
  1703. WASMExecEnv *exec_env_main, uint32 stack_size,
  1704. uint32 heap_size, char *error_buf, uint32 error_buf_size)
  1705. {
  1706. WASMModuleInstance *module_inst;
  1707. WASMGlobalInstance *globals = NULL, *global;
  1708. WASMTableInstance *first_table;
  1709. uint32 global_count, i;
  1710. uint32 base_offset, length, extra_info_offset;
  1711. uint32 module_inst_struct_size =
  1712. offsetof(WASMModuleInstance, global_table_data.bytes);
  1713. uint64 module_inst_mem_inst_size;
  1714. uint64 total_size, table_size = 0;
  1715. uint8 *global_data, *global_data_end;
  1716. #if WASM_ENABLE_MULTI_MODULE != 0
  1717. bool ret = false;
  1718. #endif
  1719. const bool is_sub_inst = parent != NULL;
  1720. if (!module)
  1721. return NULL;
  1722. /* Check the heap size */
  1723. heap_size = align_uint(heap_size, 8);
  1724. if (heap_size > APP_HEAP_SIZE_MAX)
  1725. heap_size = APP_HEAP_SIZE_MAX;
  1726. module_inst_mem_inst_size =
  1727. (uint64)sizeof(WASMMemoryInstance)
  1728. * (module->import_memory_count + module->memory_count);
  1729. #if WASM_ENABLE_JIT != 0
  1730. /* If the module dosen't have memory, reserve one mem_info space
  1731. with empty content to align with llvm jit compiler */
  1732. if (module_inst_mem_inst_size == 0)
  1733. module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance);
  1734. #endif
  1735. /* Size of module inst, memory instances and global data */
  1736. total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size
  1737. + module->global_data_size;
  1738. /* Calculate the size of table data */
  1739. for (i = 0; i < module->import_table_count; i++) {
  1740. WASMTableImport *import_table = &module->import_tables[i].u.table;
  1741. table_size += offsetof(WASMTableInstance, elems);
  1742. #if WASM_ENABLE_MULTI_MODULE != 0
  1743. table_size +=
  1744. (uint64)sizeof(table_elem_type_t) * import_table->max_size;
  1745. #else
  1746. table_size += (uint64)sizeof(table_elem_type_t)
  1747. * (import_table->possible_grow ? import_table->max_size
  1748. : import_table->init_size);
  1749. #endif
  1750. }
  1751. for (i = 0; i < module->table_count; i++) {
  1752. WASMTable *table = module->tables + i;
  1753. table_size += offsetof(WASMTableInstance, elems);
  1754. #if WASM_ENABLE_MULTI_MODULE != 0
  1755. table_size += (uint64)sizeof(table_elem_type_t) * table->max_size;
  1756. #else
  1757. table_size +=
  1758. (uint64)sizeof(table_elem_type_t)
  1759. * (table->possible_grow ? table->max_size : table->init_size);
  1760. #endif
  1761. }
  1762. total_size += table_size;
  1763. /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */
  1764. total_size = (total_size + 7LL) & ~7LL;
  1765. extra_info_offset = (uint32)total_size;
  1766. total_size += sizeof(WASMModuleInstanceExtra);
  1767. /* Allocate the memory for module instance with memory instances,
  1768. global data, table data appended at the end */
  1769. if (!(module_inst =
  1770. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1771. return NULL;
  1772. }
  1773. module_inst->module_type = Wasm_Module_Bytecode;
  1774. module_inst->module = module;
  1775. module_inst->e =
  1776. (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset);
  1777. #if WASM_ENABLE_MULTI_MODULE != 0
  1778. module_inst->e->sub_module_inst_list =
  1779. &module_inst->e->sub_module_inst_list_head;
  1780. ret = wasm_runtime_sub_module_instantiate(
  1781. (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst,
  1782. stack_size, heap_size, error_buf, error_buf_size);
  1783. if (!ret) {
  1784. LOG_DEBUG("build a sub module list failed");
  1785. goto fail;
  1786. }
  1787. #endif
  1788. #if WASM_ENABLE_BULK_MEMORY != 0
  1789. if (module->data_seg_count > 0) {
  1790. module_inst->e->common.data_dropped =
  1791. bh_bitmap_new(0, module->data_seg_count);
  1792. if (module_inst->e->common.data_dropped == NULL) {
  1793. LOG_DEBUG("failed to allocate bitmaps");
  1794. set_error_buf(error_buf, error_buf_size,
  1795. "failed to allocate bitmaps");
  1796. goto fail;
  1797. }
  1798. for (i = 0; i < module->data_seg_count; i++) {
  1799. if (!module->data_segments[i]->is_passive)
  1800. bh_bitmap_set_bit(module_inst->e->common.data_dropped, i);
  1801. }
  1802. }
  1803. #endif
  1804. #if WASM_ENABLE_REF_TYPES != 0
  1805. if (module->table_seg_count > 0) {
  1806. module_inst->e->common.elem_dropped =
  1807. bh_bitmap_new(0, module->table_seg_count);
  1808. if (module_inst->e->common.elem_dropped == NULL) {
  1809. LOG_DEBUG("failed to allocate bitmaps");
  1810. set_error_buf(error_buf, error_buf_size,
  1811. "failed to allocate bitmaps");
  1812. goto fail;
  1813. }
  1814. for (i = 0; i < module->table_seg_count; i++) {
  1815. if (wasm_elem_is_active(module->table_segments[i].mode))
  1816. bh_bitmap_set_bit(module_inst->e->common.elem_dropped, i);
  1817. }
  1818. }
  1819. #endif
  1820. #if WASM_ENABLE_GC != 0
  1821. if (!is_sub_inst) {
  1822. uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default();
  1823. if (gc_heap_size < GC_HEAP_SIZE_MIN)
  1824. gc_heap_size = GC_HEAP_SIZE_MIN;
  1825. if (gc_heap_size > GC_HEAP_SIZE_MAX)
  1826. gc_heap_size = GC_HEAP_SIZE_MAX;
  1827. module_inst->e->common.gc_heap_pool =
  1828. runtime_malloc(gc_heap_size, error_buf, error_buf_size);
  1829. if (!module_inst->e->common.gc_heap_pool)
  1830. goto fail;
  1831. module_inst->e->common.gc_heap_handle = mem_allocator_create(
  1832. module_inst->e->common.gc_heap_pool, gc_heap_size);
  1833. if (!module_inst->e->common.gc_heap_handle)
  1834. goto fail;
  1835. }
  1836. #endif
  1837. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  1838. if (!(module_inst->frames = runtime_malloc((uint64)sizeof(Vector),
  1839. error_buf, error_buf_size))) {
  1840. goto fail;
  1841. }
  1842. #endif
  1843. /* Instantiate global firstly to get the mutable data size */
  1844. global_count = module->import_global_count + module->global_count;
  1845. if (global_count
  1846. && !(globals = globals_instantiate(module, module_inst, error_buf,
  1847. error_buf_size))) {
  1848. goto fail;
  1849. }
  1850. module_inst->e->global_count = global_count;
  1851. module_inst->e->globals = globals;
  1852. module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size
  1853. + module_inst_mem_inst_size;
  1854. module_inst->global_data_size = module->global_data_size;
  1855. first_table = (WASMTableInstance *)(module_inst->global_data
  1856. + module->global_data_size);
  1857. module_inst->memory_count =
  1858. module->import_memory_count + module->memory_count;
  1859. module_inst->table_count = module->import_table_count + module->table_count;
  1860. module_inst->e->function_count =
  1861. module->import_function_count + module->function_count;
  1862. #if WASM_ENABLE_TAGS != 0
  1863. module_inst->e->tag_count = module->import_tag_count + module->tag_count;
  1864. #endif
  1865. /* export */
  1866. module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC);
  1867. #if WASM_ENABLE_MULTI_MODULE != 0
  1868. module_inst->export_table_count =
  1869. get_export_count(module, EXPORT_KIND_TABLE);
  1870. module_inst->export_memory_count =
  1871. get_export_count(module, EXPORT_KIND_MEMORY);
  1872. #if WASM_ENABLE_TAGS != 0
  1873. module_inst->e->export_tag_count =
  1874. get_export_count(module, EXPORT_KIND_TAG);
  1875. #endif
  1876. module_inst->export_global_count =
  1877. get_export_count(module, EXPORT_KIND_GLOBAL);
  1878. #endif
  1879. /* Instantiate memories/tables/functions/tags */
  1880. if ((module_inst->memory_count > 0
  1881. && !(module_inst->memories =
  1882. memories_instantiate(module, module_inst, parent, heap_size,
  1883. error_buf, error_buf_size)))
  1884. || (module_inst->table_count > 0
  1885. && !(module_inst->tables =
  1886. tables_instantiate(module, module_inst, first_table,
  1887. error_buf, error_buf_size)))
  1888. || (module_inst->e->function_count > 0
  1889. && !(module_inst->e->functions = functions_instantiate(
  1890. module, module_inst, error_buf, error_buf_size)))
  1891. || (module_inst->export_func_count > 0
  1892. && !(module_inst->export_functions = export_functions_instantiate(
  1893. module, module_inst, module_inst->export_func_count,
  1894. error_buf, error_buf_size)))
  1895. #if WASM_ENABLE_TAGS != 0
  1896. || (module_inst->e->tag_count > 0
  1897. && !(module_inst->e->tags = tags_instantiate(
  1898. module, module_inst, error_buf, error_buf_size)))
  1899. || (module_inst->e->export_tag_count > 0
  1900. && !(module_inst->e->export_tags = export_tags_instantiate(
  1901. module, module_inst, module_inst->e->export_tag_count,
  1902. error_buf, error_buf_size)))
  1903. #endif
  1904. #if WASM_ENABLE_MULTI_MODULE != 0
  1905. || (module_inst->export_global_count > 0
  1906. && !(module_inst->export_globals = export_globals_instantiate(
  1907. module, module_inst, module_inst->export_global_count,
  1908. error_buf, error_buf_size)))
  1909. #endif
  1910. #if WASM_ENABLE_JIT != 0
  1911. || (module_inst->e->function_count > 0
  1912. && !init_func_ptrs(module_inst, module, error_buf, error_buf_size))
  1913. #endif
  1914. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  1915. || (module_inst->e->function_count > 0
  1916. && !init_func_type_indexes(module_inst, error_buf, error_buf_size))
  1917. #endif
  1918. ) {
  1919. goto fail;
  1920. }
  1921. if (global_count > 0) {
  1922. /* Initialize the global data */
  1923. global_data = module_inst->global_data;
  1924. global_data_end = global_data + module->global_data_size;
  1925. global = globals;
  1926. for (i = 0; i < global_count; i++, global++) {
  1927. switch (global->type) {
  1928. case VALUE_TYPE_I32:
  1929. case VALUE_TYPE_F32:
  1930. #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
  1931. case VALUE_TYPE_FUNCREF:
  1932. case VALUE_TYPE_EXTERNREF:
  1933. #endif
  1934. *(int32 *)global_data = global->initial_value.i32;
  1935. global_data += sizeof(int32);
  1936. break;
  1937. case VALUE_TYPE_I64:
  1938. case VALUE_TYPE_F64:
  1939. bh_memcpy_s(global_data,
  1940. (uint32)(global_data_end - global_data),
  1941. &global->initial_value.i64, sizeof(int64));
  1942. global_data += sizeof(int64);
  1943. break;
  1944. #if WASM_ENABLE_SIMD != 0
  1945. case VALUE_TYPE_V128:
  1946. bh_memcpy_s(global_data, (uint32)sizeof(V128),
  1947. &global->initial_value.v128, sizeof(V128));
  1948. global_data += sizeof(V128);
  1949. break;
  1950. #endif
  1951. #if WASM_ENABLE_GC != 0
  1952. case VALUE_TYPE_EXTERNREF:
  1953. /* the initial value should be a null reference */
  1954. bh_assert(global->initial_value.gc_obj == NULL_REF);
  1955. STORE_PTR((void **)global_data, NULL_REF);
  1956. global_data += sizeof(void *);
  1957. break;
  1958. #endif
  1959. default:
  1960. {
  1961. #if WASM_ENABLE_GC != 0
  1962. InitializerExpression *global_init = NULL;
  1963. bh_assert(wasm_is_type_reftype(global->type));
  1964. if (i >= module->import_global_count) {
  1965. global_init =
  1966. &module->globals[i - module->import_global_count]
  1967. .init_expr;
  1968. }
  1969. if (global->type == REF_TYPE_NULLFUNCREF
  1970. || global->type == REF_TYPE_NULLEXTERNREF
  1971. || global->type == REF_TYPE_NULLREF) {
  1972. STORE_PTR((void **)global_data, NULL_REF);
  1973. global_data += sizeof(void *);
  1974. break;
  1975. }
  1976. /* We can't create funcref obj during global instantiation
  1977. * since the functions are not instantiated yet, so we need
  1978. * to defer the initialization here */
  1979. if (global_init
  1980. && (global_init->init_expr_type
  1981. == INIT_EXPR_TYPE_FUNCREF_CONST)
  1982. && wasm_reftype_is_subtype_of(
  1983. global->type, global->ref_type, REF_TYPE_FUNCREF,
  1984. NULL, module_inst->module->types,
  1985. module_inst->module->type_count)) {
  1986. WASMFuncObjectRef func_obj = NULL;
  1987. /* UINT32_MAX indicates that it is a null reference */
  1988. if ((uint32)global->initial_value.i32 != UINT32_MAX) {
  1989. if (!(func_obj = wasm_create_func_obj(
  1990. module_inst, global->initial_value.i32,
  1991. false, error_buf, error_buf_size)))
  1992. goto fail;
  1993. }
  1994. STORE_PTR((void **)global_data, func_obj);
  1995. global_data += sizeof(void *);
  1996. /* Also update the inital_value since other globals may
  1997. * refer to this */
  1998. global->initial_value.gc_obj = (wasm_obj_t)func_obj;
  1999. break;
  2000. }
  2001. else {
  2002. STORE_PTR((void **)global_data,
  2003. global->initial_value.gc_obj);
  2004. global_data += sizeof(void *);
  2005. break;
  2006. }
  2007. #endif
  2008. bh_assert(0);
  2009. break;
  2010. }
  2011. }
  2012. }
  2013. bh_assert(global_data == global_data_end);
  2014. }
  2015. if (!check_linked_symbol(module_inst, error_buf, error_buf_size)) {
  2016. goto fail;
  2017. }
  2018. /* Initialize the memory data with data segment section */
  2019. for (i = 0; i < module->data_seg_count; i++) {
  2020. WASMMemoryInstance *memory = NULL;
  2021. uint8 *memory_data = NULL;
  2022. uint64 memory_size = 0;
  2023. WASMDataSeg *data_seg = module->data_segments[i];
  2024. #if WASM_ENABLE_BULK_MEMORY != 0
  2025. if (data_seg->is_passive)
  2026. continue;
  2027. #endif
  2028. if (is_sub_inst)
  2029. /* Ignore setting memory init data if the memory has been
  2030. initialized */
  2031. continue;
  2032. /* has check it in loader */
  2033. memory = module_inst->memories[data_seg->memory_index];
  2034. bh_assert(memory);
  2035. memory_data = memory->memory_data;
  2036. memory_size =
  2037. (uint64)memory->num_bytes_per_page * memory->cur_page_count;
  2038. bh_assert(memory_data || memory_size == 0);
  2039. bh_assert(data_seg->base_offset.init_expr_type
  2040. == INIT_EXPR_TYPE_I32_CONST
  2041. || data_seg->base_offset.init_expr_type
  2042. == INIT_EXPR_TYPE_GET_GLOBAL);
  2043. if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
  2044. if (!check_global_init_expr(module,
  2045. data_seg->base_offset.u.global_index,
  2046. error_buf, error_buf_size)) {
  2047. goto fail;
  2048. }
  2049. if (!globals
  2050. || globals[data_seg->base_offset.u.global_index].type
  2051. != VALUE_TYPE_I32) {
  2052. set_error_buf(error_buf, error_buf_size,
  2053. "data segment does not fit");
  2054. goto fail;
  2055. }
  2056. base_offset =
  2057. globals[data_seg->base_offset.u.global_index].initial_value.i32;
  2058. }
  2059. else {
  2060. base_offset = (uint32)data_seg->base_offset.u.i32;
  2061. }
  2062. /* check offset */
  2063. if (base_offset > memory_size) {
  2064. LOG_DEBUG("base_offset(%d) > memory_size(%d)", base_offset,
  2065. memory_size);
  2066. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2067. set_error_buf(error_buf, error_buf_size,
  2068. "out of bounds memory access");
  2069. #else
  2070. set_error_buf(error_buf, error_buf_size,
  2071. "data segment does not fit");
  2072. #endif
  2073. goto fail;
  2074. }
  2075. /* check offset + length(could be zero) */
  2076. length = data_seg->data_length;
  2077. if ((uint64)base_offset + length > memory_size) {
  2078. LOG_DEBUG("base_offset(%d) + length(%d) > memory_size(%d)",
  2079. base_offset, length, memory_size);
  2080. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2081. set_error_buf(error_buf, error_buf_size,
  2082. "out of bounds memory access");
  2083. #else
  2084. set_error_buf(error_buf, error_buf_size,
  2085. "data segment does not fit");
  2086. #endif
  2087. goto fail;
  2088. }
  2089. if (memory_data) {
  2090. bh_memcpy_s(memory_data + base_offset,
  2091. (uint32)memory_size - base_offset, data_seg->data,
  2092. length);
  2093. }
  2094. }
  2095. #if WASM_ENABLE_GC != 0
  2096. /* Initialize the table data with init expr */
  2097. for (i = 0; i < module->table_count; i++) {
  2098. WASMTable *table = module->tables + i;
  2099. WASMTableInstance *table_inst = module_inst->tables[i];
  2100. table_elem_type_t *table_data;
  2101. uint32 j;
  2102. if (table->init_expr.init_expr_type == 0) {
  2103. /* No table initializer */
  2104. continue;
  2105. }
  2106. table_data = table_inst->elems;
  2107. bh_assert(
  2108. table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL
  2109. || table->init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST
  2110. || table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST);
  2111. if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
  2112. if (!check_global_init_expr(module, table->init_expr.u.global_index,
  2113. error_buf, error_buf_size)) {
  2114. goto fail;
  2115. }
  2116. table->init_expr.u.gc_obj =
  2117. globals[table->init_expr.u.global_index].initial_value.gc_obj;
  2118. }
  2119. else if (table->init_expr.init_expr_type
  2120. == INIT_EXPR_TYPE_FUNCREF_CONST) {
  2121. uint32 func_idx = table->init_expr.u.ref_index;
  2122. if (func_idx != UINT32_MAX) {
  2123. if (!(table->init_expr.u.gc_obj =
  2124. wasm_create_func_obj(module_inst, func_idx, false,
  2125. error_buf, error_buf_size)))
  2126. goto fail;
  2127. }
  2128. else {
  2129. table->init_expr.u.gc_obj = NULL_REF;
  2130. }
  2131. }
  2132. else if (table->init_expr.init_expr_type
  2133. == INIT_EXPR_TYPE_REFNULL_CONST) {
  2134. table->init_expr.u.gc_obj = NULL_REF;
  2135. }
  2136. LOG_DEBUG("Init table [%d] elements from [%d] to [%d] as: %p", i, 0,
  2137. table_inst->cur_size, (void *)table->init_expr.u.gc_obj);
  2138. for (j = 0; j < table_inst->cur_size; j++) {
  2139. *(table_data + j) = table->init_expr.u.gc_obj;
  2140. }
  2141. }
  2142. #endif /* end of WASM_ENABLE_GC != 0 */
  2143. /* Initialize the table data with table segment section */
  2144. for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count;
  2145. i++) {
  2146. WASMTableSeg *table_seg = module->table_segments + i;
  2147. /* has check it in loader */
  2148. WASMTableInstance *table = module_inst->tables[table_seg->table_index];
  2149. table_elem_type_t *table_data;
  2150. uint32 j;
  2151. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2152. uint8 tbl_elem_type;
  2153. uint32 tbl_init_size, tbl_max_size;
  2154. #endif
  2155. #if WASM_ENABLE_GC != 0
  2156. WASMRefType *tbl_elem_ref_type;
  2157. #endif
  2158. bh_assert(table);
  2159. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2160. (void)wasm_runtime_get_table_inst_elem_type(
  2161. (WASMModuleInstanceCommon *)module_inst, table_seg->table_index,
  2162. &tbl_elem_type,
  2163. #if WASM_ENABLE_GC != 0
  2164. &tbl_elem_ref_type,
  2165. #endif
  2166. &tbl_init_size, &tbl_max_size);
  2167. #if WASM_ENABLE_GC == 0
  2168. if (tbl_elem_type != VALUE_TYPE_FUNCREF
  2169. && tbl_elem_type != VALUE_TYPE_EXTERNREF) {
  2170. set_error_buf(error_buf, error_buf_size,
  2171. "type mismatch: elements segment does not fit");
  2172. goto fail;
  2173. }
  2174. #elif WASM_ENABLE_GC != 0
  2175. if (!wasm_elem_is_declarative(table_seg->mode)
  2176. && !wasm_reftype_is_subtype_of(
  2177. table_seg->elem_type, table_seg->elem_ref_type,
  2178. table->elem_type, table->elem_ref_type.elem_ref_type,
  2179. module->types, module->type_count)) {
  2180. set_error_buf(error_buf, error_buf_size,
  2181. "type mismatch: elements segment does not fit");
  2182. goto fail;
  2183. }
  2184. #endif
  2185. (void)tbl_init_size;
  2186. (void)tbl_max_size;
  2187. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  2188. table_data = table->elems;
  2189. #if WASM_ENABLE_MULTI_MODULE != 0
  2190. if (table_seg->table_index < module->import_table_count
  2191. && module_inst->e->table_insts_linked[table_seg->table_index]) {
  2192. table_data =
  2193. module_inst->e->table_insts_linked[table_seg->table_index]
  2194. ->elems;
  2195. }
  2196. #endif
  2197. bh_assert(table_data);
  2198. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2199. if (!wasm_elem_is_active(table_seg->mode))
  2200. continue;
  2201. #endif
  2202. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2203. bh_assert(table_seg->base_offset.init_expr_type
  2204. == INIT_EXPR_TYPE_I32_CONST
  2205. || table_seg->base_offset.init_expr_type
  2206. == INIT_EXPR_TYPE_GET_GLOBAL
  2207. || table_seg->base_offset.init_expr_type
  2208. == INIT_EXPR_TYPE_FUNCREF_CONST
  2209. || table_seg->base_offset.init_expr_type
  2210. == INIT_EXPR_TYPE_REFNULL_CONST);
  2211. #else
  2212. bh_assert(table_seg->base_offset.init_expr_type
  2213. == INIT_EXPR_TYPE_I32_CONST
  2214. || table_seg->base_offset.init_expr_type
  2215. == INIT_EXPR_TYPE_GET_GLOBAL);
  2216. #endif
  2217. /* init vec(funcidx) or vec(expr) */
  2218. if (table_seg->base_offset.init_expr_type
  2219. == INIT_EXPR_TYPE_GET_GLOBAL) {
  2220. if (!check_global_init_expr(module,
  2221. table_seg->base_offset.u.global_index,
  2222. error_buf, error_buf_size)) {
  2223. goto fail;
  2224. }
  2225. if (!globals
  2226. || globals[table_seg->base_offset.u.global_index].type
  2227. != VALUE_TYPE_I32) {
  2228. set_error_buf(error_buf, error_buf_size,
  2229. "type mismatch: elements segment does not fit");
  2230. goto fail;
  2231. }
  2232. table_seg->base_offset.u.i32 =
  2233. globals[table_seg->base_offset.u.global_index]
  2234. .initial_value.i32;
  2235. }
  2236. /* check offset since length might negative */
  2237. if ((uint32)table_seg->base_offset.u.i32 > table->cur_size) {
  2238. LOG_DEBUG("base_offset(%d) > table->cur_size(%d)",
  2239. table_seg->base_offset.u.i32, table->cur_size);
  2240. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2241. set_error_buf(error_buf, error_buf_size,
  2242. "out of bounds table access");
  2243. #else
  2244. set_error_buf(error_buf, error_buf_size,
  2245. "type mismatch: elements segment does not fit");
  2246. #endif
  2247. goto fail;
  2248. }
  2249. /* check offset + length(could be zero) */
  2250. length = table_seg->value_count;
  2251. if ((uint32)table_seg->base_offset.u.i32 + length > table->cur_size) {
  2252. LOG_DEBUG("base_offset(%d) + length(%d)> table->cur_size(%d)",
  2253. table_seg->base_offset.u.i32, length, table->cur_size);
  2254. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2255. set_error_buf(error_buf, error_buf_size,
  2256. "out of bounds table access");
  2257. #else
  2258. set_error_buf(error_buf, error_buf_size,
  2259. "type mismatch: elements segment does not fit");
  2260. #endif
  2261. goto fail;
  2262. }
  2263. for (j = 0; j < length; j++) {
  2264. InitializerExpression *init_expr = &table_seg->init_values[j];
  2265. uint8 flag = init_expr->init_expr_type;
  2266. void *ref = NULL;
  2267. /* const and get global init values should be resolved during
  2268. * loading */
  2269. bh_assert((flag == INIT_EXPR_TYPE_GET_GLOBAL)
  2270. || (flag == INIT_EXPR_TYPE_REFNULL_CONST)
  2271. || ((flag >= INIT_EXPR_TYPE_FUNCREF_CONST)
  2272. && (flag <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY)));
  2273. switch (flag) {
  2274. case INIT_EXPR_TYPE_REFNULL_CONST:
  2275. ref = NULL;
  2276. break;
  2277. case INIT_EXPR_TYPE_FUNCREF_CONST:
  2278. {
  2279. #if WASM_ENABLE_GC == 0
  2280. ref = (void *)(uintptr_t)init_expr->u.ref_index;
  2281. #else
  2282. WASMFuncObjectRef func_obj;
  2283. uint32 func_idx = init_expr->u.ref_index;
  2284. /* UINT32_MAX indicates that it is a null reference */
  2285. if (func_idx != UINT32_MAX) {
  2286. if (!(func_obj = wasm_create_func_obj(
  2287. module_inst, func_idx, false, error_buf,
  2288. error_buf_size))) {
  2289. goto fail;
  2290. }
  2291. ref = func_obj;
  2292. }
  2293. else {
  2294. ref = NULL_REF;
  2295. }
  2296. #endif /* end of WASM_ENABLE_GC != 0 */
  2297. break;
  2298. }
  2299. #if WASM_ENABLE_GC != 0
  2300. case INIT_EXPR_TYPE_GET_GLOBAL:
  2301. {
  2302. if (!check_global_init_expr(module,
  2303. init_expr->u.global_index,
  2304. error_buf, error_buf_size)) {
  2305. goto fail;
  2306. }
  2307. ref =
  2308. globals[init_expr->u.global_index].initial_value.gc_obj;
  2309. break;
  2310. }
  2311. case INIT_EXPR_TYPE_STRUCT_NEW:
  2312. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  2313. {
  2314. WASMRttType *rtt_type;
  2315. WASMStructObjectRef struct_obj;
  2316. WASMStructType *struct_type;
  2317. WASMStructNewInitValues *init_values = NULL;
  2318. uint32 type_idx;
  2319. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  2320. init_values =
  2321. (WASMStructNewInitValues *)init_expr->u.data;
  2322. type_idx = init_values->type_idx;
  2323. }
  2324. else {
  2325. type_idx = init_expr->u.type_index;
  2326. }
  2327. struct_type = (WASMStructType *)module->types[type_idx];
  2328. if (!(rtt_type = wasm_rtt_type_new(
  2329. (WASMType *)struct_type, type_idx,
  2330. module->rtt_types, module->type_count,
  2331. &module->rtt_type_lock))) {
  2332. set_error_buf(error_buf, error_buf_size,
  2333. "create rtt object failed");
  2334. goto fail;
  2335. }
  2336. if (!(struct_obj = wasm_struct_obj_new_internal(
  2337. module_inst->e->common.gc_heap_handle,
  2338. rtt_type))) {
  2339. set_error_buf(error_buf, error_buf_size,
  2340. "create struct object failed");
  2341. goto fail;
  2342. }
  2343. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  2344. uint32 field_idx;
  2345. bh_assert(init_values->count
  2346. == struct_type->field_count);
  2347. for (field_idx = 0; field_idx < init_values->count;
  2348. field_idx++) {
  2349. wasm_struct_obj_set_field(
  2350. struct_obj, field_idx,
  2351. &init_values->fields[field_idx]);
  2352. }
  2353. }
  2354. ref = struct_obj;
  2355. break;
  2356. }
  2357. case INIT_EXPR_TYPE_ARRAY_NEW:
  2358. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  2359. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  2360. {
  2361. WASMRttType *rtt_type;
  2362. WASMArrayObjectRef array_obj;
  2363. WASMArrayType *array_type;
  2364. WASMArrayNewInitValues *init_values = NULL;
  2365. WASMValue *arr_init_val = NULL, empty_val = { 0 };
  2366. uint32 type_idx, len;
  2367. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) {
  2368. type_idx = init_expr->u.array_new_default.type_index;
  2369. len = init_expr->u.array_new_default.length;
  2370. arr_init_val = &empty_val;
  2371. }
  2372. else {
  2373. init_values =
  2374. (WASMArrayNewInitValues *)init_expr->u.data;
  2375. type_idx = init_values->type_idx;
  2376. len = init_values->length;
  2377. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  2378. arr_init_val = init_values->elem_data;
  2379. }
  2380. }
  2381. array_type = (WASMArrayType *)module->types[type_idx];
  2382. if (!(rtt_type = wasm_rtt_type_new(
  2383. (WASMType *)array_type, type_idx,
  2384. module->rtt_types, module->type_count,
  2385. &module->rtt_type_lock))) {
  2386. set_error_buf(error_buf, error_buf_size,
  2387. "create rtt object failed");
  2388. goto fail;
  2389. }
  2390. if (!(array_obj = wasm_array_obj_new_internal(
  2391. module_inst->e->common.gc_heap_handle, rtt_type,
  2392. len, arr_init_val))) {
  2393. set_error_buf(error_buf, error_buf_size,
  2394. "create array object failed");
  2395. goto fail;
  2396. }
  2397. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  2398. uint32 elem_idx;
  2399. bh_assert(init_values);
  2400. for (elem_idx = 0; elem_idx < len; elem_idx++) {
  2401. wasm_array_obj_set_elem(
  2402. array_obj, elem_idx,
  2403. &init_values->elem_data[elem_idx]);
  2404. }
  2405. }
  2406. ref = array_obj;
  2407. break;
  2408. }
  2409. case INIT_EXPR_TYPE_I31_NEW:
  2410. {
  2411. ref = (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32);
  2412. break;
  2413. }
  2414. #endif /* end of WASM_ENABLE_GC != 0 */
  2415. }
  2416. *(table_data + table_seg->base_offset.u.i32 + j) =
  2417. (table_elem_type_t)ref;
  2418. }
  2419. }
  2420. /* Initialize the thread related data */
  2421. if (stack_size == 0)
  2422. stack_size = DEFAULT_WASM_STACK_SIZE;
  2423. #if WASM_ENABLE_SPEC_TEST != 0
  2424. #if WASM_ENABLE_TAIL_CALL == 0
  2425. if (stack_size < 128 * 1024)
  2426. stack_size = 128 * 1024;
  2427. #else
  2428. /* Some tail-call cases require large operand stack */
  2429. if (stack_size < 10 * 1024 * 1024)
  2430. stack_size = 10 * 1024 * 1024;
  2431. #endif
  2432. #endif
  2433. module_inst->default_wasm_stack_size = stack_size;
  2434. if (module->malloc_function != (uint32)-1) {
  2435. module_inst->e->malloc_function =
  2436. &module_inst->e->functions[module->malloc_function];
  2437. }
  2438. if (module->free_function != (uint32)-1) {
  2439. module_inst->e->free_function =
  2440. &module_inst->e->functions[module->free_function];
  2441. }
  2442. if (module->retain_function != (uint32)-1) {
  2443. module_inst->e->retain_function =
  2444. &module_inst->e->functions[module->retain_function];
  2445. }
  2446. #if WASM_ENABLE_LIBC_WASI != 0
  2447. /* The sub-instance will get the wasi_ctx from main-instance */
  2448. if (!is_sub_inst) {
  2449. if (!wasm_runtime_init_wasi(
  2450. (WASMModuleInstanceCommon *)module_inst,
  2451. module->wasi_args.dir_list, module->wasi_args.dir_count,
  2452. module->wasi_args.map_dir_list, module->wasi_args.map_dir_count,
  2453. module->wasi_args.env, module->wasi_args.env_count,
  2454. module->wasi_args.addr_pool, module->wasi_args.addr_count,
  2455. module->wasi_args.ns_lookup_pool,
  2456. module->wasi_args.ns_lookup_count, module->wasi_args.argv,
  2457. module->wasi_args.argc, module->wasi_args.stdio[0],
  2458. module->wasi_args.stdio[1], module->wasi_args.stdio[2],
  2459. error_buf, error_buf_size)) {
  2460. goto fail;
  2461. }
  2462. }
  2463. #endif
  2464. #if WASM_ENABLE_DEBUG_INTERP != 0
  2465. if (!is_sub_inst) {
  2466. /* Add module instance into module's instance list */
  2467. os_mutex_lock(&module->instance_list_lock);
  2468. if (module->instance_list) {
  2469. LOG_WARNING(
  2470. "warning: multiple instances referencing to the same module "
  2471. "may cause unexpected behaviour during debugging");
  2472. }
  2473. module_inst->e->next = module->instance_list;
  2474. module->instance_list = module_inst;
  2475. os_mutex_unlock(&module->instance_list_lock);
  2476. }
  2477. #endif
  2478. /* Set running mode before executing wasm functions */
  2479. if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(),
  2480. true)) {
  2481. set_error_buf(error_buf, error_buf_size,
  2482. "set instance running mode failed");
  2483. goto fail;
  2484. }
  2485. if (module->start_function != (uint32)-1) {
  2486. /* TODO: fix start function can be import function issue */
  2487. if (module->start_function >= module->import_function_count)
  2488. module_inst->e->start_function =
  2489. &module_inst->e->functions[module->start_function];
  2490. }
  2491. if (!execute_post_instantiate_functions(module_inst, is_sub_inst,
  2492. exec_env_main)) {
  2493. set_error_buf(error_buf, error_buf_size, module_inst->cur_exception);
  2494. goto fail;
  2495. }
  2496. #if WASM_ENABLE_MEMORY_TRACING != 0
  2497. wasm_runtime_dump_module_inst_mem_consumption(
  2498. (WASMModuleInstanceCommon *)module_inst);
  2499. #endif
  2500. (void)global_data_end;
  2501. return module_inst;
  2502. fail:
  2503. wasm_deinstantiate(module_inst, false);
  2504. return NULL;
  2505. }
  2506. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2507. static void
  2508. destroy_c_api_frames(Vector *frames)
  2509. {
  2510. WASMCApiFrame frame = { 0 };
  2511. uint32 i, total_frames, ret;
  2512. total_frames = (uint32)bh_vector_size(frames);
  2513. for (i = 0; i < total_frames; i++) {
  2514. ret = bh_vector_get(frames, i, &frame);
  2515. bh_assert(ret);
  2516. if (frame.lp)
  2517. wasm_runtime_free(frame.lp);
  2518. }
  2519. ret = bh_vector_destroy(frames);
  2520. bh_assert(ret);
  2521. (void)ret;
  2522. }
  2523. #endif
  2524. void
  2525. wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
  2526. {
  2527. if (!module_inst)
  2528. return;
  2529. if (module_inst->exec_env_singleton) {
  2530. /* wasm_exec_env_destroy will call
  2531. wasm_cluster_wait_for_all_except_self to wait for other
  2532. threads, so as to destroy their exec_envs and module
  2533. instances first, and avoid accessing the shared resources
  2534. of current module instance after it is deinstantiated. */
  2535. wasm_exec_env_destroy(module_inst->exec_env_singleton);
  2536. }
  2537. #if WASM_ENABLE_DEBUG_INTERP != 0 \
  2538. || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  2539. && WASM_ENABLE_LAZY_JIT != 0)
  2540. /* Remove instance from module's instance list before freeing
  2541. func_ptrs and fast_jit_func_ptrs of the instance, to avoid
  2542. accessing the freed memory in the jit backend compilation
  2543. threads */
  2544. {
  2545. WASMModule *module = module_inst->module;
  2546. WASMModuleInstance *instance_prev = NULL, *instance;
  2547. os_mutex_lock(&module->instance_list_lock);
  2548. instance = module->instance_list;
  2549. while (instance) {
  2550. if (instance == module_inst) {
  2551. if (!instance_prev)
  2552. module->instance_list = instance->e->next;
  2553. else
  2554. instance_prev->e->next = instance->e->next;
  2555. break;
  2556. }
  2557. instance_prev = instance;
  2558. instance = instance->e->next;
  2559. }
  2560. os_mutex_unlock(&module->instance_list_lock);
  2561. }
  2562. #endif
  2563. #if WASM_ENABLE_JIT != 0
  2564. if (module_inst->func_ptrs)
  2565. wasm_runtime_free(module_inst->func_ptrs);
  2566. #endif
  2567. #if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  2568. && WASM_ENABLE_LAZY_JIT != 0
  2569. if (module_inst->fast_jit_func_ptrs
  2570. && module_inst->fast_jit_func_ptrs
  2571. != module_inst->module->fast_jit_func_ptrs)
  2572. wasm_runtime_free(module_inst->fast_jit_func_ptrs);
  2573. #endif
  2574. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  2575. if (module_inst->func_type_indexes)
  2576. wasm_runtime_free(module_inst->func_type_indexes);
  2577. #endif
  2578. #if WASM_ENABLE_MULTI_MODULE != 0
  2579. wasm_runtime_sub_module_deinstantiate(
  2580. (WASMModuleInstanceCommon *)module_inst);
  2581. #endif
  2582. if (module_inst->memory_count > 0)
  2583. memories_deinstantiate(module_inst, module_inst->memories,
  2584. module_inst->memory_count);
  2585. if (module_inst->import_func_ptrs) {
  2586. wasm_runtime_free(module_inst->import_func_ptrs);
  2587. }
  2588. tables_deinstantiate(module_inst);
  2589. functions_deinstantiate(module_inst->e->functions);
  2590. #if WASM_ENABLE_TAGS != 0
  2591. tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs);
  2592. #endif
  2593. globals_deinstantiate(module_inst->e->globals);
  2594. export_functions_deinstantiate(module_inst->export_functions);
  2595. #if WASM_ENABLE_TAGS != 0
  2596. export_tags_deinstantiate(module_inst->e->export_tags);
  2597. #endif
  2598. #if WASM_ENABLE_MULTI_MODULE != 0
  2599. export_globals_deinstantiate(module_inst->export_globals);
  2600. #endif
  2601. #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
  2602. wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst);
  2603. #endif
  2604. #if WASM_ENABLE_GC != 0
  2605. if (!is_sub_inst) {
  2606. if (module_inst->e->common.gc_heap_handle)
  2607. mem_allocator_destroy(module_inst->e->common.gc_heap_handle);
  2608. if (module_inst->e->common.gc_heap_pool)
  2609. wasm_runtime_free(module_inst->e->common.gc_heap_pool);
  2610. }
  2611. #endif
  2612. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2613. if (module_inst->frames) {
  2614. destroy_c_api_frames(module_inst->frames);
  2615. wasm_runtime_free(module_inst->frames);
  2616. module_inst->frames = NULL;
  2617. }
  2618. #endif
  2619. if (module_inst->e->common.c_api_func_imports)
  2620. wasm_runtime_free(module_inst->e->common.c_api_func_imports);
  2621. if (!is_sub_inst) {
  2622. #if WASM_ENABLE_WASI_NN != 0
  2623. wasi_nn_destroy(module_inst);
  2624. #endif
  2625. wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst);
  2626. }
  2627. #if WASM_ENABLE_BULK_MEMORY != 0
  2628. bh_bitmap_delete(module_inst->e->common.data_dropped);
  2629. #endif
  2630. #if WASM_ENABLE_REF_TYPES != 0
  2631. bh_bitmap_delete(module_inst->e->common.elem_dropped);
  2632. #endif
  2633. wasm_runtime_free(module_inst);
  2634. }
  2635. WASMFunctionInstance *
  2636. wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name,
  2637. const char *signature)
  2638. {
  2639. uint32 i;
  2640. for (i = 0; i < module_inst->export_func_count; i++)
  2641. if (!strcmp(module_inst->export_functions[i].name, name))
  2642. return module_inst->export_functions[i].function;
  2643. (void)signature;
  2644. return NULL;
  2645. }
  2646. #if WASM_ENABLE_MULTI_MODULE != 0
  2647. WASMGlobalInstance *
  2648. wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name)
  2649. {
  2650. uint32 i;
  2651. for (i = 0; i < module_inst->export_global_count; i++)
  2652. if (!strcmp(module_inst->export_globals[i].name, name))
  2653. return module_inst->export_globals[i].global;
  2654. return NULL;
  2655. }
  2656. WASMMemoryInstance *
  2657. wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name)
  2658. {
  2659. /**
  2660. * using a strong assumption that one module instance only has
  2661. * one memory instance
  2662. */
  2663. (void)module_inst->export_memories;
  2664. return module_inst->memories[0];
  2665. }
  2666. WASMTableInstance *
  2667. wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name)
  2668. {
  2669. /**
  2670. * using a strong assumption that one module instance only has
  2671. * one table instance
  2672. */
  2673. (void)module_inst->export_tables;
  2674. return module_inst->tables[0];
  2675. }
  2676. #if WASM_ENABLE_TAGS != 0
  2677. WASMTagInstance *
  2678. wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name,
  2679. const char *signature)
  2680. {
  2681. uint32 i;
  2682. for (i = 0; i < module_inst->e->export_tag_count; i++)
  2683. if (!strcmp(module_inst->e->export_tags[i].name, name))
  2684. return module_inst->e->export_tags[i].tag;
  2685. (void)signature;
  2686. return NULL;
  2687. }
  2688. #endif
  2689. #endif
  2690. #ifdef OS_ENABLE_HW_BOUND_CHECK
  2691. static void
  2692. call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst,
  2693. WASMExecEnv *exec_env,
  2694. WASMFunctionInstance *function, unsigned argc,
  2695. uint32 argv[])
  2696. {
  2697. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  2698. WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop;
  2699. uint32 page_size = os_getpagesize();
  2700. uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
  2701. WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env);
  2702. uint8 *prev_top = exec_env->wasm_stack.top;
  2703. #ifdef BH_PLATFORM_WINDOWS
  2704. int result;
  2705. bool has_exception;
  2706. char exception[EXCEPTION_BUF_LEN];
  2707. #endif
  2708. bool ret = true;
  2709. /* Check native stack overflow firstly to ensure we have enough
  2710. native stack to run the following codes before actually calling
  2711. the aot function in invokeNative function. */
  2712. RECORD_STACK_USAGE(exec_env, (uint8 *)&exec_env_tls);
  2713. if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary
  2714. + page_size * (guard_page_count + 1)) {
  2715. wasm_set_exception(module_inst, "native stack overflow");
  2716. return;
  2717. }
  2718. if (!exec_env_tls) {
  2719. if (!os_thread_signal_inited()) {
  2720. wasm_set_exception(module_inst, "thread signal env not inited");
  2721. return;
  2722. }
  2723. /* Set thread handle and stack boundary if they haven't been set */
  2724. wasm_exec_env_set_thread_info(exec_env);
  2725. wasm_runtime_set_exec_env_tls(exec_env);
  2726. }
  2727. else {
  2728. if (exec_env_tls != exec_env) {
  2729. wasm_set_exception(module_inst, "invalid exec env");
  2730. return;
  2731. }
  2732. }
  2733. wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node);
  2734. if (os_setjmp(jmpbuf_node.jmpbuf) == 0) {
  2735. #ifndef BH_PLATFORM_WINDOWS
  2736. wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2737. #else
  2738. __try {
  2739. wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2740. } __except (wasm_copy_exception(module_inst, NULL)
  2741. ? EXCEPTION_EXECUTE_HANDLER
  2742. : EXCEPTION_CONTINUE_SEARCH) {
  2743. /* Exception was thrown in wasm_exception_handler */
  2744. ret = false;
  2745. }
  2746. has_exception = wasm_copy_exception(module_inst, exception);
  2747. if (has_exception && strstr(exception, "native stack overflow")) {
  2748. /* After a stack overflow, the stack was left
  2749. in a damaged state, let the CRT repair it */
  2750. result = _resetstkoflw();
  2751. bh_assert(result != 0);
  2752. }
  2753. #endif
  2754. }
  2755. else {
  2756. /* Exception has been set in signal handler before calling longjmp */
  2757. ret = false;
  2758. }
  2759. /* Note: can't check wasm_get_exception(module_inst) here, there may be
  2760. * exception which is not caught by hardware (e.g. uninitialized elements),
  2761. * then the stack-frame is already freed inside wasm_interp_call_wasm */
  2762. if (!ret) {
  2763. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2764. if (wasm_interp_create_call_stack(exec_env)) {
  2765. wasm_interp_dump_call_stack(exec_env, true, NULL, 0);
  2766. }
  2767. #endif
  2768. /* Restore operand frames */
  2769. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  2770. exec_env->wasm_stack.top = prev_top;
  2771. }
  2772. jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env);
  2773. bh_assert(&jmpbuf_node == jmpbuf_node_pop);
  2774. if (!exec_env->jmpbuf_stack_top) {
  2775. wasm_runtime_set_exec_env_tls(NULL);
  2776. }
  2777. if (!ret) {
  2778. os_sigreturn();
  2779. os_signal_unmask();
  2780. }
  2781. (void)jmpbuf_node_pop;
  2782. }
  2783. #define interp_call_wasm call_wasm_with_hw_bound_check
  2784. #else
  2785. #define interp_call_wasm wasm_interp_call_wasm
  2786. #endif
  2787. bool
  2788. wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function,
  2789. unsigned argc, uint32 argv[])
  2790. {
  2791. WASMModuleInstance *module_inst =
  2792. (WASMModuleInstance *)exec_env->module_inst;
  2793. #ifndef OS_ENABLE_HW_BOUND_CHECK
  2794. /* Set thread handle and stack boundary */
  2795. wasm_exec_env_set_thread_info(exec_env);
  2796. #else
  2797. /* Set thread info in call_wasm_with_hw_bound_check when
  2798. hw bound check is enabled */
  2799. #endif
  2800. /* Set exec env so it can be later retrieved from instance */
  2801. module_inst->e->common.cur_exec_env = exec_env;
  2802. interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2803. return !wasm_copy_exception(module_inst, NULL);
  2804. }
  2805. #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0
  2806. /* look for the function name */
  2807. static char *
  2808. get_func_name_from_index(const WASMModuleInstance *inst, uint32 func_index)
  2809. {
  2810. char *func_name = NULL;
  2811. WASMFunctionInstance *func_inst = inst->e->functions + func_index;
  2812. if (func_inst->is_import_func) {
  2813. func_name = func_inst->u.func_import->field_name;
  2814. }
  2815. else {
  2816. #if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
  2817. func_name = func_inst->u.func->field_name;
  2818. #endif
  2819. /* if custom name section is not generated,
  2820. search symbols from export table */
  2821. if (!func_name) {
  2822. unsigned j;
  2823. for (j = 0; j < inst->export_func_count; j++) {
  2824. WASMExportFuncInstance *export_func =
  2825. inst->export_functions + j;
  2826. if (export_func->function == func_inst) {
  2827. func_name = export_func->name;
  2828. break;
  2829. }
  2830. }
  2831. }
  2832. }
  2833. return func_name;
  2834. }
  2835. #endif /*WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0*/
  2836. #if WASM_ENABLE_PERF_PROFILING != 0
  2837. void
  2838. wasm_dump_perf_profiling(const WASMModuleInstance *module_inst)
  2839. {
  2840. WASMFunctionInstance *func_inst;
  2841. char *func_name;
  2842. uint32 i;
  2843. os_printf("Performance profiler data:\n");
  2844. for (i = 0; i < module_inst->e->function_count; i++) {
  2845. func_inst = module_inst->e->functions + i;
  2846. if (func_inst->total_exec_cnt == 0)
  2847. continue;
  2848. func_name = get_func_name_from_index(module_inst, i);
  2849. if (func_name)
  2850. os_printf(
  2851. " func %s, execution time: %.3f ms, execution count: %" PRIu32
  2852. " times, children execution time: %.3f ms\n",
  2853. func_name, func_inst->total_exec_time / 1000.0f,
  2854. func_inst->total_exec_cnt,
  2855. func_inst->children_exec_time / 1000.0f);
  2856. else
  2857. os_printf(" func %" PRIu32
  2858. ", execution time: %.3f ms, execution count: %" PRIu32
  2859. " times, children execution time: %.3f ms\n",
  2860. i, func_inst->total_exec_time / 1000.0f,
  2861. func_inst->total_exec_cnt,
  2862. func_inst->children_exec_time / 1000.0f);
  2863. }
  2864. }
  2865. double
  2866. wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst)
  2867. {
  2868. double ret = 0;
  2869. unsigned i;
  2870. for (i = 0; i < inst->e->function_count; i++) {
  2871. WASMFunctionInstance *func = inst->e->functions + i;
  2872. ret += (func->total_exec_time - func->children_exec_time) / 1000.0f;
  2873. }
  2874. return ret;
  2875. }
  2876. double
  2877. wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst,
  2878. const char *func_name)
  2879. {
  2880. unsigned i;
  2881. for (i = 0; i < inst->e->function_count; i++) {
  2882. char *name_in_wasm = get_func_name_from_index(inst, i);
  2883. if (name_in_wasm && strcmp(name_in_wasm, func_name) == 0) {
  2884. WASMFunctionInstance *func = inst->e->functions + i;
  2885. return (func->total_exec_time - func->children_exec_time) / 1000.0f;
  2886. }
  2887. }
  2888. return -1.0;
  2889. }
  2890. #endif /*WASM_ENABLE_PERF_PROFILING != 0*/
  2891. uint32
  2892. wasm_module_malloc_internal(WASMModuleInstance *module_inst,
  2893. WASMExecEnv *exec_env, uint32 size,
  2894. void **p_native_addr)
  2895. {
  2896. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2897. uint8 *addr = NULL;
  2898. uint32 offset = 0;
  2899. if (!memory) {
  2900. wasm_set_exception(module_inst, "uninitialized memory");
  2901. return 0;
  2902. }
  2903. if (memory->heap_handle) {
  2904. addr = mem_allocator_malloc(memory->heap_handle, size);
  2905. }
  2906. else if (module_inst->e->malloc_function && module_inst->e->free_function) {
  2907. if (!execute_malloc_function(
  2908. module_inst, exec_env, module_inst->e->malloc_function,
  2909. module_inst->e->retain_function, size, &offset)) {
  2910. return 0;
  2911. }
  2912. /* If we use app's malloc function,
  2913. the default memory may be changed while memory growing */
  2914. memory = wasm_get_default_memory(module_inst);
  2915. addr = offset ? memory->memory_data + offset : NULL;
  2916. }
  2917. if (!addr) {
  2918. if (memory->heap_handle
  2919. && mem_allocator_is_heap_corrupted(memory->heap_handle)) {
  2920. wasm_runtime_show_app_heap_corrupted_prompt();
  2921. wasm_set_exception(module_inst, "app heap corrupted");
  2922. }
  2923. else {
  2924. LOG_WARNING("warning: allocate %u bytes memory failed", size);
  2925. }
  2926. return 0;
  2927. }
  2928. if (p_native_addr)
  2929. *p_native_addr = addr;
  2930. return (uint32)(addr - memory->memory_data);
  2931. }
  2932. uint32
  2933. wasm_module_realloc_internal(WASMModuleInstance *module_inst,
  2934. WASMExecEnv *exec_env, uint32 ptr, uint32 size,
  2935. void **p_native_addr)
  2936. {
  2937. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2938. uint8 *addr = NULL;
  2939. if (!memory) {
  2940. wasm_set_exception(module_inst, "uninitialized memory");
  2941. return 0;
  2942. }
  2943. if (memory->heap_handle) {
  2944. addr = mem_allocator_realloc(
  2945. memory->heap_handle, ptr ? memory->memory_data + ptr : NULL, size);
  2946. }
  2947. /* Only support realloc in WAMR's app heap */
  2948. (void)exec_env;
  2949. if (!addr) {
  2950. if (memory->heap_handle
  2951. && mem_allocator_is_heap_corrupted(memory->heap_handle)) {
  2952. wasm_set_exception(module_inst, "app heap corrupted");
  2953. }
  2954. else {
  2955. wasm_set_exception(module_inst, "out of memory");
  2956. }
  2957. return 0;
  2958. }
  2959. if (p_native_addr)
  2960. *p_native_addr = addr;
  2961. return (uint32)(addr - memory->memory_data);
  2962. }
  2963. void
  2964. wasm_module_free_internal(WASMModuleInstance *module_inst,
  2965. WASMExecEnv *exec_env, uint32 ptr)
  2966. {
  2967. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2968. if (!memory) {
  2969. return;
  2970. }
  2971. if (ptr) {
  2972. uint8 *addr = memory->memory_data + ptr;
  2973. uint8 *memory_data_end;
  2974. /* memory->memory_data_end may be changed in memory grow */
  2975. SHARED_MEMORY_LOCK(memory);
  2976. memory_data_end = memory->memory_data_end;
  2977. SHARED_MEMORY_UNLOCK(memory);
  2978. if (memory->heap_handle && memory->heap_data <= addr
  2979. && addr < memory->heap_data_end) {
  2980. mem_allocator_free(memory->heap_handle, addr);
  2981. }
  2982. else if (module_inst->e->malloc_function
  2983. && module_inst->e->free_function && memory->memory_data <= addr
  2984. && addr < memory_data_end) {
  2985. execute_free_function(module_inst, exec_env,
  2986. module_inst->e->free_function, ptr);
  2987. }
  2988. }
  2989. }
  2990. uint32
  2991. wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size,
  2992. void **p_native_addr)
  2993. {
  2994. return wasm_module_malloc_internal(module_inst, NULL, size, p_native_addr);
  2995. }
  2996. uint32
  2997. wasm_module_realloc(WASMModuleInstance *module_inst, uint32 ptr, uint32 size,
  2998. void **p_native_addr)
  2999. {
  3000. return wasm_module_realloc_internal(module_inst, NULL, ptr, size,
  3001. p_native_addr);
  3002. }
  3003. void
  3004. wasm_module_free(WASMModuleInstance *module_inst, uint32 ptr)
  3005. {
  3006. wasm_module_free_internal(module_inst, NULL, ptr);
  3007. }
  3008. uint32
  3009. wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src,
  3010. uint32 size)
  3011. {
  3012. char *buffer;
  3013. uint32 buffer_offset =
  3014. wasm_module_malloc(module_inst, size, (void **)&buffer);
  3015. if (buffer_offset != 0) {
  3016. buffer = wasm_runtime_addr_app_to_native(
  3017. (WASMModuleInstanceCommon *)module_inst, buffer_offset);
  3018. bh_memcpy_s(buffer, size, src, size);
  3019. }
  3020. return buffer_offset;
  3021. }
  3022. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  3023. bool
  3024. wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx,
  3025. uint32 inc_size, table_elem_type_t init_val)
  3026. {
  3027. uint32 total_size, i;
  3028. table_elem_type_t *new_table_data_start;
  3029. WASMTableInstance *table_inst;
  3030. if (!inc_size) {
  3031. return true;
  3032. }
  3033. bh_assert(table_idx < module_inst->table_count);
  3034. table_inst = wasm_get_table_inst(module_inst, table_idx);
  3035. if (!table_inst) {
  3036. return false;
  3037. }
  3038. if (inc_size > UINT32_MAX - table_inst->cur_size) {
  3039. return false;
  3040. }
  3041. total_size = table_inst->cur_size + inc_size;
  3042. if (total_size > table_inst->max_size) {
  3043. return false;
  3044. }
  3045. /* fill in */
  3046. new_table_data_start = table_inst->elems + table_inst->cur_size;
  3047. for (i = 0; i < inc_size; ++i) {
  3048. new_table_data_start[i] = init_val;
  3049. }
  3050. table_inst->cur_size = total_size;
  3051. return true;
  3052. }
  3053. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  3054. static bool
  3055. call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 tbl_elem_idx,
  3056. uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx)
  3057. {
  3058. WASMModuleInstance *module_inst = NULL;
  3059. WASMTableInstance *table_inst = NULL;
  3060. table_elem_type_t tbl_elem_val = NULL_REF;
  3061. uint32 func_idx = 0;
  3062. WASMFunctionInstance *func_inst = NULL;
  3063. module_inst = (WASMModuleInstance *)exec_env->module_inst;
  3064. bh_assert(module_inst);
  3065. table_inst = module_inst->tables[tbl_idx];
  3066. if (!table_inst) {
  3067. wasm_set_exception(module_inst, "unknown table");
  3068. goto got_exception;
  3069. }
  3070. if (tbl_elem_idx >= table_inst->cur_size) {
  3071. wasm_set_exception(module_inst, "undefined element");
  3072. goto got_exception;
  3073. }
  3074. tbl_elem_val = ((table_elem_type_t *)table_inst->elems)[tbl_elem_idx];
  3075. if (tbl_elem_val == NULL_REF) {
  3076. wasm_set_exception(module_inst, "uninitialized element");
  3077. goto got_exception;
  3078. }
  3079. #if WASM_ENABLE_GC == 0
  3080. func_idx = tbl_elem_val;
  3081. #else
  3082. func_idx =
  3083. wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val);
  3084. #endif
  3085. /**
  3086. * we insist to call functions owned by the module itself
  3087. **/
  3088. if (func_idx >= module_inst->e->function_count) {
  3089. wasm_set_exception(module_inst, "unknown function");
  3090. goto got_exception;
  3091. }
  3092. func_inst = module_inst->e->functions + func_idx;
  3093. if (check_type_idx) {
  3094. WASMType *cur_type = module_inst->module->types[type_idx];
  3095. WASMType *cur_func_type;
  3096. if (func_inst->is_import_func)
  3097. cur_func_type = (WASMType *)func_inst->u.func_import->func_type;
  3098. else
  3099. cur_func_type = (WASMType *)func_inst->u.func->func_type;
  3100. if (cur_type != cur_func_type) {
  3101. wasm_set_exception(module_inst, "indirect call type mismatch");
  3102. goto got_exception;
  3103. }
  3104. }
  3105. interp_call_wasm(module_inst, exec_env, func_inst, argc, argv);
  3106. return !wasm_copy_exception(module_inst, NULL);
  3107. got_exception:
  3108. return false;
  3109. }
  3110. bool
  3111. wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3112. uint32 argc, uint32 argv[])
  3113. {
  3114. return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
  3115. }
  3116. #if WASM_ENABLE_THREAD_MGR != 0
  3117. bool
  3118. wasm_set_aux_stack(WASMExecEnv *exec_env, uint32 start_offset, uint32 size)
  3119. {
  3120. WASMModuleInstance *module_inst =
  3121. (WASMModuleInstance *)exec_env->module_inst;
  3122. uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index;
  3123. #if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
  3124. /* Check the aux stack space */
  3125. uint32 data_end = module_inst->module->aux_data_end;
  3126. uint32 stack_bottom = module_inst->module->aux_stack_bottom;
  3127. bool is_stack_before_data = stack_bottom < data_end ? true : false;
  3128. if ((is_stack_before_data && (size > start_offset))
  3129. || ((!is_stack_before_data) && (start_offset - data_end < size)))
  3130. return false;
  3131. #endif
  3132. if (stack_top_idx != (uint32)-1) {
  3133. /* The aux stack top is a wasm global,
  3134. set the initial value for the global */
  3135. uint8 *global_addr =
  3136. module_inst->global_data
  3137. + module_inst->e->globals[stack_top_idx].data_offset;
  3138. *(int32 *)global_addr = start_offset;
  3139. /* The aux stack boundary is a constant value,
  3140. set the value to exec_env */
  3141. exec_env->aux_stack_boundary.boundary = start_offset - size;
  3142. exec_env->aux_stack_bottom.bottom = start_offset;
  3143. return true;
  3144. }
  3145. return false;
  3146. }
  3147. bool
  3148. wasm_get_aux_stack(WASMExecEnv *exec_env, uint32 *start_offset, uint32 *size)
  3149. {
  3150. WASMModuleInstance *module_inst =
  3151. (WASMModuleInstance *)exec_env->module_inst;
  3152. /* The aux stack information is resolved in loader
  3153. and store in module */
  3154. uint32 stack_bottom = module_inst->module->aux_stack_bottom;
  3155. uint32 total_aux_stack_size = module_inst->module->aux_stack_size;
  3156. if (stack_bottom != 0 && total_aux_stack_size != 0) {
  3157. if (start_offset)
  3158. *start_offset = stack_bottom;
  3159. if (size)
  3160. *size = total_aux_stack_size;
  3161. return true;
  3162. }
  3163. return false;
  3164. }
  3165. #endif
  3166. #if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0)
  3167. void
  3168. wasm_get_module_mem_consumption(const WASMModule *module,
  3169. WASMModuleMemConsumption *mem_conspn)
  3170. {
  3171. uint32 i, size;
  3172. memset(mem_conspn, 0, sizeof(*mem_conspn));
  3173. mem_conspn->module_struct_size = sizeof(WASMModule);
  3174. mem_conspn->types_size = sizeof(WASMFuncType *) * module->type_count;
  3175. for (i = 0; i < module->type_count; i++) {
  3176. WASMFuncType *type = module->types[i];
  3177. size = offsetof(WASMFuncType, types)
  3178. + sizeof(uint8) * (type->param_count + type->result_count);
  3179. mem_conspn->types_size += size;
  3180. }
  3181. mem_conspn->imports_size = sizeof(WASMImport) * module->import_count;
  3182. mem_conspn->functions_size =
  3183. sizeof(WASMFunction *) * module->function_count;
  3184. for (i = 0; i < module->function_count; i++) {
  3185. WASMFunction *func = module->functions[i];
  3186. WASMFuncType *type = func->func_type;
  3187. size = sizeof(WASMFunction) + func->local_count
  3188. + sizeof(uint16) * (type->param_count + func->local_count);
  3189. #if WASM_ENABLE_FAST_INTERP != 0
  3190. size +=
  3191. func->code_compiled_size + sizeof(uint32) * func->const_cell_num;
  3192. #endif
  3193. mem_conspn->functions_size += size;
  3194. }
  3195. mem_conspn->tables_size = sizeof(WASMTable) * module->table_count;
  3196. mem_conspn->memories_size = sizeof(WASMMemory) * module->memory_count;
  3197. mem_conspn->globals_size = sizeof(WASMGlobal) * module->global_count;
  3198. mem_conspn->exports_size = sizeof(WASMExport) * module->export_count;
  3199. mem_conspn->table_segs_size =
  3200. sizeof(WASMTableSeg) * module->table_seg_count;
  3201. for (i = 0; i < module->table_seg_count; i++) {
  3202. WASMTableSeg *table_seg = &module->table_segments[i];
  3203. mem_conspn->tables_size +=
  3204. sizeof(InitializerExpression *) * table_seg->value_count;
  3205. }
  3206. mem_conspn->data_segs_size = sizeof(WASMDataSeg *) * module->data_seg_count;
  3207. for (i = 0; i < module->data_seg_count; i++) {
  3208. mem_conspn->data_segs_size += sizeof(WASMDataSeg);
  3209. }
  3210. if (module->const_str_list) {
  3211. StringNode *node = module->const_str_list, *node_next;
  3212. while (node) {
  3213. node_next = node->next;
  3214. mem_conspn->const_strs_size +=
  3215. sizeof(StringNode) + strlen(node->str) + 1;
  3216. node = node_next;
  3217. }
  3218. }
  3219. mem_conspn->total_size += mem_conspn->module_struct_size;
  3220. mem_conspn->total_size += mem_conspn->types_size;
  3221. mem_conspn->total_size += mem_conspn->imports_size;
  3222. mem_conspn->total_size += mem_conspn->functions_size;
  3223. mem_conspn->total_size += mem_conspn->tables_size;
  3224. mem_conspn->total_size += mem_conspn->memories_size;
  3225. mem_conspn->total_size += mem_conspn->globals_size;
  3226. mem_conspn->total_size += mem_conspn->exports_size;
  3227. mem_conspn->total_size += mem_conspn->table_segs_size;
  3228. mem_conspn->total_size += mem_conspn->data_segs_size;
  3229. mem_conspn->total_size += mem_conspn->const_strs_size;
  3230. }
  3231. void
  3232. wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst,
  3233. WASMModuleInstMemConsumption *mem_conspn)
  3234. {
  3235. uint32 i, size;
  3236. memset(mem_conspn, 0, sizeof(*mem_conspn));
  3237. mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e
  3238. - (uint8 *)module_inst
  3239. + sizeof(WASMModuleInstanceExtra);
  3240. mem_conspn->memories_size =
  3241. sizeof(WASMMemoryInstance *) * module_inst->memory_count;
  3242. for (i = 0; i < module_inst->memory_count; i++) {
  3243. WASMMemoryInstance *memory = module_inst->memories[i];
  3244. size = memory->num_bytes_per_page * memory->cur_page_count;
  3245. mem_conspn->memories_size += size;
  3246. mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data;
  3247. /* size of app heap structure */
  3248. mem_conspn->memories_size += mem_allocator_get_heap_struct_size();
  3249. /* Module instance structures have been appened into the end of
  3250. module instance */
  3251. }
  3252. mem_conspn->tables_size =
  3253. sizeof(WASMTableInstance *) * module_inst->table_count;
  3254. /* Table instance structures and table elements have been appened into
  3255. the end of module instance */
  3256. mem_conspn->functions_size =
  3257. sizeof(WASMFunctionInstance) * module_inst->e->function_count;
  3258. mem_conspn->globals_size =
  3259. sizeof(WASMGlobalInstance) * module_inst->e->global_count;
  3260. /* Global data has been appened into the end of module instance */
  3261. mem_conspn->exports_size =
  3262. sizeof(WASMExportFuncInstance) * module_inst->export_func_count;
  3263. mem_conspn->total_size += mem_conspn->module_inst_struct_size;
  3264. mem_conspn->total_size += mem_conspn->memories_size;
  3265. mem_conspn->total_size += mem_conspn->functions_size;
  3266. mem_conspn->total_size += mem_conspn->tables_size;
  3267. mem_conspn->total_size += mem_conspn->globals_size;
  3268. mem_conspn->total_size += mem_conspn->exports_size;
  3269. }
  3270. #endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \
  3271. || (WASM_ENABLE_MEMORY_TRACING != 0) */
  3272. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  3273. bool
  3274. wasm_interp_create_call_stack(struct WASMExecEnv *exec_env)
  3275. {
  3276. WASMModuleInstance *module_inst =
  3277. (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env);
  3278. WASMModule *module = module_inst->module;
  3279. WASMInterpFrame *first_frame,
  3280. *cur_frame = wasm_exec_env_get_cur_frame(exec_env);
  3281. uint32 n = 0;
  3282. /* count frames includes a function */
  3283. first_frame = cur_frame;
  3284. while (cur_frame) {
  3285. if (cur_frame->function) {
  3286. n++;
  3287. }
  3288. cur_frame = cur_frame->prev_frame;
  3289. }
  3290. /* release previous stack frames and create new ones */
  3291. destroy_c_api_frames(module_inst->frames);
  3292. if (!bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) {
  3293. return false;
  3294. }
  3295. cur_frame = first_frame;
  3296. n = 0;
  3297. while (cur_frame) {
  3298. WASMCApiFrame frame = { 0 };
  3299. WASMFunctionInstance *func_inst = cur_frame->function;
  3300. const char *func_name = NULL;
  3301. const uint8 *func_code_base = NULL;
  3302. uint32 max_local_cell_num, max_stack_cell_num;
  3303. uint32 all_cell_num, lp_size;
  3304. if (!func_inst) {
  3305. cur_frame = cur_frame->prev_frame;
  3306. continue;
  3307. }
  3308. /* place holder, will overwrite it in wasm_c_api */
  3309. frame.instance = module_inst;
  3310. frame.module_offset = 0;
  3311. frame.func_index = (uint32)(func_inst - module_inst->e->functions);
  3312. func_code_base = wasm_get_func_code(func_inst);
  3313. if (!cur_frame->ip || !func_code_base) {
  3314. frame.func_offset = 0;
  3315. }
  3316. else {
  3317. #if WASM_ENABLE_FAST_INTERP == 0
  3318. frame.func_offset = (uint32)(cur_frame->ip - module->load_addr);
  3319. #else
  3320. frame.func_offset = (uint32)(cur_frame->ip - func_code_base);
  3321. #endif
  3322. }
  3323. func_name = get_func_name_from_index(module_inst, frame.func_index);
  3324. frame.func_name_wp = func_name;
  3325. if (frame.func_index >= module->import_function_count) {
  3326. uint32 wasm_func_idx =
  3327. frame.func_index - module->import_function_count;
  3328. max_local_cell_num =
  3329. module->functions[wasm_func_idx]->param_cell_num
  3330. + module->functions[wasm_func_idx]->local_cell_num;
  3331. max_stack_cell_num =
  3332. module->functions[wasm_func_idx]->max_stack_cell_num;
  3333. all_cell_num = max_local_cell_num + max_stack_cell_num;
  3334. #if WASM_ENABLE_FAST_INTERP != 0
  3335. all_cell_num += module->functions[wasm_func_idx]->const_cell_num;
  3336. #endif
  3337. }
  3338. else {
  3339. WASMFuncType *func_type =
  3340. module->import_functions[frame.func_index].u.function.func_type;
  3341. max_local_cell_num =
  3342. func_type->param_cell_num > 2 ? func_type->param_cell_num : 2;
  3343. max_stack_cell_num = 0;
  3344. all_cell_num = max_local_cell_num + max_stack_cell_num;
  3345. }
  3346. #if WASM_ENABLE_GC == 0
  3347. lp_size = all_cell_num * 4;
  3348. #else
  3349. lp_size = align_uint(all_cell_num * 5, 4);
  3350. #endif
  3351. if (lp_size > 0) {
  3352. if (!(frame.lp = wasm_runtime_malloc(lp_size))) {
  3353. destroy_c_api_frames(module_inst->frames);
  3354. return false;
  3355. }
  3356. bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size);
  3357. #if WASM_ENABLE_GC != 0
  3358. #if WASM_ENABLE_FAST_INTERP == 0
  3359. frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp);
  3360. #else
  3361. /* for fast-interp, let frame sp point to the end of the frame */
  3362. frame.sp = frame.lp + all_cell_num;
  3363. #endif
  3364. frame.frame_ref = (uint8 *)frame.lp
  3365. + (wasm_interp_get_frame_ref(cur_frame)
  3366. - (uint8 *)cur_frame->lp);
  3367. #endif
  3368. }
  3369. if (!bh_vector_append(module_inst->frames, &frame)) {
  3370. if (frame.lp)
  3371. wasm_runtime_free(frame.lp);
  3372. destroy_c_api_frames(module_inst->frames);
  3373. return false;
  3374. }
  3375. cur_frame = cur_frame->prev_frame;
  3376. n++;
  3377. }
  3378. return true;
  3379. }
  3380. #define PRINT_OR_DUMP() \
  3381. do { \
  3382. total_len += \
  3383. wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \
  3384. if ((!print) && buf && (len == 0)) { \
  3385. exception_unlock(module_inst); \
  3386. return total_len; \
  3387. } \
  3388. } while (0)
  3389. uint32
  3390. wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf,
  3391. uint32 len)
  3392. {
  3393. WASMModuleInstance *module_inst =
  3394. (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env);
  3395. uint32 n = 0, total_len = 0, total_frames;
  3396. /* reserve 256 bytes for line buffer, any line longer than 256 bytes
  3397. * will be truncated */
  3398. char line_buf[256];
  3399. if (!module_inst->frames) {
  3400. return 0;
  3401. }
  3402. total_frames = (uint32)bh_vector_size(module_inst->frames);
  3403. if (total_frames == 0) {
  3404. return 0;
  3405. }
  3406. exception_lock(module_inst);
  3407. snprintf(line_buf, sizeof(line_buf), "\n");
  3408. PRINT_OR_DUMP();
  3409. while (n < total_frames) {
  3410. WASMCApiFrame frame = { 0 };
  3411. uint32 line_length, i;
  3412. if (!bh_vector_get(module_inst->frames, n, &frame)) {
  3413. exception_unlock(module_inst);
  3414. return 0;
  3415. }
  3416. #if WASM_ENABLE_FAST_JIT != 0
  3417. /* Fast JIT doesn't support committing ip (instruction pointer) yet */
  3418. if (module_inst->e->running_mode == Mode_Fast_JIT
  3419. || module_inst->e->running_mode == Mode_Multi_Tier_JIT) {
  3420. /* function name not exported, print number instead */
  3421. if (frame.func_name_wp == NULL) {
  3422. line_length = snprintf(line_buf, sizeof(line_buf),
  3423. "#%02" PRIu32 " $f%" PRIu32 "\n", n,
  3424. frame.func_index);
  3425. }
  3426. else {
  3427. line_length =
  3428. snprintf(line_buf, sizeof(line_buf), "#%02" PRIu32 " %s\n",
  3429. n, frame.func_name_wp);
  3430. }
  3431. }
  3432. else
  3433. #endif
  3434. {
  3435. /* function name not exported, print number instead */
  3436. if (frame.func_name_wp == NULL) {
  3437. line_length =
  3438. snprintf(line_buf, sizeof(line_buf),
  3439. "#%02" PRIu32 ": 0x%04x - $f%" PRIu32 "\n", n,
  3440. frame.func_offset, frame.func_index);
  3441. }
  3442. else {
  3443. line_length = snprintf(line_buf, sizeof(line_buf),
  3444. "#%02" PRIu32 ": 0x%04x - %s\n", n,
  3445. frame.func_offset, frame.func_name_wp);
  3446. }
  3447. }
  3448. if (line_length >= sizeof(line_buf)) {
  3449. uint32 line_buffer_len = sizeof(line_buf);
  3450. /* If line too long, ensure the last character is '\n' */
  3451. for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) {
  3452. line_buf[i] = '.';
  3453. }
  3454. line_buf[line_buffer_len - 2] = '\n';
  3455. }
  3456. PRINT_OR_DUMP();
  3457. n++;
  3458. }
  3459. snprintf(line_buf, sizeof(line_buf), "\n");
  3460. PRINT_OR_DUMP();
  3461. exception_unlock(module_inst);
  3462. return total_len + 1;
  3463. }
  3464. #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */
  3465. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
  3466. || WASM_ENABLE_WAMR_COMPILER != 0
  3467. void
  3468. jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id)
  3469. {
  3470. if (id != EXCE_ALREADY_THROWN)
  3471. wasm_set_exception_with_id(module_inst, id);
  3472. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3473. wasm_runtime_access_exce_check_guard_page();
  3474. #endif
  3475. }
  3476. bool
  3477. jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
  3478. uint32 app_buf_addr, uint32 app_buf_size,
  3479. void **p_native_addr)
  3480. {
  3481. bool ret = wasm_check_app_addr_and_convert(
  3482. module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr);
  3483. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3484. if (!ret)
  3485. wasm_runtime_access_exce_check_guard_page();
  3486. #endif
  3487. return ret;
  3488. }
  3489. #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
  3490. || WASM_ENABLE_WAMR_COMPILER != 0 */
  3491. #if WASM_ENABLE_FAST_JIT != 0
  3492. bool
  3493. fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3494. uint32 type_idx, uint32 argc, uint32 *argv)
  3495. {
  3496. return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true,
  3497. type_idx);
  3498. }
  3499. #endif /* end of WASM_ENABLE_FAST_JIT != 0 */
  3500. #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
  3501. bool
  3502. llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3503. uint32 argc, uint32 *argv)
  3504. {
  3505. bool ret;
  3506. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  3507. ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
  3508. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3509. if (!ret)
  3510. wasm_runtime_access_exce_check_guard_page();
  3511. #endif
  3512. return ret;
  3513. }
  3514. bool
  3515. llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
  3516. uint32 *argv)
  3517. {
  3518. WASMModuleInstance *module_inst;
  3519. WASMModule *module;
  3520. uint32 *func_type_indexes;
  3521. uint32 func_type_idx;
  3522. WASMFuncType *func_type;
  3523. void *func_ptr;
  3524. WASMFunctionImport *import_func;
  3525. CApiFuncImport *c_api_func_import = NULL;
  3526. const char *signature;
  3527. void *attachment;
  3528. char buf[96];
  3529. bool ret = false;
  3530. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  3531. module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env);
  3532. module = module_inst->module;
  3533. func_type_indexes = module_inst->func_type_indexes;
  3534. func_type_idx = func_type_indexes[func_idx];
  3535. func_type = (WASMFuncType *)module->types[func_type_idx];
  3536. func_ptr = module_inst->func_ptrs[func_idx];
  3537. bh_assert(func_idx < module->import_function_count);
  3538. import_func = &module->import_functions[func_idx].u.function;
  3539. if (import_func->call_conv_wasm_c_api) {
  3540. if (module_inst->e->common.c_api_func_imports) {
  3541. c_api_func_import =
  3542. module_inst->e->common.c_api_func_imports + func_idx;
  3543. func_ptr = c_api_func_import->func_ptr_linked;
  3544. }
  3545. else {
  3546. c_api_func_import = NULL;
  3547. func_ptr = NULL;
  3548. }
  3549. }
  3550. if (!func_ptr) {
  3551. snprintf(buf, sizeof(buf),
  3552. "failed to call unlinked import function (%s, %s)",
  3553. import_func->module_name, import_func->field_name);
  3554. wasm_set_exception(module_inst, buf);
  3555. goto fail;
  3556. }
  3557. attachment = import_func->attachment;
  3558. if (import_func->call_conv_wasm_c_api) {
  3559. ret = wasm_runtime_invoke_c_api_native(
  3560. (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
  3561. argv, c_api_func_import->with_env_arg, c_api_func_import->env_arg);
  3562. }
  3563. else if (!import_func->call_conv_raw) {
  3564. signature = import_func->signature;
  3565. ret =
  3566. wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
  3567. attachment, argv, argc, argv);
  3568. }
  3569. else {
  3570. signature = import_func->signature;
  3571. ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
  3572. signature, attachment, argv, argc,
  3573. argv);
  3574. }
  3575. fail:
  3576. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3577. if (!ret)
  3578. wasm_runtime_access_exce_check_guard_page();
  3579. #endif
  3580. return ret;
  3581. }
  3582. #if WASM_ENABLE_BULK_MEMORY != 0
  3583. bool
  3584. llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index,
  3585. uint32 offset, uint32 len, uint32 dst)
  3586. {
  3587. WASMMemoryInstance *memory_inst;
  3588. WASMModule *module;
  3589. uint8 *data;
  3590. uint8 *maddr;
  3591. uint64 seg_len;
  3592. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3593. memory_inst = wasm_get_default_memory(module_inst);
  3594. if (bh_bitmap_get_bit(module_inst->e->common.data_dropped, seg_index)) {
  3595. seg_len = 0;
  3596. data = NULL;
  3597. }
  3598. else {
  3599. module = module_inst->module;
  3600. seg_len = module->data_segments[seg_index]->data_length;
  3601. data = module->data_segments[seg_index]->data;
  3602. }
  3603. if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst,
  3604. dst, len))
  3605. return false;
  3606. if ((uint64)offset + (uint64)len > seg_len) {
  3607. wasm_set_exception(module_inst, "out of bounds memory access");
  3608. return false;
  3609. }
  3610. maddr = wasm_runtime_addr_app_to_native(
  3611. (WASMModuleInstanceCommon *)module_inst, dst);
  3612. SHARED_MEMORY_LOCK(memory_inst);
  3613. bh_memcpy_s(maddr, memory_inst->memory_data_size - dst, data + offset, len);
  3614. SHARED_MEMORY_UNLOCK(memory_inst);
  3615. return true;
  3616. }
  3617. bool
  3618. llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index)
  3619. {
  3620. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3621. bh_bitmap_set_bit(module_inst->e->common.data_dropped, seg_index);
  3622. /* Currently we can't free the dropped data segment
  3623. as they are stored in wasm bytecode */
  3624. return true;
  3625. }
  3626. #endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */
  3627. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  3628. void
  3629. llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx)
  3630. {
  3631. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3632. bh_bitmap_set_bit(module_inst->e->common.elem_dropped, tbl_seg_idx);
  3633. }
  3634. void
  3635. llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3636. uint32 tbl_seg_idx, uint32 length, uint32 src_offset,
  3637. uint32 dst_offset)
  3638. {
  3639. WASMTableInstance *tbl_inst;
  3640. WASMTableSeg *tbl_seg;
  3641. table_elem_type_t *table_elems;
  3642. InitializerExpression *tbl_seg_init_values = NULL, *init_values;
  3643. uint32 i, tbl_seg_len = 0;
  3644. #if WASM_ENABLE_GC != 0
  3645. void *func_obj;
  3646. #endif
  3647. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3648. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3649. tbl_seg = module_inst->module->table_segments + tbl_seg_idx;
  3650. bh_assert(tbl_inst);
  3651. bh_assert(tbl_seg);
  3652. if (!bh_bitmap_get_bit(module_inst->e->common.elem_dropped, tbl_seg_idx)) {
  3653. /* table segment isn't dropped */
  3654. tbl_seg_init_values = tbl_seg->init_values;
  3655. tbl_seg_len = tbl_seg->value_count;
  3656. }
  3657. if (offset_len_out_of_bounds(src_offset, length, tbl_seg_len)
  3658. || offset_len_out_of_bounds(dst_offset, length, tbl_inst->cur_size)) {
  3659. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3660. return;
  3661. }
  3662. if (!length) {
  3663. return;
  3664. }
  3665. table_elems = tbl_inst->elems + dst_offset;
  3666. init_values = tbl_seg_init_values + src_offset;
  3667. for (i = 0; i < length; i++) {
  3668. #if WASM_ENABLE_GC != 0
  3669. /* UINT32_MAX indicates that it is a null ref */
  3670. if (init_values[i].u.ref_index != UINT32_MAX) {
  3671. if (!(func_obj = wasm_create_func_obj(module_inst,
  3672. init_values[i].u.ref_index,
  3673. true, NULL, 0))) {
  3674. wasm_set_exception(module_inst, "null function object");
  3675. return;
  3676. }
  3677. table_elems[i] = func_obj;
  3678. }
  3679. else {
  3680. table_elems[i] = NULL_REF;
  3681. }
  3682. #else
  3683. table_elems[i] = init_values[i].u.ref_index;
  3684. #endif
  3685. }
  3686. }
  3687. void
  3688. llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx,
  3689. uint32 dst_tbl_idx, uint32 length, uint32 src_offset,
  3690. uint32 dst_offset)
  3691. {
  3692. WASMTableInstance *src_tbl_inst;
  3693. WASMTableInstance *dst_tbl_inst;
  3694. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3695. src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx);
  3696. dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx);
  3697. bh_assert(src_tbl_inst);
  3698. bh_assert(dst_tbl_inst);
  3699. if (offset_len_out_of_bounds(dst_offset, length, dst_tbl_inst->cur_size)
  3700. || offset_len_out_of_bounds(src_offset, length,
  3701. src_tbl_inst->cur_size)) {
  3702. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3703. return;
  3704. }
  3705. /* if src_offset >= dst_offset, copy from front to back */
  3706. /* if src_offset < dst_offset, copy from back to front */
  3707. /* merge all together */
  3708. bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems)
  3709. + sizeof(table_elem_type_t) * dst_offset,
  3710. (uint32)sizeof(table_elem_type_t)
  3711. * (dst_tbl_inst->cur_size - dst_offset),
  3712. (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems)
  3713. + sizeof(table_elem_type_t) * src_offset,
  3714. (uint32)sizeof(table_elem_type_t) * length);
  3715. }
  3716. void
  3717. llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3718. uint32 length, uintptr_t val, uint32 data_offset)
  3719. {
  3720. WASMTableInstance *tbl_inst;
  3721. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3722. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3723. bh_assert(tbl_inst);
  3724. if (offset_len_out_of_bounds(data_offset, length, tbl_inst->cur_size)) {
  3725. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3726. return;
  3727. }
  3728. for (; length != 0; data_offset++, length--) {
  3729. tbl_inst->elems[data_offset] = (table_elem_type_t)val;
  3730. }
  3731. }
  3732. uint32
  3733. llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3734. uint32 inc_size, uintptr_t init_val)
  3735. {
  3736. WASMTableInstance *tbl_inst;
  3737. uint32 i, orig_size, total_size;
  3738. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3739. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3740. if (!tbl_inst) {
  3741. return (uint32)-1;
  3742. }
  3743. orig_size = tbl_inst->cur_size;
  3744. if (!inc_size) {
  3745. return orig_size;
  3746. }
  3747. if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */
  3748. return (uint32)-1;
  3749. }
  3750. total_size = tbl_inst->cur_size + inc_size;
  3751. if (total_size > tbl_inst->max_size) {
  3752. return (uint32)-1;
  3753. }
  3754. /* fill in */
  3755. for (i = 0; i < inc_size; ++i) {
  3756. tbl_inst->elems[tbl_inst->cur_size + i] = (table_elem_type_t)init_val;
  3757. }
  3758. tbl_inst->cur_size = total_size;
  3759. return orig_size;
  3760. }
  3761. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  3762. #if WASM_ENABLE_GC != 0
  3763. void *
  3764. llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
  3765. bool throw_exce, char *error_buf,
  3766. uint32 error_buf_size)
  3767. {
  3768. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3769. return wasm_create_func_obj(module_inst, func_idx, throw_exce, error_buf,
  3770. error_buf_size);
  3771. }
  3772. bool
  3773. llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst,
  3774. WASMObjectRef gc_obj, uint32 type_index)
  3775. {
  3776. WASMModule *module = module_inst->module;
  3777. WASMType **types = module->types;
  3778. uint32 type_count = module->type_count;
  3779. return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count);
  3780. }
  3781. WASMRttTypeRef
  3782. llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index)
  3783. {
  3784. WASMModule *module = module_inst->module;
  3785. WASMType *defined_type = module->types[type_index];
  3786. WASMRttType **rtt_types = module->rtt_types;
  3787. uint32 rtt_type_count = module->type_count;
  3788. korp_mutex *rtt_type_lock = &module->rtt_type_lock;
  3789. return wasm_rtt_type_new(defined_type, type_index, rtt_types,
  3790. rtt_type_count, rtt_type_lock);
  3791. }
  3792. bool
  3793. llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index,
  3794. uint32 data_seg_offset, WASMArrayObjectRef array_obj,
  3795. uint32 elem_size, uint32 array_len)
  3796. {
  3797. WASMModule *wasm_module = module_inst->module;
  3798. WASMDataSeg *data_seg;
  3799. uint8 *array_elem_base;
  3800. uint64 total_size;
  3801. data_seg = wasm_module->data_segments[seg_index];
  3802. total_size = (int64)elem_size * array_len;
  3803. if (data_seg_offset >= data_seg->data_length
  3804. || total_size > data_seg->data_length - data_seg_offset) {
  3805. wasm_set_exception(module_inst, "out of bounds memory access");
  3806. return false;
  3807. }
  3808. array_elem_base = (uint8 *)wasm_array_obj_first_elem_addr(array_obj);
  3809. bh_memcpy_s(array_elem_base, (uint32)total_size,
  3810. data_seg->data + data_seg_offset, (uint32)total_size);
  3811. return true;
  3812. }
  3813. #endif /* end of WASM_ENABLE_GC != 0 */
  3814. #endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */
  3815. #if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0
  3816. void
  3817. wasm_propagate_wasi_args(WASMModule *module)
  3818. {
  3819. if (!module->import_count)
  3820. return;
  3821. bh_assert(&module->import_module_list_head);
  3822. WASMRegisteredModule *node =
  3823. bh_list_first_elem(&module->import_module_list_head);
  3824. while (node) {
  3825. WASIArguments *wasi_args_impt_mod =
  3826. &((WASMModule *)(node->module))->wasi_args;
  3827. bh_assert(wasi_args_impt_mod);
  3828. bh_memcpy_s(wasi_args_impt_mod, sizeof(WASIArguments),
  3829. &module->wasi_args, sizeof(WASIArguments));
  3830. node = bh_list_elem_next(node);
  3831. }
  3832. }
  3833. #endif