wasm_runtime.c 153 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553
  1. /*
  2. * Copyright (C) 2019 Intel Corporation. All rights reserved.
  3. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  4. */
  5. #include "wasm_runtime.h"
  6. #include "wasm.h"
  7. #include "wasm_loader.h"
  8. #include "wasm_interp.h"
  9. #include "bh_common.h"
  10. #include "bh_log.h"
  11. #include "mem_alloc.h"
  12. #include "../common/wasm_runtime_common.h"
  13. #include "../common/wasm_memory.h"
  14. #if WASM_ENABLE_GC != 0
  15. #include "../common/gc/gc_object.h"
  16. #endif
  17. #if WASM_ENABLE_SHARED_MEMORY != 0
  18. #include "../common/wasm_shared_memory.h"
  19. #endif
  20. #if WASM_ENABLE_THREAD_MGR != 0
  21. #include "../libraries/thread-mgr/thread_manager.h"
  22. #endif
  23. #if WASM_ENABLE_DEBUG_INTERP != 0
  24. #include "../libraries/debug-engine/debug_engine.h"
  25. #endif
  26. #if WASM_ENABLE_FAST_JIT != 0
  27. #include "../fast-jit/jit_compiler.h"
  28. #endif
  29. #if WASM_ENABLE_JIT != 0
  30. #include "../aot/aot_runtime.h"
  31. #endif
  32. static void
  33. set_error_buf(char *error_buf, uint32 error_buf_size, const char *string)
  34. {
  35. if (error_buf != NULL) {
  36. snprintf(error_buf, error_buf_size,
  37. "WASM module instantiate failed: %s", string);
  38. }
  39. }
  40. static void
  41. set_error_buf_v(char *error_buf, uint32 error_buf_size, const char *format, ...)
  42. {
  43. va_list args;
  44. char buf[128];
  45. if (error_buf != NULL) {
  46. va_start(args, format);
  47. vsnprintf(buf, sizeof(buf), format, args);
  48. va_end(args);
  49. snprintf(error_buf, error_buf_size,
  50. "WASM module instantiate failed: %s", buf);
  51. }
  52. }
  53. WASMModule *
  54. wasm_load(uint8 *buf, uint32 size,
  55. #if WASM_ENABLE_MULTI_MODULE != 0
  56. bool main_module,
  57. #endif
  58. char *error_buf, uint32 error_buf_size)
  59. {
  60. return wasm_loader_load(buf, size,
  61. #if WASM_ENABLE_MULTI_MODULE != 0
  62. main_module,
  63. #endif
  64. error_buf, error_buf_size);
  65. }
  66. WASMModule *
  67. wasm_load_from_sections(WASMSection *section_list, char *error_buf,
  68. uint32 error_buf_size)
  69. {
  70. return wasm_loader_load_from_sections(section_list, error_buf,
  71. error_buf_size);
  72. }
  73. void
  74. wasm_unload(WASMModule *module)
  75. {
  76. wasm_loader_unload(module);
  77. }
  78. static void *
  79. runtime_malloc(uint64 size, char *error_buf, uint32 error_buf_size)
  80. {
  81. void *mem;
  82. if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
  83. set_error_buf(error_buf, error_buf_size, "allocate memory failed");
  84. return NULL;
  85. }
  86. memset(mem, 0, (uint32)size);
  87. return mem;
  88. }
  89. #if WASM_ENABLE_MULTI_MODULE != 0
  90. static WASMModuleInstance *
  91. get_sub_module_inst(const WASMModuleInstance *parent_module_inst,
  92. const WASMModule *sub_module)
  93. {
  94. bh_list *sub_module_inst_list = parent_module_inst->e->sub_module_inst_list;
  95. WASMSubModInstNode *node = bh_list_first_elem(sub_module_inst_list);
  96. while (node && sub_module != node->module_inst->module) {
  97. node = bh_list_elem_next(node);
  98. }
  99. return node ? node->module_inst : NULL;
  100. }
  101. #endif
  102. /**
  103. * Destroy memory instances.
  104. */
  105. static void
  106. memories_deinstantiate(WASMModuleInstance *module_inst,
  107. WASMMemoryInstance **memories, uint32 count)
  108. {
  109. uint32 i;
  110. if (memories) {
  111. for (i = 0; i < count; i++) {
  112. if (memories[i]) {
  113. #if WASM_ENABLE_MULTI_MODULE != 0
  114. WASMModule *module = module_inst->module;
  115. if (i < module->import_memory_count
  116. && module->import_memories[i].u.memory.import_module) {
  117. continue;
  118. }
  119. #endif
  120. #if WASM_ENABLE_SHARED_MEMORY != 0
  121. if (shared_memory_is_shared(memories[i])) {
  122. uint32 ref_count = shared_memory_dec_reference(memories[i]);
  123. /* if the reference count is not zero,
  124. don't free the memory */
  125. if (ref_count > 0)
  126. continue;
  127. }
  128. #endif
  129. if (memories[i]->heap_handle) {
  130. mem_allocator_destroy(memories[i]->heap_handle);
  131. wasm_runtime_free(memories[i]->heap_handle);
  132. memories[i]->heap_handle = NULL;
  133. }
  134. if (memories[i]->memory_data) {
  135. wasm_deallocate_linear_memory(memories[i]);
  136. }
  137. }
  138. }
  139. wasm_runtime_free(memories);
  140. }
  141. (void)module_inst;
  142. }
  143. static WASMMemoryInstance *
  144. memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent,
  145. WASMMemoryInstance *memory, uint32 memory_idx,
  146. uint32 num_bytes_per_page, uint32 init_page_count,
  147. uint32 max_page_count, uint32 heap_size, uint32 flags,
  148. char *error_buf, uint32 error_buf_size)
  149. {
  150. WASMModule *module = module_inst->module;
  151. uint32 inc_page_count, global_idx;
  152. uint32 bytes_of_last_page, bytes_to_page_end;
  153. uint64 aux_heap_base,
  154. heap_offset = (uint64)num_bytes_per_page * init_page_count;
  155. uint64 memory_data_size, max_memory_data_size;
  156. uint8 *global_addr;
  157. bool is_shared_memory = false;
  158. #if WASM_ENABLE_SHARED_MEMORY != 0
  159. is_shared_memory = flags & 0x02 ? true : false;
  160. /* shared memory */
  161. if (is_shared_memory && parent != NULL) {
  162. bh_assert(parent->memory_count > memory_idx);
  163. memory = parent->memories[memory_idx];
  164. shared_memory_inc_reference(memory);
  165. return memory;
  166. }
  167. #else
  168. (void)parent;
  169. (void)memory_idx;
  170. (void)flags;
  171. #endif /* end of WASM_ENABLE_SHARED_MEMORY */
  172. if (heap_size > 0 && module_inst->module->malloc_function != (uint32)-1
  173. && module_inst->module->free_function != (uint32)-1) {
  174. /* Disable app heap, use malloc/free function exported
  175. by wasm app to allocate/free memory instead */
  176. heap_size = 0;
  177. }
  178. /* If initial memory is the largest size allowed, disallowing insert host
  179. * managed heap */
  180. if (heap_size > 0 && heap_offset == MAX_LINEAR_MEMORY_SIZE) {
  181. set_error_buf(error_buf, error_buf_size,
  182. "failed to insert app heap into linear memory, "
  183. "try using `--heap-size=0` option");
  184. return NULL;
  185. }
  186. if (init_page_count == max_page_count && init_page_count == 1) {
  187. /* If only one page and at most one page, we just append
  188. the app heap to the end of linear memory, enlarge the
  189. num_bytes_per_page, and don't change the page count */
  190. heap_offset = num_bytes_per_page;
  191. num_bytes_per_page += heap_size;
  192. if (num_bytes_per_page < heap_size) {
  193. set_error_buf(error_buf, error_buf_size,
  194. "failed to insert app heap into linear memory, "
  195. "try using `--heap-size=0` option");
  196. return NULL;
  197. }
  198. }
  199. else if (heap_size > 0) {
  200. if (init_page_count == max_page_count && init_page_count == 0) {
  201. /* If the memory data size is always 0, we resize it to
  202. one page for app heap */
  203. num_bytes_per_page = heap_size;
  204. heap_offset = 0;
  205. inc_page_count = 1;
  206. }
  207. else if (module->aux_heap_base_global_index != (uint32)-1
  208. && module->aux_heap_base
  209. < (uint64)num_bytes_per_page * init_page_count) {
  210. /* Insert app heap before __heap_base */
  211. aux_heap_base = module->aux_heap_base;
  212. bytes_of_last_page = aux_heap_base % num_bytes_per_page;
  213. if (bytes_of_last_page == 0)
  214. bytes_of_last_page = num_bytes_per_page;
  215. bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
  216. inc_page_count =
  217. (heap_size - bytes_to_page_end + num_bytes_per_page - 1)
  218. / num_bytes_per_page;
  219. heap_offset = aux_heap_base;
  220. aux_heap_base += heap_size;
  221. bytes_of_last_page = aux_heap_base % num_bytes_per_page;
  222. if (bytes_of_last_page == 0)
  223. bytes_of_last_page = num_bytes_per_page;
  224. bytes_to_page_end = num_bytes_per_page - bytes_of_last_page;
  225. if (bytes_to_page_end < 1 * BH_KB) {
  226. aux_heap_base += 1 * BH_KB;
  227. inc_page_count++;
  228. }
  229. /* Adjust __heap_base global value */
  230. global_idx = module->aux_heap_base_global_index;
  231. bh_assert(module_inst->e->globals
  232. && global_idx < module_inst->e->global_count);
  233. global_addr = module_inst->global_data
  234. + module_inst->e->globals[global_idx].data_offset;
  235. *(uint32 *)global_addr = (uint32)aux_heap_base;
  236. LOG_VERBOSE("Reset __heap_base global to %u", aux_heap_base);
  237. }
  238. else {
  239. /* Insert app heap before new page */
  240. inc_page_count =
  241. (heap_size + num_bytes_per_page - 1) / num_bytes_per_page;
  242. heap_offset = (uint64)num_bytes_per_page * init_page_count;
  243. heap_size = (uint64)num_bytes_per_page * inc_page_count;
  244. if (heap_size > 0)
  245. heap_size -= 1 * BH_KB;
  246. }
  247. init_page_count += inc_page_count;
  248. max_page_count += inc_page_count;
  249. if (init_page_count > DEFAULT_MAX_PAGES) {
  250. set_error_buf(error_buf, error_buf_size,
  251. "failed to insert app heap into linear memory, "
  252. "try using `--heap-size=0` option");
  253. return NULL;
  254. }
  255. if (max_page_count > DEFAULT_MAX_PAGES)
  256. max_page_count = DEFAULT_MAX_PAGES;
  257. }
  258. LOG_VERBOSE("Memory instantiate:");
  259. LOG_VERBOSE(" page bytes: %u, init pages: %u, max pages: %u",
  260. num_bytes_per_page, init_page_count, max_page_count);
  261. LOG_VERBOSE(" heap offset: %u, heap size: %d\n", heap_offset, heap_size);
  262. max_memory_data_size = (uint64)num_bytes_per_page * max_page_count;
  263. bh_assert(max_memory_data_size <= MAX_LINEAR_MEMORY_SIZE);
  264. (void)max_memory_data_size;
  265. bh_assert(memory != NULL);
  266. if (wasm_allocate_linear_memory(&memory->memory_data, is_shared_memory,
  267. num_bytes_per_page, init_page_count,
  268. max_page_count, &memory_data_size)
  269. != BHT_OK) {
  270. set_error_buf(error_buf, error_buf_size,
  271. "allocate linear memory failed");
  272. return NULL;
  273. }
  274. memory->module_type = Wasm_Module_Bytecode;
  275. memory->num_bytes_per_page = num_bytes_per_page;
  276. memory->cur_page_count = init_page_count;
  277. memory->max_page_count = max_page_count;
  278. memory->memory_data_size = memory_data_size;
  279. memory->heap_data = memory->memory_data + heap_offset;
  280. memory->heap_data_end = memory->heap_data + heap_size;
  281. memory->memory_data_end = memory->memory_data + memory_data_size;
  282. /* Initialize heap */
  283. if (heap_size > 0) {
  284. uint32 heap_struct_size = mem_allocator_get_heap_struct_size();
  285. if (!(memory->heap_handle = runtime_malloc(
  286. (uint64)heap_struct_size, error_buf, error_buf_size))) {
  287. goto fail1;
  288. }
  289. if (!mem_allocator_create_with_struct_and_pool(
  290. memory->heap_handle, heap_struct_size, memory->heap_data,
  291. heap_size)) {
  292. set_error_buf(error_buf, error_buf_size, "init app heap failed");
  293. goto fail2;
  294. }
  295. }
  296. if (memory_data_size > 0) {
  297. wasm_runtime_set_mem_bound_check_bytes(memory, memory_data_size);
  298. }
  299. #if WASM_ENABLE_SHARED_MEMORY != 0
  300. if (is_shared_memory) {
  301. memory->is_shared_memory = 1;
  302. memory->ref_count = 1;
  303. }
  304. #endif
  305. LOG_VERBOSE("Memory instantiate success.");
  306. return memory;
  307. fail2:
  308. if (heap_size > 0)
  309. wasm_runtime_free(memory->heap_handle);
  310. fail1:
  311. if (memory->memory_data)
  312. wasm_deallocate_linear_memory(memory);
  313. return NULL;
  314. }
  315. /**
  316. * Instantiate memories in a module.
  317. */
  318. static WASMMemoryInstance **
  319. memories_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  320. WASMModuleInstance *parent, uint32 heap_size,
  321. uint32 max_memory_pages, char *error_buf,
  322. uint32 error_buf_size)
  323. {
  324. WASMImport *import;
  325. uint32 mem_index = 0, i,
  326. memory_count = module->import_memory_count + module->memory_count;
  327. uint64 total_size;
  328. WASMMemoryInstance **memories, *memory;
  329. total_size = sizeof(WASMMemoryInstance *) * (uint64)memory_count;
  330. if (!(memories = runtime_malloc(total_size, error_buf, error_buf_size))) {
  331. return NULL;
  332. }
  333. memory = module_inst->global_table_data.memory_instances;
  334. /* instantiate memories from import section */
  335. import = module->import_memories;
  336. for (i = 0; i < module->import_memory_count; i++, import++, memory++) {
  337. uint32 num_bytes_per_page = import->u.memory.num_bytes_per_page;
  338. uint32 init_page_count = import->u.memory.init_page_count;
  339. uint32 max_page_count = wasm_runtime_get_max_mem(
  340. max_memory_pages, import->u.memory.init_page_count,
  341. import->u.memory.max_page_count);
  342. uint32 flags = import->u.memory.flags;
  343. uint32 actual_heap_size = heap_size;
  344. #if WASM_ENABLE_MULTI_MODULE != 0
  345. if (import->u.memory.import_module != NULL) {
  346. WASMModuleInstance *module_inst_linked;
  347. if (!(module_inst_linked = get_sub_module_inst(
  348. module_inst, import->u.memory.import_module))) {
  349. set_error_buf(error_buf, error_buf_size, "unknown memory");
  350. memories_deinstantiate(module_inst, memories, memory_count);
  351. return NULL;
  352. }
  353. if (!(memories[mem_index++] = wasm_lookup_memory(
  354. module_inst_linked, import->u.memory.field_name))) {
  355. set_error_buf(error_buf, error_buf_size, "unknown memory");
  356. memories_deinstantiate(module_inst, memories, memory_count);
  357. return NULL;
  358. }
  359. }
  360. else
  361. #endif
  362. {
  363. if (!(memories[mem_index] = memory_instantiate(
  364. module_inst, parent, memory, mem_index,
  365. num_bytes_per_page, init_page_count, max_page_count,
  366. actual_heap_size, flags, error_buf, error_buf_size))) {
  367. memories_deinstantiate(module_inst, memories, memory_count);
  368. return NULL;
  369. }
  370. mem_index++;
  371. }
  372. }
  373. /* instantiate memories from memory section */
  374. for (i = 0; i < module->memory_count; i++, memory++) {
  375. uint32 max_page_count = wasm_runtime_get_max_mem(
  376. max_memory_pages, module->memories[i].init_page_count,
  377. module->memories[i].max_page_count);
  378. if (!(memories[mem_index] = memory_instantiate(
  379. module_inst, parent, memory, mem_index,
  380. module->memories[i].num_bytes_per_page,
  381. module->memories[i].init_page_count, max_page_count,
  382. heap_size, module->memories[i].flags, error_buf,
  383. error_buf_size))) {
  384. memories_deinstantiate(module_inst, memories, memory_count);
  385. return NULL;
  386. }
  387. mem_index++;
  388. }
  389. bh_assert(mem_index == memory_count);
  390. (void)module_inst;
  391. return memories;
  392. }
  393. /**
  394. * Destroy table instances.
  395. */
  396. static void
  397. tables_deinstantiate(WASMModuleInstance *module_inst)
  398. {
  399. if (module_inst->tables) {
  400. wasm_runtime_free(module_inst->tables);
  401. }
  402. #if WASM_ENABLE_MULTI_MODULE != 0
  403. if (module_inst->e->table_insts_linked) {
  404. wasm_runtime_free(module_inst->e->table_insts_linked);
  405. }
  406. #endif
  407. }
  408. /**
  409. * Instantiate tables in a module.
  410. */
  411. static WASMTableInstance **
  412. tables_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  413. WASMTableInstance *first_table, char *error_buf,
  414. uint32 error_buf_size)
  415. {
  416. WASMImport *import;
  417. uint32 table_index = 0, i;
  418. uint32 table_count = module->import_table_count + module->table_count;
  419. uint64 total_size = (uint64)sizeof(WASMTableInstance *) * table_count;
  420. WASMTableInstance **tables, *table = first_table;
  421. #if WASM_ENABLE_MULTI_MODULE != 0
  422. uint64 total_size_of_tables_linked =
  423. (uint64)sizeof(WASMTableInstance *) * module->import_table_count;
  424. WASMTableInstance **table_linked = NULL;
  425. #endif
  426. if (!(tables = runtime_malloc(total_size, error_buf, error_buf_size))) {
  427. return NULL;
  428. }
  429. #if WASM_ENABLE_MULTI_MODULE != 0
  430. if (module->import_table_count > 0
  431. && !(module_inst->e->table_insts_linked = table_linked = runtime_malloc(
  432. total_size_of_tables_linked, error_buf, error_buf_size))) {
  433. goto fail;
  434. }
  435. #endif
  436. /* instantiate tables from import section */
  437. import = module->import_tables;
  438. for (i = 0; i < module->import_table_count; i++, import++) {
  439. uint32 max_size_fixed = 0;
  440. #if WASM_ENABLE_MULTI_MODULE != 0
  441. WASMTableInstance *table_inst_linked = NULL;
  442. WASMModuleInstance *module_inst_linked = NULL;
  443. if (import->u.table.import_module) {
  444. if (!(module_inst_linked = get_sub_module_inst(
  445. module_inst, import->u.table.import_module))) {
  446. set_error_buf(error_buf, error_buf_size, "unknown table");
  447. goto fail;
  448. }
  449. if (!(table_inst_linked = wasm_lookup_table(
  450. module_inst_linked, import->u.table.field_name))) {
  451. set_error_buf(error_buf, error_buf_size, "unknown table");
  452. goto fail;
  453. }
  454. total_size = offsetof(WASMTableInstance, elems);
  455. }
  456. else
  457. #endif
  458. {
  459. /* in order to save memory, alloc resource as few as possible */
  460. max_size_fixed = import->u.table.possible_grow
  461. ? import->u.table.max_size
  462. : import->u.table.init_size;
  463. /* it is a built-in table, every module has its own */
  464. total_size = offsetof(WASMTableInstance, elems);
  465. /* store function indexes for non-gc, object pointers for gc */
  466. total_size += (uint64)sizeof(table_elem_type_t) * max_size_fixed;
  467. }
  468. tables[table_index++] = table;
  469. #if WASM_ENABLE_GC == 0
  470. /* Set all elements to -1 to mark them as uninitialized elements */
  471. memset(table, -1, (uint32)total_size);
  472. #else
  473. /* For GC, all elements have already been set to NULL_REF (0) as
  474. uninitialized elements */
  475. #endif
  476. #if WASM_ENABLE_MULTI_MODULE != 0
  477. *table_linked = table_inst_linked;
  478. if (table_inst_linked != NULL) {
  479. #if WASM_ENABLE_GC != 0
  480. table->elem_type = table_inst_linked->elem_type;
  481. table->elem_ref_type = table_inst_linked->elem_ref_type;
  482. #endif
  483. table->cur_size = table_inst_linked->cur_size;
  484. table->max_size = table_inst_linked->max_size;
  485. }
  486. else
  487. #endif
  488. {
  489. #if WASM_ENABLE_GC != 0
  490. table->elem_type = import->u.table.elem_type;
  491. table->elem_ref_type.elem_ref_type = import->u.table.elem_ref_type;
  492. #endif
  493. table->cur_size = import->u.table.init_size;
  494. table->max_size = max_size_fixed;
  495. }
  496. table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size);
  497. #if WASM_ENABLE_MULTI_MODULE != 0
  498. table_linked++;
  499. #endif
  500. }
  501. /* instantiate tables from table section */
  502. for (i = 0; i < module->table_count; i++) {
  503. uint32 max_size_fixed = 0;
  504. total_size = offsetof(WASMTableInstance, elems);
  505. #if WASM_ENABLE_MULTI_MODULE != 0
  506. /* in case, a module which imports this table will grow it */
  507. max_size_fixed = module->tables[i].max_size;
  508. #else
  509. max_size_fixed = module->tables[i].possible_grow
  510. ? module->tables[i].max_size
  511. : module->tables[i].init_size;
  512. #endif
  513. #if WASM_ENABLE_GC == 0
  514. /* Store function indexes */
  515. total_size += sizeof(uintptr_t) * (uint64)max_size_fixed;
  516. #else
  517. /* Store object pointers */
  518. total_size += sizeof(uintptr_t) * (uint64)max_size_fixed;
  519. #endif
  520. tables[table_index++] = table;
  521. #if WASM_ENABLE_GC == 0
  522. /* Set all elements to -1 to mark them as uninitialized elements */
  523. memset(table, -1, (uint32)total_size);
  524. #else
  525. /* For GC, all elements have already been set to NULL_REF (0) as
  526. uninitialized elements */
  527. #endif
  528. #if WASM_ENABLE_GC != 0
  529. table->elem_type = module->tables[i].elem_type;
  530. table->elem_ref_type.elem_ref_type = module->tables[i].elem_ref_type;
  531. #endif
  532. table->cur_size = module->tables[i].init_size;
  533. table->max_size = max_size_fixed;
  534. table = (WASMTableInstance *)((uint8 *)table + (uint32)total_size);
  535. }
  536. bh_assert(table_index == table_count);
  537. (void)module_inst;
  538. return tables;
  539. #if WASM_ENABLE_MULTI_MODULE != 0
  540. fail:
  541. wasm_runtime_free(tables);
  542. return NULL;
  543. #endif
  544. }
  545. /**
  546. * Destroy function instances.
  547. */
  548. static void
  549. functions_deinstantiate(WASMFunctionInstance *functions)
  550. {
  551. if (functions) {
  552. wasm_runtime_free(functions);
  553. }
  554. }
  555. /**
  556. * Instantiate functions in a module.
  557. */
  558. static WASMFunctionInstance *
  559. functions_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  560. char *error_buf, uint32 error_buf_size)
  561. {
  562. WASMImport *import;
  563. uint32 i,
  564. function_count = module->import_function_count + module->function_count;
  565. uint64 total_size = sizeof(WASMFunctionInstance) * (uint64)function_count;
  566. WASMFunctionInstance *functions, *function;
  567. if (!(functions = runtime_malloc(total_size, error_buf, error_buf_size))) {
  568. return NULL;
  569. }
  570. total_size = sizeof(void *) * (uint64)module->import_function_count;
  571. if (total_size > 0
  572. && !(module_inst->import_func_ptrs =
  573. runtime_malloc(total_size, error_buf, error_buf_size))) {
  574. wasm_runtime_free(functions);
  575. return NULL;
  576. }
  577. /* instantiate functions from import section */
  578. function = functions;
  579. import = module->import_functions;
  580. for (i = 0; i < module->import_function_count; i++, import++) {
  581. function->is_import_func = true;
  582. #if WASM_ENABLE_MULTI_MODULE != 0
  583. if (import->u.function.import_module) {
  584. function->import_module_inst = get_sub_module_inst(
  585. module_inst, import->u.function.import_module);
  586. if (function->import_module_inst) {
  587. function->import_func_inst =
  588. wasm_lookup_function(function->import_module_inst,
  589. import->u.function.field_name);
  590. }
  591. }
  592. #endif /* WASM_ENABLE_MULTI_MODULE */
  593. function->u.func_import = &import->u.function;
  594. function->param_cell_num = import->u.function.func_type->param_cell_num;
  595. function->ret_cell_num = import->u.function.func_type->ret_cell_num;
  596. function->param_count =
  597. (uint16)function->u.func_import->func_type->param_count;
  598. function->param_types = function->u.func_import->func_type->types;
  599. function->local_cell_num = 0;
  600. function->local_count = 0;
  601. function->local_types = NULL;
  602. /* Copy the function pointer to current instance */
  603. module_inst->import_func_ptrs[i] =
  604. function->u.func_import->func_ptr_linked;
  605. function++;
  606. }
  607. /* instantiate functions from function section */
  608. for (i = 0; i < module->function_count; i++) {
  609. function->is_import_func = false;
  610. function->u.func = module->functions[i];
  611. function->param_cell_num = function->u.func->param_cell_num;
  612. function->ret_cell_num = function->u.func->ret_cell_num;
  613. function->local_cell_num = function->u.func->local_cell_num;
  614. function->param_count =
  615. (uint16)function->u.func->func_type->param_count;
  616. function->local_count = (uint16)function->u.func->local_count;
  617. function->param_types = function->u.func->func_type->types;
  618. function->local_types = function->u.func->local_types;
  619. function->local_offsets = function->u.func->local_offsets;
  620. #if WASM_ENABLE_FAST_INTERP != 0
  621. function->const_cell_num = function->u.func->const_cell_num;
  622. #endif
  623. function++;
  624. }
  625. bh_assert((uint32)(function - functions) == function_count);
  626. #if WASM_ENABLE_FAST_JIT != 0
  627. module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
  628. #endif
  629. return functions;
  630. }
  631. #if WASM_ENABLE_TAGS != 0
  632. /**
  633. * Destroy tags instances.
  634. */
  635. static void
  636. tags_deinstantiate(WASMTagInstance *tags, void **import_tag_ptrs)
  637. {
  638. if (tags) {
  639. wasm_runtime_free(tags);
  640. }
  641. if (import_tag_ptrs) {
  642. wasm_runtime_free(import_tag_ptrs);
  643. }
  644. }
  645. /**
  646. * Instantiate tags in a module.
  647. */
  648. static WASMTagInstance *
  649. tags_instantiate(const WASMModule *module, WASMModuleInstance *module_inst,
  650. char *error_buf, uint32 error_buf_size)
  651. {
  652. WASMImport *import;
  653. uint32 i, tag_count = module->import_tag_count + module->tag_count;
  654. uint64 total_size = sizeof(WASMTagInstance) * (uint64)tag_count;
  655. WASMTagInstance *tags, *tag;
  656. if (!(tags = runtime_malloc(total_size, error_buf, error_buf_size))) {
  657. return NULL;
  658. }
  659. total_size = sizeof(void *) * (uint64)module->import_tag_count;
  660. if (total_size > 0
  661. && !(module_inst->e->import_tag_ptrs =
  662. runtime_malloc(total_size, error_buf, error_buf_size))) {
  663. wasm_runtime_free(tags);
  664. return NULL;
  665. }
  666. /* instantiate tags from import section */
  667. tag = tags;
  668. import = module->import_tags;
  669. for (i = 0; i < module->import_tag_count; i++, import++) {
  670. tag->is_import_tag = true;
  671. tag->u.tag_import = &import->u.tag;
  672. tag->type = import->u.tag.type;
  673. tag->attribute = import->u.tag.attribute;
  674. #if WASM_ENABLE_MULTI_MODULE != 0
  675. if (import->u.tag.import_module) {
  676. if (!(tag->import_module_inst = get_sub_module_inst(
  677. module_inst, import->u.tag.import_module))) {
  678. set_error_buf(error_buf, error_buf_size, "unknown tag");
  679. goto fail;
  680. }
  681. if (!(tag->import_tag_inst =
  682. wasm_lookup_tag(tag->import_module_inst,
  683. import->u.tag.field_name, NULL))) {
  684. set_error_buf(error_buf, error_buf_size, "unknown tag");
  685. goto fail;
  686. }
  687. /* Copy the imported tag to current instance */
  688. module_inst->e->import_tag_ptrs[i] =
  689. tag->u.tag_import->import_tag_linked;
  690. }
  691. #endif
  692. tag++;
  693. }
  694. /* instantiate tags from tag section */
  695. for (i = 0; i < module->tag_count; i++) {
  696. tag->is_import_tag = false;
  697. tag->type = module->tags[i]->type;
  698. tag->u.tag = module->tags[i];
  699. #if WASM_ENABLE_FAST_INTERP != 0
  700. /* tag->const_cell_num = function->u.func->const_cell_num; */
  701. #endif
  702. tag++;
  703. }
  704. bh_assert((uint32)(tag - tags) == tag_count);
  705. return tags;
  706. #if WASM_ENABLE_MULTI_MODULE != 0
  707. fail:
  708. tags_deinstantiate(tags, module_inst->e->import_tag_ptrs);
  709. /* clean up */
  710. module_inst->e->import_tag_ptrs = NULL;
  711. return NULL;
  712. #endif
  713. }
  714. #endif /* end of WASM_ENABLE_TAGS != 0 */
  715. /**
  716. * Destroy global instances.
  717. */
  718. static void
  719. globals_deinstantiate(WASMGlobalInstance *globals)
  720. {
  721. if (globals)
  722. wasm_runtime_free(globals);
  723. }
  724. static bool
  725. check_global_init_expr(const WASMModule *module, uint32 global_index,
  726. char *error_buf, uint32 error_buf_size)
  727. {
  728. if (global_index >= module->import_global_count + module->global_count) {
  729. set_error_buf_v(error_buf, error_buf_size, "unknown global %d",
  730. global_index);
  731. return false;
  732. }
  733. #if WASM_ENABLE_GC == 0
  734. /**
  735. * Currently, constant expressions occurring as initializers of
  736. * globals are further constrained in that contained global.get
  737. * instructions are only allowed to refer to imported globals.
  738. *
  739. * And initializer expression cannot reference a mutable global.
  740. */
  741. if (global_index >= module->import_global_count
  742. || (module->import_globals + global_index)->u.global.is_mutable) {
  743. set_error_buf(error_buf, error_buf_size,
  744. "constant expression required");
  745. return false;
  746. }
  747. #endif
  748. return true;
  749. }
  750. /**
  751. * Instantiate globals in a module.
  752. */
  753. static WASMGlobalInstance *
  754. globals_instantiate(WASMModule *module, WASMModuleInstance *module_inst,
  755. char *error_buf, uint32 error_buf_size)
  756. {
  757. WASMImport *import;
  758. uint32 global_data_offset = 0;
  759. uint32 i, global_count = module->import_global_count + module->global_count;
  760. uint64 total_size = sizeof(WASMGlobalInstance) * (uint64)global_count;
  761. WASMGlobalInstance *globals, *global;
  762. if (!(globals = runtime_malloc(total_size, error_buf, error_buf_size))) {
  763. return NULL;
  764. }
  765. /* instantiate globals from import section */
  766. global = globals;
  767. import = module->import_globals;
  768. for (i = 0; i < module->import_global_count; i++, import++) {
  769. WASMGlobalImport *global_import = &import->u.global;
  770. global->type = global_import->type;
  771. global->is_mutable = global_import->is_mutable;
  772. #if WASM_ENABLE_GC != 0
  773. global->ref_type = global_import->ref_type;
  774. #endif
  775. #if WASM_ENABLE_MULTI_MODULE != 0
  776. if (global_import->import_module) {
  777. if (!(global->import_module_inst = get_sub_module_inst(
  778. module_inst, global_import->import_module))) {
  779. set_error_buf(error_buf, error_buf_size, "unknown global");
  780. goto fail;
  781. }
  782. if (!(global->import_global_inst = wasm_lookup_global(
  783. global->import_module_inst, global_import->field_name))) {
  784. set_error_buf(error_buf, error_buf_size, "unknown global");
  785. goto fail;
  786. }
  787. /* The linked global instance has been initialized, we
  788. just need to copy the value. */
  789. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  790. &(global_import->import_global_linked->init_expr),
  791. sizeof(WASMValue));
  792. }
  793. else
  794. #endif
  795. {
  796. /* native globals share their initial_values in one module */
  797. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  798. &(global_import->global_data_linked),
  799. sizeof(WASMValue));
  800. }
  801. #if WASM_ENABLE_FAST_JIT != 0
  802. bh_assert(global_data_offset == global_import->data_offset);
  803. #endif
  804. global->data_offset = global_data_offset;
  805. global_data_offset += wasm_value_type_size(global->type);
  806. global++;
  807. }
  808. /* instantiate globals from global section */
  809. for (i = 0; i < module->global_count; i++) {
  810. InitializerExpression *init_expr = &(module->globals[i].init_expr);
  811. uint8 flag = init_expr->init_expr_type;
  812. global->type = module->globals[i].type;
  813. global->is_mutable = module->globals[i].is_mutable;
  814. #if WASM_ENABLE_FAST_JIT != 0
  815. bh_assert(global_data_offset == module->globals[i].data_offset);
  816. #endif
  817. global->data_offset = global_data_offset;
  818. global_data_offset += wasm_value_type_size(global->type);
  819. #if WASM_ENABLE_GC != 0
  820. global->ref_type = module->globals[i].ref_type;
  821. #endif
  822. switch (flag) {
  823. case INIT_EXPR_TYPE_GET_GLOBAL:
  824. {
  825. if (!check_global_init_expr(module, init_expr->u.global_index,
  826. error_buf, error_buf_size)) {
  827. goto fail;
  828. }
  829. bh_memcpy_s(
  830. &(global->initial_value), sizeof(WASMValue),
  831. &(globals[init_expr->u.global_index].initial_value),
  832. sizeof(globals[init_expr->u.global_index].initial_value));
  833. break;
  834. }
  835. #if WASM_ENABLE_GC != 0
  836. case INIT_EXPR_TYPE_STRUCT_NEW:
  837. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  838. {
  839. WASMRttType *rtt_type;
  840. WASMStructObjectRef struct_obj;
  841. WASMStructType *struct_type;
  842. WASMStructNewInitValues *init_values = NULL;
  843. uint32 type_idx;
  844. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  845. init_values = (WASMStructNewInitValues *)init_expr->u.data;
  846. type_idx = init_values->type_idx;
  847. }
  848. else {
  849. type_idx = init_expr->u.type_index;
  850. }
  851. struct_type = (WASMStructType *)module->types[type_idx];
  852. if (!(rtt_type = wasm_rtt_type_new(
  853. (WASMType *)struct_type, type_idx, module->rtt_types,
  854. module->type_count, &module->rtt_type_lock))) {
  855. set_error_buf(error_buf, error_buf_size,
  856. "create rtt object failed");
  857. goto fail;
  858. }
  859. if (!(struct_obj = wasm_struct_obj_new_internal(
  860. module_inst->e->common.gc_heap_handle, rtt_type))) {
  861. set_error_buf(error_buf, error_buf_size,
  862. "create struct object failed");
  863. goto fail;
  864. }
  865. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  866. uint32 field_idx;
  867. bh_assert(init_values->count == struct_type->field_count);
  868. for (field_idx = 0; field_idx < init_values->count;
  869. field_idx++) {
  870. wasm_struct_obj_set_field(
  871. struct_obj, field_idx,
  872. &init_values->fields[field_idx]);
  873. }
  874. }
  875. global->initial_value.gc_obj = (void *)struct_obj;
  876. break;
  877. }
  878. case INIT_EXPR_TYPE_ARRAY_NEW:
  879. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  880. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  881. {
  882. WASMRttType *rtt_type;
  883. WASMArrayObjectRef array_obj;
  884. WASMArrayType *array_type;
  885. WASMArrayNewInitValues *init_values = NULL;
  886. WASMValue *arr_init_val = NULL, empty_val = { 0 };
  887. uint32 type_idx, len;
  888. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) {
  889. type_idx = init_expr->u.array_new_default.type_index;
  890. len = init_expr->u.array_new_default.length;
  891. arr_init_val = &empty_val;
  892. }
  893. else {
  894. init_values = (WASMArrayNewInitValues *)init_expr->u.data;
  895. type_idx = init_values->type_idx;
  896. len = init_values->length;
  897. if (flag == INIT_EXPR_TYPE_ARRAY_NEW) {
  898. arr_init_val = init_values->elem_data;
  899. }
  900. }
  901. array_type = (WASMArrayType *)module->types[type_idx];
  902. if (!(rtt_type = wasm_rtt_type_new(
  903. (WASMType *)array_type, type_idx, module->rtt_types,
  904. module->type_count, &module->rtt_type_lock))) {
  905. set_error_buf(error_buf, error_buf_size,
  906. "create rtt object failed");
  907. goto fail;
  908. }
  909. if (!(array_obj = wasm_array_obj_new_internal(
  910. module_inst->e->common.gc_heap_handle, rtt_type, len,
  911. arr_init_val))) {
  912. set_error_buf(error_buf, error_buf_size,
  913. "create array object failed");
  914. goto fail;
  915. }
  916. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  917. uint32 elem_idx;
  918. bh_assert(init_values);
  919. for (elem_idx = 0; elem_idx < len; elem_idx++) {
  920. wasm_array_obj_set_elem(
  921. array_obj, elem_idx,
  922. &init_values->elem_data[elem_idx]);
  923. }
  924. }
  925. global->initial_value.gc_obj = (void *)array_obj;
  926. break;
  927. }
  928. case INIT_EXPR_TYPE_I31_NEW:
  929. {
  930. global->initial_value.gc_obj =
  931. (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32);
  932. break;
  933. }
  934. #endif /* end of WASM_ENABLE_GC != 0 */
  935. default:
  936. bh_memcpy_s(&(global->initial_value), sizeof(WASMValue),
  937. &(init_expr->u), sizeof(init_expr->u));
  938. break;
  939. }
  940. global++;
  941. }
  942. bh_assert((uint32)(global - globals) == global_count);
  943. bh_assert(global_data_offset == module->global_data_size);
  944. (void)module_inst;
  945. return globals;
  946. fail:
  947. wasm_runtime_free(globals);
  948. return NULL;
  949. }
  950. /**
  951. * Return export function count in module export section.
  952. */
  953. static uint32
  954. get_export_count(const WASMModule *module, uint8 kind)
  955. {
  956. WASMExport *export = module->exports;
  957. uint32 count = 0, i;
  958. for (i = 0; i < module->export_count; i++, export ++)
  959. if (export->kind == kind)
  960. count++;
  961. return count;
  962. }
  963. /**
  964. * Destroy export function instances.
  965. */
  966. static void
  967. export_functions_deinstantiate(WASMExportFuncInstance *functions)
  968. {
  969. if (functions)
  970. wasm_runtime_free(functions);
  971. }
  972. /**
  973. * Instantiate export functions in a module.
  974. */
  975. static WASMExportFuncInstance *
  976. export_functions_instantiate(const WASMModule *module,
  977. WASMModuleInstance *module_inst,
  978. uint32 export_func_count, char *error_buf,
  979. uint32 error_buf_size)
  980. {
  981. WASMExportFuncInstance *export_funcs, *export_func;
  982. WASMExport *export = module->exports;
  983. uint32 i;
  984. uint64 total_size =
  985. sizeof(WASMExportFuncInstance) * (uint64)export_func_count;
  986. if (!(export_func = export_funcs =
  987. runtime_malloc(total_size, error_buf, error_buf_size))) {
  988. return NULL;
  989. }
  990. for (i = 0; i < module->export_count; i++, export ++)
  991. if (export->kind == EXPORT_KIND_FUNC) {
  992. export_func->name = export->name;
  993. export_func->function = &module_inst->e->functions[export->index];
  994. export_func++;
  995. }
  996. bh_assert((uint32)(export_func - export_funcs) == export_func_count);
  997. return export_funcs;
  998. }
  999. #if WASM_ENABLE_TAGS != 0
  1000. /**
  1001. * Destroy export function instances.
  1002. */
  1003. static void
  1004. export_tags_deinstantiate(WASMExportTagInstance *tags)
  1005. {
  1006. if (tags)
  1007. wasm_runtime_free(tags);
  1008. }
  1009. /**
  1010. * Instantiate export functions in a module.
  1011. */
  1012. static WASMExportTagInstance *
  1013. export_tags_instantiate(const WASMModule *module,
  1014. WASMModuleInstance *module_inst,
  1015. uint32 export_tag_count, char *error_buf,
  1016. uint32 error_buf_size)
  1017. {
  1018. WASMExportTagInstance *export_tags, *export_tag;
  1019. WASMExport *export = module->exports;
  1020. uint32 i;
  1021. uint64 total_size =
  1022. sizeof(WASMExportTagInstance) * (uint64)export_tag_count;
  1023. if (!(export_tag = export_tags =
  1024. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1025. return NULL;
  1026. }
  1027. for (i = 0; i < module->export_count; i++, export ++)
  1028. if (export->kind == EXPORT_KIND_TAG) {
  1029. export_tag->name = export->name;
  1030. bh_assert(module_inst->e->tags);
  1031. export_tag->tag = &module_inst->e->tags[export->index];
  1032. export_tag++;
  1033. }
  1034. bh_assert((uint32)(export_tag - export_tags) == export_tag_count);
  1035. return export_tags;
  1036. }
  1037. #endif /* end of WASM_ENABLE_TAGS != 0 */
  1038. #if WASM_ENABLE_MULTI_MODULE != 0
  1039. static void
  1040. export_globals_deinstantiate(WASMExportGlobInstance *globals)
  1041. {
  1042. if (globals)
  1043. wasm_runtime_free(globals);
  1044. }
  1045. static WASMExportGlobInstance *
  1046. export_globals_instantiate(const WASMModule *module,
  1047. WASMModuleInstance *module_inst,
  1048. uint32 export_glob_count, char *error_buf,
  1049. uint32 error_buf_size)
  1050. {
  1051. WASMExportGlobInstance *export_globals, *export_global;
  1052. WASMExport *export = module->exports;
  1053. uint32 i;
  1054. uint64 total_size =
  1055. sizeof(WASMExportGlobInstance) * (uint64)export_glob_count;
  1056. if (!(export_global = export_globals =
  1057. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1058. return NULL;
  1059. }
  1060. for (i = 0; i < module->export_count; i++, export ++)
  1061. if (export->kind == EXPORT_KIND_GLOBAL) {
  1062. export_global->name = export->name;
  1063. export_global->global = &module_inst->e->globals[export->index];
  1064. export_global++;
  1065. }
  1066. bh_assert((uint32)(export_global - export_globals) == export_glob_count);
  1067. return export_globals;
  1068. }
  1069. #endif
  1070. static WASMFunctionInstance *
  1071. lookup_post_instantiate_func(WASMModuleInstance *module_inst,
  1072. const char *func_name)
  1073. {
  1074. WASMFunctionInstance *func;
  1075. WASMFuncType *func_type;
  1076. if (!(func = wasm_lookup_function(module_inst, func_name)))
  1077. /* Not found */
  1078. return NULL;
  1079. func_type = func->u.func->func_type;
  1080. if (!(func_type->param_count == 0 && func_type->result_count == 0))
  1081. /* Not a valid function type, ignore it */
  1082. return NULL;
  1083. return func;
  1084. }
  1085. static bool
  1086. execute_post_instantiate_functions(WASMModuleInstance *module_inst,
  1087. bool is_sub_inst, WASMExecEnv *exec_env_main)
  1088. {
  1089. WASMFunctionInstance *start_func = module_inst->e->start_function;
  1090. WASMFunctionInstance *initialize_func = NULL;
  1091. WASMFunctionInstance *post_inst_func = NULL;
  1092. WASMFunctionInstance *call_ctors_func = NULL;
  1093. #if WASM_ENABLE_LIBC_WASI != 0
  1094. WASMModule *module = module_inst->module;
  1095. #endif
  1096. WASMModuleInstanceCommon *module_inst_main = NULL;
  1097. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1098. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1099. #endif
  1100. WASMExecEnv *exec_env = NULL, *exec_env_created = NULL;
  1101. bool ret = false;
  1102. #if WASM_ENABLE_LIBC_WASI != 0
  1103. /*
  1104. * WASI reactor instances may assume that _initialize will be called by
  1105. * the environment at most once, and that none of their other exports
  1106. * are accessed before that call.
  1107. */
  1108. if (!is_sub_inst && module->import_wasi_api) {
  1109. initialize_func =
  1110. lookup_post_instantiate_func(module_inst, "_initialize");
  1111. }
  1112. #endif
  1113. /* Execute possible "__post_instantiate" function if wasm app is
  1114. compiled by emsdk's early version */
  1115. if (!is_sub_inst) {
  1116. post_inst_func =
  1117. lookup_post_instantiate_func(module_inst, "__post_instantiate");
  1118. }
  1119. #if WASM_ENABLE_BULK_MEMORY != 0
  1120. /* Only execute the memory init function for main instance since
  1121. the data segments will be dropped once initialized */
  1122. if (!is_sub_inst
  1123. #if WASM_ENABLE_LIBC_WASI != 0
  1124. && !module->import_wasi_api
  1125. #endif
  1126. ) {
  1127. call_ctors_func =
  1128. lookup_post_instantiate_func(module_inst, "__wasm_call_ctors");
  1129. }
  1130. #endif
  1131. if (!start_func && !initialize_func && !post_inst_func
  1132. && !call_ctors_func) {
  1133. /* No post instantiation functions to call */
  1134. return true;
  1135. }
  1136. if (is_sub_inst) {
  1137. bh_assert(exec_env_main);
  1138. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1139. bh_assert(exec_env_tls == exec_env_main);
  1140. (void)exec_env_tls;
  1141. #endif
  1142. exec_env = exec_env_main;
  1143. /* Temporarily replace parent exec_env's module inst to current
  1144. module inst to avoid checking failure when calling the
  1145. wasm functions, and ensure that the exec_env's module inst
  1146. is the correct one. */
  1147. module_inst_main = exec_env_main->module_inst;
  1148. wasm_exec_env_set_module_inst(exec_env,
  1149. (WASMModuleInstanceCommon *)module_inst);
  1150. }
  1151. else {
  1152. /* Try using the existing exec_env */
  1153. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1154. exec_env = exec_env_tls;
  1155. #endif
  1156. #if WASM_ENABLE_THREAD_MGR != 0
  1157. if (!exec_env)
  1158. exec_env = wasm_clusters_search_exec_env(
  1159. (WASMModuleInstanceCommon *)module_inst);
  1160. #endif
  1161. if (!exec_env) {
  1162. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1163. (WASMModuleInstanceCommon *)module_inst,
  1164. module_inst->default_wasm_stack_size))) {
  1165. wasm_set_exception(module_inst, "allocate memory failed");
  1166. return false;
  1167. }
  1168. }
  1169. else {
  1170. /* Temporarily replace exec_env's module inst with current
  1171. module inst to ensure that the exec_env's module inst
  1172. is the correct one. */
  1173. module_inst_main = exec_env->module_inst;
  1174. wasm_exec_env_set_module_inst(
  1175. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1176. }
  1177. }
  1178. /* Execute start function for both main insance and sub instance */
  1179. if (start_func && !wasm_call_function(exec_env, start_func, 0, NULL)) {
  1180. goto fail;
  1181. }
  1182. #if WASM_ENABLE_LIBC_WASI != 0
  1183. if (initialize_func
  1184. && !wasm_call_function(exec_env, initialize_func, 0, NULL)) {
  1185. goto fail;
  1186. }
  1187. #else
  1188. (void)initialize_func;
  1189. #endif
  1190. if (post_inst_func
  1191. && !wasm_call_function(exec_env, post_inst_func, 0, NULL)) {
  1192. goto fail;
  1193. }
  1194. if (call_ctors_func
  1195. && !wasm_call_function(exec_env, call_ctors_func, 0, NULL)) {
  1196. goto fail;
  1197. }
  1198. ret = true;
  1199. fail:
  1200. if (is_sub_inst) {
  1201. /* Restore the parent exec_env's module inst */
  1202. wasm_exec_env_restore_module_inst(exec_env_main, module_inst_main);
  1203. }
  1204. else {
  1205. if (module_inst_main)
  1206. /* Restore the existing exec_env's module inst */
  1207. wasm_exec_env_restore_module_inst(exec_env, module_inst_main);
  1208. if (exec_env_created)
  1209. wasm_exec_env_destroy(exec_env_created);
  1210. }
  1211. return ret;
  1212. }
  1213. static bool
  1214. execute_malloc_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
  1215. WASMFunctionInstance *malloc_func,
  1216. WASMFunctionInstance *retain_func, uint32 size,
  1217. uint32 *p_result)
  1218. {
  1219. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1220. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1221. #endif
  1222. WASMExecEnv *exec_env_created = NULL;
  1223. WASMModuleInstanceCommon *module_inst_old = NULL;
  1224. uint32 argv[2], argc;
  1225. bool ret;
  1226. argv[0] = size;
  1227. argc = 1;
  1228. /* if __retain is exported, then this module is compiled by
  1229. assemblyscript, the memory should be managed by as's runtime,
  1230. in this case we need to call the retain function after malloc
  1231. the memory */
  1232. if (retain_func) {
  1233. /* the malloc functino from assemblyscript is:
  1234. function __new(size: usize, id: u32)
  1235. id = 0 means this is an ArrayBuffer object */
  1236. argv[1] = 0;
  1237. argc = 2;
  1238. }
  1239. if (exec_env) {
  1240. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1241. if (exec_env_tls) {
  1242. bh_assert(exec_env_tls == exec_env);
  1243. }
  1244. #endif
  1245. bh_assert(exec_env->module_inst
  1246. == (WASMModuleInstanceCommon *)module_inst);
  1247. }
  1248. else {
  1249. /* Try using the existing exec_env */
  1250. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1251. exec_env = exec_env_tls;
  1252. #endif
  1253. #if WASM_ENABLE_THREAD_MGR != 0
  1254. if (!exec_env)
  1255. exec_env = wasm_clusters_search_exec_env(
  1256. (WASMModuleInstanceCommon *)module_inst);
  1257. #endif
  1258. if (!exec_env) {
  1259. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1260. (WASMModuleInstanceCommon *)module_inst,
  1261. module_inst->default_wasm_stack_size))) {
  1262. wasm_set_exception(module_inst, "allocate memory failed");
  1263. return false;
  1264. }
  1265. }
  1266. else {
  1267. /* Temporarily replace exec_env's module inst with current
  1268. module inst to ensure that the exec_env's module inst
  1269. is the correct one. */
  1270. module_inst_old = exec_env->module_inst;
  1271. wasm_exec_env_set_module_inst(
  1272. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1273. }
  1274. }
  1275. ret = wasm_call_function(exec_env, malloc_func, argc, argv);
  1276. if (retain_func && ret)
  1277. ret = wasm_call_function(exec_env, retain_func, 1, argv);
  1278. if (module_inst_old)
  1279. /* Restore the existing exec_env's module inst */
  1280. wasm_exec_env_restore_module_inst(exec_env, module_inst_old);
  1281. if (exec_env_created)
  1282. wasm_exec_env_destroy(exec_env_created);
  1283. if (ret)
  1284. *p_result = argv[0];
  1285. return ret;
  1286. }
  1287. static bool
  1288. execute_free_function(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
  1289. WASMFunctionInstance *free_func, uint32 offset)
  1290. {
  1291. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1292. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  1293. #endif
  1294. WASMExecEnv *exec_env_created = NULL;
  1295. WASMModuleInstanceCommon *module_inst_old = NULL;
  1296. uint32 argv[2];
  1297. bool ret;
  1298. argv[0] = offset;
  1299. if (exec_env) {
  1300. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1301. if (exec_env_tls) {
  1302. bh_assert(exec_env_tls == exec_env);
  1303. }
  1304. #endif
  1305. bh_assert(exec_env->module_inst
  1306. == (WASMModuleInstanceCommon *)module_inst);
  1307. }
  1308. else {
  1309. /* Try using the existing exec_env */
  1310. #ifdef OS_ENABLE_HW_BOUND_CHECK
  1311. exec_env = exec_env_tls;
  1312. #endif
  1313. #if WASM_ENABLE_THREAD_MGR != 0
  1314. if (!exec_env)
  1315. exec_env = wasm_clusters_search_exec_env(
  1316. (WASMModuleInstanceCommon *)module_inst);
  1317. #endif
  1318. if (!exec_env) {
  1319. if (!(exec_env = exec_env_created = wasm_exec_env_create(
  1320. (WASMModuleInstanceCommon *)module_inst,
  1321. module_inst->default_wasm_stack_size))) {
  1322. wasm_set_exception(module_inst, "allocate memory failed");
  1323. return false;
  1324. }
  1325. }
  1326. else {
  1327. /* Temporarily replace exec_env's module inst with current
  1328. module inst to ensure that the exec_env's module inst
  1329. is the correct one. */
  1330. module_inst_old = exec_env->module_inst;
  1331. wasm_exec_env_set_module_inst(
  1332. exec_env, (WASMModuleInstanceCommon *)module_inst);
  1333. }
  1334. }
  1335. ret = wasm_call_function(exec_env, free_func, 1, argv);
  1336. if (module_inst_old)
  1337. /* Restore the existing exec_env's module inst */
  1338. wasm_exec_env_restore_module_inst(exec_env, module_inst_old);
  1339. if (exec_env_created)
  1340. wasm_exec_env_destroy(exec_env_created);
  1341. return ret;
  1342. }
  1343. static bool
  1344. check_linked_symbol(WASMModuleInstance *module_inst, char *error_buf,
  1345. uint32 error_buf_size)
  1346. {
  1347. WASMModule *module = module_inst->module;
  1348. uint32 i;
  1349. for (i = 0; i < module->import_function_count; i++) {
  1350. WASMFunctionImport *func =
  1351. &((module->import_functions + i)->u.function);
  1352. if (!func->func_ptr_linked
  1353. #if WASM_ENABLE_MULTI_MODULE != 0
  1354. && !func->import_func_linked
  1355. #endif
  1356. ) {
  1357. #if WASM_ENABLE_WAMR_COMPILER == 0
  1358. LOG_WARNING("warning: failed to link import function (%s, %s)",
  1359. func->module_name, func->field_name);
  1360. /* will throw exception only if calling */
  1361. #else
  1362. /* do nothing to avoid confused message */
  1363. #endif /* WASM_ENABLE_WAMR_COMPILER == 0 */
  1364. }
  1365. }
  1366. for (i = 0; i < module->import_global_count; i++) {
  1367. WASMGlobalImport *global = &((module->import_globals + i)->u.global);
  1368. if (!global->is_linked) {
  1369. #if WASM_ENABLE_SPEC_TEST != 0
  1370. set_error_buf(error_buf, error_buf_size,
  1371. "unknown import or incompatible import type");
  1372. return false;
  1373. #else
  1374. #if WASM_ENABLE_WAMR_COMPILER == 0
  1375. set_error_buf_v(error_buf, error_buf_size,
  1376. "failed to link import global (%s, %s)",
  1377. global->module_name, global->field_name);
  1378. return false;
  1379. #else
  1380. /* do nothing to avoid confused message */
  1381. #endif /* WASM_ENABLE_WAMR_COMPILER == 0 */
  1382. #endif /* WASM_ENABLE_SPEC_TEST != 0 */
  1383. }
  1384. }
  1385. return true;
  1386. }
  1387. #if WASM_ENABLE_JIT != 0
  1388. static bool
  1389. init_func_ptrs(WASMModuleInstance *module_inst, WASMModule *module,
  1390. char *error_buf, uint32 error_buf_size)
  1391. {
  1392. uint32 i;
  1393. void **func_ptrs;
  1394. uint64 total_size = (uint64)sizeof(void *) * module_inst->e->function_count;
  1395. /* Allocate memory */
  1396. if (!(func_ptrs = module_inst->func_ptrs =
  1397. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1398. return false;
  1399. }
  1400. /* Set import function pointers */
  1401. for (i = 0; i < module->import_function_count; i++, func_ptrs++) {
  1402. WASMFunctionImport *import_func =
  1403. &module->import_functions[i].u.function;
  1404. /* TODO: handle multi module */
  1405. *func_ptrs = import_func->func_ptr_linked;
  1406. }
  1407. /* The defined function pointers will be set in
  1408. wasm_runtime_set_running_mode, no need to set them here */
  1409. return true;
  1410. }
  1411. #endif /* end of WASM_ENABLE_JIT != 0 */
  1412. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  1413. static uint32
  1414. get_smallest_type_idx(WASMModule *module, WASMFuncType *func_type)
  1415. {
  1416. uint32 i;
  1417. for (i = 0; i < module->type_count; i++) {
  1418. if (func_type == (WASMFuncType *)module->types[i])
  1419. return i;
  1420. }
  1421. bh_assert(0);
  1422. return -1;
  1423. }
  1424. static bool
  1425. init_func_type_indexes(WASMModuleInstance *module_inst, char *error_buf,
  1426. uint32 error_buf_size)
  1427. {
  1428. uint32 i;
  1429. uint64 total_size = (uint64)sizeof(uint32) * module_inst->e->function_count;
  1430. /* Allocate memory */
  1431. if (!(module_inst->func_type_indexes =
  1432. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1433. return false;
  1434. }
  1435. for (i = 0; i < module_inst->e->function_count; i++) {
  1436. WASMFunctionInstance *func_inst = module_inst->e->functions + i;
  1437. WASMFuncType *func_type = func_inst->is_import_func
  1438. ? func_inst->u.func_import->func_type
  1439. : func_inst->u.func->func_type;
  1440. module_inst->func_type_indexes[i] =
  1441. get_smallest_type_idx(module_inst->module, func_type);
  1442. }
  1443. return true;
  1444. }
  1445. #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 */
  1446. #if WASM_ENABLE_GC != 0
  1447. void *
  1448. wasm_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
  1449. bool throw_exce, char *error_buf, uint32 error_buf_size)
  1450. {
  1451. WASMModule *module = module_inst->module;
  1452. WASMRttTypeRef rtt_type;
  1453. WASMFuncObjectRef func_obj;
  1454. WASMFunctionInstance *func_inst;
  1455. WASMFuncType *func_type;
  1456. uint32 type_idx;
  1457. if (throw_exce) {
  1458. error_buf = module_inst->cur_exception;
  1459. error_buf_size = sizeof(module_inst->cur_exception);
  1460. }
  1461. if (func_idx >= module_inst->e->function_count) {
  1462. set_error_buf_v(error_buf, error_buf_size, "unknown function %d",
  1463. func_idx);
  1464. return NULL;
  1465. }
  1466. func_inst = &module_inst->e->functions[func_idx];
  1467. func_type = func_inst->is_import_func ? func_inst->u.func_import->func_type
  1468. : func_inst->u.func->func_type;
  1469. type_idx = func_inst->is_import_func ? func_inst->u.func_import->type_idx
  1470. : func_inst->u.func->type_idx;
  1471. if (!(rtt_type = wasm_rtt_type_new((WASMType *)func_type, type_idx,
  1472. module->rtt_types, module->type_count,
  1473. &module->rtt_type_lock))) {
  1474. set_error_buf(error_buf, error_buf_size, "create rtt object failed");
  1475. return NULL;
  1476. }
  1477. if (!(func_obj = wasm_func_obj_new_internal(
  1478. module_inst->e->common.gc_heap_handle, rtt_type, func_idx))) {
  1479. set_error_buf(error_buf, error_buf_size, "create func object failed");
  1480. return NULL;
  1481. }
  1482. return func_obj;
  1483. }
  1484. static bool
  1485. wasm_global_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap)
  1486. {
  1487. WASMGlobalInstance *global = module_inst->e->globals;
  1488. WASMGlobalInstance *global_end = global + module_inst->e->global_count;
  1489. uint8 *global_data = module_inst->global_data;
  1490. WASMObjectRef gc_obj;
  1491. while (global < global_end) {
  1492. if (wasm_is_type_reftype(global->type)) {
  1493. gc_obj = GET_REF_FROM_ADDR(
  1494. (uint32 *)(global_data + global->data_offset));
  1495. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1496. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1497. return false;
  1498. }
  1499. }
  1500. global++;
  1501. }
  1502. return true;
  1503. }
  1504. static bool
  1505. wasm_table_traverse_gc_rootset(WASMModuleInstance *module_inst, void *heap)
  1506. {
  1507. WASMTableInstance **tables = module_inst->tables, *table;
  1508. uint32 table_count = module_inst->table_count, i, j;
  1509. WASMObjectRef gc_obj, *table_elems;
  1510. for (i = 0; i < table_count; i++) {
  1511. table = tables[i];
  1512. table_elems = (WASMObjectRef *)table->elems;
  1513. for (j = 0; j < table->cur_size; j++) {
  1514. gc_obj = table_elems[j];
  1515. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1516. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1517. return false;
  1518. }
  1519. }
  1520. }
  1521. return true;
  1522. }
  1523. static bool
  1524. local_object_refs_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap)
  1525. {
  1526. WASMLocalObjectRef *r;
  1527. WASMObjectRef gc_obj;
  1528. for (r = exec_env->cur_local_object_ref; r; r = r->prev) {
  1529. gc_obj = r->val;
  1530. if (wasm_obj_is_created_from_heap(gc_obj)) {
  1531. if (0 != mem_allocator_add_root((mem_allocator_t)heap, gc_obj))
  1532. return false;
  1533. }
  1534. }
  1535. return true;
  1536. }
  1537. bool
  1538. wasm_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap)
  1539. {
  1540. WASMModuleInstance *module_inst =
  1541. (WASMModuleInstance *)exec_env->module_inst;
  1542. bool ret;
  1543. ret = wasm_global_traverse_gc_rootset(module_inst, heap);
  1544. if (!ret)
  1545. return ret;
  1546. ret = wasm_table_traverse_gc_rootset(module_inst, heap);
  1547. if (!ret)
  1548. return ret;
  1549. ret = local_object_refs_traverse_gc_rootset(exec_env, heap);
  1550. if (!ret)
  1551. return ret;
  1552. return wasm_interp_traverse_gc_rootset(exec_env, heap);
  1553. }
  1554. #endif /* end of WASM_ENABLE_GC != 0 */
  1555. static bool
  1556. set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode,
  1557. bool first_time_set)
  1558. {
  1559. WASMModule *module = module_inst->module;
  1560. if (running_mode == Mode_Default) {
  1561. #if WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT == 0
  1562. running_mode = Mode_Interp;
  1563. #elif WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT == 0
  1564. running_mode = Mode_Fast_JIT;
  1565. #elif WASM_ENABLE_FAST_JIT == 0 && WASM_ENABLE_JIT != 0
  1566. running_mode = Mode_LLVM_JIT;
  1567. #else /* WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 */
  1568. #if WASM_ENABLE_LAZY_JIT == 0
  1569. running_mode = Mode_LLVM_JIT;
  1570. #else
  1571. running_mode = Mode_Multi_Tier_JIT;
  1572. #endif
  1573. #endif
  1574. }
  1575. if (!wasm_runtime_is_running_mode_supported(running_mode))
  1576. return false;
  1577. #if !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  1578. && WASM_ENABLE_LAZY_JIT != 0) /* No possible multi-tier JIT */
  1579. (void)first_time_set;
  1580. module_inst->e->running_mode = running_mode;
  1581. if (running_mode == Mode_Interp) {
  1582. /* Do nothing for Mode_Interp */
  1583. }
  1584. else if (running_mode == Mode_Fast_JIT) {
  1585. /* Do nothing for Mode_Fast_JIT since
  1586. module_inst->fast_jit_func_ptrs is same as
  1587. module->fast_jit_func_ptrs */
  1588. }
  1589. #if WASM_ENABLE_JIT != 0
  1590. else if (running_mode == Mode_LLVM_JIT) {
  1591. /* Set defined function pointers */
  1592. bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
  1593. sizeof(void *) * module->function_count, module->func_ptrs,
  1594. sizeof(void *) * module->function_count);
  1595. }
  1596. #endif
  1597. else {
  1598. bh_assert(0);
  1599. }
  1600. #else /* Possible multi-tier JIT */
  1601. os_mutex_lock(&module->instance_list_lock);
  1602. module_inst->e->running_mode = running_mode;
  1603. if (running_mode == Mode_Interp) {
  1604. /* Do nothing for Mode_Interp */
  1605. }
  1606. #if WASM_ENABLE_FAST_JIT != 0
  1607. else if (running_mode == Mode_Fast_JIT) {
  1608. JitGlobals *jit_globals = jit_compiler_get_jit_globals();
  1609. uint32 i;
  1610. /* Allocate memory for fast_jit_func_ptrs if needed */
  1611. if (!module_inst->fast_jit_func_ptrs
  1612. || module_inst->fast_jit_func_ptrs == module->fast_jit_func_ptrs) {
  1613. uint64 total_size = (uint64)sizeof(void *) * module->function_count;
  1614. if (!(module_inst->fast_jit_func_ptrs =
  1615. runtime_malloc(total_size, NULL, 0))) {
  1616. os_mutex_unlock(&module->instance_list_lock);
  1617. return false;
  1618. }
  1619. }
  1620. for (i = 0; i < module->function_count; i++) {
  1621. if (module->functions[i]->fast_jit_jitted_code) {
  1622. /* current fast jit function has been compiled */
  1623. module_inst->fast_jit_func_ptrs[i] =
  1624. module->functions[i]->fast_jit_jitted_code;
  1625. }
  1626. else {
  1627. module_inst->fast_jit_func_ptrs[i] =
  1628. jit_globals->compile_fast_jit_and_then_call;
  1629. }
  1630. }
  1631. }
  1632. #endif
  1633. #if WASM_ENABLE_JIT != 0
  1634. else if (running_mode == Mode_LLVM_JIT) {
  1635. void **llvm_jit_func_ptrs;
  1636. uint32 i;
  1637. /* Notify backend threads to start llvm jit compilation */
  1638. module->enable_llvm_jit_compilation = true;
  1639. /* Wait until llvm jit finishes initialization */
  1640. os_mutex_lock(&module->tierup_wait_lock);
  1641. while (!module->llvm_jit_inited) {
  1642. os_cond_reltimedwait(&module->tierup_wait_cond,
  1643. &module->tierup_wait_lock, 10000);
  1644. if (module->orcjit_stop_compiling) {
  1645. /* init_llvm_jit_functions_stage2 failed */
  1646. os_mutex_unlock(&module->tierup_wait_lock);
  1647. os_mutex_unlock(&module->instance_list_lock);
  1648. return false;
  1649. }
  1650. }
  1651. os_mutex_unlock(&module->tierup_wait_lock);
  1652. llvm_jit_func_ptrs =
  1653. module_inst->func_ptrs + module->import_function_count;
  1654. for (i = 0; i < module->function_count; i++) {
  1655. llvm_jit_func_ptrs[i] = module->functions[i]->llvm_jit_func_ptr;
  1656. }
  1657. }
  1658. #endif
  1659. else if (running_mode == Mode_Multi_Tier_JIT) {
  1660. /* Notify backend threads to start llvm jit compilation */
  1661. module->enable_llvm_jit_compilation = true;
  1662. /* Free fast_jit_func_ptrs if it is allocated before */
  1663. if (module_inst->fast_jit_func_ptrs
  1664. && module_inst->fast_jit_func_ptrs != module->fast_jit_func_ptrs) {
  1665. wasm_runtime_free(module_inst->fast_jit_func_ptrs);
  1666. }
  1667. module_inst->fast_jit_func_ptrs = module->fast_jit_func_ptrs;
  1668. /* Copy all llvm jit func ptrs from the module */
  1669. bh_memcpy_s(module_inst->func_ptrs + module->import_function_count,
  1670. sizeof(void *) * module->function_count, module->func_ptrs,
  1671. sizeof(void *) * module->function_count);
  1672. }
  1673. else {
  1674. bh_assert(0);
  1675. }
  1676. /* Add module instance into module's instance list if not added */
  1677. if (first_time_set) {
  1678. bool found = false;
  1679. WASMModuleInstance *node = module->instance_list;
  1680. while (node) {
  1681. if (node == module_inst) {
  1682. found = true;
  1683. break;
  1684. }
  1685. node = node->e->next;
  1686. }
  1687. if (!found) {
  1688. module_inst->e->next = module->instance_list;
  1689. module->instance_list = module_inst;
  1690. }
  1691. }
  1692. os_mutex_unlock(&module->instance_list_lock);
  1693. #endif /* end of !(WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  1694. && WASM_ENABLE_LAZY_JIT != 0) */
  1695. (void)module;
  1696. return true;
  1697. }
  1698. bool
  1699. wasm_set_running_mode(WASMModuleInstance *module_inst, RunningMode running_mode)
  1700. {
  1701. return set_running_mode(module_inst, running_mode, false);
  1702. }
  1703. /**
  1704. * Instantiate module
  1705. */
  1706. WASMModuleInstance *
  1707. wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
  1708. WASMExecEnv *exec_env_main, uint32 stack_size,
  1709. uint32 heap_size, uint32 max_memory_pages, char *error_buf,
  1710. uint32 error_buf_size)
  1711. {
  1712. WASMModuleInstance *module_inst;
  1713. WASMGlobalInstance *globals = NULL, *global;
  1714. WASMTableInstance *first_table;
  1715. uint32 global_count, i;
  1716. uint32 base_offset, length, extra_info_offset;
  1717. uint32 module_inst_struct_size =
  1718. offsetof(WASMModuleInstance, global_table_data.bytes);
  1719. uint64 module_inst_mem_inst_size;
  1720. uint64 total_size, table_size = 0;
  1721. uint8 *global_data, *global_data_end;
  1722. #if WASM_ENABLE_MULTI_MODULE != 0
  1723. bool ret = false;
  1724. #endif
  1725. const bool is_sub_inst = parent != NULL;
  1726. if (!module)
  1727. return NULL;
  1728. /* Check the heap size */
  1729. heap_size = align_uint(heap_size, 8);
  1730. if (heap_size > APP_HEAP_SIZE_MAX)
  1731. heap_size = APP_HEAP_SIZE_MAX;
  1732. module_inst_mem_inst_size =
  1733. (uint64)sizeof(WASMMemoryInstance)
  1734. * (module->import_memory_count + module->memory_count);
  1735. #if WASM_ENABLE_JIT != 0
  1736. /* If the module dosen't have memory, reserve one mem_info space
  1737. with empty content to align with llvm jit compiler */
  1738. if (module_inst_mem_inst_size == 0)
  1739. module_inst_mem_inst_size = (uint64)sizeof(WASMMemoryInstance);
  1740. #endif
  1741. /* Size of module inst, memory instances and global data */
  1742. total_size = (uint64)module_inst_struct_size + module_inst_mem_inst_size
  1743. + module->global_data_size;
  1744. /* Calculate the size of table data */
  1745. for (i = 0; i < module->import_table_count; i++) {
  1746. WASMTableImport *import_table = &module->import_tables[i].u.table;
  1747. table_size += offsetof(WASMTableInstance, elems);
  1748. #if WASM_ENABLE_MULTI_MODULE != 0
  1749. table_size +=
  1750. (uint64)sizeof(table_elem_type_t) * import_table->max_size;
  1751. #else
  1752. table_size += (uint64)sizeof(table_elem_type_t)
  1753. * (import_table->possible_grow ? import_table->max_size
  1754. : import_table->init_size);
  1755. #endif
  1756. }
  1757. for (i = 0; i < module->table_count; i++) {
  1758. WASMTable *table = module->tables + i;
  1759. table_size += offsetof(WASMTableInstance, elems);
  1760. #if WASM_ENABLE_MULTI_MODULE != 0
  1761. table_size += (uint64)sizeof(table_elem_type_t) * table->max_size;
  1762. #else
  1763. table_size +=
  1764. (uint64)sizeof(table_elem_type_t)
  1765. * (table->possible_grow ? table->max_size : table->init_size);
  1766. #endif
  1767. }
  1768. total_size += table_size;
  1769. /* The offset of WASMModuleInstanceExtra, make it 8-byte aligned */
  1770. total_size = (total_size + 7LL) & ~7LL;
  1771. extra_info_offset = (uint32)total_size;
  1772. total_size += sizeof(WASMModuleInstanceExtra);
  1773. /* Allocate the memory for module instance with memory instances,
  1774. global data, table data appended at the end */
  1775. if (!(module_inst =
  1776. runtime_malloc(total_size, error_buf, error_buf_size))) {
  1777. return NULL;
  1778. }
  1779. module_inst->module_type = Wasm_Module_Bytecode;
  1780. module_inst->module = module;
  1781. module_inst->e =
  1782. (WASMModuleInstanceExtra *)((uint8 *)module_inst + extra_info_offset);
  1783. #if WASM_ENABLE_MULTI_MODULE != 0
  1784. module_inst->e->sub_module_inst_list =
  1785. &module_inst->e->sub_module_inst_list_head;
  1786. ret = wasm_runtime_sub_module_instantiate(
  1787. (WASMModuleCommon *)module, (WASMModuleInstanceCommon *)module_inst,
  1788. stack_size, heap_size, max_memory_pages, error_buf, error_buf_size);
  1789. if (!ret) {
  1790. LOG_DEBUG("build a sub module list failed");
  1791. goto fail;
  1792. }
  1793. #endif
  1794. #if WASM_ENABLE_BULK_MEMORY != 0
  1795. if (module->data_seg_count > 0) {
  1796. module_inst->e->common.data_dropped =
  1797. bh_bitmap_new(0, module->data_seg_count);
  1798. if (module_inst->e->common.data_dropped == NULL) {
  1799. LOG_DEBUG("failed to allocate bitmaps");
  1800. set_error_buf(error_buf, error_buf_size,
  1801. "failed to allocate bitmaps");
  1802. goto fail;
  1803. }
  1804. for (i = 0; i < module->data_seg_count; i++) {
  1805. if (!module->data_segments[i]->is_passive)
  1806. bh_bitmap_set_bit(module_inst->e->common.data_dropped, i);
  1807. }
  1808. }
  1809. #endif
  1810. #if WASM_ENABLE_REF_TYPES != 0
  1811. if (module->table_seg_count > 0) {
  1812. module_inst->e->common.elem_dropped =
  1813. bh_bitmap_new(0, module->table_seg_count);
  1814. if (module_inst->e->common.elem_dropped == NULL) {
  1815. LOG_DEBUG("failed to allocate bitmaps");
  1816. set_error_buf(error_buf, error_buf_size,
  1817. "failed to allocate bitmaps");
  1818. goto fail;
  1819. }
  1820. for (i = 0; i < module->table_seg_count; i++) {
  1821. if (wasm_elem_is_active(module->table_segments[i].mode))
  1822. bh_bitmap_set_bit(module_inst->e->common.elem_dropped, i);
  1823. }
  1824. }
  1825. #endif
  1826. #if WASM_ENABLE_GC != 0
  1827. if (!is_sub_inst) {
  1828. uint32 gc_heap_size = wasm_runtime_get_gc_heap_size_default();
  1829. if (gc_heap_size < GC_HEAP_SIZE_MIN)
  1830. gc_heap_size = GC_HEAP_SIZE_MIN;
  1831. if (gc_heap_size > GC_HEAP_SIZE_MAX)
  1832. gc_heap_size = GC_HEAP_SIZE_MAX;
  1833. module_inst->e->common.gc_heap_pool =
  1834. runtime_malloc(gc_heap_size, error_buf, error_buf_size);
  1835. if (!module_inst->e->common.gc_heap_pool)
  1836. goto fail;
  1837. module_inst->e->common.gc_heap_handle = mem_allocator_create(
  1838. module_inst->e->common.gc_heap_pool, gc_heap_size);
  1839. if (!module_inst->e->common.gc_heap_handle)
  1840. goto fail;
  1841. }
  1842. #endif
  1843. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  1844. if (!(module_inst->frames = runtime_malloc((uint64)sizeof(Vector),
  1845. error_buf, error_buf_size))) {
  1846. goto fail;
  1847. }
  1848. #endif
  1849. /* Instantiate global firstly to get the mutable data size */
  1850. global_count = module->import_global_count + module->global_count;
  1851. if (global_count
  1852. && !(globals = globals_instantiate(module, module_inst, error_buf,
  1853. error_buf_size))) {
  1854. goto fail;
  1855. }
  1856. module_inst->e->global_count = global_count;
  1857. module_inst->e->globals = globals;
  1858. module_inst->global_data = (uint8 *)module_inst + module_inst_struct_size
  1859. + module_inst_mem_inst_size;
  1860. module_inst->global_data_size = module->global_data_size;
  1861. first_table = (WASMTableInstance *)(module_inst->global_data
  1862. + module->global_data_size);
  1863. module_inst->memory_count =
  1864. module->import_memory_count + module->memory_count;
  1865. module_inst->table_count = module->import_table_count + module->table_count;
  1866. module_inst->e->function_count =
  1867. module->import_function_count + module->function_count;
  1868. #if WASM_ENABLE_TAGS != 0
  1869. module_inst->e->tag_count = module->import_tag_count + module->tag_count;
  1870. #endif
  1871. /* export */
  1872. module_inst->export_func_count = get_export_count(module, EXPORT_KIND_FUNC);
  1873. #if WASM_ENABLE_MULTI_MODULE != 0
  1874. module_inst->export_table_count =
  1875. get_export_count(module, EXPORT_KIND_TABLE);
  1876. module_inst->export_memory_count =
  1877. get_export_count(module, EXPORT_KIND_MEMORY);
  1878. #if WASM_ENABLE_TAGS != 0
  1879. module_inst->e->export_tag_count =
  1880. get_export_count(module, EXPORT_KIND_TAG);
  1881. #endif
  1882. module_inst->export_global_count =
  1883. get_export_count(module, EXPORT_KIND_GLOBAL);
  1884. #endif
  1885. /* Instantiate memories/tables/functions/tags */
  1886. if ((module_inst->memory_count > 0
  1887. && !(module_inst->memories = memories_instantiate(
  1888. module, module_inst, parent, heap_size, max_memory_pages,
  1889. error_buf, error_buf_size)))
  1890. || (module_inst->table_count > 0
  1891. && !(module_inst->tables =
  1892. tables_instantiate(module, module_inst, first_table,
  1893. error_buf, error_buf_size)))
  1894. || (module_inst->e->function_count > 0
  1895. && !(module_inst->e->functions = functions_instantiate(
  1896. module, module_inst, error_buf, error_buf_size)))
  1897. || (module_inst->export_func_count > 0
  1898. && !(module_inst->export_functions = export_functions_instantiate(
  1899. module, module_inst, module_inst->export_func_count,
  1900. error_buf, error_buf_size)))
  1901. #if WASM_ENABLE_TAGS != 0
  1902. || (module_inst->e->tag_count > 0
  1903. && !(module_inst->e->tags = tags_instantiate(
  1904. module, module_inst, error_buf, error_buf_size)))
  1905. || (module_inst->e->export_tag_count > 0
  1906. && !(module_inst->e->export_tags = export_tags_instantiate(
  1907. module, module_inst, module_inst->e->export_tag_count,
  1908. error_buf, error_buf_size)))
  1909. #endif
  1910. #if WASM_ENABLE_MULTI_MODULE != 0
  1911. || (module_inst->export_global_count > 0
  1912. && !(module_inst->export_globals = export_globals_instantiate(
  1913. module, module_inst, module_inst->export_global_count,
  1914. error_buf, error_buf_size)))
  1915. #endif
  1916. #if WASM_ENABLE_JIT != 0
  1917. || (module_inst->e->function_count > 0
  1918. && !init_func_ptrs(module_inst, module, error_buf, error_buf_size))
  1919. #endif
  1920. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  1921. || (module_inst->e->function_count > 0
  1922. && !init_func_type_indexes(module_inst, error_buf, error_buf_size))
  1923. #endif
  1924. ) {
  1925. goto fail;
  1926. }
  1927. if (global_count > 0) {
  1928. /* Initialize the global data */
  1929. global_data = module_inst->global_data;
  1930. global_data_end = global_data + module->global_data_size;
  1931. global = globals;
  1932. for (i = 0; i < global_count; i++, global++) {
  1933. switch (global->type) {
  1934. case VALUE_TYPE_I32:
  1935. case VALUE_TYPE_F32:
  1936. #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
  1937. case VALUE_TYPE_FUNCREF:
  1938. case VALUE_TYPE_EXTERNREF:
  1939. #endif
  1940. *(int32 *)global_data = global->initial_value.i32;
  1941. global_data += sizeof(int32);
  1942. break;
  1943. case VALUE_TYPE_I64:
  1944. case VALUE_TYPE_F64:
  1945. bh_memcpy_s(global_data,
  1946. (uint32)(global_data_end - global_data),
  1947. &global->initial_value.i64, sizeof(int64));
  1948. global_data += sizeof(int64);
  1949. break;
  1950. #if WASM_ENABLE_SIMD != 0
  1951. case VALUE_TYPE_V128:
  1952. bh_memcpy_s(global_data, (uint32)sizeof(V128),
  1953. &global->initial_value.v128, sizeof(V128));
  1954. global_data += sizeof(V128);
  1955. break;
  1956. #endif
  1957. #if WASM_ENABLE_GC != 0
  1958. case VALUE_TYPE_EXTERNREF:
  1959. /* the initial value should be a null reference */
  1960. bh_assert(global->initial_value.gc_obj == NULL_REF);
  1961. STORE_PTR((void **)global_data, NULL_REF);
  1962. global_data += sizeof(void *);
  1963. break;
  1964. #endif
  1965. default:
  1966. {
  1967. #if WASM_ENABLE_GC != 0
  1968. InitializerExpression *global_init = NULL;
  1969. bh_assert(wasm_is_type_reftype(global->type));
  1970. if (i >= module->import_global_count) {
  1971. global_init =
  1972. &module->globals[i - module->import_global_count]
  1973. .init_expr;
  1974. }
  1975. if (global->type == REF_TYPE_NULLFUNCREF
  1976. || global->type == REF_TYPE_NULLEXTERNREF
  1977. || global->type == REF_TYPE_NULLREF) {
  1978. STORE_PTR((void **)global_data, NULL_REF);
  1979. global_data += sizeof(void *);
  1980. break;
  1981. }
  1982. /* We can't create funcref obj during global instantiation
  1983. * since the functions are not instantiated yet, so we need
  1984. * to defer the initialization here */
  1985. if (global_init
  1986. && (global_init->init_expr_type
  1987. == INIT_EXPR_TYPE_FUNCREF_CONST)
  1988. && wasm_reftype_is_subtype_of(
  1989. global->type, global->ref_type, REF_TYPE_FUNCREF,
  1990. NULL, module_inst->module->types,
  1991. module_inst->module->type_count)) {
  1992. WASMFuncObjectRef func_obj = NULL;
  1993. /* UINT32_MAX indicates that it is a null reference */
  1994. if ((uint32)global->initial_value.i32 != UINT32_MAX) {
  1995. if (!(func_obj = wasm_create_func_obj(
  1996. module_inst, global->initial_value.i32,
  1997. false, error_buf, error_buf_size)))
  1998. goto fail;
  1999. }
  2000. STORE_PTR((void **)global_data, func_obj);
  2001. global_data += sizeof(void *);
  2002. /* Also update the inital_value since other globals may
  2003. * refer to this */
  2004. global->initial_value.gc_obj = (wasm_obj_t)func_obj;
  2005. break;
  2006. }
  2007. else {
  2008. STORE_PTR((void **)global_data,
  2009. global->initial_value.gc_obj);
  2010. global_data += sizeof(void *);
  2011. break;
  2012. }
  2013. #endif
  2014. bh_assert(0);
  2015. break;
  2016. }
  2017. }
  2018. }
  2019. bh_assert(global_data == global_data_end);
  2020. }
  2021. if (!check_linked_symbol(module_inst, error_buf, error_buf_size)) {
  2022. goto fail;
  2023. }
  2024. /* Initialize the memory data with data segment section */
  2025. for (i = 0; i < module->data_seg_count; i++) {
  2026. WASMMemoryInstance *memory = NULL;
  2027. uint8 *memory_data = NULL;
  2028. uint64 memory_size = 0;
  2029. WASMDataSeg *data_seg = module->data_segments[i];
  2030. #if WASM_ENABLE_BULK_MEMORY != 0
  2031. if (data_seg->is_passive)
  2032. continue;
  2033. #endif
  2034. if (is_sub_inst)
  2035. /* Ignore setting memory init data if the memory has been
  2036. initialized */
  2037. continue;
  2038. /* has check it in loader */
  2039. memory = module_inst->memories[data_seg->memory_index];
  2040. bh_assert(memory);
  2041. memory_data = memory->memory_data;
  2042. memory_size =
  2043. (uint64)memory->num_bytes_per_page * memory->cur_page_count;
  2044. bh_assert(memory_data || memory_size == 0);
  2045. bh_assert(data_seg->base_offset.init_expr_type
  2046. == INIT_EXPR_TYPE_I32_CONST
  2047. || data_seg->base_offset.init_expr_type
  2048. == INIT_EXPR_TYPE_GET_GLOBAL);
  2049. if (data_seg->base_offset.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
  2050. if (!check_global_init_expr(module,
  2051. data_seg->base_offset.u.global_index,
  2052. error_buf, error_buf_size)) {
  2053. goto fail;
  2054. }
  2055. if (!globals
  2056. || globals[data_seg->base_offset.u.global_index].type
  2057. != VALUE_TYPE_I32) {
  2058. set_error_buf(error_buf, error_buf_size,
  2059. "data segment does not fit");
  2060. goto fail;
  2061. }
  2062. base_offset =
  2063. globals[data_seg->base_offset.u.global_index].initial_value.i32;
  2064. }
  2065. else {
  2066. base_offset = (uint32)data_seg->base_offset.u.i32;
  2067. }
  2068. /* check offset */
  2069. if (base_offset > memory_size) {
  2070. LOG_DEBUG("base_offset(%d) > memory_size(%d)", base_offset,
  2071. memory_size);
  2072. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2073. set_error_buf(error_buf, error_buf_size,
  2074. "out of bounds memory access");
  2075. #else
  2076. set_error_buf(error_buf, error_buf_size,
  2077. "data segment does not fit");
  2078. #endif
  2079. goto fail;
  2080. }
  2081. /* check offset + length(could be zero) */
  2082. length = data_seg->data_length;
  2083. if ((uint64)base_offset + length > memory_size) {
  2084. LOG_DEBUG("base_offset(%d) + length(%d) > memory_size(%d)",
  2085. base_offset, length, memory_size);
  2086. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2087. set_error_buf(error_buf, error_buf_size,
  2088. "out of bounds memory access");
  2089. #else
  2090. set_error_buf(error_buf, error_buf_size,
  2091. "data segment does not fit");
  2092. #endif
  2093. goto fail;
  2094. }
  2095. if (memory_data) {
  2096. bh_memcpy_s(memory_data + base_offset,
  2097. (uint32)memory_size - base_offset, data_seg->data,
  2098. length);
  2099. }
  2100. }
  2101. #if WASM_ENABLE_GC != 0
  2102. /* Initialize the table data with init expr */
  2103. for (i = 0; i < module->table_count; i++) {
  2104. WASMTable *table = module->tables + i;
  2105. WASMTableInstance *table_inst = module_inst->tables[i];
  2106. table_elem_type_t *table_data;
  2107. uint32 j;
  2108. if (table->init_expr.init_expr_type == 0) {
  2109. /* No table initializer */
  2110. continue;
  2111. }
  2112. table_data = table_inst->elems;
  2113. bh_assert(
  2114. table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL
  2115. || table->init_expr.init_expr_type == INIT_EXPR_TYPE_FUNCREF_CONST
  2116. || table->init_expr.init_expr_type == INIT_EXPR_TYPE_REFNULL_CONST);
  2117. if (table->init_expr.init_expr_type == INIT_EXPR_TYPE_GET_GLOBAL) {
  2118. if (!check_global_init_expr(module, table->init_expr.u.global_index,
  2119. error_buf, error_buf_size)) {
  2120. goto fail;
  2121. }
  2122. table->init_expr.u.gc_obj =
  2123. globals[table->init_expr.u.global_index].initial_value.gc_obj;
  2124. }
  2125. else if (table->init_expr.init_expr_type
  2126. == INIT_EXPR_TYPE_FUNCREF_CONST) {
  2127. uint32 func_idx = table->init_expr.u.ref_index;
  2128. if (func_idx != UINT32_MAX) {
  2129. if (!(table->init_expr.u.gc_obj =
  2130. wasm_create_func_obj(module_inst, func_idx, false,
  2131. error_buf, error_buf_size)))
  2132. goto fail;
  2133. }
  2134. else {
  2135. table->init_expr.u.gc_obj = NULL_REF;
  2136. }
  2137. }
  2138. else if (table->init_expr.init_expr_type
  2139. == INIT_EXPR_TYPE_REFNULL_CONST) {
  2140. table->init_expr.u.gc_obj = NULL_REF;
  2141. }
  2142. LOG_DEBUG("Init table [%d] elements from [%d] to [%d] as: %p", i, 0,
  2143. table_inst->cur_size, (void *)table->init_expr.u.gc_obj);
  2144. for (j = 0; j < table_inst->cur_size; j++) {
  2145. *(table_data + j) = table->init_expr.u.gc_obj;
  2146. }
  2147. }
  2148. #endif /* end of WASM_ENABLE_GC != 0 */
  2149. /* Initialize the table data with table segment section */
  2150. for (i = 0; module_inst->table_count > 0 && i < module->table_seg_count;
  2151. i++) {
  2152. WASMTableSeg *table_seg = module->table_segments + i;
  2153. /* has check it in loader */
  2154. WASMTableInstance *table = module_inst->tables[table_seg->table_index];
  2155. table_elem_type_t *table_data;
  2156. uint32 j;
  2157. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2158. uint8 tbl_elem_type;
  2159. uint32 tbl_init_size, tbl_max_size;
  2160. #endif
  2161. #if WASM_ENABLE_GC != 0
  2162. WASMRefType *tbl_elem_ref_type;
  2163. #endif
  2164. bh_assert(table);
  2165. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2166. (void)wasm_runtime_get_table_inst_elem_type(
  2167. (WASMModuleInstanceCommon *)module_inst, table_seg->table_index,
  2168. &tbl_elem_type,
  2169. #if WASM_ENABLE_GC != 0
  2170. &tbl_elem_ref_type,
  2171. #endif
  2172. &tbl_init_size, &tbl_max_size);
  2173. #if WASM_ENABLE_GC == 0
  2174. if (tbl_elem_type != VALUE_TYPE_FUNCREF
  2175. && tbl_elem_type != VALUE_TYPE_EXTERNREF) {
  2176. set_error_buf(error_buf, error_buf_size,
  2177. "type mismatch: elements segment does not fit");
  2178. goto fail;
  2179. }
  2180. #elif WASM_ENABLE_GC != 0
  2181. if (!wasm_elem_is_declarative(table_seg->mode)
  2182. && !wasm_reftype_is_subtype_of(
  2183. table_seg->elem_type, table_seg->elem_ref_type,
  2184. table->elem_type, table->elem_ref_type.elem_ref_type,
  2185. module->types, module->type_count)) {
  2186. set_error_buf(error_buf, error_buf_size,
  2187. "type mismatch: elements segment does not fit");
  2188. goto fail;
  2189. }
  2190. #endif
  2191. (void)tbl_init_size;
  2192. (void)tbl_max_size;
  2193. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  2194. table_data = table->elems;
  2195. #if WASM_ENABLE_MULTI_MODULE != 0
  2196. if (table_seg->table_index < module->import_table_count
  2197. && module_inst->e->table_insts_linked[table_seg->table_index]) {
  2198. table_data =
  2199. module_inst->e->table_insts_linked[table_seg->table_index]
  2200. ->elems;
  2201. }
  2202. #endif
  2203. bh_assert(table_data);
  2204. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2205. if (!wasm_elem_is_active(table_seg->mode))
  2206. continue;
  2207. #endif
  2208. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2209. bh_assert(table_seg->base_offset.init_expr_type
  2210. == INIT_EXPR_TYPE_I32_CONST
  2211. || table_seg->base_offset.init_expr_type
  2212. == INIT_EXPR_TYPE_GET_GLOBAL
  2213. || table_seg->base_offset.init_expr_type
  2214. == INIT_EXPR_TYPE_FUNCREF_CONST
  2215. || table_seg->base_offset.init_expr_type
  2216. == INIT_EXPR_TYPE_REFNULL_CONST);
  2217. #else
  2218. bh_assert(table_seg->base_offset.init_expr_type
  2219. == INIT_EXPR_TYPE_I32_CONST
  2220. || table_seg->base_offset.init_expr_type
  2221. == INIT_EXPR_TYPE_GET_GLOBAL);
  2222. #endif
  2223. /* init vec(funcidx) or vec(expr) */
  2224. if (table_seg->base_offset.init_expr_type
  2225. == INIT_EXPR_TYPE_GET_GLOBAL) {
  2226. if (!check_global_init_expr(module,
  2227. table_seg->base_offset.u.global_index,
  2228. error_buf, error_buf_size)) {
  2229. goto fail;
  2230. }
  2231. if (!globals
  2232. || globals[table_seg->base_offset.u.global_index].type
  2233. != VALUE_TYPE_I32) {
  2234. set_error_buf(error_buf, error_buf_size,
  2235. "type mismatch: elements segment does not fit");
  2236. goto fail;
  2237. }
  2238. table_seg->base_offset.u.i32 =
  2239. globals[table_seg->base_offset.u.global_index]
  2240. .initial_value.i32;
  2241. }
  2242. /* check offset since length might negative */
  2243. if ((uint32)table_seg->base_offset.u.i32 > table->cur_size) {
  2244. LOG_DEBUG("base_offset(%d) > table->cur_size(%d)",
  2245. table_seg->base_offset.u.i32, table->cur_size);
  2246. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2247. set_error_buf(error_buf, error_buf_size,
  2248. "out of bounds table access");
  2249. #else
  2250. set_error_buf(error_buf, error_buf_size,
  2251. "type mismatch: elements segment does not fit");
  2252. #endif
  2253. goto fail;
  2254. }
  2255. /* check offset + length(could be zero) */
  2256. length = table_seg->value_count;
  2257. if ((uint32)table_seg->base_offset.u.i32 + length > table->cur_size) {
  2258. LOG_DEBUG("base_offset(%d) + length(%d)> table->cur_size(%d)",
  2259. table_seg->base_offset.u.i32, length, table->cur_size);
  2260. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2261. set_error_buf(error_buf, error_buf_size,
  2262. "out of bounds table access");
  2263. #else
  2264. set_error_buf(error_buf, error_buf_size,
  2265. "type mismatch: elements segment does not fit");
  2266. #endif
  2267. goto fail;
  2268. }
  2269. for (j = 0; j < length; j++) {
  2270. InitializerExpression *init_expr = &table_seg->init_values[j];
  2271. uint8 flag = init_expr->init_expr_type;
  2272. void *ref = NULL;
  2273. /* const and get global init values should be resolved during
  2274. * loading */
  2275. bh_assert((flag == INIT_EXPR_TYPE_GET_GLOBAL)
  2276. || (flag == INIT_EXPR_TYPE_REFNULL_CONST)
  2277. || ((flag >= INIT_EXPR_TYPE_FUNCREF_CONST)
  2278. && (flag <= INIT_EXPR_TYPE_EXTERN_CONVERT_ANY)));
  2279. switch (flag) {
  2280. case INIT_EXPR_TYPE_REFNULL_CONST:
  2281. ref = NULL;
  2282. break;
  2283. case INIT_EXPR_TYPE_FUNCREF_CONST:
  2284. {
  2285. #if WASM_ENABLE_GC == 0
  2286. ref = (void *)(uintptr_t)init_expr->u.ref_index;
  2287. #else
  2288. WASMFuncObjectRef func_obj;
  2289. uint32 func_idx = init_expr->u.ref_index;
  2290. /* UINT32_MAX indicates that it is a null reference */
  2291. if (func_idx != UINT32_MAX) {
  2292. if (!(func_obj = wasm_create_func_obj(
  2293. module_inst, func_idx, false, error_buf,
  2294. error_buf_size))) {
  2295. goto fail;
  2296. }
  2297. ref = func_obj;
  2298. }
  2299. else {
  2300. ref = NULL_REF;
  2301. }
  2302. #endif /* end of WASM_ENABLE_GC != 0 */
  2303. break;
  2304. }
  2305. #if WASM_ENABLE_GC != 0
  2306. case INIT_EXPR_TYPE_GET_GLOBAL:
  2307. {
  2308. if (!check_global_init_expr(module,
  2309. init_expr->u.global_index,
  2310. error_buf, error_buf_size)) {
  2311. goto fail;
  2312. }
  2313. ref =
  2314. globals[init_expr->u.global_index].initial_value.gc_obj;
  2315. break;
  2316. }
  2317. case INIT_EXPR_TYPE_STRUCT_NEW:
  2318. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  2319. {
  2320. WASMRttType *rtt_type;
  2321. WASMStructObjectRef struct_obj;
  2322. WASMStructType *struct_type;
  2323. WASMStructNewInitValues *init_values = NULL;
  2324. uint32 type_idx;
  2325. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  2326. init_values =
  2327. (WASMStructNewInitValues *)init_expr->u.data;
  2328. type_idx = init_values->type_idx;
  2329. }
  2330. else {
  2331. type_idx = init_expr->u.type_index;
  2332. }
  2333. struct_type = (WASMStructType *)module->types[type_idx];
  2334. if (!(rtt_type = wasm_rtt_type_new(
  2335. (WASMType *)struct_type, type_idx,
  2336. module->rtt_types, module->type_count,
  2337. &module->rtt_type_lock))) {
  2338. set_error_buf(error_buf, error_buf_size,
  2339. "create rtt object failed");
  2340. goto fail;
  2341. }
  2342. if (!(struct_obj = wasm_struct_obj_new_internal(
  2343. module_inst->e->common.gc_heap_handle,
  2344. rtt_type))) {
  2345. set_error_buf(error_buf, error_buf_size,
  2346. "create struct object failed");
  2347. goto fail;
  2348. }
  2349. if (flag == INIT_EXPR_TYPE_STRUCT_NEW) {
  2350. uint32 field_idx;
  2351. bh_assert(init_values->count
  2352. == struct_type->field_count);
  2353. for (field_idx = 0; field_idx < init_values->count;
  2354. field_idx++) {
  2355. wasm_struct_obj_set_field(
  2356. struct_obj, field_idx,
  2357. &init_values->fields[field_idx]);
  2358. }
  2359. }
  2360. ref = struct_obj;
  2361. break;
  2362. }
  2363. case INIT_EXPR_TYPE_ARRAY_NEW:
  2364. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  2365. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  2366. {
  2367. WASMRttType *rtt_type;
  2368. WASMArrayObjectRef array_obj;
  2369. WASMArrayType *array_type;
  2370. WASMArrayNewInitValues *init_values = NULL;
  2371. WASMValue *arr_init_val = NULL, empty_val = { 0 };
  2372. uint32 type_idx, len;
  2373. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT) {
  2374. type_idx = init_expr->u.array_new_default.type_index;
  2375. len = init_expr->u.array_new_default.length;
  2376. arr_init_val = &empty_val;
  2377. }
  2378. else {
  2379. init_values =
  2380. (WASMArrayNewInitValues *)init_expr->u.data;
  2381. type_idx = init_values->type_idx;
  2382. len = init_values->length;
  2383. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  2384. arr_init_val = init_values->elem_data;
  2385. }
  2386. }
  2387. array_type = (WASMArrayType *)module->types[type_idx];
  2388. if (!(rtt_type = wasm_rtt_type_new(
  2389. (WASMType *)array_type, type_idx,
  2390. module->rtt_types, module->type_count,
  2391. &module->rtt_type_lock))) {
  2392. set_error_buf(error_buf, error_buf_size,
  2393. "create rtt object failed");
  2394. goto fail;
  2395. }
  2396. if (!(array_obj = wasm_array_obj_new_internal(
  2397. module_inst->e->common.gc_heap_handle, rtt_type,
  2398. len, arr_init_val))) {
  2399. set_error_buf(error_buf, error_buf_size,
  2400. "create array object failed");
  2401. goto fail;
  2402. }
  2403. if (flag == INIT_EXPR_TYPE_ARRAY_NEW_FIXED) {
  2404. uint32 elem_idx;
  2405. bh_assert(init_values);
  2406. for (elem_idx = 0; elem_idx < len; elem_idx++) {
  2407. wasm_array_obj_set_elem(
  2408. array_obj, elem_idx,
  2409. &init_values->elem_data[elem_idx]);
  2410. }
  2411. }
  2412. ref = array_obj;
  2413. break;
  2414. }
  2415. case INIT_EXPR_TYPE_I31_NEW:
  2416. {
  2417. ref = (wasm_obj_t)wasm_i31_obj_new(init_expr->u.i32);
  2418. break;
  2419. }
  2420. #endif /* end of WASM_ENABLE_GC != 0 */
  2421. }
  2422. *(table_data + table_seg->base_offset.u.i32 + j) =
  2423. (table_elem_type_t)ref;
  2424. }
  2425. }
  2426. /* Initialize the thread related data */
  2427. if (stack_size == 0)
  2428. stack_size = DEFAULT_WASM_STACK_SIZE;
  2429. #if WASM_ENABLE_SPEC_TEST != 0
  2430. #if WASM_ENABLE_TAIL_CALL == 0
  2431. if (stack_size < 128 * 1024)
  2432. stack_size = 128 * 1024;
  2433. #else
  2434. /* Some tail-call cases require large operand stack */
  2435. if (stack_size < 10 * 1024 * 1024)
  2436. stack_size = 10 * 1024 * 1024;
  2437. #endif
  2438. #endif
  2439. module_inst->default_wasm_stack_size = stack_size;
  2440. if (module->malloc_function != (uint32)-1) {
  2441. module_inst->e->malloc_function =
  2442. &module_inst->e->functions[module->malloc_function];
  2443. }
  2444. if (module->free_function != (uint32)-1) {
  2445. module_inst->e->free_function =
  2446. &module_inst->e->functions[module->free_function];
  2447. }
  2448. if (module->retain_function != (uint32)-1) {
  2449. module_inst->e->retain_function =
  2450. &module_inst->e->functions[module->retain_function];
  2451. }
  2452. #if WASM_ENABLE_LIBC_WASI != 0
  2453. /* The sub-instance will get the wasi_ctx from main-instance */
  2454. if (!is_sub_inst) {
  2455. if (!wasm_runtime_init_wasi(
  2456. (WASMModuleInstanceCommon *)module_inst,
  2457. module->wasi_args.dir_list, module->wasi_args.dir_count,
  2458. module->wasi_args.map_dir_list, module->wasi_args.map_dir_count,
  2459. module->wasi_args.env, module->wasi_args.env_count,
  2460. module->wasi_args.addr_pool, module->wasi_args.addr_count,
  2461. module->wasi_args.ns_lookup_pool,
  2462. module->wasi_args.ns_lookup_count, module->wasi_args.argv,
  2463. module->wasi_args.argc, module->wasi_args.stdio[0],
  2464. module->wasi_args.stdio[1], module->wasi_args.stdio[2],
  2465. error_buf, error_buf_size)) {
  2466. goto fail;
  2467. }
  2468. }
  2469. #endif
  2470. #if WASM_ENABLE_DEBUG_INTERP != 0
  2471. if (!is_sub_inst) {
  2472. /* Add module instance into module's instance list */
  2473. os_mutex_lock(&module->instance_list_lock);
  2474. if (module->instance_list) {
  2475. LOG_WARNING(
  2476. "warning: multiple instances referencing to the same module "
  2477. "may cause unexpected behaviour during debugging");
  2478. }
  2479. module_inst->e->next = module->instance_list;
  2480. module->instance_list = module_inst;
  2481. os_mutex_unlock(&module->instance_list_lock);
  2482. }
  2483. #endif
  2484. /* Set running mode before executing wasm functions */
  2485. if (!set_running_mode(module_inst, wasm_runtime_get_default_running_mode(),
  2486. true)) {
  2487. set_error_buf(error_buf, error_buf_size,
  2488. "set instance running mode failed");
  2489. goto fail;
  2490. }
  2491. if (module->start_function != (uint32)-1) {
  2492. /* TODO: fix start function can be import function issue */
  2493. if (module->start_function >= module->import_function_count)
  2494. module_inst->e->start_function =
  2495. &module_inst->e->functions[module->start_function];
  2496. }
  2497. if (!execute_post_instantiate_functions(module_inst, is_sub_inst,
  2498. exec_env_main)) {
  2499. set_error_buf(error_buf, error_buf_size, module_inst->cur_exception);
  2500. goto fail;
  2501. }
  2502. #if WASM_ENABLE_MEMORY_TRACING != 0
  2503. wasm_runtime_dump_module_inst_mem_consumption(
  2504. (WASMModuleInstanceCommon *)module_inst);
  2505. #endif
  2506. (void)global_data_end;
  2507. return module_inst;
  2508. fail:
  2509. wasm_deinstantiate(module_inst, false);
  2510. return NULL;
  2511. }
  2512. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2513. static void
  2514. destroy_c_api_frames(Vector *frames)
  2515. {
  2516. WASMCApiFrame frame = { 0 };
  2517. uint32 i, total_frames, ret;
  2518. total_frames = (uint32)bh_vector_size(frames);
  2519. for (i = 0; i < total_frames; i++) {
  2520. ret = bh_vector_get(frames, i, &frame);
  2521. bh_assert(ret);
  2522. if (frame.lp)
  2523. wasm_runtime_free(frame.lp);
  2524. }
  2525. ret = bh_vector_destroy(frames);
  2526. bh_assert(ret);
  2527. (void)ret;
  2528. }
  2529. #endif
  2530. void
  2531. wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst)
  2532. {
  2533. if (!module_inst)
  2534. return;
  2535. if (module_inst->exec_env_singleton) {
  2536. /* wasm_exec_env_destroy will call
  2537. wasm_cluster_wait_for_all_except_self to wait for other
  2538. threads, so as to destroy their exec_envs and module
  2539. instances first, and avoid accessing the shared resources
  2540. of current module instance after it is deinstantiated. */
  2541. wasm_exec_env_destroy(module_inst->exec_env_singleton);
  2542. }
  2543. #if WASM_ENABLE_DEBUG_INTERP != 0 \
  2544. || (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  2545. && WASM_ENABLE_LAZY_JIT != 0)
  2546. /* Remove instance from module's instance list before freeing
  2547. func_ptrs and fast_jit_func_ptrs of the instance, to avoid
  2548. accessing the freed memory in the jit backend compilation
  2549. threads */
  2550. {
  2551. WASMModule *module = module_inst->module;
  2552. WASMModuleInstance *instance_prev = NULL, *instance;
  2553. os_mutex_lock(&module->instance_list_lock);
  2554. instance = module->instance_list;
  2555. while (instance) {
  2556. if (instance == module_inst) {
  2557. if (!instance_prev)
  2558. module->instance_list = instance->e->next;
  2559. else
  2560. instance_prev->e->next = instance->e->next;
  2561. break;
  2562. }
  2563. instance_prev = instance;
  2564. instance = instance->e->next;
  2565. }
  2566. os_mutex_unlock(&module->instance_list_lock);
  2567. }
  2568. #endif
  2569. #if WASM_ENABLE_JIT != 0
  2570. if (module_inst->func_ptrs)
  2571. wasm_runtime_free(module_inst->func_ptrs);
  2572. #endif
  2573. #if WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
  2574. && WASM_ENABLE_LAZY_JIT != 0
  2575. if (module_inst->fast_jit_func_ptrs
  2576. && module_inst->fast_jit_func_ptrs
  2577. != module_inst->module->fast_jit_func_ptrs)
  2578. wasm_runtime_free(module_inst->fast_jit_func_ptrs);
  2579. #endif
  2580. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0
  2581. if (module_inst->func_type_indexes)
  2582. wasm_runtime_free(module_inst->func_type_indexes);
  2583. #endif
  2584. #if WASM_ENABLE_MULTI_MODULE != 0
  2585. wasm_runtime_sub_module_deinstantiate(
  2586. (WASMModuleInstanceCommon *)module_inst);
  2587. #endif
  2588. if (module_inst->memory_count > 0)
  2589. memories_deinstantiate(module_inst, module_inst->memories,
  2590. module_inst->memory_count);
  2591. if (module_inst->import_func_ptrs) {
  2592. wasm_runtime_free(module_inst->import_func_ptrs);
  2593. }
  2594. tables_deinstantiate(module_inst);
  2595. functions_deinstantiate(module_inst->e->functions);
  2596. #if WASM_ENABLE_TAGS != 0
  2597. tags_deinstantiate(module_inst->e->tags, module_inst->e->import_tag_ptrs);
  2598. #endif
  2599. globals_deinstantiate(module_inst->e->globals);
  2600. export_functions_deinstantiate(module_inst->export_functions);
  2601. #if WASM_ENABLE_TAGS != 0
  2602. export_tags_deinstantiate(module_inst->e->export_tags);
  2603. #endif
  2604. #if WASM_ENABLE_MULTI_MODULE != 0
  2605. export_globals_deinstantiate(module_inst->export_globals);
  2606. #endif
  2607. #if WASM_ENABLE_GC == 0 && WASM_ENABLE_REF_TYPES != 0
  2608. wasm_externref_cleanup((WASMModuleInstanceCommon *)module_inst);
  2609. #endif
  2610. #if WASM_ENABLE_GC != 0
  2611. if (!is_sub_inst) {
  2612. if (module_inst->e->common.gc_heap_handle)
  2613. mem_allocator_destroy(module_inst->e->common.gc_heap_handle);
  2614. if (module_inst->e->common.gc_heap_pool)
  2615. wasm_runtime_free(module_inst->e->common.gc_heap_pool);
  2616. }
  2617. #endif
  2618. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2619. if (module_inst->frames) {
  2620. destroy_c_api_frames(module_inst->frames);
  2621. wasm_runtime_free(module_inst->frames);
  2622. module_inst->frames = NULL;
  2623. }
  2624. #endif
  2625. if (module_inst->c_api_func_imports)
  2626. wasm_runtime_free(module_inst->c_api_func_imports);
  2627. if (!is_sub_inst) {
  2628. #if WASM_ENABLE_WASI_NN != 0
  2629. wasi_nn_destroy(module_inst);
  2630. #endif
  2631. wasm_native_call_context_dtors((WASMModuleInstanceCommon *)module_inst);
  2632. }
  2633. #if WASM_ENABLE_BULK_MEMORY != 0
  2634. bh_bitmap_delete(module_inst->e->common.data_dropped);
  2635. #endif
  2636. #if WASM_ENABLE_REF_TYPES != 0
  2637. bh_bitmap_delete(module_inst->e->common.elem_dropped);
  2638. #endif
  2639. wasm_runtime_free(module_inst);
  2640. }
  2641. WASMFunctionInstance *
  2642. wasm_lookup_function(const WASMModuleInstance *module_inst, const char *name)
  2643. {
  2644. uint32 i;
  2645. for (i = 0; i < module_inst->export_func_count; i++)
  2646. if (!strcmp(module_inst->export_functions[i].name, name))
  2647. return module_inst->export_functions[i].function;
  2648. return NULL;
  2649. }
  2650. #if WASM_ENABLE_MULTI_MODULE != 0
  2651. WASMGlobalInstance *
  2652. wasm_lookup_global(const WASMModuleInstance *module_inst, const char *name)
  2653. {
  2654. uint32 i;
  2655. for (i = 0; i < module_inst->export_global_count; i++)
  2656. if (!strcmp(module_inst->export_globals[i].name, name))
  2657. return module_inst->export_globals[i].global;
  2658. return NULL;
  2659. }
  2660. WASMMemoryInstance *
  2661. wasm_lookup_memory(const WASMModuleInstance *module_inst, const char *name)
  2662. {
  2663. /**
  2664. * using a strong assumption that one module instance only has
  2665. * one memory instance
  2666. */
  2667. (void)module_inst->export_memories;
  2668. return module_inst->memories[0];
  2669. }
  2670. WASMTableInstance *
  2671. wasm_lookup_table(const WASMModuleInstance *module_inst, const char *name)
  2672. {
  2673. /**
  2674. * using a strong assumption that one module instance only has
  2675. * one table instance
  2676. */
  2677. (void)module_inst->export_tables;
  2678. return module_inst->tables[0];
  2679. }
  2680. #if WASM_ENABLE_TAGS != 0
  2681. WASMTagInstance *
  2682. wasm_lookup_tag(const WASMModuleInstance *module_inst, const char *name,
  2683. const char *signature)
  2684. {
  2685. uint32 i;
  2686. for (i = 0; i < module_inst->e->export_tag_count; i++)
  2687. if (!strcmp(module_inst->e->export_tags[i].name, name))
  2688. return module_inst->e->export_tags[i].tag;
  2689. (void)signature;
  2690. return NULL;
  2691. }
  2692. #endif
  2693. #endif
  2694. #ifdef OS_ENABLE_HW_BOUND_CHECK
  2695. static void
  2696. call_wasm_with_hw_bound_check(WASMModuleInstance *module_inst,
  2697. WASMExecEnv *exec_env,
  2698. WASMFunctionInstance *function, unsigned argc,
  2699. uint32 argv[])
  2700. {
  2701. WASMExecEnv *exec_env_tls = wasm_runtime_get_exec_env_tls();
  2702. WASMJmpBuf jmpbuf_node = { 0 }, *jmpbuf_node_pop;
  2703. uint32 page_size = os_getpagesize();
  2704. uint32 guard_page_count = STACK_OVERFLOW_CHECK_GUARD_PAGE_COUNT;
  2705. WASMRuntimeFrame *prev_frame = wasm_exec_env_get_cur_frame(exec_env);
  2706. uint8 *prev_top = exec_env->wasm_stack.top;
  2707. #ifdef BH_PLATFORM_WINDOWS
  2708. int result;
  2709. bool has_exception;
  2710. char exception[EXCEPTION_BUF_LEN];
  2711. #endif
  2712. bool ret = true;
  2713. /* Check native stack overflow firstly to ensure we have enough
  2714. native stack to run the following codes before actually calling
  2715. the aot function in invokeNative function. */
  2716. RECORD_STACK_USAGE(exec_env, (uint8 *)&exec_env_tls);
  2717. if ((uint8 *)&exec_env_tls < exec_env->native_stack_boundary
  2718. + page_size * (guard_page_count + 1)) {
  2719. wasm_set_exception(module_inst, "native stack overflow");
  2720. return;
  2721. }
  2722. if (!exec_env_tls) {
  2723. if (!os_thread_signal_inited()) {
  2724. wasm_set_exception(module_inst, "thread signal env not inited");
  2725. return;
  2726. }
  2727. /* Set thread handle and stack boundary if they haven't been set */
  2728. wasm_exec_env_set_thread_info(exec_env);
  2729. wasm_runtime_set_exec_env_tls(exec_env);
  2730. }
  2731. else {
  2732. if (exec_env_tls != exec_env) {
  2733. wasm_set_exception(module_inst, "invalid exec env");
  2734. return;
  2735. }
  2736. }
  2737. wasm_exec_env_push_jmpbuf(exec_env, &jmpbuf_node);
  2738. if (os_setjmp(jmpbuf_node.jmpbuf) == 0) {
  2739. #ifndef BH_PLATFORM_WINDOWS
  2740. wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2741. #else
  2742. __try {
  2743. wasm_interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2744. } __except (wasm_copy_exception(module_inst, NULL)
  2745. ? EXCEPTION_EXECUTE_HANDLER
  2746. : EXCEPTION_CONTINUE_SEARCH) {
  2747. /* Exception was thrown in wasm_exception_handler */
  2748. ret = false;
  2749. }
  2750. has_exception = wasm_copy_exception(module_inst, exception);
  2751. if (has_exception && strstr(exception, "native stack overflow")) {
  2752. /* After a stack overflow, the stack was left
  2753. in a damaged state, let the CRT repair it */
  2754. result = _resetstkoflw();
  2755. bh_assert(result != 0);
  2756. }
  2757. #endif
  2758. }
  2759. else {
  2760. /* Exception has been set in signal handler before calling longjmp */
  2761. ret = false;
  2762. }
  2763. /* Note: can't check wasm_get_exception(module_inst) here, there may be
  2764. * exception which is not caught by hardware (e.g. uninitialized elements),
  2765. * then the stack-frame is already freed inside wasm_interp_call_wasm */
  2766. if (!ret) {
  2767. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  2768. if (wasm_interp_create_call_stack(exec_env)) {
  2769. wasm_interp_dump_call_stack(exec_env, true, NULL, 0);
  2770. }
  2771. #endif
  2772. /* Restore operand frames */
  2773. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  2774. exec_env->wasm_stack.top = prev_top;
  2775. }
  2776. jmpbuf_node_pop = wasm_exec_env_pop_jmpbuf(exec_env);
  2777. bh_assert(&jmpbuf_node == jmpbuf_node_pop);
  2778. if (!exec_env->jmpbuf_stack_top) {
  2779. wasm_runtime_set_exec_env_tls(NULL);
  2780. }
  2781. if (!ret) {
  2782. os_sigreturn();
  2783. os_signal_unmask();
  2784. }
  2785. (void)jmpbuf_node_pop;
  2786. }
  2787. #define interp_call_wasm call_wasm_with_hw_bound_check
  2788. #else
  2789. #define interp_call_wasm wasm_interp_call_wasm
  2790. #endif
  2791. bool
  2792. wasm_call_function(WASMExecEnv *exec_env, WASMFunctionInstance *function,
  2793. unsigned argc, uint32 argv[])
  2794. {
  2795. WASMModuleInstance *module_inst =
  2796. (WASMModuleInstance *)exec_env->module_inst;
  2797. #ifndef OS_ENABLE_HW_BOUND_CHECK
  2798. /* Set thread handle and stack boundary */
  2799. wasm_exec_env_set_thread_info(exec_env);
  2800. #else
  2801. /* Set thread info in call_wasm_with_hw_bound_check when
  2802. hw bound check is enabled */
  2803. #endif
  2804. /* Set exec env, so it can be later retrieved from instance */
  2805. module_inst->cur_exec_env = exec_env;
  2806. interp_call_wasm(module_inst, exec_env, function, argc, argv);
  2807. return !wasm_copy_exception(module_inst, NULL);
  2808. }
  2809. #if WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0
  2810. /* look for the function name */
  2811. static char *
  2812. get_func_name_from_index(const WASMModuleInstance *inst, uint32 func_index)
  2813. {
  2814. char *func_name = NULL;
  2815. WASMFunctionInstance *func_inst = inst->e->functions + func_index;
  2816. if (func_inst->is_import_func) {
  2817. func_name = func_inst->u.func_import->field_name;
  2818. }
  2819. else {
  2820. #if WASM_ENABLE_CUSTOM_NAME_SECTION != 0
  2821. func_name = func_inst->u.func->field_name;
  2822. #endif
  2823. /* if custom name section is not generated,
  2824. search symbols from export table */
  2825. if (!func_name) {
  2826. unsigned j;
  2827. for (j = 0; j < inst->export_func_count; j++) {
  2828. WASMExportFuncInstance *export_func =
  2829. inst->export_functions + j;
  2830. if (export_func->function == func_inst) {
  2831. func_name = export_func->name;
  2832. break;
  2833. }
  2834. }
  2835. }
  2836. }
  2837. return func_name;
  2838. }
  2839. #endif /*WASM_ENABLE_PERF_PROFILING != 0 || WASM_ENABLE_DUMP_CALL_STACK != 0*/
  2840. #if WASM_ENABLE_PERF_PROFILING != 0
  2841. void
  2842. wasm_dump_perf_profiling(const WASMModuleInstance *module_inst)
  2843. {
  2844. WASMFunctionInstance *func_inst;
  2845. char *func_name;
  2846. uint32 i;
  2847. os_printf("Performance profiler data:\n");
  2848. for (i = 0; i < module_inst->e->function_count; i++) {
  2849. func_inst = module_inst->e->functions + i;
  2850. if (func_inst->total_exec_cnt == 0)
  2851. continue;
  2852. func_name = get_func_name_from_index(module_inst, i);
  2853. if (func_name)
  2854. os_printf(
  2855. " func %s, execution time: %.3f ms, execution count: %" PRIu32
  2856. " times, children execution time: %.3f ms\n",
  2857. func_name, func_inst->total_exec_time / 1000.0f,
  2858. func_inst->total_exec_cnt,
  2859. func_inst->children_exec_time / 1000.0f);
  2860. else
  2861. os_printf(" func %" PRIu32
  2862. ", execution time: %.3f ms, execution count: %" PRIu32
  2863. " times, children execution time: %.3f ms\n",
  2864. i, func_inst->total_exec_time / 1000.0f,
  2865. func_inst->total_exec_cnt,
  2866. func_inst->children_exec_time / 1000.0f);
  2867. }
  2868. }
  2869. double
  2870. wasm_summarize_wasm_execute_time(const WASMModuleInstance *inst)
  2871. {
  2872. double ret = 0;
  2873. unsigned i;
  2874. for (i = 0; i < inst->e->function_count; i++) {
  2875. WASMFunctionInstance *func = inst->e->functions + i;
  2876. ret += (func->total_exec_time - func->children_exec_time) / 1000.0f;
  2877. }
  2878. return ret;
  2879. }
  2880. double
  2881. wasm_get_wasm_func_exec_time(const WASMModuleInstance *inst,
  2882. const char *func_name)
  2883. {
  2884. unsigned i;
  2885. for (i = 0; i < inst->e->function_count; i++) {
  2886. char *name_in_wasm = get_func_name_from_index(inst, i);
  2887. if (name_in_wasm && strcmp(name_in_wasm, func_name) == 0) {
  2888. WASMFunctionInstance *func = inst->e->functions + i;
  2889. return (func->total_exec_time - func->children_exec_time) / 1000.0f;
  2890. }
  2891. }
  2892. return -1.0;
  2893. }
  2894. #endif /*WASM_ENABLE_PERF_PROFILING != 0*/
  2895. uint64
  2896. wasm_module_malloc_internal(WASMModuleInstance *module_inst,
  2897. WASMExecEnv *exec_env, uint64 size,
  2898. void **p_native_addr)
  2899. {
  2900. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2901. uint8 *addr = NULL;
  2902. uint32 offset = 0;
  2903. /* TODO: Memory64 size check based on memory idx type */
  2904. bh_assert(size <= UINT32_MAX);
  2905. if (!memory) {
  2906. wasm_set_exception(module_inst, "uninitialized memory");
  2907. return 0;
  2908. }
  2909. if (memory->heap_handle) {
  2910. addr = mem_allocator_malloc(memory->heap_handle, (uint32)size);
  2911. }
  2912. else if (module_inst->e->malloc_function && module_inst->e->free_function) {
  2913. if (!execute_malloc_function(
  2914. module_inst, exec_env, module_inst->e->malloc_function,
  2915. module_inst->e->retain_function, (uint32)size, &offset)) {
  2916. return 0;
  2917. }
  2918. /* If we use app's malloc function,
  2919. the default memory may be changed while memory growing */
  2920. memory = wasm_get_default_memory(module_inst);
  2921. addr = offset ? memory->memory_data + offset : NULL;
  2922. }
  2923. if (!addr) {
  2924. if (memory->heap_handle
  2925. && mem_allocator_is_heap_corrupted(memory->heap_handle)) {
  2926. wasm_runtime_show_app_heap_corrupted_prompt();
  2927. wasm_set_exception(module_inst, "app heap corrupted");
  2928. }
  2929. else {
  2930. LOG_WARNING("warning: allocate %u bytes memory failed", size);
  2931. }
  2932. return 0;
  2933. }
  2934. if (p_native_addr)
  2935. *p_native_addr = addr;
  2936. return (uint64)(addr - memory->memory_data);
  2937. }
  2938. uint64
  2939. wasm_module_realloc_internal(WASMModuleInstance *module_inst,
  2940. WASMExecEnv *exec_env, uint64 ptr, uint64 size,
  2941. void **p_native_addr)
  2942. {
  2943. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2944. uint8 *addr = NULL;
  2945. /* TODO: Memory64 ptr and size check based on memory idx type */
  2946. bh_assert(ptr <= UINT32_MAX);
  2947. bh_assert(size <= UINT32_MAX);
  2948. if (!memory) {
  2949. wasm_set_exception(module_inst, "uninitialized memory");
  2950. return 0;
  2951. }
  2952. if (memory->heap_handle) {
  2953. addr = mem_allocator_realloc(
  2954. memory->heap_handle,
  2955. (uint32)ptr ? memory->memory_data + (uint32)ptr : NULL,
  2956. (uint32)size);
  2957. }
  2958. /* Only support realloc in WAMR's app heap */
  2959. (void)exec_env;
  2960. if (!addr) {
  2961. if (memory->heap_handle
  2962. && mem_allocator_is_heap_corrupted(memory->heap_handle)) {
  2963. wasm_set_exception(module_inst, "app heap corrupted");
  2964. }
  2965. else {
  2966. wasm_set_exception(module_inst, "out of memory");
  2967. }
  2968. return 0;
  2969. }
  2970. if (p_native_addr)
  2971. *p_native_addr = addr;
  2972. return (uint64)(addr - memory->memory_data);
  2973. }
  2974. void
  2975. wasm_module_free_internal(WASMModuleInstance *module_inst,
  2976. WASMExecEnv *exec_env, uint64 ptr)
  2977. {
  2978. WASMMemoryInstance *memory = wasm_get_default_memory(module_inst);
  2979. /* TODO: Memory64 ptr and size check based on memory idx type */
  2980. bh_assert(ptr <= UINT32_MAX);
  2981. if (!memory) {
  2982. return;
  2983. }
  2984. if (ptr) {
  2985. uint8 *addr = memory->memory_data + (uint32)ptr;
  2986. uint8 *memory_data_end;
  2987. /* memory->memory_data_end may be changed in memory grow */
  2988. SHARED_MEMORY_LOCK(memory);
  2989. memory_data_end = memory->memory_data_end;
  2990. SHARED_MEMORY_UNLOCK(memory);
  2991. if (memory->heap_handle && memory->heap_data <= addr
  2992. && addr < memory->heap_data_end) {
  2993. mem_allocator_free(memory->heap_handle, addr);
  2994. }
  2995. else if (module_inst->e->malloc_function
  2996. && module_inst->e->free_function && memory->memory_data <= addr
  2997. && addr < memory_data_end) {
  2998. execute_free_function(module_inst, exec_env,
  2999. module_inst->e->free_function, (uint32)ptr);
  3000. }
  3001. }
  3002. }
  3003. uint64
  3004. wasm_module_malloc(WASMModuleInstance *module_inst, uint64 size,
  3005. void **p_native_addr)
  3006. {
  3007. return wasm_module_malloc_internal(module_inst, NULL, size, p_native_addr);
  3008. }
  3009. uint64
  3010. wasm_module_realloc(WASMModuleInstance *module_inst, uint64 ptr, uint64 size,
  3011. void **p_native_addr)
  3012. {
  3013. return wasm_module_realloc_internal(module_inst, NULL, ptr, size,
  3014. p_native_addr);
  3015. }
  3016. void
  3017. wasm_module_free(WASMModuleInstance *module_inst, uint64 ptr)
  3018. {
  3019. wasm_module_free_internal(module_inst, NULL, ptr);
  3020. }
  3021. uint64
  3022. wasm_module_dup_data(WASMModuleInstance *module_inst, const char *src,
  3023. uint64 size)
  3024. {
  3025. char *buffer;
  3026. uint64 buffer_offset;
  3027. /* TODO: Memory64 size check based on memory idx type */
  3028. bh_assert(size <= UINT32_MAX);
  3029. buffer_offset = wasm_module_malloc(module_inst, size, (void **)&buffer);
  3030. if (buffer_offset != 0) {
  3031. buffer = wasm_runtime_addr_app_to_native(
  3032. (WASMModuleInstanceCommon *)module_inst, buffer_offset);
  3033. bh_memcpy_s(buffer, (uint32)size, src, (uint32)size);
  3034. }
  3035. return buffer_offset;
  3036. }
  3037. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  3038. bool
  3039. wasm_enlarge_table(WASMModuleInstance *module_inst, uint32 table_idx,
  3040. uint32 inc_size, table_elem_type_t init_val)
  3041. {
  3042. uint32 total_size, i;
  3043. table_elem_type_t *new_table_data_start;
  3044. WASMTableInstance *table_inst;
  3045. if (!inc_size) {
  3046. return true;
  3047. }
  3048. bh_assert(table_idx < module_inst->table_count);
  3049. table_inst = wasm_get_table_inst(module_inst, table_idx);
  3050. if (!table_inst) {
  3051. return false;
  3052. }
  3053. if (inc_size > UINT32_MAX - table_inst->cur_size) {
  3054. return false;
  3055. }
  3056. total_size = table_inst->cur_size + inc_size;
  3057. if (total_size > table_inst->max_size) {
  3058. return false;
  3059. }
  3060. /* fill in */
  3061. new_table_data_start = table_inst->elems + table_inst->cur_size;
  3062. for (i = 0; i < inc_size; ++i) {
  3063. new_table_data_start[i] = init_val;
  3064. }
  3065. table_inst->cur_size = total_size;
  3066. return true;
  3067. }
  3068. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  3069. static bool
  3070. call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 tbl_elem_idx,
  3071. uint32 argc, uint32 argv[], bool check_type_idx, uint32 type_idx)
  3072. {
  3073. WASMModuleInstance *module_inst = NULL;
  3074. WASMTableInstance *table_inst = NULL;
  3075. table_elem_type_t tbl_elem_val = NULL_REF;
  3076. uint32 func_idx = 0;
  3077. WASMFunctionInstance *func_inst = NULL;
  3078. module_inst = (WASMModuleInstance *)exec_env->module_inst;
  3079. bh_assert(module_inst);
  3080. table_inst = module_inst->tables[tbl_idx];
  3081. if (!table_inst) {
  3082. wasm_set_exception(module_inst, "unknown table");
  3083. goto got_exception;
  3084. }
  3085. if (tbl_elem_idx >= table_inst->cur_size) {
  3086. wasm_set_exception(module_inst, "undefined element");
  3087. goto got_exception;
  3088. }
  3089. tbl_elem_val = ((table_elem_type_t *)table_inst->elems)[tbl_elem_idx];
  3090. if (tbl_elem_val == NULL_REF) {
  3091. wasm_set_exception(module_inst, "uninitialized element");
  3092. goto got_exception;
  3093. }
  3094. #if WASM_ENABLE_GC == 0
  3095. func_idx = tbl_elem_val;
  3096. #else
  3097. func_idx =
  3098. wasm_func_obj_get_func_idx_bound((WASMFuncObjectRef)tbl_elem_val);
  3099. #endif
  3100. /**
  3101. * we insist to call functions owned by the module itself
  3102. **/
  3103. if (func_idx >= module_inst->e->function_count) {
  3104. wasm_set_exception(module_inst, "unknown function");
  3105. goto got_exception;
  3106. }
  3107. func_inst = module_inst->e->functions + func_idx;
  3108. if (check_type_idx) {
  3109. WASMType *cur_type = module_inst->module->types[type_idx];
  3110. WASMType *cur_func_type;
  3111. if (func_inst->is_import_func)
  3112. cur_func_type = (WASMType *)func_inst->u.func_import->func_type;
  3113. else
  3114. cur_func_type = (WASMType *)func_inst->u.func->func_type;
  3115. if (cur_type != cur_func_type) {
  3116. wasm_set_exception(module_inst, "indirect call type mismatch");
  3117. goto got_exception;
  3118. }
  3119. }
  3120. interp_call_wasm(module_inst, exec_env, func_inst, argc, argv);
  3121. return !wasm_copy_exception(module_inst, NULL);
  3122. got_exception:
  3123. return false;
  3124. }
  3125. bool
  3126. wasm_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3127. uint32 argc, uint32 argv[])
  3128. {
  3129. return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
  3130. }
  3131. #if WASM_ENABLE_THREAD_MGR != 0
  3132. bool
  3133. wasm_set_aux_stack(WASMExecEnv *exec_env, uint64 start_offset, uint32 size)
  3134. {
  3135. WASMModuleInstance *module_inst =
  3136. (WASMModuleInstance *)exec_env->module_inst;
  3137. uint32 stack_top_idx = module_inst->module->aux_stack_top_global_index;
  3138. #if WASM_ENABLE_HEAP_AUX_STACK_ALLOCATION == 0
  3139. /* Check the aux stack space */
  3140. uint64 data_end = module_inst->module->aux_data_end;
  3141. uint64 stack_bottom = module_inst->module->aux_stack_bottom;
  3142. bool is_stack_before_data = stack_bottom < data_end ? true : false;
  3143. if ((is_stack_before_data && (size > start_offset))
  3144. || ((!is_stack_before_data) && (start_offset - data_end < size)))
  3145. return false;
  3146. #endif
  3147. if (stack_top_idx != (uint32)-1) {
  3148. /* The aux stack top is a wasm global,
  3149. set the initial value for the global */
  3150. uint8 *global_addr =
  3151. module_inst->global_data
  3152. + module_inst->e->globals[stack_top_idx].data_offset;
  3153. *(int32 *)global_addr = (uint32)start_offset;
  3154. /* The aux stack boundary is a constant value,
  3155. set the value to exec_env */
  3156. exec_env->aux_stack_boundary = (uintptr_t)start_offset - size;
  3157. exec_env->aux_stack_bottom = (uintptr_t)start_offset;
  3158. return true;
  3159. }
  3160. return false;
  3161. }
  3162. bool
  3163. wasm_get_aux_stack(WASMExecEnv *exec_env, uint64 *start_offset, uint32 *size)
  3164. {
  3165. WASMModuleInstance *module_inst =
  3166. (WASMModuleInstance *)exec_env->module_inst;
  3167. /* The aux stack information is resolved in loader
  3168. and store in module */
  3169. uint64 stack_bottom = module_inst->module->aux_stack_bottom;
  3170. uint32 total_aux_stack_size = module_inst->module->aux_stack_size;
  3171. if (stack_bottom != 0 && total_aux_stack_size != 0) {
  3172. if (start_offset)
  3173. *start_offset = stack_bottom;
  3174. if (size)
  3175. *size = total_aux_stack_size;
  3176. return true;
  3177. }
  3178. return false;
  3179. }
  3180. #endif
  3181. #if (WASM_ENABLE_MEMORY_PROFILING != 0) || (WASM_ENABLE_MEMORY_TRACING != 0)
  3182. void
  3183. wasm_get_module_mem_consumption(const WASMModule *module,
  3184. WASMModuleMemConsumption *mem_conspn)
  3185. {
  3186. uint32 i, size;
  3187. memset(mem_conspn, 0, sizeof(*mem_conspn));
  3188. mem_conspn->module_struct_size = sizeof(WASMModule);
  3189. mem_conspn->types_size = sizeof(WASMFuncType *) * module->type_count;
  3190. for (i = 0; i < module->type_count; i++) {
  3191. WASMFuncType *type = module->types[i];
  3192. size = offsetof(WASMFuncType, types)
  3193. + sizeof(uint8) * (type->param_count + type->result_count);
  3194. mem_conspn->types_size += size;
  3195. }
  3196. mem_conspn->imports_size = sizeof(WASMImport) * module->import_count;
  3197. mem_conspn->functions_size =
  3198. sizeof(WASMFunction *) * module->function_count;
  3199. for (i = 0; i < module->function_count; i++) {
  3200. WASMFunction *func = module->functions[i];
  3201. WASMFuncType *type = func->func_type;
  3202. size = sizeof(WASMFunction) + func->local_count
  3203. + sizeof(uint16) * (type->param_count + func->local_count);
  3204. #if WASM_ENABLE_FAST_INTERP != 0
  3205. size +=
  3206. func->code_compiled_size + sizeof(uint32) * func->const_cell_num;
  3207. #endif
  3208. mem_conspn->functions_size += size;
  3209. }
  3210. mem_conspn->tables_size = sizeof(WASMTable) * module->table_count;
  3211. mem_conspn->memories_size = sizeof(WASMMemory) * module->memory_count;
  3212. mem_conspn->globals_size = sizeof(WASMGlobal) * module->global_count;
  3213. mem_conspn->exports_size = sizeof(WASMExport) * module->export_count;
  3214. mem_conspn->table_segs_size =
  3215. sizeof(WASMTableSeg) * module->table_seg_count;
  3216. for (i = 0; i < module->table_seg_count; i++) {
  3217. WASMTableSeg *table_seg = &module->table_segments[i];
  3218. mem_conspn->tables_size +=
  3219. sizeof(InitializerExpression *) * table_seg->value_count;
  3220. }
  3221. mem_conspn->data_segs_size = sizeof(WASMDataSeg *) * module->data_seg_count;
  3222. for (i = 0; i < module->data_seg_count; i++) {
  3223. mem_conspn->data_segs_size += sizeof(WASMDataSeg);
  3224. }
  3225. if (module->const_str_list) {
  3226. StringNode *node = module->const_str_list, *node_next;
  3227. while (node) {
  3228. node_next = node->next;
  3229. mem_conspn->const_strs_size +=
  3230. sizeof(StringNode) + strlen(node->str) + 1;
  3231. node = node_next;
  3232. }
  3233. }
  3234. mem_conspn->total_size += mem_conspn->module_struct_size;
  3235. mem_conspn->total_size += mem_conspn->types_size;
  3236. mem_conspn->total_size += mem_conspn->imports_size;
  3237. mem_conspn->total_size += mem_conspn->functions_size;
  3238. mem_conspn->total_size += mem_conspn->tables_size;
  3239. mem_conspn->total_size += mem_conspn->memories_size;
  3240. mem_conspn->total_size += mem_conspn->globals_size;
  3241. mem_conspn->total_size += mem_conspn->exports_size;
  3242. mem_conspn->total_size += mem_conspn->table_segs_size;
  3243. mem_conspn->total_size += mem_conspn->data_segs_size;
  3244. mem_conspn->total_size += mem_conspn->const_strs_size;
  3245. }
  3246. void
  3247. wasm_get_module_inst_mem_consumption(const WASMModuleInstance *module_inst,
  3248. WASMModuleInstMemConsumption *mem_conspn)
  3249. {
  3250. uint32 i;
  3251. uint64 size;
  3252. memset(mem_conspn, 0, sizeof(*mem_conspn));
  3253. mem_conspn->module_inst_struct_size = (uint8 *)module_inst->e
  3254. - (uint8 *)module_inst
  3255. + sizeof(WASMModuleInstanceExtra);
  3256. mem_conspn->memories_size =
  3257. sizeof(WASMMemoryInstance *) * module_inst->memory_count;
  3258. for (i = 0; i < module_inst->memory_count; i++) {
  3259. WASMMemoryInstance *memory = module_inst->memories[i];
  3260. size = memory->num_bytes_per_page * memory->cur_page_count;
  3261. mem_conspn->memories_size += size;
  3262. mem_conspn->app_heap_size += memory->heap_data_end - memory->heap_data;
  3263. /* size of app heap structure */
  3264. mem_conspn->memories_size += mem_allocator_get_heap_struct_size();
  3265. /* Module instance structures have been appened into the end of
  3266. module instance */
  3267. }
  3268. mem_conspn->tables_size =
  3269. sizeof(WASMTableInstance *) * module_inst->table_count;
  3270. /* Table instance structures and table elements have been appened into
  3271. the end of module instance */
  3272. mem_conspn->functions_size =
  3273. sizeof(WASMFunctionInstance) * module_inst->e->function_count;
  3274. mem_conspn->globals_size =
  3275. sizeof(WASMGlobalInstance) * module_inst->e->global_count;
  3276. /* Global data has been appened into the end of module instance */
  3277. mem_conspn->exports_size =
  3278. sizeof(WASMExportFuncInstance) * module_inst->export_func_count;
  3279. mem_conspn->total_size += mem_conspn->module_inst_struct_size;
  3280. mem_conspn->total_size += mem_conspn->memories_size;
  3281. mem_conspn->total_size += mem_conspn->functions_size;
  3282. mem_conspn->total_size += mem_conspn->tables_size;
  3283. mem_conspn->total_size += mem_conspn->globals_size;
  3284. mem_conspn->total_size += mem_conspn->exports_size;
  3285. }
  3286. #endif /* end of (WASM_ENABLE_MEMORY_PROFILING != 0) \
  3287. || (WASM_ENABLE_MEMORY_TRACING != 0) */
  3288. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  3289. bool
  3290. wasm_interp_create_call_stack(struct WASMExecEnv *exec_env)
  3291. {
  3292. WASMModuleInstance *module_inst =
  3293. (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env);
  3294. WASMModule *module = module_inst->module;
  3295. WASMInterpFrame *first_frame,
  3296. *cur_frame = wasm_exec_env_get_cur_frame(exec_env);
  3297. uint32 n = 0;
  3298. /* count frames includes a function */
  3299. first_frame = cur_frame;
  3300. while (cur_frame) {
  3301. if (cur_frame->function) {
  3302. n++;
  3303. }
  3304. cur_frame = cur_frame->prev_frame;
  3305. }
  3306. /* release previous stack frames and create new ones */
  3307. destroy_c_api_frames(module_inst->frames);
  3308. if (!bh_vector_init(module_inst->frames, n, sizeof(WASMCApiFrame), false)) {
  3309. return false;
  3310. }
  3311. cur_frame = first_frame;
  3312. n = 0;
  3313. while (cur_frame) {
  3314. WASMCApiFrame frame = { 0 };
  3315. WASMFunctionInstance *func_inst = cur_frame->function;
  3316. const char *func_name = NULL;
  3317. const uint8 *func_code_base = NULL;
  3318. uint32 max_local_cell_num, max_stack_cell_num;
  3319. uint32 all_cell_num, lp_size;
  3320. if (!func_inst) {
  3321. cur_frame = cur_frame->prev_frame;
  3322. continue;
  3323. }
  3324. /* place holder, will overwrite it in wasm_c_api */
  3325. frame.instance = module_inst;
  3326. frame.module_offset = 0;
  3327. frame.func_index = (uint32)(func_inst - module_inst->e->functions);
  3328. func_code_base = wasm_get_func_code(func_inst);
  3329. if (!cur_frame->ip || !func_code_base) {
  3330. frame.func_offset = 0;
  3331. }
  3332. else {
  3333. #if WASM_ENABLE_FAST_INTERP == 0
  3334. frame.func_offset = (uint32)(cur_frame->ip - module->load_addr);
  3335. #else
  3336. frame.func_offset = (uint32)(cur_frame->ip - func_code_base);
  3337. #endif
  3338. }
  3339. func_name = get_func_name_from_index(module_inst, frame.func_index);
  3340. frame.func_name_wp = func_name;
  3341. if (frame.func_index >= module->import_function_count) {
  3342. uint32 wasm_func_idx =
  3343. frame.func_index - module->import_function_count;
  3344. max_local_cell_num =
  3345. module->functions[wasm_func_idx]->param_cell_num
  3346. + module->functions[wasm_func_idx]->local_cell_num;
  3347. max_stack_cell_num =
  3348. module->functions[wasm_func_idx]->max_stack_cell_num;
  3349. all_cell_num = max_local_cell_num + max_stack_cell_num;
  3350. #if WASM_ENABLE_FAST_INTERP != 0
  3351. all_cell_num += module->functions[wasm_func_idx]->const_cell_num;
  3352. #endif
  3353. }
  3354. else {
  3355. WASMFuncType *func_type =
  3356. module->import_functions[frame.func_index].u.function.func_type;
  3357. max_local_cell_num =
  3358. func_type->param_cell_num > 2 ? func_type->param_cell_num : 2;
  3359. max_stack_cell_num = 0;
  3360. all_cell_num = max_local_cell_num + max_stack_cell_num;
  3361. }
  3362. #if WASM_ENABLE_GC == 0
  3363. lp_size = all_cell_num * 4;
  3364. #else
  3365. lp_size = align_uint(all_cell_num * 5, 4);
  3366. #endif
  3367. if (lp_size > 0) {
  3368. if (!(frame.lp = wasm_runtime_malloc(lp_size))) {
  3369. destroy_c_api_frames(module_inst->frames);
  3370. return false;
  3371. }
  3372. bh_memcpy_s(frame.lp, lp_size, cur_frame->lp, lp_size);
  3373. #if WASM_ENABLE_GC != 0
  3374. #if WASM_ENABLE_FAST_INTERP == 0
  3375. frame.sp = frame.lp + (cur_frame->sp - cur_frame->lp);
  3376. #else
  3377. /* for fast-interp, let frame sp point to the end of the frame */
  3378. frame.sp = frame.lp + all_cell_num;
  3379. #endif
  3380. frame.frame_ref = (uint8 *)frame.lp
  3381. + (wasm_interp_get_frame_ref(cur_frame)
  3382. - (uint8 *)cur_frame->lp);
  3383. #endif
  3384. }
  3385. if (!bh_vector_append(module_inst->frames, &frame)) {
  3386. if (frame.lp)
  3387. wasm_runtime_free(frame.lp);
  3388. destroy_c_api_frames(module_inst->frames);
  3389. return false;
  3390. }
  3391. cur_frame = cur_frame->prev_frame;
  3392. n++;
  3393. }
  3394. return true;
  3395. }
  3396. #define PRINT_OR_DUMP() \
  3397. do { \
  3398. total_len += \
  3399. wasm_runtime_dump_line_buf_impl(line_buf, print, &buf, &len); \
  3400. if ((!print) && buf && (len == 0)) { \
  3401. exception_unlock(module_inst); \
  3402. return total_len; \
  3403. } \
  3404. } while (0)
  3405. uint32
  3406. wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf,
  3407. uint32 len)
  3408. {
  3409. WASMModuleInstance *module_inst =
  3410. (WASMModuleInstance *)wasm_exec_env_get_module_inst(exec_env);
  3411. uint32 n = 0, total_len = 0, total_frames;
  3412. /* reserve 256 bytes for line buffer, any line longer than 256 bytes
  3413. * will be truncated */
  3414. char line_buf[256];
  3415. if (!module_inst->frames) {
  3416. return 0;
  3417. }
  3418. total_frames = (uint32)bh_vector_size(module_inst->frames);
  3419. if (total_frames == 0) {
  3420. return 0;
  3421. }
  3422. exception_lock(module_inst);
  3423. snprintf(line_buf, sizeof(line_buf), "\n");
  3424. PRINT_OR_DUMP();
  3425. while (n < total_frames) {
  3426. WASMCApiFrame frame = { 0 };
  3427. uint32 line_length, i;
  3428. if (!bh_vector_get(module_inst->frames, n, &frame)) {
  3429. exception_unlock(module_inst);
  3430. return 0;
  3431. }
  3432. #if WASM_ENABLE_FAST_JIT != 0
  3433. /* Fast JIT doesn't support committing ip (instruction pointer) yet */
  3434. if (module_inst->e->running_mode == Mode_Fast_JIT
  3435. || module_inst->e->running_mode == Mode_Multi_Tier_JIT) {
  3436. /* function name not exported, print number instead */
  3437. if (frame.func_name_wp == NULL) {
  3438. line_length = snprintf(line_buf, sizeof(line_buf),
  3439. "#%02" PRIu32 " $f%" PRIu32 "\n", n,
  3440. frame.func_index);
  3441. }
  3442. else {
  3443. line_length =
  3444. snprintf(line_buf, sizeof(line_buf), "#%02" PRIu32 " %s\n",
  3445. n, frame.func_name_wp);
  3446. }
  3447. }
  3448. else
  3449. #endif
  3450. {
  3451. /* function name not exported, print number instead */
  3452. if (frame.func_name_wp == NULL) {
  3453. line_length =
  3454. snprintf(line_buf, sizeof(line_buf),
  3455. "#%02" PRIu32 ": 0x%04x - $f%" PRIu32 "\n", n,
  3456. frame.func_offset, frame.func_index);
  3457. }
  3458. else {
  3459. line_length = snprintf(line_buf, sizeof(line_buf),
  3460. "#%02" PRIu32 ": 0x%04x - %s\n", n,
  3461. frame.func_offset, frame.func_name_wp);
  3462. }
  3463. }
  3464. if (line_length >= sizeof(line_buf)) {
  3465. uint32 line_buffer_len = sizeof(line_buf);
  3466. /* If line too long, ensure the last character is '\n' */
  3467. for (i = line_buffer_len - 5; i < line_buffer_len - 2; i++) {
  3468. line_buf[i] = '.';
  3469. }
  3470. line_buf[line_buffer_len - 2] = '\n';
  3471. }
  3472. PRINT_OR_DUMP();
  3473. n++;
  3474. }
  3475. snprintf(line_buf, sizeof(line_buf), "\n");
  3476. PRINT_OR_DUMP();
  3477. exception_unlock(module_inst);
  3478. return total_len + 1;
  3479. }
  3480. #endif /* end of WASM_ENABLE_DUMP_CALL_STACK */
  3481. #if WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
  3482. || WASM_ENABLE_WAMR_COMPILER != 0
  3483. void
  3484. jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id)
  3485. {
  3486. if (id != EXCE_ALREADY_THROWN)
  3487. wasm_set_exception_with_id(module_inst, id);
  3488. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3489. wasm_runtime_access_exce_check_guard_page();
  3490. #endif
  3491. }
  3492. bool
  3493. jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
  3494. uint64 app_buf_addr, uint64 app_buf_size,
  3495. void **p_native_addr)
  3496. {
  3497. bool ret = wasm_check_app_addr_and_convert(
  3498. module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr);
  3499. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3500. if (!ret)
  3501. wasm_runtime_access_exce_check_guard_page();
  3502. #endif
  3503. return ret;
  3504. }
  3505. #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \
  3506. || WASM_ENABLE_WAMR_COMPILER != 0 */
  3507. #if WASM_ENABLE_FAST_JIT != 0
  3508. bool
  3509. fast_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3510. uint32 type_idx, uint32 argc, uint32 *argv)
  3511. {
  3512. return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, true,
  3513. type_idx);
  3514. }
  3515. #endif /* end of WASM_ENABLE_FAST_JIT != 0 */
  3516. #if WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0
  3517. bool
  3518. llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx,
  3519. uint32 argc, uint32 *argv)
  3520. {
  3521. bool ret;
  3522. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  3523. ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0);
  3524. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3525. if (!ret)
  3526. wasm_runtime_access_exce_check_guard_page();
  3527. #endif
  3528. return ret;
  3529. }
  3530. bool
  3531. llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
  3532. uint32 *argv)
  3533. {
  3534. WASMModuleInstance *module_inst;
  3535. WASMModule *module;
  3536. uint32 *func_type_indexes;
  3537. uint32 func_type_idx;
  3538. WASMFuncType *func_type;
  3539. void *func_ptr;
  3540. WASMFunctionImport *import_func;
  3541. CApiFuncImport *c_api_func_import = NULL;
  3542. const char *signature;
  3543. void *attachment;
  3544. char buf[96];
  3545. bool ret = false;
  3546. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  3547. module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env);
  3548. module = module_inst->module;
  3549. func_type_indexes = module_inst->func_type_indexes;
  3550. func_type_idx = func_type_indexes[func_idx];
  3551. func_type = (WASMFuncType *)module->types[func_type_idx];
  3552. func_ptr = module_inst->func_ptrs[func_idx];
  3553. bh_assert(func_idx < module->import_function_count);
  3554. import_func = &module->import_functions[func_idx].u.function;
  3555. if (import_func->call_conv_wasm_c_api) {
  3556. if (module_inst->c_api_func_imports) {
  3557. c_api_func_import = module_inst->c_api_func_imports + func_idx;
  3558. func_ptr = c_api_func_import->func_ptr_linked;
  3559. }
  3560. else {
  3561. c_api_func_import = NULL;
  3562. func_ptr = NULL;
  3563. }
  3564. }
  3565. if (!func_ptr) {
  3566. snprintf(buf, sizeof(buf),
  3567. "failed to call unlinked import function (%s, %s)",
  3568. import_func->module_name, import_func->field_name);
  3569. wasm_set_exception(module_inst, buf);
  3570. goto fail;
  3571. }
  3572. attachment = import_func->attachment;
  3573. if (import_func->call_conv_wasm_c_api) {
  3574. ret = wasm_runtime_invoke_c_api_native(
  3575. (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
  3576. argv, c_api_func_import->with_env_arg, c_api_func_import->env_arg);
  3577. }
  3578. else if (!import_func->call_conv_raw) {
  3579. signature = import_func->signature;
  3580. ret =
  3581. wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
  3582. attachment, argv, argc, argv);
  3583. }
  3584. else {
  3585. signature = import_func->signature;
  3586. ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
  3587. signature, attachment, argv, argc,
  3588. argv);
  3589. }
  3590. fail:
  3591. #ifdef OS_ENABLE_HW_BOUND_CHECK
  3592. if (!ret)
  3593. wasm_runtime_access_exce_check_guard_page();
  3594. #endif
  3595. return ret;
  3596. }
  3597. #if WASM_ENABLE_BULK_MEMORY != 0
  3598. bool
  3599. llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index,
  3600. uint32 offset, uint32 len, uint32 dst)
  3601. {
  3602. WASMMemoryInstance *memory_inst;
  3603. WASMModule *module;
  3604. uint8 *data;
  3605. uint8 *maddr;
  3606. uint64 seg_len;
  3607. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3608. memory_inst = wasm_get_default_memory(module_inst);
  3609. if (bh_bitmap_get_bit(module_inst->e->common.data_dropped, seg_index)) {
  3610. seg_len = 0;
  3611. data = NULL;
  3612. }
  3613. else {
  3614. module = module_inst->module;
  3615. seg_len = module->data_segments[seg_index]->data_length;
  3616. data = module->data_segments[seg_index]->data;
  3617. }
  3618. if (!wasm_runtime_validate_app_addr((WASMModuleInstanceCommon *)module_inst,
  3619. (uint64)dst, (uint64)len))
  3620. return false;
  3621. if ((uint64)offset + (uint64)len > seg_len) {
  3622. wasm_set_exception(module_inst, "out of bounds memory access");
  3623. return false;
  3624. }
  3625. maddr = wasm_runtime_addr_app_to_native(
  3626. (WASMModuleInstanceCommon *)module_inst, (uint64)dst);
  3627. SHARED_MEMORY_LOCK(memory_inst);
  3628. bh_memcpy_s(maddr, (uint32)(memory_inst->memory_data_size - dst),
  3629. data + offset, len);
  3630. SHARED_MEMORY_UNLOCK(memory_inst);
  3631. return true;
  3632. }
  3633. bool
  3634. llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index)
  3635. {
  3636. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3637. bh_bitmap_set_bit(module_inst->e->common.data_dropped, seg_index);
  3638. /* Currently we can't free the dropped data segment
  3639. as they are stored in wasm bytecode */
  3640. return true;
  3641. }
  3642. #endif /* end of WASM_ENABLE_BULK_MEMORY != 0 */
  3643. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  3644. void
  3645. llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx)
  3646. {
  3647. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3648. bh_bitmap_set_bit(module_inst->e->common.elem_dropped, tbl_seg_idx);
  3649. }
  3650. void
  3651. llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3652. uint32 tbl_seg_idx, uint32 length, uint32 src_offset,
  3653. uint32 dst_offset)
  3654. {
  3655. WASMTableInstance *tbl_inst;
  3656. WASMTableSeg *tbl_seg;
  3657. table_elem_type_t *table_elems;
  3658. InitializerExpression *tbl_seg_init_values = NULL, *init_values;
  3659. uint32 i, tbl_seg_len = 0;
  3660. #if WASM_ENABLE_GC != 0
  3661. void *func_obj;
  3662. #endif
  3663. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3664. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3665. tbl_seg = module_inst->module->table_segments + tbl_seg_idx;
  3666. bh_assert(tbl_inst);
  3667. bh_assert(tbl_seg);
  3668. if (!bh_bitmap_get_bit(module_inst->e->common.elem_dropped, tbl_seg_idx)) {
  3669. /* table segment isn't dropped */
  3670. tbl_seg_init_values = tbl_seg->init_values;
  3671. tbl_seg_len = tbl_seg->value_count;
  3672. }
  3673. if (offset_len_out_of_bounds(src_offset, length, tbl_seg_len)
  3674. || offset_len_out_of_bounds(dst_offset, length, tbl_inst->cur_size)) {
  3675. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3676. return;
  3677. }
  3678. if (!length) {
  3679. return;
  3680. }
  3681. table_elems = tbl_inst->elems + dst_offset;
  3682. init_values = tbl_seg_init_values + src_offset;
  3683. for (i = 0; i < length; i++) {
  3684. #if WASM_ENABLE_GC != 0
  3685. /* UINT32_MAX indicates that it is a null ref */
  3686. if (init_values[i].u.ref_index != UINT32_MAX) {
  3687. if (!(func_obj = wasm_create_func_obj(module_inst,
  3688. init_values[i].u.ref_index,
  3689. true, NULL, 0))) {
  3690. wasm_set_exception(module_inst, "null function object");
  3691. return;
  3692. }
  3693. table_elems[i] = func_obj;
  3694. }
  3695. else {
  3696. table_elems[i] = NULL_REF;
  3697. }
  3698. #else
  3699. table_elems[i] = init_values[i].u.ref_index;
  3700. #endif
  3701. }
  3702. }
  3703. void
  3704. llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx,
  3705. uint32 dst_tbl_idx, uint32 length, uint32 src_offset,
  3706. uint32 dst_offset)
  3707. {
  3708. WASMTableInstance *src_tbl_inst;
  3709. WASMTableInstance *dst_tbl_inst;
  3710. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3711. src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx);
  3712. dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx);
  3713. bh_assert(src_tbl_inst);
  3714. bh_assert(dst_tbl_inst);
  3715. if (offset_len_out_of_bounds(dst_offset, length, dst_tbl_inst->cur_size)
  3716. || offset_len_out_of_bounds(src_offset, length,
  3717. src_tbl_inst->cur_size)) {
  3718. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3719. return;
  3720. }
  3721. /* if src_offset >= dst_offset, copy from front to back */
  3722. /* if src_offset < dst_offset, copy from back to front */
  3723. /* merge all together */
  3724. bh_memmove_s((uint8 *)dst_tbl_inst + offsetof(WASMTableInstance, elems)
  3725. + sizeof(table_elem_type_t) * dst_offset,
  3726. (uint32)sizeof(table_elem_type_t)
  3727. * (dst_tbl_inst->cur_size - dst_offset),
  3728. (uint8 *)src_tbl_inst + offsetof(WASMTableInstance, elems)
  3729. + sizeof(table_elem_type_t) * src_offset,
  3730. (uint32)sizeof(table_elem_type_t) * length);
  3731. }
  3732. void
  3733. llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3734. uint32 length, uintptr_t val, uint32 data_offset)
  3735. {
  3736. WASMTableInstance *tbl_inst;
  3737. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3738. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3739. bh_assert(tbl_inst);
  3740. if (offset_len_out_of_bounds(data_offset, length, tbl_inst->cur_size)) {
  3741. jit_set_exception_with_id(module_inst, EXCE_OUT_OF_BOUNDS_TABLE_ACCESS);
  3742. return;
  3743. }
  3744. for (; length != 0; data_offset++, length--) {
  3745. tbl_inst->elems[data_offset] = (table_elem_type_t)val;
  3746. }
  3747. }
  3748. uint32
  3749. llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx,
  3750. uint32 inc_size, uintptr_t init_val)
  3751. {
  3752. WASMTableInstance *tbl_inst;
  3753. uint32 i, orig_size, total_size;
  3754. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3755. tbl_inst = wasm_get_table_inst(module_inst, tbl_idx);
  3756. if (!tbl_inst) {
  3757. return (uint32)-1;
  3758. }
  3759. orig_size = tbl_inst->cur_size;
  3760. if (!inc_size) {
  3761. return orig_size;
  3762. }
  3763. if (tbl_inst->cur_size > UINT32_MAX - inc_size) { /* integer overflow */
  3764. return (uint32)-1;
  3765. }
  3766. total_size = tbl_inst->cur_size + inc_size;
  3767. if (total_size > tbl_inst->max_size) {
  3768. return (uint32)-1;
  3769. }
  3770. /* fill in */
  3771. for (i = 0; i < inc_size; ++i) {
  3772. tbl_inst->elems[tbl_inst->cur_size + i] = (table_elem_type_t)init_val;
  3773. }
  3774. tbl_inst->cur_size = total_size;
  3775. return orig_size;
  3776. }
  3777. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  3778. #if WASM_ENABLE_GC != 0
  3779. void *
  3780. llvm_jit_create_func_obj(WASMModuleInstance *module_inst, uint32 func_idx,
  3781. bool throw_exce, char *error_buf,
  3782. uint32 error_buf_size)
  3783. {
  3784. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  3785. return wasm_create_func_obj(module_inst, func_idx, throw_exce, error_buf,
  3786. error_buf_size);
  3787. }
  3788. bool
  3789. llvm_jit_obj_is_instance_of(WASMModuleInstance *module_inst,
  3790. WASMObjectRef gc_obj, uint32 type_index)
  3791. {
  3792. WASMModule *module = module_inst->module;
  3793. WASMType **types = module->types;
  3794. uint32 type_count = module->type_count;
  3795. return wasm_obj_is_instance_of(gc_obj, type_index, types, type_count);
  3796. }
  3797. WASMRttTypeRef
  3798. llvm_jit_rtt_type_new(WASMModuleInstance *module_inst, uint32 type_index)
  3799. {
  3800. WASMModule *module = module_inst->module;
  3801. WASMType *defined_type = module->types[type_index];
  3802. WASMRttType **rtt_types = module->rtt_types;
  3803. uint32 rtt_type_count = module->type_count;
  3804. korp_mutex *rtt_type_lock = &module->rtt_type_lock;
  3805. return wasm_rtt_type_new(defined_type, type_index, rtt_types,
  3806. rtt_type_count, rtt_type_lock);
  3807. }
  3808. bool
  3809. llvm_array_init_with_data(WASMModuleInstance *module_inst, uint32 seg_index,
  3810. uint32 data_seg_offset, WASMArrayObjectRef array_obj,
  3811. uint32 elem_size, uint32 array_len)
  3812. {
  3813. WASMModule *wasm_module = module_inst->module;
  3814. WASMDataSeg *data_seg;
  3815. uint8 *array_elem_base;
  3816. uint64 total_size;
  3817. data_seg = wasm_module->data_segments[seg_index];
  3818. total_size = (int64)elem_size * array_len;
  3819. if (data_seg_offset >= data_seg->data_length
  3820. || total_size > data_seg->data_length - data_seg_offset) {
  3821. wasm_set_exception(module_inst, "out of bounds memory access");
  3822. return false;
  3823. }
  3824. array_elem_base = (uint8 *)wasm_array_obj_first_elem_addr(array_obj);
  3825. bh_memcpy_s(array_elem_base, (uint32)total_size,
  3826. data_seg->data + data_seg_offset, (uint32)total_size);
  3827. return true;
  3828. }
  3829. #endif /* end of WASM_ENABLE_GC != 0 */
  3830. #endif /* end of WASM_ENABLE_JIT != 0 || WASM_ENABLE_WAMR_COMPILER != 0 */
  3831. #if WASM_ENABLE_LIBC_WASI != 0 && WASM_ENABLE_MULTI_MODULE != 0
  3832. void
  3833. wasm_propagate_wasi_args(WASMModule *module)
  3834. {
  3835. if (!module->import_count)
  3836. return;
  3837. bh_assert(&module->import_module_list_head);
  3838. WASMRegisteredModule *node =
  3839. bh_list_first_elem(&module->import_module_list_head);
  3840. while (node) {
  3841. WASIArguments *wasi_args_impt_mod =
  3842. &((WASMModule *)(node->module))->wasi_args;
  3843. bh_assert(wasi_args_impt_mod);
  3844. bh_memcpy_s(wasi_args_impt_mod, sizeof(WASIArguments),
  3845. &module->wasi_args, sizeof(WASIArguments));
  3846. node = bh_list_elem_next(node);
  3847. }
  3848. }
  3849. #endif
  3850. bool
  3851. wasm_check_utf8_str(const uint8 *str, uint32 len)
  3852. {
  3853. /* The valid ranges are taken from page 125, below link
  3854. https://www.unicode.org/versions/Unicode9.0.0/ch03.pdf */
  3855. const uint8 *p = str, *p_end = str + len;
  3856. uint8 chr;
  3857. while (p < p_end) {
  3858. chr = *p;
  3859. if (chr == 0) {
  3860. LOG_WARNING(
  3861. "LIMITATION: a string which contains '\\00' is unsupported");
  3862. return false;
  3863. }
  3864. else if (chr < 0x80) {
  3865. p++;
  3866. }
  3867. else if (chr >= 0xC2 && chr <= 0xDF && p + 1 < p_end) {
  3868. if (p[1] < 0x80 || p[1] > 0xBF) {
  3869. return false;
  3870. }
  3871. p += 2;
  3872. }
  3873. else if (chr >= 0xE0 && chr <= 0xEF && p + 2 < p_end) {
  3874. if (chr == 0xE0) {
  3875. if (p[1] < 0xA0 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) {
  3876. return false;
  3877. }
  3878. }
  3879. else if (chr == 0xED) {
  3880. if (p[1] < 0x80 || p[1] > 0x9F || p[2] < 0x80 || p[2] > 0xBF) {
  3881. return false;
  3882. }
  3883. }
  3884. else { /* chr >= 0xE1 && chr <= 0xEF */
  3885. if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF) {
  3886. return false;
  3887. }
  3888. }
  3889. p += 3;
  3890. }
  3891. else if (chr >= 0xF0 && chr <= 0xF4 && p + 3 < p_end) {
  3892. if (chr == 0xF0) {
  3893. if (p[1] < 0x90 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF
  3894. || p[3] < 0x80 || p[3] > 0xBF) {
  3895. return false;
  3896. }
  3897. }
  3898. else if (chr <= 0xF3) { /* and also chr >= 0xF1 */
  3899. if (p[1] < 0x80 || p[1] > 0xBF || p[2] < 0x80 || p[2] > 0xBF
  3900. || p[3] < 0x80 || p[3] > 0xBF) {
  3901. return false;
  3902. }
  3903. }
  3904. else { /* chr == 0xF4 */
  3905. if (p[1] < 0x80 || p[1] > 0x8F || p[2] < 0x80 || p[2] > 0xBF
  3906. || p[3] < 0x80 || p[3] > 0xBF) {
  3907. return false;
  3908. }
  3909. }
  3910. p += 4;
  3911. }
  3912. else {
  3913. return false;
  3914. }
  3915. }
  3916. return (p == p_end);
  3917. }
  3918. char *
  3919. wasm_const_str_list_insert(const uint8 *str, uint32 len, WASMModule *module,
  3920. bool is_load_from_file_buf, char *error_buf,
  3921. uint32 error_buf_size)
  3922. {
  3923. StringNode *node, *node_next;
  3924. if (!wasm_check_utf8_str(str, len)) {
  3925. set_error_buf(error_buf, error_buf_size, "invalid UTF-8 encoding");
  3926. return NULL;
  3927. }
  3928. if (len == 0) {
  3929. return "";
  3930. }
  3931. else if (is_load_from_file_buf) {
  3932. /* As the file buffer can be referred to after loading, we use
  3933. the previous byte of leb encoded size to adjust the string:
  3934. move string 1 byte backward and then append '\0' */
  3935. char *c_str = (char *)str - 1;
  3936. bh_memmove_s(c_str, len + 1, c_str + 1, len);
  3937. c_str[len] = '\0';
  3938. return c_str;
  3939. }
  3940. /* Search const str list */
  3941. node = module->const_str_list;
  3942. while (node) {
  3943. node_next = node->next;
  3944. if (strlen(node->str) == len && !memcmp(node->str, str, len))
  3945. break;
  3946. node = node_next;
  3947. }
  3948. if (node) {
  3949. return node->str;
  3950. }
  3951. if (!(node = runtime_malloc(sizeof(StringNode) + len + 1, error_buf,
  3952. error_buf_size))) {
  3953. return NULL;
  3954. }
  3955. node->str = ((char *)node) + sizeof(StringNode);
  3956. bh_memcpy_s(node->str, len + 1, str, len);
  3957. node->str[len] = '\0';
  3958. if (!module->const_str_list) {
  3959. /* set as head */
  3960. module->const_str_list = node;
  3961. node->next = NULL;
  3962. }
  3963. else {
  3964. /* insert it */
  3965. node->next = module->const_str_list;
  3966. module->const_str_list = node;
  3967. }
  3968. return node->str;
  3969. }
  3970. bool
  3971. wasm_set_module_name(WASMModule *module, const char *name, char *error_buf,
  3972. uint32_t error_buf_size)
  3973. {
  3974. if (!name)
  3975. return false;
  3976. module->name =
  3977. wasm_const_str_list_insert((const uint8 *)name, strlen(name), module,
  3978. false, error_buf, error_buf_size);
  3979. return module->name != NULL;
  3980. }
  3981. const char *
  3982. wasm_get_module_name(WASMModule *module)
  3983. {
  3984. return module->name;
  3985. }