wasm_interp_classic.c 284 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565
  1. /*
  2. * Copyright (C) 2019 Intel Corporation. All rights reserved.
  3. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  4. */
  5. #include "wasm_interp.h"
  6. #include "bh_log.h"
  7. #include "wasm_runtime.h"
  8. #include "wasm_opcode.h"
  9. #include "wasm_loader.h"
  10. #include "wasm_memory.h"
  11. #include "../common/wasm_exec_env.h"
  12. #if WASM_ENABLE_GC != 0
  13. #include "../common/gc/gc_object.h"
  14. #include "mem_alloc.h"
  15. #if WASM_ENABLE_STRINGREF != 0
  16. #include "string_object.h"
  17. #endif
  18. #endif
  19. #if WASM_ENABLE_SHARED_MEMORY != 0
  20. #include "../common/wasm_shared_memory.h"
  21. #endif
  22. #if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
  23. #include "../libraries/thread-mgr/thread_manager.h"
  24. #include "../libraries/debug-engine/debug_engine.h"
  25. #endif
  26. #if WASM_ENABLE_FAST_JIT != 0
  27. #include "../fast-jit/jit_compiler.h"
  28. #endif
  29. typedef int32 CellType_I32;
  30. typedef int64 CellType_I64;
  31. typedef float32 CellType_F32;
  32. typedef float64 CellType_F64;
  33. #define BR_TABLE_TMP_BUF_LEN 32
  34. #if WASM_ENABLE_THREAD_MGR == 0
  35. #define get_linear_mem_size() linear_mem_size
  36. #else
  37. /**
  38. * Load memory data size in each time boundary check in
  39. * multi-threading mode since it may be changed by other
  40. * threads in memory.grow
  41. */
  42. #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
  43. #endif
  44. #if WASM_ENABLE_MEMORY64 == 0
  45. #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \
  46. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0)
  47. #define CHECK_MEMORY_OVERFLOW(bytes) \
  48. do { \
  49. uint64 offset1 = (uint64)offset + (uint64)addr; \
  50. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  51. if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
  52. /* If offset1 is in valid range, maddr must also \
  53. be in valid range, no need to check it again. */ \
  54. maddr = memory->memory_data + offset1; \
  55. else \
  56. goto out_of_bounds; \
  57. } while (0)
  58. #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
  59. do { \
  60. uint64 offset1 = (uint32)(start); \
  61. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  62. if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
  63. /* App heap space is not valid space for \
  64. bulk memory operation */ \
  65. maddr = memory->memory_data + offset1; \
  66. else \
  67. goto out_of_bounds; \
  68. } while (0)
  69. #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
  70. WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
  71. #define CHECK_MEMORY_OVERFLOW(bytes) \
  72. do { \
  73. uint64 offset1 = (uint64)offset + (uint64)addr; \
  74. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  75. maddr = memory->memory_data + offset1; \
  76. } while (0)
  77. #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
  78. do { \
  79. uint64 offset1 = (uint32)(start); \
  80. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  81. maddr = memory->memory_data + offset1; \
  82. } while (0)
  83. #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
  84. WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
  85. #else /* else of WASM_ENABLE_MEMORY64 == 0 */
  86. #define CHECK_MEMORY_OVERFLOW(bytes) \
  87. do { \
  88. uint64 offset1 = (uint64)offset + (uint64)addr; \
  89. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  90. /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \
  91. if (disable_bounds_checks \
  92. || (offset1 >= offset && offset1 + bytes >= offset1 \
  93. && offset1 + bytes <= get_linear_mem_size())) \
  94. maddr = memory->memory_data + offset1; \
  95. else \
  96. goto out_of_bounds; \
  97. } while (0)
  98. #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
  99. do { \
  100. uint64 offset1 = (uint64)(start); \
  101. CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
  102. /* If memory64 is enabled, offset1 + bytes can overflow */ \
  103. if (disable_bounds_checks \
  104. || (offset1 + bytes >= offset1 \
  105. && offset1 + bytes <= get_linear_mem_size())) \
  106. /* App heap space is not valid space for \
  107. bulk memory operation */ \
  108. maddr = memory->memory_data + offset1; \
  109. else \
  110. goto out_of_bounds; \
  111. } while (0)
  112. #endif /* end of WASM_ENABLE_MEMORY64 == 0 */
  113. #define CHECK_ATOMIC_MEMORY_ACCESS() \
  114. do { \
  115. if (((uintptr_t)maddr & (((uintptr_t)1 << align) - 1)) != 0) \
  116. goto unaligned_atomic; \
  117. } while (0)
  118. #if WASM_ENABLE_DEBUG_INTERP != 0
  119. #define TRIGGER_WATCHPOINT_SIGTRAP() \
  120. do { \
  121. wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP); \
  122. CHECK_SUSPEND_FLAGS(); \
  123. } while (0)
  124. #define CHECK_WATCHPOINT(list, current_addr) \
  125. do { \
  126. WASMDebugWatchPoint *watchpoint = bh_list_first_elem(list); \
  127. while (watchpoint) { \
  128. WASMDebugWatchPoint *next = bh_list_elem_next(watchpoint); \
  129. if (watchpoint->addr <= current_addr \
  130. && watchpoint->addr + watchpoint->length > current_addr) { \
  131. TRIGGER_WATCHPOINT_SIGTRAP(); \
  132. } \
  133. watchpoint = next; \
  134. } \
  135. } while (0)
  136. #define CHECK_READ_WATCHPOINT(addr, offset) \
  137. CHECK_WATCHPOINT(watch_point_list_read, WASM_ADDR_OFFSET(addr + offset))
  138. #define CHECK_WRITE_WATCHPOINT(addr, offset) \
  139. CHECK_WATCHPOINT(watch_point_list_write, WASM_ADDR_OFFSET(addr + offset))
  140. #else
  141. #define CHECK_READ_WATCHPOINT(addr, offset) (void)0
  142. #define CHECK_WRITE_WATCHPOINT(addr, offset) (void)0
  143. #endif
  144. static inline uint32
  145. rotl32(uint32 n, uint32 c)
  146. {
  147. const uint32 mask = (31);
  148. c = c % 32;
  149. c &= mask;
  150. return (n << c) | (n >> ((0 - c) & mask));
  151. }
  152. static inline uint32
  153. rotr32(uint32 n, uint32 c)
  154. {
  155. const uint32 mask = (31);
  156. c = c % 32;
  157. c &= mask;
  158. return (n >> c) | (n << ((0 - c) & mask));
  159. }
  160. static inline uint64
  161. rotl64(uint64 n, uint64 c)
  162. {
  163. const uint64 mask = (63);
  164. c = c % 64;
  165. c &= mask;
  166. return (n << c) | (n >> ((0 - c) & mask));
  167. }
  168. static inline uint64
  169. rotr64(uint64 n, uint64 c)
  170. {
  171. const uint64 mask = (63);
  172. c = c % 64;
  173. c &= mask;
  174. return (n >> c) | (n << ((0 - c) & mask));
  175. }
  176. static inline float32
  177. f32_min(float32 a, float32 b)
  178. {
  179. if (isnan(a) || isnan(b))
  180. return NAN;
  181. else if (a == 0 && a == b)
  182. return signbit(a) ? a : b;
  183. else
  184. return a > b ? b : a;
  185. }
  186. static inline float32
  187. f32_max(float32 a, float32 b)
  188. {
  189. if (isnan(a) || isnan(b))
  190. return NAN;
  191. else if (a == 0 && a == b)
  192. return signbit(a) ? b : a;
  193. else
  194. return a > b ? a : b;
  195. }
  196. static inline float64
  197. f64_min(float64 a, float64 b)
  198. {
  199. if (isnan(a) || isnan(b))
  200. return (float64)NAN;
  201. else if (a == 0 && a == b)
  202. return signbit(a) ? a : b;
  203. else
  204. return a > b ? b : a;
  205. }
  206. static inline float64
  207. f64_max(float64 a, float64 b)
  208. {
  209. if (isnan(a) || isnan(b))
  210. return (float64)NAN;
  211. else if (a == 0 && a == b)
  212. return signbit(a) ? b : a;
  213. else
  214. return a > b ? a : b;
  215. }
  216. static inline uint32
  217. clz32(uint32 type)
  218. {
  219. uint32 num = 0;
  220. if (type == 0)
  221. return 32;
  222. while (!(type & 0x80000000)) {
  223. num++;
  224. type <<= 1;
  225. }
  226. return num;
  227. }
  228. static inline uint32
  229. clz64(uint64 type)
  230. {
  231. uint32 num = 0;
  232. if (type == 0)
  233. return 64;
  234. while (!(type & 0x8000000000000000LL)) {
  235. num++;
  236. type <<= 1;
  237. }
  238. return num;
  239. }
  240. static inline uint32
  241. ctz32(uint32 type)
  242. {
  243. uint32 num = 0;
  244. if (type == 0)
  245. return 32;
  246. while (!(type & 1)) {
  247. num++;
  248. type >>= 1;
  249. }
  250. return num;
  251. }
  252. static inline uint32
  253. ctz64(uint64 type)
  254. {
  255. uint32 num = 0;
  256. if (type == 0)
  257. return 64;
  258. while (!(type & 1)) {
  259. num++;
  260. type >>= 1;
  261. }
  262. return num;
  263. }
  264. static inline uint32
  265. popcount32(uint32 u)
  266. {
  267. uint32 ret = 0;
  268. while (u) {
  269. u = (u & (u - 1));
  270. ret++;
  271. }
  272. return ret;
  273. }
  274. static inline uint32
  275. popcount64(uint64 u)
  276. {
  277. uint32 ret = 0;
  278. while (u) {
  279. u = (u & (u - 1));
  280. ret++;
  281. }
  282. return ret;
  283. }
  284. static float
  285. local_copysignf(float x, float y)
  286. {
  287. union {
  288. float f;
  289. uint32 i;
  290. } ux = { x }, uy = { y };
  291. ux.i &= 0x7fffffff;
  292. ux.i |= uy.i & 0x80000000;
  293. return ux.f;
  294. }
  295. static double
  296. local_copysign(double x, double y)
  297. {
  298. union {
  299. double f;
  300. uint64 i;
  301. } ux = { x }, uy = { y };
  302. ux.i &= UINT64_MAX / 2;
  303. ux.i |= uy.i & 1ULL << 63;
  304. return ux.f;
  305. }
  306. static uint64
  307. read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign)
  308. {
  309. uint64 result = 0, byte;
  310. uint32 offset = *p_offset;
  311. uint32 shift = 0;
  312. while (true) {
  313. byte = buf[offset++];
  314. result |= ((byte & 0x7f) << shift);
  315. shift += 7;
  316. if ((byte & 0x80) == 0) {
  317. break;
  318. }
  319. }
  320. if (sign && (shift < maxbits) && (byte & 0x40)) {
  321. /* Sign extend */
  322. result |= (~((uint64)0)) << shift;
  323. }
  324. *p_offset = offset;
  325. return result;
  326. }
  327. #if WASM_ENABLE_GC != 0
  328. static uint8 *
  329. get_frame_ref(WASMInterpFrame *frame)
  330. {
  331. WASMFunctionInstance *cur_func = frame->function;
  332. uint32 all_cell_num;
  333. if (!cur_func) {
  334. /* it's a glue frame created in wasm_interp_call_wasm,
  335. no GC object will be traversed */
  336. return (uint8 *)frame->lp;
  337. }
  338. else if (!frame->ip) {
  339. /* it's a native method frame created in
  340. wasm_interp_call_func_native */
  341. all_cell_num =
  342. cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2;
  343. return (uint8 *)(frame->lp + all_cell_num);
  344. }
  345. else {
  346. #if WASM_ENABLE_JIT == 0
  347. /* it's a wasm bytecode function frame */
  348. return (uint8 *)frame->csp_boundary;
  349. #else
  350. return (uint8 *)(frame->lp + cur_func->param_cell_num
  351. + cur_func->local_cell_num
  352. + cur_func->u.func->max_stack_cell_num);
  353. #endif
  354. }
  355. }
  356. static void
  357. init_frame_refs(uint8 *frame_ref, uint32 cell_num, WASMFunctionInstance *func)
  358. {
  359. uint32 i, j;
  360. memset(frame_ref, 0, cell_num);
  361. for (i = 0, j = 0; i < func->param_count; i++) {
  362. if (wasm_is_type_reftype(func->param_types[i])
  363. && !wasm_is_reftype_i31ref(func->param_types[i])) {
  364. frame_ref[j++] = 1;
  365. #if UINTPTR_MAX == UINT64_MAX
  366. frame_ref[j++] = 1;
  367. #endif
  368. }
  369. else {
  370. j += wasm_value_type_cell_num(func->param_types[i]);
  371. }
  372. }
  373. for (i = 0; i < func->local_count; i++) {
  374. if (wasm_is_type_reftype(func->local_types[i])
  375. && !wasm_is_reftype_i31ref(func->local_types[i])) {
  376. frame_ref[j++] = 1;
  377. #if UINTPTR_MAX == UINT64_MAX
  378. frame_ref[j++] = 1;
  379. #endif
  380. }
  381. else {
  382. j += wasm_value_type_cell_num(func->local_types[i]);
  383. }
  384. }
  385. }
  386. uint8 *
  387. wasm_interp_get_frame_ref(WASMInterpFrame *frame)
  388. {
  389. return get_frame_ref(frame);
  390. }
  391. /* Return the corresponding ref slot of the given address of local
  392. variable or stack pointer. */
  393. #define COMPUTE_FRAME_REF(ref, lp, p) (ref + (unsigned)((uint32 *)p - lp))
  394. #define FRAME_REF(p) COMPUTE_FRAME_REF(frame_ref, frame_lp, p)
  395. #define FRAME_REF_FOR(frame, p) \
  396. COMPUTE_FRAME_REF(get_frame_ref(frame), frame->lp, p)
  397. #define CLEAR_FRAME_REF(p, n) \
  398. do { \
  399. int32 ref_i, ref_n = (int32)(n); \
  400. uint8 *ref = FRAME_REF(p); \
  401. for (ref_i = 0; ref_i < ref_n; ref_i++) \
  402. ref[ref_i] = 0; \
  403. } while (0)
  404. #else
  405. #define CLEAR_FRAME_REF(p, n) (void)0
  406. #endif /* end of WASM_ENABLE_GC != 0 */
  407. #define skip_leb(p) while (*p++ & 0x80)
  408. #define PUSH_I32(value) \
  409. do { \
  410. *(int32 *)frame_sp++ = (int32)(value); \
  411. } while (0)
  412. #define PUSH_F32(value) \
  413. do { \
  414. *(float32 *)frame_sp++ = (float32)(value); \
  415. } while (0)
  416. #define PUSH_I64(value) \
  417. do { \
  418. PUT_I64_TO_ADDR(frame_sp, value); \
  419. frame_sp += 2; \
  420. } while (0)
  421. #define PUSH_F64(value) \
  422. do { \
  423. PUT_F64_TO_ADDR(frame_sp, value); \
  424. frame_sp += 2; \
  425. } while (0)
  426. #if UINTPTR_MAX == UINT64_MAX
  427. #define PUSH_REF(value) \
  428. do { \
  429. PUT_REF_TO_ADDR(frame_sp, value); \
  430. frame_ref_tmp = FRAME_REF(frame_sp); \
  431. *frame_ref_tmp = *(frame_ref_tmp + 1) = 1; \
  432. frame_sp += 2; \
  433. } while (0)
  434. #define PUSH_I31REF(value) \
  435. do { \
  436. PUT_REF_TO_ADDR(frame_sp, value); \
  437. frame_sp += 2; \
  438. } while (0)
  439. #else
  440. #define PUSH_REF(value) \
  441. do { \
  442. PUT_REF_TO_ADDR(frame_sp, value); \
  443. frame_ref_tmp = FRAME_REF(frame_sp); \
  444. *frame_ref_tmp = 1; \
  445. frame_sp++; \
  446. } while (0)
  447. #define PUSH_I31REF(value) \
  448. do { \
  449. PUT_REF_TO_ADDR(frame_sp, value); \
  450. frame_sp++; \
  451. } while (0)
  452. #endif
  453. #if UINTPTR_MAX == UINT64_MAX
  454. #define PUSH_PTR(value) PUSH_I64(value)
  455. #else
  456. #define PUSH_PTR(value) PUSH_I32(value)
  457. #endif
  458. /* in exception handling, label_type needs to be stored to lookup exception
  459. * handlers */
  460. #if WASM_ENABLE_EXCE_HANDLING != 0
  461. #define SET_LABEL_TYPE(_label_type) frame_csp->label_type = _label_type
  462. #else
  463. #define SET_LABEL_TYPE(_label_type) (void)0
  464. #endif
  465. #if WASM_ENABLE_MEMORY64 != 0
  466. #define COND_PUSH_TEMPLATE(cond, value) \
  467. do { \
  468. if (cond) { \
  469. PUT_I64_TO_ADDR(frame_sp, value); \
  470. frame_sp += 2; \
  471. } \
  472. else { \
  473. *(int32 *)frame_sp++ = (int32)(value); \
  474. } \
  475. } while (0)
  476. #define PUSH_MEM_OFFSET(value) COND_PUSH_TEMPLATE(is_memory64, value)
  477. #define PUSH_TBL_ELEM_IDX(value) COND_PUSH_TEMPLATE(is_table64, value)
  478. #else
  479. #define PUSH_MEM_OFFSET(value) PUSH_I32(value)
  480. #define PUSH_TBL_ELEM_IDX(value) PUSH_I32(value)
  481. #endif
  482. #define PUSH_PAGE_COUNT(value) PUSH_MEM_OFFSET(value)
  483. #define PUSH_CSP(_label_type, param_cell_num, cell_num, _target_addr) \
  484. do { \
  485. bh_assert(frame_csp < frame->csp_boundary); \
  486. SET_LABEL_TYPE(_label_type); \
  487. frame_csp->cell_num = cell_num; \
  488. frame_csp->begin_addr = frame_ip; \
  489. frame_csp->target_addr = _target_addr; \
  490. frame_csp->frame_sp = frame_sp - param_cell_num; \
  491. frame_csp++; \
  492. } while (0)
  493. #define POP_I32() (--frame_sp, *(int32 *)frame_sp)
  494. #define POP_F32() (--frame_sp, *(float32 *)frame_sp)
  495. #define POP_I64() (frame_sp -= 2, GET_I64_FROM_ADDR(frame_sp))
  496. #define POP_F64() (frame_sp -= 2, GET_F64_FROM_ADDR(frame_sp))
  497. #if UINTPTR_MAX == UINT64_MAX
  498. #define POP_REF() \
  499. (frame_sp -= 2, frame_ref_tmp = FRAME_REF(frame_sp), \
  500. *frame_ref_tmp = *(frame_ref_tmp + 1) = 0, GET_REF_FROM_ADDR(frame_sp))
  501. #else
  502. #define POP_REF() \
  503. (frame_sp--, frame_ref_tmp = FRAME_REF(frame_sp), *frame_ref_tmp = 0, \
  504. GET_REF_FROM_ADDR(frame_sp))
  505. #endif
  506. #if WASM_ENABLE_MEMORY64 != 0
  507. #define POP_MEM_OFFSET() (is_memory64 ? POP_I64() : (uint32)POP_I32())
  508. #define POP_TBL_ELEM_IDX() (is_table64 ? POP_I64() : (uint32)POP_I32())
  509. #else
  510. #define POP_MEM_OFFSET() POP_I32()
  511. #define POP_TBL_ELEM_IDX() POP_I32()
  512. #endif
  513. #define POP_PAGE_COUNT() POP_MEM_OFFSET()
  514. #define POP_CSP_CHECK_OVERFLOW(n) \
  515. do { \
  516. bh_assert(frame_csp - n >= frame->csp_bottom); \
  517. } while (0)
  518. #define POP_CSP() \
  519. do { \
  520. POP_CSP_CHECK_OVERFLOW(1); \
  521. --frame_csp; \
  522. } while (0)
  523. #define POP_CSP_N(n) \
  524. do { \
  525. uint32 *frame_sp_old = frame_sp; \
  526. uint32 cell_num_to_copy; \
  527. POP_CSP_CHECK_OVERFLOW(n + 1); \
  528. frame_csp -= n; \
  529. frame_ip = (frame_csp - 1)->target_addr; \
  530. /* copy arity values of block */ \
  531. frame_sp = (frame_csp - 1)->frame_sp; \
  532. cell_num_to_copy = (frame_csp - 1)->cell_num; \
  533. if (cell_num_to_copy > 0) { \
  534. word_copy(frame_sp, frame_sp_old - cell_num_to_copy, \
  535. cell_num_to_copy); \
  536. frame_ref_copy(FRAME_REF(frame_sp), \
  537. FRAME_REF(frame_sp_old - cell_num_to_copy), \
  538. cell_num_to_copy); \
  539. } \
  540. frame_sp += cell_num_to_copy; \
  541. CLEAR_FRAME_REF(frame_sp, frame_sp_old - frame_sp); \
  542. } while (0)
  543. /* Pop the given number of elements from the given frame's stack. */
  544. #define POP(N) \
  545. do { \
  546. int n = (N); \
  547. frame_sp -= n; \
  548. CLEAR_FRAME_REF(frame_sp, n); \
  549. } while (0)
  550. #if WASM_ENABLE_EXCE_HANDLING != 0
  551. /* unwind the CSP to a given label and optionally modify the labeltype */
  552. #define UNWIND_CSP(N, T) \
  553. do { \
  554. /* unwind to function frame */ \
  555. frame_csp -= N; \
  556. /* drop handlers and values pushd in try block */ \
  557. frame_sp = (frame_csp - 1)->frame_sp; \
  558. (frame_csp - 1)->label_type = T ? T : (frame_csp - 1)->label_type; \
  559. } while (0)
  560. #endif
  561. #define SYNC_ALL_TO_FRAME() \
  562. do { \
  563. frame->sp = frame_sp; \
  564. frame->ip = frame_ip; \
  565. frame->csp = frame_csp; \
  566. } while (0)
  567. #define UPDATE_ALL_FROM_FRAME() \
  568. do { \
  569. frame_sp = frame->sp; \
  570. frame_ip = frame->ip; \
  571. frame_csp = frame->csp; \
  572. } while (0)
  573. #define read_leb_int64(p, p_end, res) \
  574. do { \
  575. uint8 _val = *p; \
  576. if (!(_val & 0x80)) { \
  577. res = (int64)_val; \
  578. if (_val & 0x40) \
  579. /* sign extend */ \
  580. res |= 0xFFFFFFFFFFFFFF80LL; \
  581. p++; \
  582. } \
  583. else { \
  584. uint32 _off = 0; \
  585. res = (int64)read_leb(p, &_off, 64, true); \
  586. p += _off; \
  587. } \
  588. } while (0)
  589. #define read_leb_uint32(p, p_end, res) \
  590. do { \
  591. uint8 _val = *p; \
  592. if (!(_val & 0x80)) { \
  593. res = _val; \
  594. p++; \
  595. } \
  596. else { \
  597. uint32 _off = 0; \
  598. res = (uint32)read_leb(p, &_off, 32, false); \
  599. p += _off; \
  600. } \
  601. } while (0)
  602. #define read_leb_int32(p, p_end, res) \
  603. do { \
  604. uint8 _val = *p; \
  605. if (!(_val & 0x80)) { \
  606. res = (int32)_val; \
  607. if (_val & 0x40) \
  608. /* sign extend */ \
  609. res |= 0xFFFFFF80; \
  610. p++; \
  611. } \
  612. else { \
  613. uint32 _off = 0; \
  614. res = (int32)read_leb(p, &_off, 32, true); \
  615. p += _off; \
  616. } \
  617. } while (0)
  618. #if WASM_ENABLE_MEMORY64 != 0
  619. #define read_leb_mem_offset(p, p_end, res) \
  620. do { \
  621. uint8 _val = *p; \
  622. if (!(_val & 0x80)) { \
  623. res = (mem_offset_t)_val; \
  624. p++; \
  625. } \
  626. else { \
  627. uint32 _off = 0; \
  628. res = (mem_offset_t)read_leb(p, &_off, is_memory64 ? 64 : 32, \
  629. false); \
  630. p += _off; \
  631. } \
  632. } while (0)
  633. #else
  634. #define read_leb_mem_offset(p, p_end, res) read_leb_uint32(p, p_end, res)
  635. #endif
  636. #if WASM_ENABLE_MULTI_MEMORY != 0
  637. /* If the current memidx differs than the last cached one,
  638. * update memory related information */
  639. #define read_leb_memidx(p, p_end, res) \
  640. do { \
  641. read_leb_uint32(p, p_end, res); \
  642. if (res != memidx_cached) { \
  643. memory = wasm_get_memory_with_idx(module, res); \
  644. linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); \
  645. memidx_cached = res; \
  646. } \
  647. } while (0)
  648. /* First read the alignment, then if it has flag indicating following memidx,
  649. * read and update memory related information, if it differs than the
  650. * last(cached) one. If it doesn't have flag reset the
  651. * memory instance to the default memories[0] */
  652. #define read_leb_memarg(p, p_end, res) \
  653. do { \
  654. read_leb_uint32(p, p_end, res); \
  655. if (!(res & OPT_MEMIDX_FLAG)) \
  656. memidx = 0; \
  657. else \
  658. read_leb_uint32(p, p_end, memidx); \
  659. if (memidx != memidx_cached) { \
  660. memory = wasm_get_memory_with_idx(module, memidx); \
  661. linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory); \
  662. memidx_cached = memidx; \
  663. } \
  664. } while (0)
  665. #else
  666. #define read_leb_memarg(p, p_end, res) \
  667. do { \
  668. read_leb_uint32(p, p_end, res); \
  669. (void)res; \
  670. } while (0)
  671. #define read_leb_memidx(p, p_end, res) read_leb_memarg(p, p_end, res)
  672. #endif
  673. #if WASM_ENABLE_LABELS_AS_VALUES == 0
  674. #define RECOVER_FRAME_IP_END() frame_ip_end = wasm_get_func_code_end(cur_func)
  675. #else
  676. #define RECOVER_FRAME_IP_END() (void)0
  677. #endif
  678. #if WASM_ENABLE_GC != 0
  679. #define RECOVER_FRAME_REF() frame_ref = (uint8 *)frame->csp_boundary
  680. #else
  681. #define RECOVER_FRAME_REF() (void)0
  682. #endif
  683. #define RECOVER_CONTEXT(new_frame) \
  684. do { \
  685. frame = (new_frame); \
  686. cur_func = frame->function; \
  687. prev_frame = frame->prev_frame; \
  688. frame_ip = frame->ip; \
  689. RECOVER_FRAME_IP_END(); \
  690. frame_lp = frame->lp; \
  691. frame_sp = frame->sp; \
  692. frame_csp = frame->csp; \
  693. RECOVER_FRAME_REF(); \
  694. } while (0)
  695. #if WASM_ENABLE_LABELS_AS_VALUES != 0
  696. #define GET_OPCODE() opcode = *(frame_ip - 1);
  697. #else
  698. #define GET_OPCODE() (void)0
  699. #endif
  700. #define DEF_OP_I_CONST(ctype, src_op_type) \
  701. do { \
  702. ctype cval; \
  703. read_leb_##ctype(frame_ip, frame_ip_end, cval); \
  704. PUSH_##src_op_type(cval); \
  705. } while (0)
  706. #define DEF_OP_EQZ(src_op_type) \
  707. do { \
  708. int32 pop_val; \
  709. pop_val = POP_##src_op_type() == 0; \
  710. PUSH_I32(pop_val); \
  711. } while (0)
  712. #define DEF_OP_CMP(src_type, src_op_type, cond) \
  713. do { \
  714. uint32 res; \
  715. src_type val1, val2; \
  716. val2 = (src_type)POP_##src_op_type(); \
  717. val1 = (src_type)POP_##src_op_type(); \
  718. res = val1 cond val2; \
  719. PUSH_I32(res); \
  720. } while (0)
  721. #define DEF_OP_BIT_COUNT(src_type, src_op_type, operation) \
  722. do { \
  723. src_type val1, val2; \
  724. val1 = (src_type)POP_##src_op_type(); \
  725. val2 = (src_type)operation(val1); \
  726. PUSH_##src_op_type(val2); \
  727. } while (0)
  728. #define DEF_OP_NUMERIC(src_type1, src_type2, src_op_type, operation) \
  729. do { \
  730. frame_sp -= sizeof(src_type2) / sizeof(uint32); \
  731. *(src_type1 *)(frame_sp - sizeof(src_type1) / sizeof(uint32)) \
  732. operation## = *(src_type2 *)(frame_sp); \
  733. } while (0)
  734. #if WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS != 0
  735. #define DEF_OP_NUMERIC_64 DEF_OP_NUMERIC
  736. #else
  737. #define DEF_OP_NUMERIC_64(src_type1, src_type2, src_op_type, operation) \
  738. do { \
  739. src_type1 val1; \
  740. src_type2 val2; \
  741. frame_sp -= 2; \
  742. val1 = (src_type1)GET_##src_op_type##_FROM_ADDR(frame_sp - 2); \
  743. val2 = (src_type2)GET_##src_op_type##_FROM_ADDR(frame_sp); \
  744. val1 operation## = val2; \
  745. PUT_##src_op_type##_TO_ADDR(frame_sp - 2, val1); \
  746. } while (0)
  747. #endif
  748. #define DEF_OP_NUMERIC2(src_type1, src_type2, src_op_type, operation) \
  749. do { \
  750. frame_sp -= sizeof(src_type2) / sizeof(uint32); \
  751. *(src_type1 *)(frame_sp - sizeof(src_type1) / sizeof(uint32)) \
  752. operation## = (*(src_type2 *)(frame_sp) % 32); \
  753. } while (0)
  754. #define DEF_OP_NUMERIC2_64(src_type1, src_type2, src_op_type, operation) \
  755. do { \
  756. src_type1 val1; \
  757. src_type2 val2; \
  758. frame_sp -= 2; \
  759. val1 = (src_type1)GET_##src_op_type##_FROM_ADDR(frame_sp - 2); \
  760. val2 = (src_type2)GET_##src_op_type##_FROM_ADDR(frame_sp); \
  761. val1 operation## = (val2 % 64); \
  762. PUT_##src_op_type##_TO_ADDR(frame_sp - 2, val1); \
  763. } while (0)
  764. #define DEF_OP_MATH(src_type, src_op_type, method) \
  765. do { \
  766. src_type src_val; \
  767. src_val = POP_##src_op_type(); \
  768. PUSH_##src_op_type(method(src_val)); \
  769. } while (0)
  770. #define TRUNC_FUNCTION(func_name, src_type, dst_type, signed_type) \
  771. static dst_type func_name(src_type src_value, src_type src_min, \
  772. src_type src_max, dst_type dst_min, \
  773. dst_type dst_max, bool is_sign) \
  774. { \
  775. dst_type dst_value = 0; \
  776. if (!isnan(src_value)) { \
  777. if (src_value <= src_min) \
  778. dst_value = dst_min; \
  779. else if (src_value >= src_max) \
  780. dst_value = dst_max; \
  781. else { \
  782. if (is_sign) \
  783. dst_value = (dst_type)(signed_type)src_value; \
  784. else \
  785. dst_value = (dst_type)src_value; \
  786. } \
  787. } \
  788. return dst_value; \
  789. }
  790. TRUNC_FUNCTION(trunc_f32_to_i32, float32, uint32, int32)
  791. TRUNC_FUNCTION(trunc_f32_to_i64, float32, uint64, int64)
  792. TRUNC_FUNCTION(trunc_f64_to_i32, float64, uint32, int32)
  793. TRUNC_FUNCTION(trunc_f64_to_i64, float64, uint64, int64)
  794. static bool
  795. trunc_f32_to_int(WASMModuleInstance *module, uint32 *frame_sp, float32 src_min,
  796. float32 src_max, bool saturating, bool is_i32, bool is_sign)
  797. {
  798. float32 src_value = POP_F32();
  799. uint64 dst_value_i64;
  800. uint32 dst_value_i32;
  801. if (!saturating) {
  802. if (isnan(src_value)) {
  803. wasm_set_exception(module, "invalid conversion to integer");
  804. return false;
  805. }
  806. else if (src_value <= src_min || src_value >= src_max) {
  807. wasm_set_exception(module, "integer overflow");
  808. return false;
  809. }
  810. }
  811. if (is_i32) {
  812. uint32 dst_min = is_sign ? INT32_MIN : 0;
  813. uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX;
  814. dst_value_i32 = trunc_f32_to_i32(src_value, src_min, src_max, dst_min,
  815. dst_max, is_sign);
  816. PUSH_I32(dst_value_i32);
  817. }
  818. else {
  819. uint64 dst_min = is_sign ? INT64_MIN : 0;
  820. uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX;
  821. dst_value_i64 = trunc_f32_to_i64(src_value, src_min, src_max, dst_min,
  822. dst_max, is_sign);
  823. PUSH_I64(dst_value_i64);
  824. }
  825. return true;
  826. }
  827. static bool
  828. trunc_f64_to_int(WASMModuleInstance *module, uint32 *frame_sp, float64 src_min,
  829. float64 src_max, bool saturating, bool is_i32, bool is_sign)
  830. {
  831. float64 src_value = POP_F64();
  832. uint64 dst_value_i64;
  833. uint32 dst_value_i32;
  834. if (!saturating) {
  835. if (isnan(src_value)) {
  836. wasm_set_exception(module, "invalid conversion to integer");
  837. return false;
  838. }
  839. else if (src_value <= src_min || src_value >= src_max) {
  840. wasm_set_exception(module, "integer overflow");
  841. return false;
  842. }
  843. }
  844. if (is_i32) {
  845. uint32 dst_min = is_sign ? INT32_MIN : 0;
  846. uint32 dst_max = is_sign ? INT32_MAX : UINT32_MAX;
  847. dst_value_i32 = trunc_f64_to_i32(src_value, src_min, src_max, dst_min,
  848. dst_max, is_sign);
  849. PUSH_I32(dst_value_i32);
  850. }
  851. else {
  852. uint64 dst_min = is_sign ? INT64_MIN : 0;
  853. uint64 dst_max = is_sign ? INT64_MAX : UINT64_MAX;
  854. dst_value_i64 = trunc_f64_to_i64(src_value, src_min, src_max, dst_min,
  855. dst_max, is_sign);
  856. PUSH_I64(dst_value_i64);
  857. }
  858. return true;
  859. }
  860. #define DEF_OP_TRUNC_F32(min, max, is_i32, is_sign) \
  861. do { \
  862. if (!trunc_f32_to_int(module, frame_sp, min, max, false, is_i32, \
  863. is_sign)) \
  864. goto got_exception; \
  865. } while (0)
  866. #define DEF_OP_TRUNC_F64(min, max, is_i32, is_sign) \
  867. do { \
  868. if (!trunc_f64_to_int(module, frame_sp, min, max, false, is_i32, \
  869. is_sign)) \
  870. goto got_exception; \
  871. } while (0)
  872. #define DEF_OP_TRUNC_SAT_F32(min, max, is_i32, is_sign) \
  873. do { \
  874. (void)trunc_f32_to_int(module, frame_sp, min, max, true, is_i32, \
  875. is_sign); \
  876. } while (0)
  877. #define DEF_OP_TRUNC_SAT_F64(min, max, is_i32, is_sign) \
  878. do { \
  879. (void)trunc_f64_to_int(module, frame_sp, min, max, true, is_i32, \
  880. is_sign); \
  881. } while (0)
  882. #define DEF_OP_CONVERT(dst_type, dst_op_type, src_type, src_op_type) \
  883. do { \
  884. dst_type value = (dst_type)(src_type)POP_##src_op_type(); \
  885. PUSH_##dst_op_type(value); \
  886. } while (0)
  887. #define GET_LOCAL_INDEX_TYPE_AND_OFFSET() \
  888. do { \
  889. uint32 param_count = cur_func->param_count; \
  890. read_leb_uint32(frame_ip, frame_ip_end, local_idx); \
  891. bh_assert(local_idx < param_count + cur_func->local_count); \
  892. local_offset = cur_func->local_offsets[local_idx]; \
  893. if (local_idx < param_count) \
  894. local_type = cur_func->param_types[local_idx]; \
  895. else \
  896. local_type = cur_func->local_types[local_idx - param_count]; \
  897. } while (0)
  898. #define DEF_ATOMIC_RMW_OPCODE(OP_NAME, op) \
  899. case WASM_OP_ATOMIC_RMW_I32_##OP_NAME: \
  900. case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U: \
  901. case WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U: \
  902. { \
  903. uint32 readv, sval; \
  904. \
  905. sval = POP_I32(); \
  906. addr = POP_MEM_OFFSET(); \
  907. \
  908. if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##8_U) { \
  909. CHECK_MEMORY_OVERFLOW(1); \
  910. CHECK_ATOMIC_MEMORY_ACCESS(); \
  911. \
  912. shared_memory_lock(memory); \
  913. readv = (uint32)(*(uint8 *)maddr); \
  914. *(uint8 *)maddr = (uint8)(readv op sval); \
  915. shared_memory_unlock(memory); \
  916. } \
  917. else if (opcode == WASM_OP_ATOMIC_RMW_I32_##OP_NAME##16_U) { \
  918. CHECK_MEMORY_OVERFLOW(2); \
  919. CHECK_ATOMIC_MEMORY_ACCESS(); \
  920. \
  921. shared_memory_lock(memory); \
  922. readv = (uint32)LOAD_U16(maddr); \
  923. STORE_U16(maddr, (uint16)(readv op sval)); \
  924. shared_memory_unlock(memory); \
  925. } \
  926. else { \
  927. CHECK_MEMORY_OVERFLOW(4); \
  928. CHECK_ATOMIC_MEMORY_ACCESS(); \
  929. \
  930. shared_memory_lock(memory); \
  931. readv = LOAD_I32(maddr); \
  932. STORE_U32(maddr, readv op sval); \
  933. shared_memory_unlock(memory); \
  934. } \
  935. PUSH_I32(readv); \
  936. break; \
  937. } \
  938. case WASM_OP_ATOMIC_RMW_I64_##OP_NAME: \
  939. case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U: \
  940. case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U: \
  941. case WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U: \
  942. { \
  943. uint64 readv, sval; \
  944. \
  945. sval = (uint64)POP_I64(); \
  946. addr = POP_MEM_OFFSET(); \
  947. \
  948. if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##8_U) { \
  949. CHECK_MEMORY_OVERFLOW(1); \
  950. CHECK_ATOMIC_MEMORY_ACCESS(); \
  951. \
  952. shared_memory_lock(memory); \
  953. readv = (uint64)(*(uint8 *)maddr); \
  954. *(uint8 *)maddr = (uint8)(readv op sval); \
  955. shared_memory_unlock(memory); \
  956. } \
  957. else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##16_U) { \
  958. CHECK_MEMORY_OVERFLOW(2); \
  959. CHECK_ATOMIC_MEMORY_ACCESS(); \
  960. \
  961. shared_memory_lock(memory); \
  962. readv = (uint64)LOAD_U16(maddr); \
  963. STORE_U16(maddr, (uint16)(readv op sval)); \
  964. shared_memory_unlock(memory); \
  965. } \
  966. else if (opcode == WASM_OP_ATOMIC_RMW_I64_##OP_NAME##32_U) { \
  967. CHECK_MEMORY_OVERFLOW(4); \
  968. CHECK_ATOMIC_MEMORY_ACCESS(); \
  969. \
  970. shared_memory_lock(memory); \
  971. readv = (uint64)LOAD_U32(maddr); \
  972. STORE_U32(maddr, (uint32)(readv op sval)); \
  973. shared_memory_unlock(memory); \
  974. } \
  975. else { \
  976. uint64 op_result; \
  977. CHECK_MEMORY_OVERFLOW(8); \
  978. CHECK_ATOMIC_MEMORY_ACCESS(); \
  979. \
  980. shared_memory_lock(memory); \
  981. readv = (uint64)LOAD_I64(maddr); \
  982. op_result = readv op sval; \
  983. STORE_I64(maddr, op_result); \
  984. shared_memory_unlock(memory); \
  985. } \
  986. PUSH_I64(readv); \
  987. break; \
  988. }
  989. static inline int32
  990. sign_ext_8_32(int8 val)
  991. {
  992. if (val & 0x80)
  993. return (int32)val | (int32)0xffffff00;
  994. return val;
  995. }
  996. static inline int32
  997. sign_ext_16_32(int16 val)
  998. {
  999. if (val & 0x8000)
  1000. return (int32)val | (int32)0xffff0000;
  1001. return val;
  1002. }
  1003. static inline int64
  1004. sign_ext_8_64(int8 val)
  1005. {
  1006. if (val & 0x80)
  1007. return (int64)val | (int64)0xffffffffffffff00LL;
  1008. return val;
  1009. }
  1010. static inline int64
  1011. sign_ext_16_64(int16 val)
  1012. {
  1013. if (val & 0x8000)
  1014. return (int64)val | (int64)0xffffffffffff0000LL;
  1015. return val;
  1016. }
  1017. static inline int64
  1018. sign_ext_32_64(int32 val)
  1019. {
  1020. if (val & (int32)0x80000000)
  1021. return (int64)val | (int64)0xffffffff00000000LL;
  1022. return val;
  1023. }
  1024. static inline void
  1025. word_copy(uint32 *dest, uint32 *src, unsigned num)
  1026. {
  1027. bh_assert(dest != NULL);
  1028. bh_assert(src != NULL);
  1029. bh_assert(num > 0);
  1030. if (dest != src) {
  1031. /* No overlap buffer */
  1032. bh_assert(!((src < dest) && (dest < src + num)));
  1033. for (; num > 0; num--)
  1034. *dest++ = *src++;
  1035. }
  1036. }
  1037. #if WASM_ENABLE_GC != 0
  1038. static inline void
  1039. frame_ref_copy(uint8 *frame_ref_dest, uint8 *frame_ref_src, unsigned num)
  1040. {
  1041. if (frame_ref_dest != frame_ref_src)
  1042. for (; num > 0; num--)
  1043. *frame_ref_dest++ = *frame_ref_src++;
  1044. }
  1045. #else
  1046. #define frame_ref_copy(frame_ref_dst, frame_ref_src, num) (void)0
  1047. #endif
  1048. static inline WASMInterpFrame *
  1049. ALLOC_FRAME(WASMExecEnv *exec_env, uint32 size, WASMInterpFrame *prev_frame)
  1050. {
  1051. WASMInterpFrame *frame = wasm_exec_env_alloc_wasm_frame(exec_env, size);
  1052. if (frame) {
  1053. frame->prev_frame = prev_frame;
  1054. #if WASM_ENABLE_PERF_PROFILING != 0
  1055. frame->time_started = os_time_thread_cputime_us();
  1056. #endif
  1057. }
  1058. else {
  1059. wasm_set_exception((WASMModuleInstance *)exec_env->module_inst,
  1060. "wasm operand stack overflow");
  1061. }
  1062. return frame;
  1063. }
  1064. static inline void
  1065. FREE_FRAME(WASMExecEnv *exec_env, WASMInterpFrame *frame)
  1066. {
  1067. #if WASM_ENABLE_PERF_PROFILING != 0
  1068. if (frame->function) {
  1069. WASMInterpFrame *prev_frame = frame->prev_frame;
  1070. uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started;
  1071. frame->function->total_exec_time += time_elapsed;
  1072. frame->function->total_exec_cnt++;
  1073. if (prev_frame && prev_frame->function)
  1074. prev_frame->function->children_exec_time += time_elapsed;
  1075. }
  1076. #endif
  1077. wasm_exec_env_free_wasm_frame(exec_env, frame);
  1078. }
  1079. static void
  1080. wasm_interp_call_func_native(WASMModuleInstance *module_inst,
  1081. WASMExecEnv *exec_env,
  1082. WASMFunctionInstance *cur_func,
  1083. WASMInterpFrame *prev_frame)
  1084. {
  1085. WASMFunctionImport *func_import = cur_func->u.func_import;
  1086. CApiFuncImport *c_api_func_import = NULL;
  1087. unsigned local_cell_num =
  1088. cur_func->param_cell_num > 2 ? cur_func->param_cell_num : 2;
  1089. unsigned all_cell_num;
  1090. WASMInterpFrame *frame;
  1091. uint32 argv_ret[2], cur_func_index;
  1092. void *native_func_pointer = NULL;
  1093. char buf[128];
  1094. bool ret;
  1095. #if WASM_ENABLE_GC != 0
  1096. WASMFuncType *func_type;
  1097. uint8 *frame_ref;
  1098. #endif
  1099. if (!wasm_runtime_detect_native_stack_overflow(exec_env)) {
  1100. return;
  1101. }
  1102. all_cell_num = local_cell_num;
  1103. #if WASM_ENABLE_GC != 0
  1104. all_cell_num += (local_cell_num + 3) / 4;
  1105. #endif
  1106. if (!(frame =
  1107. ALLOC_FRAME(exec_env, wasm_interp_interp_frame_size(all_cell_num),
  1108. prev_frame)))
  1109. return;
  1110. frame->function = cur_func;
  1111. frame->ip = NULL;
  1112. frame->sp = frame->lp + local_cell_num;
  1113. #if WASM_ENABLE_GC != 0
  1114. /* native function doesn't have operand stack and label stack */
  1115. frame_ref = (uint8 *)frame->sp;
  1116. init_frame_refs(frame_ref, local_cell_num, cur_func);
  1117. #endif
  1118. wasm_exec_env_set_cur_frame(exec_env, frame);
  1119. cur_func_index = (uint32)(cur_func - module_inst->e->functions);
  1120. bh_assert(cur_func_index < module_inst->module->import_function_count);
  1121. if (!func_import->call_conv_wasm_c_api) {
  1122. native_func_pointer = module_inst->import_func_ptrs[cur_func_index];
  1123. }
  1124. else if (module_inst->c_api_func_imports) {
  1125. c_api_func_import = module_inst->c_api_func_imports + cur_func_index;
  1126. native_func_pointer = c_api_func_import->func_ptr_linked;
  1127. }
  1128. if (!native_func_pointer) {
  1129. snprintf(buf, sizeof(buf),
  1130. "failed to call unlinked import function (%s, %s)",
  1131. func_import->module_name, func_import->field_name);
  1132. wasm_set_exception(module_inst, buf);
  1133. return;
  1134. }
  1135. if (func_import->call_conv_wasm_c_api) {
  1136. ret = wasm_runtime_invoke_c_api_native(
  1137. (WASMModuleInstanceCommon *)module_inst, native_func_pointer,
  1138. func_import->func_type, cur_func->param_cell_num, frame->lp,
  1139. c_api_func_import->with_env_arg, c_api_func_import->env_arg);
  1140. if (ret) {
  1141. argv_ret[0] = frame->lp[0];
  1142. argv_ret[1] = frame->lp[1];
  1143. }
  1144. }
  1145. else if (!func_import->call_conv_raw) {
  1146. ret = wasm_runtime_invoke_native(
  1147. exec_env, native_func_pointer, func_import->func_type,
  1148. func_import->signature, func_import->attachment, frame->lp,
  1149. cur_func->param_cell_num, argv_ret);
  1150. }
  1151. else {
  1152. ret = wasm_runtime_invoke_native_raw(
  1153. exec_env, native_func_pointer, func_import->func_type,
  1154. func_import->signature, func_import->attachment, frame->lp,
  1155. cur_func->param_cell_num, argv_ret);
  1156. }
  1157. if (!ret)
  1158. return;
  1159. #if WASM_ENABLE_GC != 0
  1160. func_type = cur_func->u.func_import->func_type;
  1161. if (func_type->result_count
  1162. && wasm_is_type_reftype(func_type->types[cur_func->param_count])) {
  1163. frame_ref = (uint8 *)prev_frame->csp_boundary
  1164. + (unsigned)(uintptr_t)(prev_frame->sp - prev_frame->lp);
  1165. if (!wasm_is_reftype_i31ref(func_type->types[cur_func->param_count])) {
  1166. #if UINTPTR_MAX == UINT64_MAX
  1167. *frame_ref = *(frame_ref + 1) = 1;
  1168. #else
  1169. *frame_ref = 1;
  1170. #endif
  1171. }
  1172. }
  1173. #endif
  1174. if (cur_func->ret_cell_num == 1) {
  1175. prev_frame->sp[0] = argv_ret[0];
  1176. prev_frame->sp++;
  1177. }
  1178. else if (cur_func->ret_cell_num == 2) {
  1179. prev_frame->sp[0] = argv_ret[0];
  1180. prev_frame->sp[1] = argv_ret[1];
  1181. prev_frame->sp += 2;
  1182. }
  1183. FREE_FRAME(exec_env, frame);
  1184. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  1185. }
  1186. #if WASM_ENABLE_FAST_JIT != 0
  1187. bool
  1188. fast_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx,
  1189. WASMInterpFrame *prev_frame)
  1190. {
  1191. WASMModuleInstance *module_inst =
  1192. (WASMModuleInstance *)exec_env->module_inst;
  1193. WASMFunctionInstance *cur_func = module_inst->e->functions + func_idx;
  1194. wasm_interp_call_func_native(module_inst, exec_env, cur_func, prev_frame);
  1195. return wasm_copy_exception(module_inst, NULL) ? false : true;
  1196. }
  1197. #endif
  1198. #if WASM_ENABLE_MULTI_MODULE != 0
  1199. static void
  1200. wasm_interp_call_func_bytecode(WASMModuleInstance *module,
  1201. WASMExecEnv *exec_env,
  1202. WASMFunctionInstance *cur_func,
  1203. WASMInterpFrame *prev_frame);
  1204. static void
  1205. wasm_interp_call_func_import(WASMModuleInstance *module_inst,
  1206. WASMExecEnv *exec_env,
  1207. WASMFunctionInstance *cur_func,
  1208. WASMInterpFrame *prev_frame)
  1209. {
  1210. WASMModuleInstance *sub_module_inst = cur_func->import_module_inst;
  1211. WASMFunctionInstance *sub_func_inst = cur_func->import_func_inst;
  1212. WASMFunctionImport *func_import = cur_func->u.func_import;
  1213. uint8 *ip = prev_frame->ip;
  1214. char buf[128];
  1215. WASMExecEnv *sub_module_exec_env = NULL;
  1216. uintptr_t aux_stack_origin_boundary = 0;
  1217. uintptr_t aux_stack_origin_bottom = 0;
  1218. /*
  1219. * perform stack overflow check before calling
  1220. * wasm_interp_call_func_bytecode recursively.
  1221. */
  1222. if (!wasm_runtime_detect_native_stack_overflow(exec_env)) {
  1223. return;
  1224. }
  1225. if (!sub_func_inst) {
  1226. snprintf(buf, sizeof(buf),
  1227. "failed to call unlinked import function (%s, %s)",
  1228. func_import->module_name, func_import->field_name);
  1229. wasm_set_exception(module_inst, buf);
  1230. return;
  1231. }
  1232. /* Switch exec_env but keep using the same one by replacing necessary
  1233. * variables */
  1234. sub_module_exec_env = wasm_runtime_get_exec_env_singleton(
  1235. (WASMModuleInstanceCommon *)sub_module_inst);
  1236. if (!sub_module_exec_env) {
  1237. wasm_set_exception(module_inst, "create singleton exec_env failed");
  1238. return;
  1239. }
  1240. /* - module_inst */
  1241. wasm_exec_env_set_module_inst(exec_env,
  1242. (WASMModuleInstanceCommon *)sub_module_inst);
  1243. /* - aux_stack_boundary */
  1244. aux_stack_origin_boundary = exec_env->aux_stack_boundary;
  1245. exec_env->aux_stack_boundary = sub_module_exec_env->aux_stack_boundary;
  1246. /* - aux_stack_bottom */
  1247. aux_stack_origin_bottom = exec_env->aux_stack_bottom;
  1248. exec_env->aux_stack_bottom = sub_module_exec_env->aux_stack_bottom;
  1249. /* set ip NULL to make call_func_bytecode return after executing
  1250. this function */
  1251. prev_frame->ip = NULL;
  1252. /* call function of sub-module*/
  1253. wasm_interp_call_func_bytecode(sub_module_inst, exec_env, sub_func_inst,
  1254. prev_frame);
  1255. /* restore ip and other replaced */
  1256. prev_frame->ip = ip;
  1257. exec_env->aux_stack_boundary = aux_stack_origin_boundary;
  1258. exec_env->aux_stack_bottom = aux_stack_origin_bottom;
  1259. wasm_exec_env_restore_module_inst(exec_env,
  1260. (WASMModuleInstanceCommon *)module_inst);
  1261. }
  1262. #endif
  1263. #if WASM_ENABLE_THREAD_MGR != 0
  1264. #if WASM_ENABLE_DEBUG_INTERP != 0
  1265. #define CHECK_SUSPEND_FLAGS() \
  1266. do { \
  1267. os_mutex_lock(&exec_env->wait_lock); \
  1268. if (IS_WAMR_TERM_SIG(exec_env->current_status->signal_flag)) { \
  1269. os_mutex_unlock(&exec_env->wait_lock); \
  1270. return; \
  1271. } \
  1272. if (IS_WAMR_STOP_SIG(exec_env->current_status->signal_flag)) { \
  1273. SYNC_ALL_TO_FRAME(); \
  1274. wasm_cluster_thread_waiting_run(exec_env); \
  1275. } \
  1276. os_mutex_unlock(&exec_env->wait_lock); \
  1277. } while (0)
  1278. #else
  1279. #if WASM_SUSPEND_FLAGS_IS_ATOMIC != 0
  1280. /* The lock is only needed when the suspend_flags is atomic; otherwise
  1281. the lock is already taken at the time when SUSPENSION_LOCK() is called. */
  1282. #define SUSPENSION_LOCK() os_mutex_lock(&exec_env->wait_lock);
  1283. #define SUSPENSION_UNLOCK() os_mutex_unlock(&exec_env->wait_lock);
  1284. #else
  1285. #define SUSPENSION_LOCK()
  1286. #define SUSPENSION_UNLOCK()
  1287. #endif
  1288. #define CHECK_SUSPEND_FLAGS() \
  1289. do { \
  1290. WASM_SUSPEND_FLAGS_LOCK(exec_env->wait_lock); \
  1291. if (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \
  1292. & WASM_SUSPEND_FLAG_TERMINATE) { \
  1293. /* terminate current thread */ \
  1294. WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \
  1295. return; \
  1296. } \
  1297. while (WASM_SUSPEND_FLAGS_GET(exec_env->suspend_flags) \
  1298. & WASM_SUSPEND_FLAG_SUSPEND) { \
  1299. /* suspend current thread */ \
  1300. SUSPENSION_LOCK() \
  1301. os_cond_wait(&exec_env->wait_cond, &exec_env->wait_lock); \
  1302. SUSPENSION_UNLOCK() \
  1303. } \
  1304. WASM_SUSPEND_FLAGS_UNLOCK(exec_env->wait_lock); \
  1305. } while (0)
  1306. #endif /* WASM_ENABLE_DEBUG_INTERP */
  1307. #endif /* WASM_ENABLE_THREAD_MGR */
  1308. #if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
  1309. #if BH_ATOMIC_32_IS_ATOMIC != 0
  1310. #define GET_SIGNAL_FLAG() \
  1311. do { \
  1312. signal_flag = \
  1313. BH_ATOMIC_32_LOAD(exec_env->current_status->signal_flag); \
  1314. } while (0)
  1315. #else
  1316. #define GET_SIGNAL_FLAG() \
  1317. do { \
  1318. os_mutex_lock(&exec_env->wait_lock); \
  1319. signal_flag = exec_env->current_status->signal_flag; \
  1320. os_mutex_unlock(&exec_env->wait_lock); \
  1321. } while (0)
  1322. #endif
  1323. #endif
  1324. #if WASM_ENABLE_LABELS_AS_VALUES != 0
  1325. #define HANDLE_OP(opcode) HANDLE_##opcode:
  1326. #define FETCH_OPCODE_AND_DISPATCH() goto *handle_table[*frame_ip++]
  1327. #if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
  1328. #define HANDLE_OP_END() \
  1329. do { \
  1330. /* Record the current frame_ip, so when exception occurs, \
  1331. debugger can know the exact opcode who caused the exception */ \
  1332. frame_ip_orig = frame_ip; \
  1333. /* Atomic load the exec_env's signal_flag first, and then handle \
  1334. more with lock if it is WAMR_SIG_SINGSTEP */ \
  1335. GET_SIGNAL_FLAG(); \
  1336. if (signal_flag == WAMR_SIG_SINGSTEP) { \
  1337. os_mutex_lock(&exec_env->wait_lock); \
  1338. while (exec_env->current_status->signal_flag == WAMR_SIG_SINGSTEP \
  1339. && exec_env->current_status->step_count++ == 1) { \
  1340. exec_env->current_status->step_count = 0; \
  1341. SYNC_ALL_TO_FRAME(); \
  1342. wasm_cluster_thread_waiting_run(exec_env); \
  1343. } \
  1344. os_mutex_unlock(&exec_env->wait_lock); \
  1345. } \
  1346. CHECK_INSTRUCTION_LIMIT(); \
  1347. goto *handle_table[*frame_ip++]; \
  1348. } while (0)
  1349. #else
  1350. #define HANDLE_OP_END() \
  1351. CHECK_INSTRUCTION_LIMIT(); \
  1352. FETCH_OPCODE_AND_DISPATCH()
  1353. #endif
  1354. #else /* else of WASM_ENABLE_LABELS_AS_VALUES */
  1355. #define HANDLE_OP(opcode) case opcode:
  1356. #if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
  1357. #define HANDLE_OP_END() \
  1358. /* Record the current frame_ip, so when exception occurs, \
  1359. debugger can know the exact opcode who caused the exception */ \
  1360. frame_ip_orig = frame_ip; \
  1361. /* Atomic load the exec_env's signal_flag first, and then handle \
  1362. more with lock if it is WAMR_SIG_SINGSTEP */ \
  1363. GET_SIGNAL_FLAG(); \
  1364. if (signal_flag == WAMR_SIG_SINGSTEP) { \
  1365. os_mutex_lock(&exec_env->wait_lock); \
  1366. while (exec_env->current_status->signal_flag == WAMR_SIG_SINGSTEP \
  1367. && exec_env->current_status->step_count++ == 1) { \
  1368. exec_env->current_status->step_count = 0; \
  1369. SYNC_ALL_TO_FRAME(); \
  1370. wasm_cluster_thread_waiting_run(exec_env); \
  1371. } \
  1372. os_mutex_unlock(&exec_env->wait_lock); \
  1373. } \
  1374. CHECK_INSTRUCTION_LIMIT(); \
  1375. continue;
  1376. #else
  1377. #define HANDLE_OP_END() \
  1378. CHECK_INSTRUCTION_LIMIT(); \
  1379. continue;
  1380. #endif
  1381. #endif /* end of WASM_ENABLE_LABELS_AS_VALUES */
  1382. static inline uint8 *
  1383. get_global_addr(uint8 *global_data, WASMGlobalInstance *global)
  1384. {
  1385. #if WASM_ENABLE_MULTI_MODULE == 0
  1386. return global_data + global->data_offset;
  1387. #else
  1388. return global->import_global_inst
  1389. ? global->import_module_inst->global_data
  1390. + global->import_global_inst->data_offset
  1391. : global_data + global->data_offset;
  1392. #endif
  1393. }
  1394. #if WASM_ENABLE_INSTRUCTION_METERING != 0
  1395. #define CHECK_INSTRUCTION_LIMIT() \
  1396. if (instructions_left == 0) { \
  1397. wasm_set_exception(module, "instruction limit exceeded"); \
  1398. goto got_exception; \
  1399. } \
  1400. else if (instructions_left > 0) \
  1401. instructions_left--;
  1402. #else
  1403. #define CHECK_INSTRUCTION_LIMIT() (void)0
  1404. #endif
  1405. static void
  1406. wasm_interp_call_func_bytecode(WASMModuleInstance *module,
  1407. WASMExecEnv *exec_env,
  1408. WASMFunctionInstance *cur_func,
  1409. WASMInterpFrame *prev_frame)
  1410. {
  1411. WASMMemoryInstance *memory = wasm_get_default_memory(module);
  1412. #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
  1413. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
  1414. || WASM_ENABLE_BULK_MEMORY_OPT != 0
  1415. uint64 linear_mem_size = 0;
  1416. if (memory)
  1417. #if WASM_ENABLE_THREAD_MGR == 0
  1418. linear_mem_size = memory->memory_data_size;
  1419. #else
  1420. linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory);
  1421. #endif
  1422. #endif
  1423. WASMFuncType **wasm_types = (WASMFuncType **)module->module->types;
  1424. WASMGlobalInstance *globals = module->e->globals, *global;
  1425. uint8 *global_data = module->global_data;
  1426. uint8 opcode_IMPDEP = WASM_OP_IMPDEP;
  1427. WASMInterpFrame *frame = NULL;
  1428. /* Points to this special opcode so as to jump to the
  1429. * call_method_from_entry. */
  1430. register uint8 *frame_ip = &opcode_IMPDEP; /* cache of frame->ip */
  1431. register uint32 *frame_lp = NULL; /* cache of frame->lp */
  1432. register uint32 *frame_sp = NULL; /* cache of frame->sp */
  1433. #if WASM_ENABLE_GC != 0
  1434. register uint8 *frame_ref = NULL; /* cache of frame->ref */
  1435. uint8 *frame_ref_tmp;
  1436. #endif
  1437. WASMBranchBlock *frame_csp = NULL;
  1438. BlockAddr *cache_items;
  1439. uint8 *frame_ip_end = frame_ip + 1;
  1440. uint8 opcode;
  1441. uint32 i, depth, cond, count, fidx, tidx, lidx, frame_size = 0;
  1442. uint32 all_cell_num = 0;
  1443. tbl_elem_idx_t val;
  1444. uint8 *else_addr, *end_addr, *maddr = NULL;
  1445. uint32 local_idx, local_offset, global_idx;
  1446. uint8 local_type, *global_addr;
  1447. uint32 cache_index, type_index, param_cell_num, cell_num;
  1448. #if WASM_ENABLE_INSTRUCTION_METERING != 0
  1449. int instructions_left = -1;
  1450. if (exec_env) {
  1451. instructions_left = exec_env->instructions_to_execute;
  1452. }
  1453. #endif
  1454. #if WASM_ENABLE_EXCE_HANDLING != 0
  1455. int32_t exception_tag_index;
  1456. #endif
  1457. uint8 value_type;
  1458. #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
  1459. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0
  1460. #if WASM_CONFIGURABLE_BOUNDS_CHECKS != 0
  1461. bool disable_bounds_checks = !wasm_runtime_is_bounds_checks_enabled(
  1462. (WASMModuleInstanceCommon *)module);
  1463. #else
  1464. bool disable_bounds_checks = false;
  1465. #endif
  1466. #endif
  1467. #if WASM_ENABLE_GC != 0
  1468. WASMObjectRef gc_obj;
  1469. WASMStructObjectRef struct_obj;
  1470. WASMArrayObjectRef array_obj;
  1471. WASMFuncObjectRef func_obj;
  1472. WASMI31ObjectRef i31_obj;
  1473. WASMExternrefObjectRef externref_obj;
  1474. #if WASM_ENABLE_STRINGREF != 0
  1475. WASMString str_obj = NULL;
  1476. WASMStringrefObjectRef stringref_obj;
  1477. WASMStringviewWTF8ObjectRef stringview_wtf8_obj;
  1478. WASMStringviewWTF16ObjectRef stringview_wtf16_obj;
  1479. WASMStringviewIterObjectRef stringview_iter_obj;
  1480. #endif
  1481. #endif
  1482. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  1483. bool is_return_call = false;
  1484. #endif
  1485. #if WASM_ENABLE_MEMORY64 != 0
  1486. /* TODO: multi-memories for now assuming the memory idx type is consistent
  1487. * across multi-memories */
  1488. bool is_memory64 = false;
  1489. bool is_table64 = false;
  1490. if (memory)
  1491. is_memory64 = memory->is_memory64;
  1492. #endif
  1493. #if WASM_ENABLE_MULTI_MEMORY != 0
  1494. uint32 memidx = 0;
  1495. uint32 memidx_cached = (uint32)-1;
  1496. #endif
  1497. #if WASM_ENABLE_DEBUG_INTERP != 0
  1498. uint8 *frame_ip_orig = NULL;
  1499. WASMDebugInstance *debug_instance = wasm_exec_env_get_instance(exec_env);
  1500. bh_list *watch_point_list_read =
  1501. debug_instance ? &debug_instance->watch_point_list_read : NULL;
  1502. bh_list *watch_point_list_write =
  1503. debug_instance ? &debug_instance->watch_point_list_write : NULL;
  1504. #if WASM_ENABLE_THREAD_MGR != 0
  1505. uint32 signal_flag;
  1506. #endif
  1507. #endif
  1508. #if WASM_ENABLE_LABELS_AS_VALUES != 0
  1509. #define HANDLE_OPCODE(op) &&HANDLE_##op
  1510. DEFINE_GOTO_TABLE(const void *, handle_table);
  1511. #undef HANDLE_OPCODE
  1512. #endif
  1513. #if WASM_ENABLE_LABELS_AS_VALUES == 0
  1514. while (frame_ip < frame_ip_end) {
  1515. opcode = *frame_ip++;
  1516. switch (opcode) {
  1517. #else
  1518. FETCH_OPCODE_AND_DISPATCH();
  1519. #endif
  1520. /* control instructions */
  1521. HANDLE_OP(WASM_OP_UNREACHABLE)
  1522. {
  1523. wasm_set_exception(module, "unreachable");
  1524. goto got_exception;
  1525. }
  1526. HANDLE_OP(WASM_OP_NOP)
  1527. {
  1528. HANDLE_OP_END();
  1529. }
  1530. #if WASM_ENABLE_EXCE_HANDLING != 0
  1531. HANDLE_OP(WASM_OP_RETHROW)
  1532. {
  1533. int32_t relative_depth;
  1534. read_leb_int32(frame_ip, frame_ip_end, relative_depth);
  1535. /* No frame found with exception handler; validation should
  1536. * catch it */
  1537. bh_assert(frame_csp >= frame->csp_bottom + relative_depth);
  1538. /* go up the frame stack */
  1539. WASMBranchBlock *tgtframe = (frame_csp - 1) - relative_depth;
  1540. bh_assert(tgtframe->label_type == LABEL_TYPE_CATCH
  1541. || tgtframe->label_type == LABEL_TYPE_CATCH_ALL);
  1542. /* tgtframe points to the frame containing a thrown
  1543. * exception */
  1544. uint32 *tgtframe_sp = tgtframe->frame_sp;
  1545. /* frame sp of tgtframe points to caught exception */
  1546. exception_tag_index = *((uint32 *)tgtframe_sp);
  1547. tgtframe_sp++;
  1548. /* get tag type */
  1549. uint8 tag_type_index =
  1550. module->module->tags[exception_tag_index]->type;
  1551. uint32 cell_num_to_copy =
  1552. wasm_types[tag_type_index]->param_cell_num;
  1553. /* move exception parameters (if there are any) onto top
  1554. * of stack */
  1555. if (cell_num_to_copy > 0) {
  1556. word_copy(frame_sp, tgtframe_sp - cell_num_to_copy,
  1557. cell_num_to_copy);
  1558. }
  1559. frame_sp += cell_num_to_copy;
  1560. goto find_a_catch_handler;
  1561. }
  1562. HANDLE_OP(WASM_OP_THROW)
  1563. {
  1564. read_leb_int32(frame_ip, frame_ip_end, exception_tag_index);
  1565. /* landing pad for the rethrow ? */
  1566. find_a_catch_handler:
  1567. {
  1568. WASMFuncType *tag_type = NULL;
  1569. uint32 cell_num_to_copy = 0;
  1570. if (IS_INVALID_TAGINDEX(exception_tag_index)) {
  1571. /*
  1572. * invalid exception index,
  1573. * generated if a submodule throws an exception
  1574. * that has not been imported here
  1575. *
  1576. * This should result in a branch to the CATCH_ALL block,
  1577. * if there is one
  1578. */
  1579. tag_type = NULL;
  1580. cell_num_to_copy = 0;
  1581. }
  1582. else {
  1583. if (module->e->tags[exception_tag_index].is_import_tag) {
  1584. tag_type = module->e->tags[exception_tag_index]
  1585. .u.tag_import->tag_type;
  1586. }
  1587. else {
  1588. tag_type = module->e->tags[exception_tag_index]
  1589. .u.tag->tag_type;
  1590. }
  1591. cell_num_to_copy = tag_type->param_cell_num;
  1592. }
  1593. /* browse through frame stack */
  1594. uint32 relative_depth = 0;
  1595. do {
  1596. POP_CSP_CHECK_OVERFLOW(relative_depth - 1);
  1597. WASMBranchBlock *tgtframe = frame_csp - relative_depth - 1;
  1598. switch (tgtframe->label_type) {
  1599. case LABEL_TYPE_BLOCK:
  1600. case LABEL_TYPE_IF:
  1601. case LABEL_TYPE_LOOP:
  1602. case LABEL_TYPE_CATCH:
  1603. case LABEL_TYPE_CATCH_ALL:
  1604. /*
  1605. * skip that blocks in search
  1606. * BLOCK, IF and LOOP do not contain handlers and
  1607. * cannot catch exceptions.
  1608. * blocks marked as CATCH or
  1609. * CATCH_ALL did already caught an exception and can
  1610. * only be a target for RETHROW, but cannot catch an
  1611. * exception again
  1612. */
  1613. break;
  1614. case LABEL_TYPE_TRY:
  1615. {
  1616. uint32 handler_number = 0;
  1617. uint8 **handlers = (uint8 **)tgtframe->frame_sp;
  1618. uint8 *handler = NULL;
  1619. while ((handler = handlers[handler_number]) != 0) {
  1620. uint8 handler_opcode = *handler;
  1621. uint8 *target_addr =
  1622. handler
  1623. + 1; /* first instruction or leb-immediate
  1624. behind the handler opcode */
  1625. switch (handler_opcode) {
  1626. case WASM_OP_CATCH:
  1627. {
  1628. int32 lookup_index = 0;
  1629. /* read the tag_index and advance
  1630. * target_addr to the first instruction
  1631. * in the block */
  1632. read_leb_int32(target_addr, 0,
  1633. lookup_index);
  1634. if (exception_tag_index
  1635. == lookup_index) {
  1636. /* set ip */
  1637. frame_ip = target_addr;
  1638. /* save frame_sp (points to
  1639. * exception values) */
  1640. uint32 *frame_sp_old = frame_sp;
  1641. UNWIND_CSP(relative_depth,
  1642. LABEL_TYPE_CATCH);
  1643. /* push exception_tag_index and
  1644. * exception values for rethrow */
  1645. PUSH_I32(exception_tag_index);
  1646. if (cell_num_to_copy > 0) {
  1647. word_copy(
  1648. frame_sp,
  1649. frame_sp_old
  1650. - cell_num_to_copy,
  1651. cell_num_to_copy);
  1652. frame_sp += cell_num_to_copy;
  1653. /* push exception values for
  1654. * catch
  1655. */
  1656. word_copy(
  1657. frame_sp,
  1658. frame_sp_old
  1659. - cell_num_to_copy,
  1660. cell_num_to_copy);
  1661. frame_sp += cell_num_to_copy;
  1662. }
  1663. /* advance to handler */
  1664. HANDLE_OP_END();
  1665. }
  1666. break;
  1667. }
  1668. case WASM_OP_DELEGATE:
  1669. {
  1670. int32 lookup_depth = 0;
  1671. /* read the depth */
  1672. read_leb_int32(target_addr, 0,
  1673. lookup_depth);
  1674. /* save frame_sp (points to exception
  1675. * values) */
  1676. uint32 *frame_sp_old = frame_sp;
  1677. UNWIND_CSP(relative_depth,
  1678. LABEL_TYPE_CATCH);
  1679. /* leave the block (the delegate is
  1680. * technically not inside the frame) */
  1681. frame_csp--;
  1682. /* unwind to delegated frame */
  1683. frame_csp -= lookup_depth;
  1684. /* push exception values for catch */
  1685. if (cell_num_to_copy > 0) {
  1686. word_copy(frame_sp,
  1687. frame_sp_old
  1688. - cell_num_to_copy,
  1689. cell_num_to_copy);
  1690. frame_sp += cell_num_to_copy;
  1691. }
  1692. /* tag_index is already stored in
  1693. * exception_tag_index */
  1694. goto find_a_catch_handler;
  1695. }
  1696. case WASM_OP_CATCH_ALL:
  1697. {
  1698. /* no immediate */
  1699. /* save frame_sp (points to exception
  1700. * values) */
  1701. uint32 *frame_sp_old = frame_sp;
  1702. /* set ip */
  1703. frame_ip = target_addr;
  1704. UNWIND_CSP(relative_depth,
  1705. LABEL_TYPE_CATCH_ALL);
  1706. /* push exception_tag_index and
  1707. * exception values for rethrow */
  1708. PUSH_I32(exception_tag_index);
  1709. if (cell_num_to_copy > 0) {
  1710. word_copy(frame_sp,
  1711. frame_sp_old
  1712. - cell_num_to_copy,
  1713. cell_num_to_copy);
  1714. frame_sp += cell_num_to_copy;
  1715. }
  1716. /* catch_all has no exception values */
  1717. /* advance to handler */
  1718. HANDLE_OP_END();
  1719. }
  1720. default:
  1721. wasm_set_exception(
  1722. module, "WASM_OP_THROW found "
  1723. "unexpected handler type");
  1724. goto got_exception;
  1725. }
  1726. handler_number++;
  1727. }
  1728. /* exception not caught in this frame */
  1729. break;
  1730. }
  1731. case LABEL_TYPE_FUNCTION:
  1732. {
  1733. /* save frame_sp (points to exception values) */
  1734. uint32 *frame_sp_old = frame_sp;
  1735. UNWIND_CSP(relative_depth, LABEL_TYPE_FUNCTION);
  1736. /* push exception values for catch
  1737. * The values are copied to the CALLER FRAME
  1738. * (prev_frame->sp) same behavior ad WASM_OP_RETURN
  1739. */
  1740. if (cell_num_to_copy > 0) {
  1741. word_copy(prev_frame->sp,
  1742. frame_sp_old - cell_num_to_copy,
  1743. cell_num_to_copy);
  1744. prev_frame->sp += cell_num_to_copy;
  1745. }
  1746. *((int32 *)(prev_frame->sp)) = exception_tag_index;
  1747. prev_frame->sp++;
  1748. /* mark frame as raised exception */
  1749. wasm_set_exception(module,
  1750. "uncaught wasm exception");
  1751. /* end of function, treat as WASM_OP_RETURN */
  1752. goto return_func;
  1753. }
  1754. default:
  1755. wasm_set_exception(
  1756. module,
  1757. "unexpected or invalid label in THROW or "
  1758. "RETHROW when searching a catch handler");
  1759. goto got_exception;
  1760. }
  1761. relative_depth++;
  1762. } while (1);
  1763. }
  1764. /* something went wrong. normally, we should always find the
  1765. * func label. if not, stop the interpreter */
  1766. wasm_set_exception(
  1767. module, "WASM_OP_THROW hit the bottom of the frame stack");
  1768. goto got_exception;
  1769. }
  1770. HANDLE_OP(EXT_OP_TRY)
  1771. {
  1772. /* read the blocktype */
  1773. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  1774. param_cell_num = wasm_types[type_index]->param_cell_num;
  1775. cell_num = wasm_types[type_index]->ret_cell_num;
  1776. goto handle_op_try;
  1777. }
  1778. HANDLE_OP(WASM_OP_TRY)
  1779. {
  1780. value_type = *frame_ip++;
  1781. param_cell_num = 0;
  1782. cell_num = wasm_value_type_cell_num(value_type);
  1783. handle_op_try:
  1784. cache_index = ((uintptr_t)frame_ip)
  1785. & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1);
  1786. cache_items = exec_env->block_addr_cache[cache_index];
  1787. if (cache_items[0].start_addr == frame_ip) {
  1788. cache_items[0].start_addr = 0;
  1789. }
  1790. if (cache_items[1].start_addr == frame_ip) {
  1791. cache_items[1].start_addr = 0;
  1792. }
  1793. /* start at the first opcode following the try and its blocktype
  1794. */
  1795. uint8 *lookup_cursor = frame_ip;
  1796. uint8 handler_opcode = WASM_OP_UNREACHABLE;
  1797. /* target_addr filled in when END or DELEGATE is found */
  1798. PUSH_CSP(LABEL_TYPE_TRY, param_cell_num, cell_num, 0);
  1799. /* reset to begin of block */
  1800. lookup_cursor = frame_ip;
  1801. do {
  1802. /* lookup the next CATCH, CATCH_ALL or END for this TRY */
  1803. if (!wasm_loader_find_block_addr(
  1804. exec_env, (BlockAddr *)exec_env->block_addr_cache,
  1805. lookup_cursor, (uint8 *)-1, LABEL_TYPE_TRY,
  1806. &else_addr, &end_addr)) {
  1807. /* something went wrong */
  1808. wasm_set_exception(module, "find block address failed");
  1809. goto got_exception;
  1810. }
  1811. /* place cursor for continuation past opcode */
  1812. lookup_cursor = end_addr + 1;
  1813. /* end_addr points to CATCH, CATCH_ALL, DELEGATE or END */
  1814. handler_opcode = *end_addr;
  1815. switch (handler_opcode) {
  1816. case WASM_OP_CATCH:
  1817. skip_leb(lookup_cursor); /* skip tag_index */
  1818. PUSH_PTR(end_addr);
  1819. break;
  1820. case WASM_OP_CATCH_ALL:
  1821. PUSH_PTR(end_addr);
  1822. break;
  1823. case WASM_OP_DELEGATE:
  1824. skip_leb(lookup_cursor); /* skip depth */
  1825. PUSH_PTR(end_addr);
  1826. /* patch target_addr */
  1827. (frame_csp - 1)->target_addr = lookup_cursor;
  1828. break;
  1829. case WASM_OP_END:
  1830. PUSH_PTR(0);
  1831. /* patch target_addr */
  1832. (frame_csp - 1)->target_addr = end_addr;
  1833. break;
  1834. default:
  1835. /* something went wrong */
  1836. wasm_set_exception(module,
  1837. "find block address returned an "
  1838. "unexpected opcode");
  1839. goto got_exception;
  1840. }
  1841. /* ... search until the returned address is the END of the
  1842. * TRY block */
  1843. } while (handler_opcode != WASM_OP_END
  1844. && handler_opcode != WASM_OP_DELEGATE);
  1845. /* handler setup on stack complete */
  1846. HANDLE_OP_END();
  1847. }
  1848. HANDLE_OP(WASM_OP_CATCH)
  1849. {
  1850. /* skip the tag_index */
  1851. skip_leb(frame_ip);
  1852. /* leave the frame */
  1853. POP_CSP_N(0);
  1854. HANDLE_OP_END();
  1855. }
  1856. HANDLE_OP(WASM_OP_CATCH_ALL)
  1857. {
  1858. /* leave the frame */
  1859. POP_CSP_N(0);
  1860. HANDLE_OP_END();
  1861. }
  1862. HANDLE_OP(WASM_OP_DELEGATE)
  1863. {
  1864. /* skip the delegate depth */
  1865. skip_leb(frame_ip);
  1866. /* leave the frame like WASM_OP_END */
  1867. POP_CSP();
  1868. HANDLE_OP_END();
  1869. }
  1870. #endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */
  1871. HANDLE_OP(EXT_OP_BLOCK)
  1872. {
  1873. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  1874. param_cell_num =
  1875. ((WASMFuncType *)wasm_types[type_index])->param_cell_num;
  1876. cell_num =
  1877. ((WASMFuncType *)wasm_types[type_index])->ret_cell_num;
  1878. goto handle_op_block;
  1879. }
  1880. HANDLE_OP(WASM_OP_BLOCK)
  1881. {
  1882. value_type = *frame_ip++;
  1883. param_cell_num = 0;
  1884. cell_num = wasm_value_type_cell_num(value_type);
  1885. handle_op_block:
  1886. cache_index = ((uintptr_t)frame_ip)
  1887. & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1);
  1888. cache_items = exec_env->block_addr_cache[cache_index];
  1889. if (cache_items[0].start_addr == frame_ip) {
  1890. end_addr = cache_items[0].end_addr;
  1891. }
  1892. else if (cache_items[1].start_addr == frame_ip) {
  1893. end_addr = cache_items[1].end_addr;
  1894. }
  1895. #if WASM_ENABLE_DEBUG_INTERP != 0
  1896. else if (!wasm_loader_find_block_addr(
  1897. exec_env, (BlockAddr *)exec_env->block_addr_cache,
  1898. frame_ip, (uint8 *)-1, LABEL_TYPE_BLOCK,
  1899. &else_addr, &end_addr)) {
  1900. wasm_set_exception(module, "find block address failed");
  1901. goto got_exception;
  1902. }
  1903. #endif
  1904. else {
  1905. end_addr = NULL;
  1906. }
  1907. PUSH_CSP(LABEL_TYPE_BLOCK, param_cell_num, cell_num, end_addr);
  1908. HANDLE_OP_END();
  1909. }
  1910. HANDLE_OP(EXT_OP_LOOP)
  1911. {
  1912. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  1913. param_cell_num =
  1914. ((WASMFuncType *)wasm_types[type_index])->param_cell_num;
  1915. cell_num =
  1916. ((WASMFuncType *)wasm_types[type_index])->param_cell_num;
  1917. goto handle_op_loop;
  1918. }
  1919. HANDLE_OP(WASM_OP_LOOP)
  1920. {
  1921. value_type = *frame_ip++;
  1922. param_cell_num = 0;
  1923. cell_num = 0;
  1924. handle_op_loop:
  1925. PUSH_CSP(LABEL_TYPE_LOOP, param_cell_num, cell_num, frame_ip);
  1926. HANDLE_OP_END();
  1927. }
  1928. HANDLE_OP(EXT_OP_IF)
  1929. {
  1930. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  1931. param_cell_num =
  1932. ((WASMFuncType *)wasm_types[type_index])->param_cell_num;
  1933. cell_num =
  1934. ((WASMFuncType *)wasm_types[type_index])->ret_cell_num;
  1935. goto handle_op_if;
  1936. }
  1937. HANDLE_OP(WASM_OP_IF)
  1938. {
  1939. value_type = *frame_ip++;
  1940. param_cell_num = 0;
  1941. cell_num = wasm_value_type_cell_num(value_type);
  1942. handle_op_if:
  1943. cache_index = ((uintptr_t)frame_ip)
  1944. & (uintptr_t)(BLOCK_ADDR_CACHE_SIZE - 1);
  1945. cache_items = exec_env->block_addr_cache[cache_index];
  1946. if (cache_items[0].start_addr == frame_ip) {
  1947. else_addr = cache_items[0].else_addr;
  1948. end_addr = cache_items[0].end_addr;
  1949. }
  1950. else if (cache_items[1].start_addr == frame_ip) {
  1951. else_addr = cache_items[1].else_addr;
  1952. end_addr = cache_items[1].end_addr;
  1953. }
  1954. else if (!wasm_loader_find_block_addr(
  1955. exec_env, (BlockAddr *)exec_env->block_addr_cache,
  1956. frame_ip, (uint8 *)-1, LABEL_TYPE_IF, &else_addr,
  1957. &end_addr)) {
  1958. wasm_set_exception(module, "find block address failed");
  1959. goto got_exception;
  1960. }
  1961. cond = (uint32)POP_I32();
  1962. if (cond) { /* if branch is met */
  1963. PUSH_CSP(LABEL_TYPE_IF, param_cell_num, cell_num, end_addr);
  1964. }
  1965. else { /* if branch is not met */
  1966. /* if there is no else branch, go to the end addr */
  1967. if (else_addr == NULL) {
  1968. frame_ip = end_addr + 1;
  1969. }
  1970. /* if there is an else branch, go to the else addr */
  1971. else {
  1972. PUSH_CSP(LABEL_TYPE_IF, param_cell_num, cell_num,
  1973. end_addr);
  1974. frame_ip = else_addr + 1;
  1975. }
  1976. }
  1977. HANDLE_OP_END();
  1978. }
  1979. HANDLE_OP(WASM_OP_ELSE)
  1980. {
  1981. /* comes from the if branch in WASM_OP_IF */
  1982. frame_ip = (frame_csp - 1)->target_addr;
  1983. HANDLE_OP_END();
  1984. }
  1985. HANDLE_OP(WASM_OP_END)
  1986. {
  1987. if (frame_csp > frame->csp_bottom + 1) {
  1988. POP_CSP();
  1989. }
  1990. else { /* end of function, treat as WASM_OP_RETURN */
  1991. frame_sp -= cur_func->ret_cell_num;
  1992. for (i = 0; i < cur_func->ret_cell_num; i++) {
  1993. #if WASM_ENABLE_GC != 0
  1994. if (prev_frame->ip) {
  1995. /* prev frame is not a glue frame and has
  1996. the frame ref area */
  1997. *FRAME_REF_FOR(prev_frame, prev_frame->sp) =
  1998. *FRAME_REF(frame_sp + i);
  1999. }
  2000. #endif
  2001. *prev_frame->sp++ = frame_sp[i];
  2002. }
  2003. goto return_func;
  2004. }
  2005. HANDLE_OP_END();
  2006. }
  2007. HANDLE_OP(WASM_OP_BR)
  2008. {
  2009. #if WASM_ENABLE_THREAD_MGR != 0
  2010. CHECK_SUSPEND_FLAGS();
  2011. #endif
  2012. read_leb_uint32(frame_ip, frame_ip_end, depth);
  2013. label_pop_csp_n:
  2014. POP_CSP_N(depth);
  2015. if (!frame_ip) { /* must be label pushed by WASM_OP_BLOCK */
  2016. if (!wasm_loader_find_block_addr(
  2017. exec_env, (BlockAddr *)exec_env->block_addr_cache,
  2018. (frame_csp - 1)->begin_addr, (uint8 *)-1,
  2019. LABEL_TYPE_BLOCK, &else_addr, &end_addr)) {
  2020. wasm_set_exception(module, "find block address failed");
  2021. goto got_exception;
  2022. }
  2023. frame_ip = end_addr;
  2024. }
  2025. HANDLE_OP_END();
  2026. }
  2027. HANDLE_OP(WASM_OP_BR_IF)
  2028. {
  2029. #if WASM_ENABLE_THREAD_MGR != 0
  2030. CHECK_SUSPEND_FLAGS();
  2031. #endif
  2032. read_leb_uint32(frame_ip, frame_ip_end, depth);
  2033. cond = (uint32)POP_I32();
  2034. if (cond)
  2035. goto label_pop_csp_n;
  2036. HANDLE_OP_END();
  2037. }
  2038. HANDLE_OP(WASM_OP_BR_TABLE)
  2039. {
  2040. #if WASM_ENABLE_THREAD_MGR != 0
  2041. CHECK_SUSPEND_FLAGS();
  2042. #endif
  2043. read_leb_uint32(frame_ip, frame_ip_end, count);
  2044. lidx = POP_I32();
  2045. if (lidx > count)
  2046. lidx = count;
  2047. depth = frame_ip[lidx];
  2048. goto label_pop_csp_n;
  2049. }
  2050. HANDLE_OP(EXT_OP_BR_TABLE_CACHE)
  2051. {
  2052. BrTableCache *node_cache =
  2053. bh_list_first_elem(module->module->br_table_cache_list);
  2054. BrTableCache *node_next;
  2055. #if WASM_ENABLE_THREAD_MGR != 0
  2056. CHECK_SUSPEND_FLAGS();
  2057. #endif
  2058. lidx = POP_I32();
  2059. while (node_cache) {
  2060. node_next = bh_list_elem_next(node_cache);
  2061. if (node_cache->br_table_op_addr == frame_ip - 1) {
  2062. if (lidx > node_cache->br_count)
  2063. lidx = node_cache->br_count;
  2064. depth = node_cache->br_depths[lidx];
  2065. goto label_pop_csp_n;
  2066. }
  2067. node_cache = node_next;
  2068. }
  2069. bh_assert(0);
  2070. HANDLE_OP_END();
  2071. }
  2072. HANDLE_OP(WASM_OP_RETURN)
  2073. {
  2074. frame_sp -= cur_func->ret_cell_num;
  2075. for (i = 0; i < cur_func->ret_cell_num; i++) {
  2076. #if WASM_ENABLE_GC != 0
  2077. if (prev_frame->ip) {
  2078. /* prev frame is not a glue frame and has
  2079. the frame ref area */
  2080. *FRAME_REF_FOR(prev_frame, prev_frame->sp) =
  2081. *FRAME_REF(frame_sp + i);
  2082. }
  2083. #endif
  2084. *prev_frame->sp++ = frame_sp[i];
  2085. }
  2086. goto return_func;
  2087. }
  2088. HANDLE_OP(WASM_OP_CALL)
  2089. {
  2090. #if WASM_ENABLE_THREAD_MGR != 0
  2091. CHECK_SUSPEND_FLAGS();
  2092. #endif
  2093. read_leb_uint32(frame_ip, frame_ip_end, fidx);
  2094. #if WASM_ENABLE_MULTI_MODULE != 0
  2095. if (fidx >= module->e->function_count) {
  2096. wasm_set_exception(module, "unknown function");
  2097. goto got_exception;
  2098. }
  2099. #endif
  2100. cur_func = module->e->functions + fidx;
  2101. goto call_func_from_interp;
  2102. }
  2103. #if WASM_ENABLE_TAIL_CALL != 0
  2104. HANDLE_OP(WASM_OP_RETURN_CALL)
  2105. {
  2106. #if WASM_ENABLE_THREAD_MGR != 0
  2107. CHECK_SUSPEND_FLAGS();
  2108. #endif
  2109. read_leb_uint32(frame_ip, frame_ip_end, fidx);
  2110. #if WASM_ENABLE_MULTI_MODULE != 0
  2111. if (fidx >= module->e->function_count) {
  2112. wasm_set_exception(module, "unknown function");
  2113. goto got_exception;
  2114. }
  2115. #endif
  2116. cur_func = module->e->functions + fidx;
  2117. goto call_func_from_return_call;
  2118. }
  2119. #endif /* WASM_ENABLE_TAIL_CALL */
  2120. HANDLE_OP(WASM_OP_CALL_INDIRECT)
  2121. #if WASM_ENABLE_TAIL_CALL != 0
  2122. HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT)
  2123. #endif
  2124. {
  2125. WASMFuncType *cur_type, *cur_func_type;
  2126. WASMTableInstance *tbl_inst;
  2127. uint32 tbl_idx;
  2128. #if WASM_ENABLE_TAIL_CALL != 0
  2129. opcode = *(frame_ip - 1);
  2130. #endif
  2131. #if WASM_ENABLE_THREAD_MGR != 0
  2132. CHECK_SUSPEND_FLAGS();
  2133. #endif
  2134. /**
  2135. * type check. compiler will make sure all like
  2136. * (call_indirect (type $x) (it.const 1))
  2137. * the function type has to be defined in the module also
  2138. * no matter it is used or not
  2139. */
  2140. read_leb_uint32(frame_ip, frame_ip_end, tidx);
  2141. bh_assert(tidx < module->module->type_count);
  2142. cur_type = wasm_types[tidx];
  2143. /* clang-format off */
  2144. #if WASM_ENABLE_CALL_INDIRECT_OVERLONG != 0 || WASM_ENABLE_GC != 0
  2145. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  2146. #else
  2147. frame_ip++;
  2148. tbl_idx = 0;
  2149. #endif
  2150. bh_assert(tbl_idx < module->table_count);
  2151. /* clang-format on */
  2152. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  2153. #if WASM_ENABLE_MEMORY64 != 0
  2154. is_table64 = tbl_inst->is_table64;
  2155. #endif
  2156. val = POP_TBL_ELEM_IDX();
  2157. if (val >= tbl_inst->cur_size) {
  2158. wasm_set_exception(module, "undefined element");
  2159. goto got_exception;
  2160. }
  2161. /* clang-format off */
  2162. #if WASM_ENABLE_GC == 0
  2163. fidx = tbl_inst->elems[val];
  2164. if (fidx == (uint32)-1) {
  2165. wasm_set_exception(module, "uninitialized element");
  2166. goto got_exception;
  2167. }
  2168. #else
  2169. func_obj = (WASMFuncObjectRef)tbl_inst->elems[val];
  2170. if (!func_obj) {
  2171. wasm_set_exception(module, "uninitialized element");
  2172. goto got_exception;
  2173. }
  2174. fidx = wasm_func_obj_get_func_idx_bound(func_obj);
  2175. #endif
  2176. /* clang-format on */
  2177. /*
  2178. * we might be using a table injected by host or
  2179. * another module. In that case, we don't validate
  2180. * the elem value while loading
  2181. */
  2182. if (fidx >= module->e->function_count) {
  2183. wasm_set_exception(module, "unknown function");
  2184. goto got_exception;
  2185. }
  2186. /* always call module own functions */
  2187. cur_func = module->e->functions + fidx;
  2188. if (cur_func->is_import_func)
  2189. cur_func_type = cur_func->u.func_import->func_type;
  2190. else
  2191. cur_func_type = cur_func->u.func->func_type;
  2192. /* clang-format off */
  2193. #if WASM_ENABLE_GC == 0
  2194. if (cur_type != cur_func_type) {
  2195. wasm_set_exception(module, "indirect call type mismatch");
  2196. goto got_exception;
  2197. }
  2198. #else
  2199. if (!wasm_func_type_is_super_of(cur_type, cur_func_type)) {
  2200. wasm_set_exception(module, "indirect call type mismatch");
  2201. goto got_exception;
  2202. }
  2203. #endif
  2204. /* clang-format on */
  2205. #if WASM_ENABLE_TAIL_CALL != 0
  2206. if (opcode == WASM_OP_RETURN_CALL_INDIRECT)
  2207. goto call_func_from_return_call;
  2208. #endif
  2209. goto call_func_from_interp;
  2210. }
  2211. /* parametric instructions */
  2212. HANDLE_OP(WASM_OP_DROP)
  2213. {
  2214. frame_sp--;
  2215. #if WASM_ENABLE_GC != 0
  2216. frame_ref_tmp = FRAME_REF(frame_sp);
  2217. *frame_ref_tmp = 0;
  2218. #endif
  2219. HANDLE_OP_END();
  2220. }
  2221. HANDLE_OP(WASM_OP_DROP_64)
  2222. {
  2223. frame_sp -= 2;
  2224. #if WASM_ENABLE_GC != 0
  2225. frame_ref_tmp = FRAME_REF(frame_sp);
  2226. *frame_ref_tmp = 0;
  2227. *(frame_ref_tmp + 1) = 0;
  2228. #endif
  2229. HANDLE_OP_END();
  2230. }
  2231. HANDLE_OP(WASM_OP_SELECT)
  2232. {
  2233. cond = (uint32)POP_I32();
  2234. frame_sp--;
  2235. if (!cond)
  2236. *(frame_sp - 1) = *frame_sp;
  2237. HANDLE_OP_END();
  2238. }
  2239. HANDLE_OP(WASM_OP_SELECT_64)
  2240. {
  2241. cond = (uint32)POP_I32();
  2242. frame_sp -= 2;
  2243. if (!cond) {
  2244. *(frame_sp - 2) = *frame_sp;
  2245. *(frame_sp - 1) = *(frame_sp + 1);
  2246. }
  2247. HANDLE_OP_END();
  2248. }
  2249. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  2250. HANDLE_OP(WASM_OP_SELECT_T)
  2251. {
  2252. uint32 vec_len;
  2253. uint8 type;
  2254. read_leb_uint32(frame_ip, frame_ip_end, vec_len);
  2255. type = *frame_ip++;
  2256. cond = (uint32)POP_I32();
  2257. if (type == VALUE_TYPE_I64 || type == VALUE_TYPE_F64
  2258. #if WASM_ENABLE_GC != 0 && UINTPTR_MAX == UINT64_MAX
  2259. || wasm_is_type_reftype(type)
  2260. #endif
  2261. ) {
  2262. frame_sp -= 2;
  2263. if (!cond) {
  2264. *(frame_sp - 2) = *frame_sp;
  2265. *(frame_sp - 1) = *(frame_sp + 1);
  2266. }
  2267. }
  2268. else {
  2269. frame_sp--;
  2270. if (!cond)
  2271. *(frame_sp - 1) = *frame_sp;
  2272. }
  2273. #if WASM_ENABLE_GC != 0
  2274. frame_ref_tmp = FRAME_REF(frame_sp);
  2275. *frame_ref_tmp = 0;
  2276. #if UINTPTR_MAX == UINT64_MAX
  2277. *(frame_ref_tmp + 1) = 0;
  2278. #endif
  2279. #endif
  2280. (void)vec_len;
  2281. HANDLE_OP_END();
  2282. }
  2283. HANDLE_OP(WASM_OP_TABLE_GET)
  2284. {
  2285. uint32 tbl_idx;
  2286. tbl_elem_idx_t elem_idx;
  2287. WASMTableInstance *tbl_inst;
  2288. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  2289. bh_assert(tbl_idx < module->table_count);
  2290. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  2291. #if WASM_ENABLE_MEMORY64 != 0
  2292. is_table64 = tbl_inst->is_table64;
  2293. #endif
  2294. elem_idx = POP_TBL_ELEM_IDX();
  2295. if (elem_idx >= tbl_inst->cur_size) {
  2296. wasm_set_exception(module, "out of bounds table access");
  2297. goto got_exception;
  2298. }
  2299. #if WASM_ENABLE_GC == 0
  2300. PUSH_I32(tbl_inst->elems[elem_idx]);
  2301. #else
  2302. PUSH_REF(tbl_inst->elems[elem_idx]);
  2303. #endif
  2304. HANDLE_OP_END();
  2305. }
  2306. HANDLE_OP(WASM_OP_TABLE_SET)
  2307. {
  2308. WASMTableInstance *tbl_inst;
  2309. uint32 tbl_idx;
  2310. tbl_elem_idx_t elem_idx;
  2311. table_elem_type_t elem_val;
  2312. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  2313. bh_assert(tbl_idx < module->table_count);
  2314. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  2315. #if WASM_ENABLE_MEMORY64 != 0
  2316. is_table64 = tbl_inst->is_table64;
  2317. #endif
  2318. #if WASM_ENABLE_GC == 0
  2319. elem_val = POP_I32();
  2320. #else
  2321. elem_val = POP_REF();
  2322. #endif
  2323. elem_idx = POP_TBL_ELEM_IDX();
  2324. if (elem_idx >= tbl_inst->cur_size) {
  2325. wasm_set_exception(module, "out of bounds table access");
  2326. goto got_exception;
  2327. }
  2328. tbl_inst->elems[elem_idx] = elem_val;
  2329. HANDLE_OP_END();
  2330. }
  2331. HANDLE_OP(WASM_OP_REF_NULL)
  2332. {
  2333. uint32 ref_type;
  2334. read_leb_uint32(frame_ip, frame_ip_end, ref_type);
  2335. #if WASM_ENABLE_GC == 0
  2336. PUSH_I32(NULL_REF);
  2337. #else
  2338. PUSH_REF(NULL_REF);
  2339. #endif
  2340. (void)ref_type;
  2341. HANDLE_OP_END();
  2342. }
  2343. HANDLE_OP(WASM_OP_REF_IS_NULL)
  2344. {
  2345. #if WASM_ENABLE_GC == 0
  2346. uint32 ref_val;
  2347. ref_val = POP_I32();
  2348. #else
  2349. void *ref_val;
  2350. ref_val = POP_REF();
  2351. #endif
  2352. PUSH_I32(ref_val == NULL_REF ? 1 : 0);
  2353. HANDLE_OP_END();
  2354. }
  2355. HANDLE_OP(WASM_OP_REF_FUNC)
  2356. {
  2357. uint32 func_idx;
  2358. read_leb_uint32(frame_ip, frame_ip_end, func_idx);
  2359. #if WASM_ENABLE_GC == 0
  2360. PUSH_I32(func_idx);
  2361. #else
  2362. SYNC_ALL_TO_FRAME();
  2363. if (!(gc_obj = wasm_create_func_obj(module, func_idx, true,
  2364. NULL, 0))) {
  2365. goto got_exception;
  2366. }
  2367. PUSH_REF(gc_obj);
  2368. #endif
  2369. HANDLE_OP_END();
  2370. }
  2371. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  2372. #if WASM_ENABLE_GC != 0
  2373. HANDLE_OP(WASM_OP_CALL_REF)
  2374. {
  2375. #if WASM_ENABLE_THREAD_MGR != 0
  2376. CHECK_SUSPEND_FLAGS();
  2377. #endif
  2378. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2379. func_obj = POP_REF();
  2380. if (!func_obj) {
  2381. wasm_set_exception(module, "null function reference");
  2382. goto got_exception;
  2383. }
  2384. fidx = wasm_func_obj_get_func_idx_bound(func_obj);
  2385. cur_func = module->e->functions + fidx;
  2386. goto call_func_from_interp;
  2387. }
  2388. HANDLE_OP(WASM_OP_RETURN_CALL_REF)
  2389. {
  2390. #if WASM_ENABLE_THREAD_MGR != 0
  2391. CHECK_SUSPEND_FLAGS();
  2392. #endif
  2393. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2394. func_obj = POP_REF();
  2395. if (!func_obj) {
  2396. wasm_set_exception(module, "null function reference");
  2397. goto got_exception;
  2398. }
  2399. fidx = wasm_func_obj_get_func_idx_bound(func_obj);
  2400. cur_func = module->e->functions + fidx;
  2401. goto call_func_from_return_call;
  2402. }
  2403. HANDLE_OP(WASM_OP_REF_EQ)
  2404. {
  2405. WASMObjectRef gc_obj1, gc_obj2;
  2406. gc_obj2 = POP_REF();
  2407. gc_obj1 = POP_REF();
  2408. val = wasm_obj_equal(gc_obj1, gc_obj2);
  2409. PUSH_I32(val);
  2410. HANDLE_OP_END();
  2411. }
  2412. HANDLE_OP(WASM_OP_REF_AS_NON_NULL)
  2413. {
  2414. gc_obj = POP_REF();
  2415. if (gc_obj == NULL_REF) {
  2416. wasm_set_exception(module, "null reference");
  2417. goto got_exception;
  2418. }
  2419. PUSH_REF(gc_obj);
  2420. HANDLE_OP_END();
  2421. }
  2422. HANDLE_OP(WASM_OP_BR_ON_NULL)
  2423. {
  2424. #if WASM_ENABLE_THREAD_MGR != 0
  2425. CHECK_SUSPEND_FLAGS();
  2426. #endif
  2427. read_leb_uint32(frame_ip, frame_ip_end, depth);
  2428. gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM);
  2429. if (gc_obj == NULL_REF) {
  2430. frame_sp -= REF_CELL_NUM;
  2431. CLEAR_FRAME_REF(frame_sp, REF_CELL_NUM);
  2432. goto label_pop_csp_n;
  2433. }
  2434. HANDLE_OP_END();
  2435. }
  2436. HANDLE_OP(WASM_OP_BR_ON_NON_NULL)
  2437. {
  2438. #if WASM_ENABLE_THREAD_MGR != 0
  2439. CHECK_SUSPEND_FLAGS();
  2440. #endif
  2441. read_leb_uint32(frame_ip, frame_ip_end, depth);
  2442. gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM);
  2443. if (gc_obj != NULL_REF) {
  2444. goto label_pop_csp_n;
  2445. }
  2446. else {
  2447. frame_sp -= REF_CELL_NUM;
  2448. CLEAR_FRAME_REF(frame_sp, REF_CELL_NUM);
  2449. }
  2450. HANDLE_OP_END();
  2451. }
  2452. HANDLE_OP(WASM_OP_GC_PREFIX)
  2453. {
  2454. uint32 opcode1;
  2455. read_leb_uint32(frame_ip, frame_ip_end, opcode1);
  2456. /* opcode1 was checked in loader and is no larger than
  2457. UINT8_MAX */
  2458. opcode = (uint8)opcode1;
  2459. switch (opcode) {
  2460. case WASM_OP_STRUCT_NEW:
  2461. case WASM_OP_STRUCT_NEW_DEFAULT:
  2462. {
  2463. WASMModule *wasm_module = module->module;
  2464. WASMStructType *struct_type;
  2465. WASMRttType *rtt_type;
  2466. WASMValue field_value = { 0 };
  2467. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2468. struct_type =
  2469. (WASMStructType *)module->module->types[type_index];
  2470. if (!(rtt_type = wasm_rtt_type_new(
  2471. (WASMType *)struct_type, type_index,
  2472. wasm_module->rtt_types,
  2473. wasm_module->type_count,
  2474. &wasm_module->rtt_type_lock))) {
  2475. wasm_set_exception(module,
  2476. "create rtt type failed");
  2477. goto got_exception;
  2478. }
  2479. SYNC_ALL_TO_FRAME();
  2480. struct_obj = wasm_struct_obj_new(exec_env, rtt_type);
  2481. if (!struct_obj) {
  2482. wasm_set_exception(module,
  2483. "create struct object failed");
  2484. goto got_exception;
  2485. }
  2486. if (opcode == WASM_OP_STRUCT_NEW) {
  2487. WASMStructFieldType *fields = struct_type->fields;
  2488. int32 field_count = (int32)struct_type->field_count;
  2489. int32 field_idx;
  2490. uint8 field_type;
  2491. for (field_idx = field_count - 1; field_idx >= 0;
  2492. field_idx--) {
  2493. field_type = fields[field_idx].field_type;
  2494. if (wasm_is_type_reftype(field_type)) {
  2495. field_value.gc_obj = POP_REF();
  2496. }
  2497. else if (field_type == VALUE_TYPE_I32
  2498. || field_type == VALUE_TYPE_F32
  2499. || field_type == PACKED_TYPE_I8
  2500. || field_type == PACKED_TYPE_I16) {
  2501. field_value.i32 = POP_I32();
  2502. }
  2503. else {
  2504. field_value.i64 = POP_I64();
  2505. }
  2506. wasm_struct_obj_set_field(struct_obj, field_idx,
  2507. &field_value);
  2508. }
  2509. }
  2510. PUSH_REF(struct_obj);
  2511. HANDLE_OP_END();
  2512. }
  2513. case WASM_OP_STRUCT_GET:
  2514. case WASM_OP_STRUCT_GET_S:
  2515. case WASM_OP_STRUCT_GET_U:
  2516. {
  2517. WASMStructType *struct_type;
  2518. WASMValue field_value = { 0 };
  2519. uint32 field_idx;
  2520. uint8 field_type;
  2521. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2522. read_leb_uint32(frame_ip, frame_ip_end, field_idx);
  2523. struct_type =
  2524. (WASMStructType *)module->module->types[type_index];
  2525. struct_obj = POP_REF();
  2526. if (!struct_obj) {
  2527. wasm_set_exception(module,
  2528. "null structure reference");
  2529. goto got_exception;
  2530. }
  2531. wasm_struct_obj_get_field(
  2532. struct_obj, field_idx,
  2533. opcode == WASM_OP_STRUCT_GET_S ? true : false,
  2534. &field_value);
  2535. field_type = struct_type->fields[field_idx].field_type;
  2536. if (wasm_is_reftype_i31ref(field_type)) {
  2537. PUSH_I31REF(field_value.gc_obj);
  2538. }
  2539. else if (wasm_is_type_reftype(field_type)) {
  2540. PUSH_REF(field_value.gc_obj);
  2541. }
  2542. else if (field_type == VALUE_TYPE_I32
  2543. || field_type == VALUE_TYPE_F32
  2544. || field_type == PACKED_TYPE_I8
  2545. || field_type == PACKED_TYPE_I16) {
  2546. PUSH_I32(field_value.i32);
  2547. }
  2548. else {
  2549. PUSH_I64(field_value.i64);
  2550. }
  2551. HANDLE_OP_END();
  2552. }
  2553. case WASM_OP_STRUCT_SET:
  2554. {
  2555. WASMStructType *struct_type;
  2556. WASMValue field_value = { 0 };
  2557. uint32 field_idx;
  2558. uint8 field_type;
  2559. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2560. read_leb_uint32(frame_ip, frame_ip_end, field_idx);
  2561. struct_type =
  2562. (WASMStructType *)module->module->types[type_index];
  2563. field_type = struct_type->fields[field_idx].field_type;
  2564. if (wasm_is_type_reftype(field_type)) {
  2565. field_value.gc_obj = POP_REF();
  2566. }
  2567. else if (field_type == VALUE_TYPE_I32
  2568. || field_type == VALUE_TYPE_F32
  2569. || field_type == PACKED_TYPE_I8
  2570. || field_type == PACKED_TYPE_I16) {
  2571. field_value.i32 = POP_I32();
  2572. }
  2573. else {
  2574. field_value.i64 = POP_I64();
  2575. }
  2576. struct_obj = POP_REF();
  2577. if (!struct_obj) {
  2578. wasm_set_exception(module,
  2579. "null structure reference");
  2580. goto got_exception;
  2581. }
  2582. wasm_struct_obj_set_field(struct_obj, field_idx,
  2583. &field_value);
  2584. HANDLE_OP_END();
  2585. }
  2586. case WASM_OP_ARRAY_NEW:
  2587. case WASM_OP_ARRAY_NEW_DEFAULT:
  2588. case WASM_OP_ARRAY_NEW_FIXED:
  2589. {
  2590. WASMModule *wasm_module = module->module;
  2591. WASMArrayType *array_type;
  2592. WASMRttType *rtt_type;
  2593. WASMValue array_elem = { 0 };
  2594. uint32 array_len;
  2595. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2596. array_type =
  2597. (WASMArrayType *)wasm_module->types[type_index];
  2598. if (!(rtt_type = wasm_rtt_type_new(
  2599. (WASMType *)array_type, type_index,
  2600. wasm_module->rtt_types,
  2601. wasm_module->type_count,
  2602. &wasm_module->rtt_type_lock))) {
  2603. wasm_set_exception(module,
  2604. "create rtt type failed");
  2605. goto got_exception;
  2606. }
  2607. if (opcode != WASM_OP_ARRAY_NEW_FIXED)
  2608. array_len = POP_I32();
  2609. else
  2610. read_leb_uint32(frame_ip, frame_ip_end, array_len);
  2611. if (opcode == WASM_OP_ARRAY_NEW) {
  2612. if (wasm_is_type_reftype(array_type->elem_type)) {
  2613. array_elem.gc_obj = POP_REF();
  2614. }
  2615. else if (array_type->elem_type == VALUE_TYPE_I32
  2616. || array_type->elem_type == VALUE_TYPE_F32
  2617. || array_type->elem_type == PACKED_TYPE_I8
  2618. || array_type->elem_type
  2619. == PACKED_TYPE_I16) {
  2620. array_elem.i32 = POP_I32();
  2621. }
  2622. else {
  2623. array_elem.i64 = POP_I64();
  2624. }
  2625. }
  2626. SYNC_ALL_TO_FRAME();
  2627. array_obj = wasm_array_obj_new(exec_env, rtt_type,
  2628. array_len, &array_elem);
  2629. if (!array_obj) {
  2630. wasm_set_exception(module,
  2631. "create array object failed");
  2632. goto got_exception;
  2633. }
  2634. if (opcode == WASM_OP_ARRAY_NEW_FIXED) {
  2635. for (i = 0; i < array_len; i++) {
  2636. if (wasm_is_type_reftype(
  2637. array_type->elem_type)) {
  2638. array_elem.gc_obj = POP_REF();
  2639. }
  2640. else if (array_type->elem_type == VALUE_TYPE_I32
  2641. || array_type->elem_type
  2642. == VALUE_TYPE_F32
  2643. || array_type->elem_type
  2644. == PACKED_TYPE_I8
  2645. || array_type->elem_type
  2646. == PACKED_TYPE_I16) {
  2647. array_elem.i32 = POP_I32();
  2648. }
  2649. else {
  2650. array_elem.i64 = POP_I64();
  2651. }
  2652. wasm_array_obj_set_elem(
  2653. array_obj, array_len - 1 - i, &array_elem);
  2654. }
  2655. }
  2656. PUSH_REF(array_obj);
  2657. HANDLE_OP_END();
  2658. }
  2659. case WASM_OP_ARRAY_NEW_DATA:
  2660. {
  2661. WASMModule *wasm_module = module->module;
  2662. WASMArrayType *array_type;
  2663. WASMRttType *rtt_type;
  2664. WASMValue array_elem = { 0 };
  2665. WASMDataSeg *data_seg;
  2666. uint8 *array_elem_base;
  2667. uint32 array_len, data_seg_idx, data_seg_offset;
  2668. uint32 elem_size = 0;
  2669. uint64 total_size;
  2670. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2671. read_leb_uint32(frame_ip, frame_ip_end, data_seg_idx);
  2672. data_seg = wasm_module->data_segments[data_seg_idx];
  2673. array_type =
  2674. (WASMArrayType *)wasm_module->types[type_index];
  2675. if (!(rtt_type = wasm_rtt_type_new(
  2676. (WASMType *)array_type, type_index,
  2677. wasm_module->rtt_types,
  2678. wasm_module->type_count,
  2679. &wasm_module->rtt_type_lock))) {
  2680. wasm_set_exception(module,
  2681. "create rtt type failed");
  2682. goto got_exception;
  2683. }
  2684. array_len = POP_I32();
  2685. data_seg_offset = POP_I32();
  2686. switch (array_type->elem_type) {
  2687. case PACKED_TYPE_I8:
  2688. elem_size = 1;
  2689. break;
  2690. case PACKED_TYPE_I16:
  2691. elem_size = 2;
  2692. break;
  2693. case VALUE_TYPE_I32:
  2694. case VALUE_TYPE_F32:
  2695. elem_size = 4;
  2696. break;
  2697. case VALUE_TYPE_I64:
  2698. case VALUE_TYPE_F64:
  2699. elem_size = 8;
  2700. break;
  2701. default:
  2702. bh_assert(0);
  2703. }
  2704. total_size = (uint64)elem_size * array_len;
  2705. if (data_seg_offset >= data_seg->data_length
  2706. || total_size
  2707. > data_seg->data_length - data_seg_offset) {
  2708. wasm_set_exception(module,
  2709. "data segment out of bounds");
  2710. goto got_exception;
  2711. }
  2712. SYNC_ALL_TO_FRAME();
  2713. array_obj = wasm_array_obj_new(exec_env, rtt_type,
  2714. array_len, &array_elem);
  2715. if (!array_obj) {
  2716. wasm_set_exception(module,
  2717. "create array object failed");
  2718. goto got_exception;
  2719. }
  2720. array_elem_base =
  2721. (uint8 *)wasm_array_obj_first_elem_addr(array_obj);
  2722. bh_memcpy_s(array_elem_base, (uint32)total_size,
  2723. data_seg->data + data_seg_offset,
  2724. (uint32)total_size);
  2725. PUSH_REF(array_obj);
  2726. HANDLE_OP_END();
  2727. }
  2728. case WASM_OP_ARRAY_NEW_ELEM:
  2729. {
  2730. /* TODO */
  2731. wasm_set_exception(module, "unsupported opcode");
  2732. goto got_exception;
  2733. }
  2734. case WASM_OP_ARRAY_GET:
  2735. case WASM_OP_ARRAY_GET_S:
  2736. case WASM_OP_ARRAY_GET_U:
  2737. {
  2738. WASMArrayType *array_type;
  2739. WASMValue array_elem = { 0 };
  2740. uint32 elem_idx, elem_size_log;
  2741. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2742. array_type =
  2743. (WASMArrayType *)module->module->types[type_index];
  2744. elem_idx = POP_I32();
  2745. array_obj = POP_REF();
  2746. if (!array_obj) {
  2747. wasm_set_exception(module, "null array reference");
  2748. goto got_exception;
  2749. }
  2750. if (elem_idx >= wasm_array_obj_length(array_obj)) {
  2751. wasm_set_exception(module,
  2752. "out of bounds array access");
  2753. goto got_exception;
  2754. }
  2755. wasm_array_obj_get_elem(
  2756. array_obj, elem_idx,
  2757. opcode == WASM_OP_ARRAY_GET_S ? true : false,
  2758. &array_elem);
  2759. elem_size_log = wasm_array_obj_elem_size_log(array_obj);
  2760. if (wasm_is_reftype_i31ref(array_type->elem_type)) {
  2761. PUSH_I31REF(array_elem.gc_obj);
  2762. }
  2763. else if (wasm_is_type_reftype(array_type->elem_type)) {
  2764. PUSH_REF(array_elem.gc_obj);
  2765. }
  2766. else if (elem_size_log < 3) {
  2767. PUSH_I32(array_elem.i32);
  2768. }
  2769. else {
  2770. PUSH_I64(array_elem.i64);
  2771. }
  2772. HANDLE_OP_END();
  2773. }
  2774. case WASM_OP_ARRAY_SET:
  2775. {
  2776. WASMArrayType *array_type;
  2777. WASMValue array_elem = { 0 };
  2778. uint32 elem_idx;
  2779. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2780. array_type =
  2781. (WASMArrayType *)module->module->types[type_index];
  2782. if (wasm_is_type_reftype(array_type->elem_type)) {
  2783. array_elem.gc_obj = POP_REF();
  2784. }
  2785. else if (array_type->elem_type == VALUE_TYPE_I32
  2786. || array_type->elem_type == VALUE_TYPE_F32
  2787. || array_type->elem_type == PACKED_TYPE_I8
  2788. || array_type->elem_type == PACKED_TYPE_I16) {
  2789. array_elem.i32 = POP_I32();
  2790. }
  2791. else {
  2792. array_elem.i64 = POP_I64();
  2793. }
  2794. elem_idx = POP_I32();
  2795. array_obj = POP_REF();
  2796. if (!array_obj) {
  2797. wasm_set_exception(module, "null array reference");
  2798. goto got_exception;
  2799. }
  2800. if (elem_idx >= wasm_array_obj_length(array_obj)) {
  2801. wasm_set_exception(module,
  2802. "out of bounds array access");
  2803. goto got_exception;
  2804. }
  2805. wasm_array_obj_set_elem(array_obj, elem_idx,
  2806. &array_elem);
  2807. HANDLE_OP_END();
  2808. }
  2809. case WASM_OP_ARRAY_LEN:
  2810. {
  2811. uint32 array_len;
  2812. array_obj = POP_REF();
  2813. if (!array_obj) {
  2814. wasm_set_exception(module, "null array reference");
  2815. goto got_exception;
  2816. }
  2817. array_len = wasm_array_obj_length(array_obj);
  2818. PUSH_I32(array_len);
  2819. HANDLE_OP_END();
  2820. }
  2821. case WASM_OP_ARRAY_FILL:
  2822. {
  2823. WASMArrayType *array_type;
  2824. WASMValue fill_value = { 0 };
  2825. uint32 start_offset, len;
  2826. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2827. array_type =
  2828. (WASMArrayType *)module->module->types[type_index];
  2829. len = POP_I32();
  2830. if (wasm_is_type_reftype(array_type->elem_type)) {
  2831. fill_value.gc_obj = POP_REF();
  2832. }
  2833. else if (array_type->elem_type == VALUE_TYPE_I32
  2834. || array_type->elem_type == VALUE_TYPE_F32
  2835. || array_type->elem_type == PACKED_TYPE_I8
  2836. || array_type->elem_type == PACKED_TYPE_I16) {
  2837. fill_value.i32 = POP_I32();
  2838. }
  2839. else {
  2840. fill_value.i64 = POP_I64();
  2841. }
  2842. start_offset = POP_I32();
  2843. array_obj = POP_REF();
  2844. if (!array_obj) {
  2845. wasm_set_exception(module, "null array reference");
  2846. goto got_exception;
  2847. }
  2848. if (len > 0) {
  2849. if ((uint64)start_offset + len
  2850. > wasm_array_obj_length(array_obj)) {
  2851. wasm_set_exception(
  2852. module, "out of bounds array access");
  2853. goto got_exception;
  2854. }
  2855. wasm_array_obj_fill(array_obj, start_offset, len,
  2856. &fill_value);
  2857. }
  2858. HANDLE_OP_END();
  2859. }
  2860. case WASM_OP_ARRAY_COPY:
  2861. {
  2862. uint32 dst_offset, src_offset, len, src_type_index;
  2863. WASMArrayObjectRef src_obj, dst_obj;
  2864. read_leb_uint32(frame_ip, frame_ip_end, type_index);
  2865. read_leb_uint32(frame_ip, frame_ip_end, src_type_index);
  2866. len = POP_I32();
  2867. src_offset = POP_I32();
  2868. src_obj = POP_REF();
  2869. dst_offset = POP_I32();
  2870. dst_obj = POP_REF();
  2871. if (!src_obj || !dst_obj) {
  2872. wasm_set_exception(module, "null array reference");
  2873. goto got_exception;
  2874. }
  2875. if (len > 0) {
  2876. if ((dst_offset > UINT32_MAX - len)
  2877. || (dst_offset + len
  2878. > wasm_array_obj_length(dst_obj))
  2879. || (src_offset > UINT32_MAX - len)
  2880. || (src_offset + len
  2881. > wasm_array_obj_length(src_obj))) {
  2882. wasm_set_exception(
  2883. module, "out of bounds array access");
  2884. goto got_exception;
  2885. }
  2886. wasm_array_obj_copy(dst_obj, dst_offset, src_obj,
  2887. src_offset, len);
  2888. }
  2889. (void)src_type_index;
  2890. HANDLE_OP_END();
  2891. }
  2892. case WASM_OP_REF_I31:
  2893. {
  2894. uint32 i31_val;
  2895. i31_val = POP_I32();
  2896. i31_obj = wasm_i31_obj_new(i31_val);
  2897. PUSH_I31REF(i31_obj);
  2898. HANDLE_OP_END();
  2899. }
  2900. case WASM_OP_I31_GET_S:
  2901. case WASM_OP_I31_GET_U:
  2902. {
  2903. uint32 i31_val;
  2904. i31_obj = (WASMI31ObjectRef)POP_REF();
  2905. if (!i31_obj) {
  2906. wasm_set_exception(module, "null i31 reference");
  2907. goto got_exception;
  2908. }
  2909. i31_val = (uint32)(((uintptr_t)i31_obj) >> 1);
  2910. if (opcode == WASM_OP_I31_GET_S
  2911. && (i31_val & 0x40000000) /* bit 30 is 1 */)
  2912. /* set bit 31 to 1 */
  2913. i31_val |= 0x80000000;
  2914. PUSH_I32(i31_val);
  2915. HANDLE_OP_END();
  2916. }
  2917. case WASM_OP_REF_TEST:
  2918. case WASM_OP_REF_CAST:
  2919. case WASM_OP_REF_TEST_NULLABLE:
  2920. case WASM_OP_REF_CAST_NULLABLE:
  2921. {
  2922. int32 heap_type;
  2923. read_leb_int32(frame_ip, frame_ip_end, heap_type);
  2924. gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM);
  2925. if (!gc_obj) {
  2926. if (opcode == WASM_OP_REF_TEST
  2927. || opcode == WASM_OP_REF_TEST_NULLABLE) {
  2928. (void)POP_REF();
  2929. if (opcode == WASM_OP_REF_TEST)
  2930. PUSH_I32(0);
  2931. else
  2932. PUSH_I32(1);
  2933. }
  2934. else if (opcode == WASM_OP_REF_CAST) {
  2935. wasm_set_exception(module, "cast failure");
  2936. goto got_exception;
  2937. }
  2938. else {
  2939. /* Do nothing for WASM_OP_REF_CAST_NULLABLE */
  2940. }
  2941. }
  2942. else {
  2943. bool castable = false;
  2944. if (heap_type >= 0) {
  2945. WASMModule *wasm_module = module->module;
  2946. castable = wasm_obj_is_instance_of(
  2947. gc_obj, (uint32)heap_type,
  2948. wasm_module->types,
  2949. wasm_module->type_count);
  2950. }
  2951. else {
  2952. castable =
  2953. wasm_obj_is_type_of(gc_obj, heap_type);
  2954. }
  2955. if (opcode == WASM_OP_REF_TEST
  2956. || opcode == WASM_OP_REF_TEST_NULLABLE) {
  2957. (void)POP_REF();
  2958. if (castable)
  2959. PUSH_I32(1);
  2960. else
  2961. PUSH_I32(0);
  2962. }
  2963. else if (!castable) {
  2964. wasm_set_exception(module, "cast failure");
  2965. goto got_exception;
  2966. }
  2967. }
  2968. HANDLE_OP_END();
  2969. }
  2970. case WASM_OP_BR_ON_CAST:
  2971. case WASM_OP_BR_ON_CAST_FAIL:
  2972. {
  2973. int32 heap_type, heap_type_dst;
  2974. uint8 castflags;
  2975. #if WASM_ENABLE_THREAD_MGR != 0
  2976. CHECK_SUSPEND_FLAGS();
  2977. #endif
  2978. castflags = *frame_ip++;
  2979. read_leb_uint32(frame_ip, frame_ip_end, depth);
  2980. read_leb_int32(frame_ip, frame_ip_end, heap_type);
  2981. read_leb_int32(frame_ip, frame_ip_end, heap_type_dst);
  2982. gc_obj = GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM);
  2983. if (!gc_obj) {
  2984. /*
  2985. * castflags should be 0~3:
  2986. * 0: (non-null, non-null)
  2987. * 1: (null, non-null)
  2988. * 2: (non-null, null)
  2989. * 3: (null, null)
  2990. */
  2991. if (
  2992. /* op is BR_ON_CAST and dst reftype is nullable
  2993. */
  2994. ((opcode1 == WASM_OP_BR_ON_CAST)
  2995. && ((castflags == 2) || (castflags == 3)))
  2996. /* op is BR_ON_CAST_FAIL and dst reftype is
  2997. non-nullable */
  2998. || ((opcode1 == WASM_OP_BR_ON_CAST_FAIL)
  2999. && ((castflags == 0) || (castflags == 1))))
  3000. goto label_pop_csp_n;
  3001. }
  3002. else {
  3003. bool castable = false;
  3004. if (heap_type_dst >= 0) {
  3005. WASMModule *wasm_module = module->module;
  3006. castable = wasm_obj_is_instance_of(
  3007. gc_obj, (uint32)heap_type_dst,
  3008. wasm_module->types,
  3009. wasm_module->type_count);
  3010. }
  3011. else {
  3012. castable =
  3013. wasm_obj_is_type_of(gc_obj, heap_type_dst);
  3014. }
  3015. if ((castable && (opcode == WASM_OP_BR_ON_CAST))
  3016. || (!castable
  3017. && (opcode == WASM_OP_BR_ON_CAST_FAIL))) {
  3018. goto label_pop_csp_n;
  3019. }
  3020. }
  3021. (void)heap_type;
  3022. HANDLE_OP_END();
  3023. }
  3024. case WASM_OP_ANY_CONVERT_EXTERN:
  3025. {
  3026. externref_obj = POP_REF();
  3027. if (externref_obj == NULL_REF)
  3028. PUSH_REF(NULL_REF);
  3029. else {
  3030. gc_obj = wasm_externref_obj_to_internal_obj(
  3031. externref_obj);
  3032. PUSH_REF(gc_obj);
  3033. }
  3034. HANDLE_OP_END();
  3035. }
  3036. case WASM_OP_EXTERN_CONVERT_ANY:
  3037. {
  3038. gc_obj = POP_REF();
  3039. if (gc_obj == NULL_REF)
  3040. PUSH_REF(NULL_REF);
  3041. else {
  3042. if (!(externref_obj =
  3043. wasm_internal_obj_to_externref_obj(
  3044. exec_env, gc_obj))) {
  3045. wasm_set_exception(
  3046. module, "create externref object failed");
  3047. goto got_exception;
  3048. }
  3049. PUSH_REF(externref_obj);
  3050. }
  3051. HANDLE_OP_END();
  3052. }
  3053. #if WASM_ENABLE_STRINGREF != 0
  3054. case WASM_OP_STRING_NEW_UTF8:
  3055. case WASM_OP_STRING_NEW_WTF16:
  3056. case WASM_OP_STRING_NEW_LOSSY_UTF8:
  3057. case WASM_OP_STRING_NEW_WTF8:
  3058. {
  3059. uint32 mem_idx, addr, bytes_length, offset = 0;
  3060. EncodingFlag flag = WTF8;
  3061. read_leb_uint32(frame_ip, frame_ip_end, mem_idx);
  3062. bytes_length = POP_I32();
  3063. addr = POP_I32();
  3064. CHECK_MEMORY_OVERFLOW(bytes_length);
  3065. if (opcode == WASM_OP_STRING_NEW_WTF16) {
  3066. flag = WTF16;
  3067. }
  3068. else if (opcode == WASM_OP_STRING_NEW_UTF8) {
  3069. flag = UTF8;
  3070. }
  3071. else if (opcode == WASM_OP_STRING_NEW_LOSSY_UTF8) {
  3072. flag = LOSSY_UTF8;
  3073. }
  3074. else if (opcode == WASM_OP_STRING_NEW_WTF8) {
  3075. flag = WTF8;
  3076. }
  3077. str_obj = wasm_string_new_with_encoding(
  3078. maddr, bytes_length, flag);
  3079. if (!str_obj) {
  3080. wasm_set_exception(module,
  3081. "create string object failed");
  3082. goto got_exception;
  3083. }
  3084. SYNC_ALL_TO_FRAME();
  3085. stringref_obj =
  3086. wasm_stringref_obj_new(exec_env, str_obj);
  3087. if (!stringref_obj) {
  3088. wasm_set_exception(module,
  3089. "create stringref failed");
  3090. goto got_exception;
  3091. }
  3092. PUSH_REF(stringref_obj);
  3093. (void)mem_idx;
  3094. HANDLE_OP_END();
  3095. }
  3096. case WASM_OP_STRING_CONST:
  3097. {
  3098. WASMModule *wasm_module = module->module;
  3099. uint32 contents;
  3100. read_leb_uint32(frame_ip, frame_ip_end, contents);
  3101. str_obj = wasm_string_new_const(
  3102. (const char *)
  3103. wasm_module->string_literal_ptrs[contents],
  3104. wasm_module->string_literal_lengths[contents]);
  3105. if (!str_obj) {
  3106. wasm_set_exception(module,
  3107. "create string object failed");
  3108. goto got_exception;
  3109. }
  3110. SYNC_ALL_TO_FRAME();
  3111. stringref_obj =
  3112. wasm_stringref_obj_new(exec_env, str_obj);
  3113. if (!stringref_obj) {
  3114. wasm_set_exception(module,
  3115. "create stringref failed");
  3116. goto got_exception;
  3117. }
  3118. PUSH_REF(stringref_obj);
  3119. HANDLE_OP_END();
  3120. }
  3121. case WASM_OP_STRING_MEASURE_UTF8:
  3122. case WASM_OP_STRING_MEASURE_WTF8:
  3123. case WASM_OP_STRING_MEASURE_WTF16:
  3124. {
  3125. int32 target_bytes_length;
  3126. EncodingFlag flag = WTF8;
  3127. stringref_obj = POP_REF();
  3128. if (opcode == WASM_OP_STRING_MEASURE_WTF16) {
  3129. flag = WTF16;
  3130. }
  3131. else if (opcode == WASM_OP_STRING_MEASURE_UTF8) {
  3132. flag = UTF8;
  3133. }
  3134. else if (opcode == WASM_OP_STRING_MEASURE_WTF8) {
  3135. flag = LOSSY_UTF8;
  3136. }
  3137. target_bytes_length = wasm_string_measure(
  3138. (WASMString)wasm_stringref_obj_get_value(
  3139. stringref_obj),
  3140. flag);
  3141. PUSH_I32(target_bytes_length);
  3142. HANDLE_OP_END();
  3143. }
  3144. case WASM_OP_STRING_ENCODE_UTF8:
  3145. case WASM_OP_STRING_ENCODE_WTF16:
  3146. case WASM_OP_STRING_ENCODE_LOSSY_UTF8:
  3147. case WASM_OP_STRING_ENCODE_WTF8:
  3148. {
  3149. uint32 mem_idx, addr;
  3150. int32 target_bytes_length;
  3151. WASMMemoryInstance *memory_inst;
  3152. EncodingFlag flag = WTF8;
  3153. read_leb_uint32(frame_ip, frame_ip_end, mem_idx);
  3154. addr = POP_I32();
  3155. stringref_obj = POP_REF();
  3156. str_obj = (WASMString)wasm_stringref_obj_get_value(
  3157. stringref_obj);
  3158. #if WASM_ENABLE_SHARED_HEAP != 0
  3159. if (app_addr_in_shared_heap((uint64)addr, 1))
  3160. shared_heap_addr_app_to_native((uint64)addr, maddr);
  3161. else
  3162. #endif
  3163. {
  3164. memory_inst = module->memories[mem_idx];
  3165. maddr = memory_inst->memory_data + addr;
  3166. }
  3167. if (opcode == WASM_OP_STRING_ENCODE_WTF16) {
  3168. flag = WTF16;
  3169. count = wasm_string_measure(str_obj, flag);
  3170. target_bytes_length = wasm_string_encode(
  3171. str_obj, 0, count, maddr, NULL, flag);
  3172. }
  3173. else {
  3174. if (opcode == WASM_OP_STRING_ENCODE_UTF8) {
  3175. flag = UTF8;
  3176. }
  3177. else if (opcode
  3178. == WASM_OP_STRING_ENCODE_LOSSY_UTF8) {
  3179. flag = LOSSY_UTF8;
  3180. }
  3181. else if (opcode == WASM_OP_STRING_ENCODE_WTF8) {
  3182. flag = WTF8;
  3183. }
  3184. count = wasm_string_measure(str_obj, flag);
  3185. target_bytes_length = wasm_string_encode(
  3186. str_obj, 0, count, maddr, NULL, flag);
  3187. if (target_bytes_length == -1) {
  3188. wasm_set_exception(
  3189. module, "isolated surrogate is seen");
  3190. goto got_exception;
  3191. }
  3192. }
  3193. if (target_bytes_length < 0) {
  3194. wasm_set_exception(module,
  3195. "stringref encode failed");
  3196. goto got_exception;
  3197. }
  3198. PUSH_I32(target_bytes_length);
  3199. HANDLE_OP_END();
  3200. }
  3201. case WASM_OP_STRING_CONCAT:
  3202. {
  3203. WASMStringrefObjectRef stringref_obj1, stringref_obj2;
  3204. stringref_obj2 = POP_REF();
  3205. stringref_obj1 = POP_REF();
  3206. str_obj = wasm_string_concat(
  3207. (WASMString)wasm_stringref_obj_get_value(
  3208. stringref_obj1),
  3209. (WASMString)wasm_stringref_obj_get_value(
  3210. stringref_obj2));
  3211. if (!str_obj) {
  3212. wasm_set_exception(module,
  3213. "create string object failed");
  3214. goto got_exception;
  3215. }
  3216. SYNC_ALL_TO_FRAME();
  3217. stringref_obj =
  3218. wasm_stringref_obj_new(exec_env, str_obj);
  3219. if (!stringref_obj) {
  3220. wasm_set_exception(module,
  3221. "create stringref failed");
  3222. goto got_exception;
  3223. }
  3224. PUSH_REF(stringref_obj);
  3225. HANDLE_OP_END();
  3226. }
  3227. case WASM_OP_STRING_EQ:
  3228. {
  3229. WASMStringrefObjectRef stringref_obj1, stringref_obj2;
  3230. int32 is_eq;
  3231. stringref_obj2 = POP_REF();
  3232. stringref_obj1 = POP_REF();
  3233. is_eq = wasm_string_eq(
  3234. (WASMString)wasm_stringref_obj_get_value(
  3235. stringref_obj1),
  3236. (WASMString)wasm_stringref_obj_get_value(
  3237. stringref_obj2));
  3238. PUSH_I32(is_eq);
  3239. HANDLE_OP_END();
  3240. }
  3241. case WASM_OP_STRING_IS_USV_SEQUENCE:
  3242. {
  3243. int32 is_usv_sequence;
  3244. stringref_obj = POP_REF();
  3245. is_usv_sequence = wasm_string_is_usv_sequence(
  3246. (WASMString)wasm_stringref_obj_get_value(
  3247. stringref_obj));
  3248. PUSH_I32(is_usv_sequence);
  3249. HANDLE_OP_END();
  3250. }
  3251. case WASM_OP_STRING_AS_WTF8:
  3252. {
  3253. stringref_obj = POP_REF();
  3254. str_obj = wasm_string_create_view(
  3255. (WASMString)wasm_stringref_obj_get_value(
  3256. stringref_obj),
  3257. STRING_VIEW_WTF8);
  3258. if (!str_obj) {
  3259. wasm_set_exception(module,
  3260. "create string object failed");
  3261. goto got_exception;
  3262. }
  3263. SYNC_ALL_TO_FRAME();
  3264. stringview_wtf8_obj =
  3265. wasm_stringview_wtf8_obj_new(exec_env, str_obj);
  3266. if (!stringview_wtf8_obj) {
  3267. wasm_set_exception(module,
  3268. "create stringview wtf8 failed");
  3269. goto got_exception;
  3270. }
  3271. PUSH_REF(stringview_wtf8_obj);
  3272. HANDLE_OP_END();
  3273. }
  3274. case WASM_OP_STRINGVIEW_WTF8_ADVANCE:
  3275. {
  3276. uint32 next_pos, bytes, pos;
  3277. bytes = POP_I32();
  3278. pos = POP_I32();
  3279. stringview_wtf8_obj = POP_REF();
  3280. next_pos = wasm_string_advance(
  3281. (WASMString)wasm_stringview_wtf8_obj_get_value(
  3282. stringview_wtf8_obj),
  3283. pos, bytes, NULL);
  3284. PUSH_I32(next_pos);
  3285. HANDLE_OP_END();
  3286. }
  3287. case WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8:
  3288. case WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8:
  3289. case WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8:
  3290. {
  3291. uint32 mem_idx, addr, pos, bytes, next_pos;
  3292. int32 bytes_written;
  3293. WASMMemoryInstance *memory_inst;
  3294. EncodingFlag flag = WTF8;
  3295. if (opcode == WASM_OP_STRINGVIEW_WTF8_ENCODE_UTF8) {
  3296. flag = UTF8;
  3297. }
  3298. else if (opcode
  3299. == WASM_OP_STRINGVIEW_WTF8_ENCODE_LOSSY_UTF8) {
  3300. flag = LOSSY_UTF8;
  3301. }
  3302. else if (opcode
  3303. == WASM_OP_STRINGVIEW_WTF8_ENCODE_WTF8) {
  3304. flag = WTF8;
  3305. }
  3306. read_leb_uint32(frame_ip, frame_ip_end, mem_idx);
  3307. bytes = POP_I32();
  3308. pos = POP_I32();
  3309. addr = POP_I32();
  3310. stringview_wtf8_obj = POP_REF();
  3311. #if WASM_ENABLE_SHARED_HEAP != 0
  3312. if (app_addr_in_shared_heap((uint64)addr, 1))
  3313. shared_heap_addr_app_to_native((uint64)addr, maddr);
  3314. else
  3315. #endif
  3316. {
  3317. memory_inst = module->memories[mem_idx];
  3318. maddr = memory_inst->memory_data + addr;
  3319. }
  3320. bytes_written = wasm_string_encode(
  3321. (WASMString)wasm_stringview_wtf8_obj_get_value(
  3322. stringview_wtf8_obj),
  3323. pos, bytes, maddr, &next_pos, flag);
  3324. if (bytes_written < 0) {
  3325. if (bytes_written == Isolated_Surrogate) {
  3326. wasm_set_exception(
  3327. module, "isolated surrogate is seen");
  3328. }
  3329. else {
  3330. wasm_set_exception(module, "encode failed");
  3331. }
  3332. goto got_exception;
  3333. }
  3334. PUSH_I32(next_pos);
  3335. PUSH_I32(bytes_written);
  3336. HANDLE_OP_END();
  3337. }
  3338. case WASM_OP_STRINGVIEW_WTF8_SLICE:
  3339. {
  3340. uint32 start, end;
  3341. end = POP_I32();
  3342. start = POP_I32();
  3343. stringview_wtf8_obj = POP_REF();
  3344. str_obj = wasm_string_slice(
  3345. (WASMString)wasm_stringview_wtf8_obj_get_value(
  3346. stringview_wtf8_obj),
  3347. start, end, STRING_VIEW_WTF8);
  3348. if (!str_obj) {
  3349. wasm_set_exception(module,
  3350. "create string object failed");
  3351. goto got_exception;
  3352. }
  3353. SYNC_ALL_TO_FRAME();
  3354. stringref_obj =
  3355. wasm_stringref_obj_new(exec_env, str_obj);
  3356. if (!stringref_obj) {
  3357. wasm_set_exception(module,
  3358. "create stringref failed");
  3359. goto got_exception;
  3360. }
  3361. PUSH_REF(stringref_obj);
  3362. HANDLE_OP_END();
  3363. }
  3364. case WASM_OP_STRING_AS_WTF16:
  3365. {
  3366. stringref_obj = POP_REF();
  3367. str_obj = wasm_string_create_view(
  3368. (WASMString)wasm_stringref_obj_get_value(
  3369. stringref_obj),
  3370. STRING_VIEW_WTF16);
  3371. if (!str_obj) {
  3372. wasm_set_exception(module,
  3373. "create string object failed");
  3374. goto got_exception;
  3375. }
  3376. SYNC_ALL_TO_FRAME();
  3377. stringview_wtf16_obj =
  3378. wasm_stringview_wtf16_obj_new(exec_env, str_obj);
  3379. if (!stringview_wtf16_obj) {
  3380. wasm_set_exception(
  3381. module, "create stringview wtf16 failed");
  3382. goto got_exception;
  3383. }
  3384. PUSH_REF(stringview_wtf16_obj);
  3385. HANDLE_OP_END();
  3386. }
  3387. case WASM_OP_STRINGVIEW_WTF16_LENGTH:
  3388. {
  3389. int32 code_units_length;
  3390. stringview_wtf16_obj = POP_REF();
  3391. code_units_length = wasm_string_wtf16_get_length(
  3392. (WASMString)wasm_stringview_wtf16_obj_get_value(
  3393. stringview_wtf16_obj));
  3394. PUSH_I32(code_units_length);
  3395. HANDLE_OP_END();
  3396. }
  3397. case WASM_OP_STRINGVIEW_WTF16_GET_CODEUNIT:
  3398. {
  3399. int32 pos;
  3400. uint32 code_unit;
  3401. pos = POP_I32();
  3402. stringview_wtf16_obj = POP_REF();
  3403. code_unit = (uint32)wasm_string_get_wtf16_codeunit(
  3404. (WASMString)wasm_stringview_wtf16_obj_get_value(
  3405. stringview_wtf16_obj),
  3406. pos);
  3407. PUSH_I32(code_unit);
  3408. HANDLE_OP_END();
  3409. }
  3410. case WASM_OP_STRINGVIEW_WTF16_ENCODE:
  3411. {
  3412. uint32 mem_idx, addr, pos, len, offset = 0;
  3413. int32 written_code_units = 0;
  3414. read_leb_uint32(frame_ip, frame_ip_end, mem_idx);
  3415. len = POP_I32();
  3416. pos = POP_I32();
  3417. addr = POP_I32();
  3418. stringview_wtf16_obj = POP_REF();
  3419. CHECK_MEMORY_OVERFLOW(len * sizeof(uint16));
  3420. /* check 2-byte alignment */
  3421. if (((uintptr_t)maddr & (((uintptr_t)1 << 2) - 1))
  3422. != 0) {
  3423. wasm_set_exception(module,
  3424. "unaligned memory access");
  3425. goto got_exception;
  3426. }
  3427. written_code_units = wasm_string_encode(
  3428. (WASMString)wasm_stringview_wtf16_obj_get_value(
  3429. stringview_wtf16_obj),
  3430. pos, len, maddr, NULL, WTF16);
  3431. if (written_code_units < 0) {
  3432. wasm_set_exception(module, "encode failed");
  3433. goto got_exception;
  3434. }
  3435. PUSH_I32(written_code_units);
  3436. (void)mem_idx;
  3437. HANDLE_OP_END();
  3438. }
  3439. case WASM_OP_STRINGVIEW_WTF16_SLICE:
  3440. {
  3441. uint32 start, end;
  3442. end = POP_I32();
  3443. start = POP_I32();
  3444. stringview_wtf16_obj = POP_REF();
  3445. str_obj = wasm_string_slice(
  3446. (WASMString)wasm_stringview_wtf16_obj_get_value(
  3447. stringview_wtf16_obj),
  3448. start, end, STRING_VIEW_WTF16);
  3449. if (!str_obj) {
  3450. wasm_set_exception(module,
  3451. "create string object failed");
  3452. goto got_exception;
  3453. }
  3454. SYNC_ALL_TO_FRAME();
  3455. stringref_obj =
  3456. wasm_stringref_obj_new(exec_env, str_obj);
  3457. if (!stringref_obj) {
  3458. wasm_set_exception(module,
  3459. "create stringref failed");
  3460. goto got_exception;
  3461. }
  3462. PUSH_REF(stringref_obj);
  3463. HANDLE_OP_END();
  3464. }
  3465. case WASM_OP_STRING_AS_ITER:
  3466. {
  3467. stringref_obj = POP_REF();
  3468. str_obj = wasm_string_create_view(
  3469. (WASMString)wasm_stringref_obj_get_value(
  3470. stringref_obj),
  3471. STRING_VIEW_ITER);
  3472. if (!str_obj) {
  3473. wasm_set_exception(module,
  3474. "create string object failed");
  3475. goto got_exception;
  3476. }
  3477. SYNC_ALL_TO_FRAME();
  3478. stringview_iter_obj =
  3479. wasm_stringview_iter_obj_new(exec_env, str_obj, 0);
  3480. if (!stringview_iter_obj) {
  3481. wasm_set_exception(module,
  3482. "create stringview iter failed");
  3483. goto got_exception;
  3484. }
  3485. PUSH_REF(stringview_iter_obj);
  3486. HANDLE_OP_END();
  3487. }
  3488. case WASM_OP_STRINGVIEW_ITER_NEXT:
  3489. {
  3490. uint32 code_point;
  3491. stringview_iter_obj = POP_REF();
  3492. code_point = wasm_string_next_codepoint(
  3493. (WASMString)wasm_stringview_iter_obj_get_value(
  3494. stringview_iter_obj),
  3495. wasm_stringview_iter_obj_get_pos(
  3496. stringview_iter_obj));
  3497. PUSH_I32(code_point);
  3498. HANDLE_OP_END();
  3499. }
  3500. case WASM_OP_STRINGVIEW_ITER_ADVANCE:
  3501. case WASM_OP_STRINGVIEW_ITER_REWIND:
  3502. {
  3503. uint32 code_points_count, code_points_consumed = 0,
  3504. cur_pos, next_pos = 0;
  3505. code_points_count = POP_I32();
  3506. stringview_iter_obj = POP_REF();
  3507. str_obj =
  3508. (WASMString)wasm_stringview_iter_obj_get_value(
  3509. stringview_iter_obj);
  3510. cur_pos = wasm_stringview_iter_obj_get_pos(
  3511. stringview_iter_obj);
  3512. if (opcode == WASM_OP_STRINGVIEW_ITER_ADVANCE) {
  3513. next_pos = wasm_string_advance(
  3514. str_obj, cur_pos, code_points_count,
  3515. &code_points_consumed);
  3516. }
  3517. else if (opcode == WASM_OP_STRINGVIEW_ITER_REWIND) {
  3518. next_pos = wasm_string_rewind(
  3519. str_obj, cur_pos, code_points_count,
  3520. &code_points_consumed);
  3521. }
  3522. wasm_stringview_iter_obj_update_pos(stringview_iter_obj,
  3523. next_pos);
  3524. PUSH_I32(code_points_consumed);
  3525. HANDLE_OP_END();
  3526. }
  3527. case WASM_OP_STRINGVIEW_ITER_SLICE:
  3528. {
  3529. uint32 code_points_count, cur_pos;
  3530. code_points_count = POP_I32();
  3531. stringview_iter_obj = POP_REF();
  3532. cur_pos = wasm_stringview_iter_obj_get_pos(
  3533. stringview_iter_obj);
  3534. str_obj = wasm_string_slice(
  3535. (WASMString)wasm_stringview_iter_obj_get_value(
  3536. stringview_iter_obj),
  3537. cur_pos, cur_pos + code_points_count,
  3538. STRING_VIEW_ITER);
  3539. if (!str_obj) {
  3540. wasm_set_exception(module,
  3541. "create string object failed");
  3542. goto got_exception;
  3543. }
  3544. SYNC_ALL_TO_FRAME();
  3545. stringref_obj =
  3546. wasm_stringref_obj_new(exec_env, str_obj);
  3547. if (!stringref_obj) {
  3548. wasm_set_exception(module,
  3549. "create stringref failed");
  3550. goto got_exception;
  3551. }
  3552. PUSH_REF(stringref_obj);
  3553. HANDLE_OP_END();
  3554. }
  3555. case WASM_OP_STRING_NEW_UTF8_ARRAY:
  3556. case WASM_OP_STRING_NEW_WTF16_ARRAY:
  3557. case WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY:
  3558. case WASM_OP_STRING_NEW_WTF8_ARRAY:
  3559. {
  3560. uint32 start, end, array_len;
  3561. EncodingFlag flag = WTF8;
  3562. WASMArrayType *array_type;
  3563. void *arr_start_addr;
  3564. end = POP_I32();
  3565. start = POP_I32();
  3566. array_obj = POP_REF();
  3567. array_type = (WASMArrayType *)wasm_obj_get_defined_type(
  3568. (WASMObjectRef)array_obj);
  3569. arr_start_addr =
  3570. wasm_array_obj_elem_addr(array_obj, start);
  3571. array_len = wasm_array_obj_length(array_obj);
  3572. if (start > end || end > array_len) {
  3573. wasm_set_exception(module,
  3574. "out of bounds array access");
  3575. goto got_exception;
  3576. }
  3577. if (opcode == WASM_OP_STRING_NEW_WTF16_ARRAY) {
  3578. if (array_type->elem_type != VALUE_TYPE_I16) {
  3579. wasm_set_exception(module,
  3580. "array type mismatch");
  3581. goto got_exception;
  3582. }
  3583. flag = WTF16;
  3584. }
  3585. else {
  3586. if (array_type->elem_type != VALUE_TYPE_I8) {
  3587. wasm_set_exception(module,
  3588. "array type mismatch");
  3589. goto got_exception;
  3590. }
  3591. if (opcode == WASM_OP_STRING_NEW_UTF8_ARRAY) {
  3592. flag = UTF8;
  3593. }
  3594. else if (opcode == WASM_OP_STRING_NEW_WTF8_ARRAY) {
  3595. flag = WTF8;
  3596. }
  3597. else if (opcode
  3598. == WASM_OP_STRING_NEW_LOSSY_UTF8_ARRAY) {
  3599. flag = LOSSY_UTF8;
  3600. }
  3601. }
  3602. str_obj = wasm_string_new_with_encoding(
  3603. arr_start_addr, (end - start), flag);
  3604. if (!str_obj) {
  3605. wasm_set_exception(module,
  3606. "create string object failed");
  3607. goto got_exception;
  3608. }
  3609. SYNC_ALL_TO_FRAME();
  3610. stringref_obj =
  3611. wasm_stringref_obj_new(exec_env, str_obj);
  3612. if (!stringref_obj) {
  3613. wasm_set_exception(module,
  3614. "create stringref failed");
  3615. goto got_exception;
  3616. }
  3617. PUSH_REF(stringref_obj);
  3618. HANDLE_OP_END();
  3619. }
  3620. case WASM_OP_STRING_ENCODE_UTF8_ARRAY:
  3621. case WASM_OP_STRING_ENCODE_WTF16_ARRAY:
  3622. case WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY:
  3623. case WASM_OP_STRING_ENCODE_WTF8_ARRAY:
  3624. {
  3625. uint32 start, array_len;
  3626. int32 bytes_written;
  3627. EncodingFlag flag = WTF8;
  3628. WASMArrayType *array_type;
  3629. void *arr_start_addr;
  3630. start = POP_I32();
  3631. array_obj = POP_REF();
  3632. stringref_obj = POP_REF();
  3633. str_obj = (WASMString)wasm_stringref_obj_get_value(
  3634. stringref_obj);
  3635. array_type = (WASMArrayType *)wasm_obj_get_defined_type(
  3636. (WASMObjectRef)array_obj);
  3637. arr_start_addr =
  3638. wasm_array_obj_elem_addr(array_obj, start);
  3639. array_len = wasm_array_obj_length(array_obj);
  3640. if (start > array_len) {
  3641. wasm_set_exception(module,
  3642. "out of bounds array access");
  3643. goto got_exception;
  3644. }
  3645. if (opcode == WASM_OP_STRING_ENCODE_WTF16_ARRAY) {
  3646. if (array_type->elem_type != VALUE_TYPE_I16) {
  3647. wasm_set_exception(module,
  3648. "array type mismatch");
  3649. goto got_exception;
  3650. }
  3651. flag = WTF16;
  3652. }
  3653. else {
  3654. if (array_type->elem_type != VALUE_TYPE_I8) {
  3655. wasm_set_exception(module,
  3656. "array type mismatch");
  3657. goto got_exception;
  3658. }
  3659. if (opcode == WASM_OP_STRING_ENCODE_UTF8_ARRAY) {
  3660. flag = UTF8;
  3661. }
  3662. else if (opcode
  3663. == WASM_OP_STRING_ENCODE_WTF8_ARRAY) {
  3664. flag = WTF8;
  3665. }
  3666. else if (
  3667. opcode
  3668. == WASM_OP_STRING_ENCODE_LOSSY_UTF8_ARRAY) {
  3669. flag = LOSSY_UTF8;
  3670. }
  3671. }
  3672. count = wasm_string_measure(str_obj, flag);
  3673. bytes_written = wasm_string_encode(
  3674. str_obj, 0, count, arr_start_addr, NULL, flag);
  3675. if (bytes_written < 0) {
  3676. if (bytes_written == Isolated_Surrogate) {
  3677. wasm_set_exception(
  3678. module, "isolated surrogate is seen");
  3679. }
  3680. else if (bytes_written == Insufficient_Space) {
  3681. wasm_set_exception(
  3682. module, "array space is insufficient");
  3683. }
  3684. else {
  3685. wasm_set_exception(module, "encode failed");
  3686. }
  3687. goto got_exception;
  3688. }
  3689. PUSH_I32(bytes_written);
  3690. HANDLE_OP_END();
  3691. }
  3692. #endif /* end of WASM_ENABLE_STRINGREF != 0 */
  3693. default:
  3694. {
  3695. wasm_set_exception(module, "unsupported opcode");
  3696. goto got_exception;
  3697. }
  3698. }
  3699. }
  3700. #endif /* end of WASM_ENABLE_GC != 0 */
  3701. /* variable instructions */
  3702. HANDLE_OP(WASM_OP_GET_LOCAL)
  3703. {
  3704. GET_LOCAL_INDEX_TYPE_AND_OFFSET();
  3705. switch (local_type) {
  3706. case VALUE_TYPE_I32:
  3707. case VALUE_TYPE_F32:
  3708. #if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0
  3709. case VALUE_TYPE_FUNCREF:
  3710. case VALUE_TYPE_EXTERNREF:
  3711. #endif
  3712. PUSH_I32(*(int32 *)(frame_lp + local_offset));
  3713. break;
  3714. case VALUE_TYPE_I64:
  3715. case VALUE_TYPE_F64:
  3716. PUSH_I64(GET_I64_FROM_ADDR(frame_lp + local_offset));
  3717. break;
  3718. default:
  3719. #if WASM_ENABLE_GC != 0
  3720. if (wasm_is_type_reftype(local_type)) {
  3721. if (wasm_is_reftype_i31ref(local_type)) {
  3722. PUSH_I31REF(
  3723. GET_REF_FROM_ADDR(frame_lp + local_offset));
  3724. }
  3725. else {
  3726. PUSH_REF(
  3727. GET_REF_FROM_ADDR(frame_lp + local_offset));
  3728. }
  3729. }
  3730. else
  3731. #endif
  3732. {
  3733. wasm_set_exception(module, "invalid local type");
  3734. goto got_exception;
  3735. }
  3736. }
  3737. HANDLE_OP_END();
  3738. }
  3739. HANDLE_OP(EXT_OP_GET_LOCAL_FAST)
  3740. {
  3741. local_offset = *frame_ip++;
  3742. if (local_offset & 0x80)
  3743. PUSH_I64(
  3744. GET_I64_FROM_ADDR(frame_lp + (local_offset & 0x7F)));
  3745. else
  3746. PUSH_I32(*(int32 *)(frame_lp + local_offset));
  3747. HANDLE_OP_END();
  3748. }
  3749. HANDLE_OP(WASM_OP_SET_LOCAL)
  3750. {
  3751. GET_LOCAL_INDEX_TYPE_AND_OFFSET();
  3752. switch (local_type) {
  3753. case VALUE_TYPE_I32:
  3754. case VALUE_TYPE_F32:
  3755. #if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0
  3756. case VALUE_TYPE_FUNCREF:
  3757. case VALUE_TYPE_EXTERNREF:
  3758. #endif
  3759. *(int32 *)(frame_lp + local_offset) = POP_I32();
  3760. break;
  3761. case VALUE_TYPE_I64:
  3762. case VALUE_TYPE_F64:
  3763. PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset),
  3764. POP_I64());
  3765. break;
  3766. default:
  3767. #if WASM_ENABLE_GC != 0
  3768. if (wasm_is_type_reftype(local_type)) {
  3769. PUT_REF_TO_ADDR(frame_lp + local_offset, POP_REF());
  3770. }
  3771. else
  3772. #endif
  3773. {
  3774. wasm_set_exception(module, "invalid local type");
  3775. goto got_exception;
  3776. }
  3777. }
  3778. HANDLE_OP_END();
  3779. }
  3780. HANDLE_OP(EXT_OP_SET_LOCAL_FAST)
  3781. {
  3782. local_offset = *frame_ip++;
  3783. if (local_offset & 0x80)
  3784. PUT_I64_TO_ADDR(
  3785. (uint32 *)(frame_lp + (local_offset & 0x7F)),
  3786. POP_I64());
  3787. else
  3788. *(int32 *)(frame_lp + local_offset) = POP_I32();
  3789. HANDLE_OP_END();
  3790. }
  3791. HANDLE_OP(WASM_OP_TEE_LOCAL)
  3792. {
  3793. GET_LOCAL_INDEX_TYPE_AND_OFFSET();
  3794. switch (local_type) {
  3795. case VALUE_TYPE_I32:
  3796. case VALUE_TYPE_F32:
  3797. #if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0
  3798. case VALUE_TYPE_FUNCREF:
  3799. case VALUE_TYPE_EXTERNREF:
  3800. #endif
  3801. *(int32 *)(frame_lp + local_offset) =
  3802. *(int32 *)(frame_sp - 1);
  3803. break;
  3804. case VALUE_TYPE_I64:
  3805. case VALUE_TYPE_F64:
  3806. PUT_I64_TO_ADDR((uint32 *)(frame_lp + local_offset),
  3807. GET_I64_FROM_ADDR(frame_sp - 2));
  3808. break;
  3809. default:
  3810. #if WASM_ENABLE_GC != 0
  3811. if (wasm_is_type_reftype(local_type)) {
  3812. PUT_REF_TO_ADDR(
  3813. frame_lp + local_offset,
  3814. GET_REF_FROM_ADDR(frame_sp - REF_CELL_NUM));
  3815. }
  3816. else
  3817. #endif
  3818. {
  3819. wasm_set_exception(module, "invalid local type");
  3820. goto got_exception;
  3821. }
  3822. }
  3823. HANDLE_OP_END();
  3824. }
  3825. HANDLE_OP(EXT_OP_TEE_LOCAL_FAST)
  3826. {
  3827. local_offset = *frame_ip++;
  3828. if (local_offset & 0x80)
  3829. PUT_I64_TO_ADDR(
  3830. (uint32 *)(frame_lp + (local_offset & 0x7F)),
  3831. GET_I64_FROM_ADDR(frame_sp - 2));
  3832. else
  3833. *(int32 *)(frame_lp + local_offset) =
  3834. *(int32 *)(frame_sp - 1);
  3835. HANDLE_OP_END();
  3836. }
  3837. HANDLE_OP(WASM_OP_GET_GLOBAL)
  3838. {
  3839. read_leb_uint32(frame_ip, frame_ip_end, global_idx);
  3840. bh_assert(global_idx < module->e->global_count);
  3841. global = globals + global_idx;
  3842. global_addr = get_global_addr(global_data, global);
  3843. /* clang-format off */
  3844. #if WASM_ENABLE_GC == 0
  3845. PUSH_I32(*(uint32 *)global_addr);
  3846. #else
  3847. if (!wasm_is_type_reftype(global->type)) {
  3848. PUSH_I32(*(uint32 *)global_addr);
  3849. }
  3850. else if (wasm_is_reftype_i31ref(global->type)) {
  3851. PUSH_I31REF(GET_REF_FROM_ADDR((uint32 *)global_addr));
  3852. }
  3853. else {
  3854. PUSH_REF(GET_REF_FROM_ADDR((uint32 *)global_addr));
  3855. }
  3856. #endif
  3857. /* clang-format on */
  3858. HANDLE_OP_END();
  3859. }
  3860. HANDLE_OP(WASM_OP_GET_GLOBAL_64)
  3861. {
  3862. read_leb_uint32(frame_ip, frame_ip_end, global_idx);
  3863. bh_assert(global_idx < module->e->global_count);
  3864. global = globals + global_idx;
  3865. global_addr = get_global_addr(global_data, global);
  3866. PUSH_I64(GET_I64_FROM_ADDR((uint32 *)global_addr));
  3867. HANDLE_OP_END();
  3868. }
  3869. HANDLE_OP(WASM_OP_SET_GLOBAL)
  3870. {
  3871. read_leb_uint32(frame_ip, frame_ip_end, global_idx);
  3872. bh_assert(global_idx < module->e->global_count);
  3873. global = globals + global_idx;
  3874. global_addr = get_global_addr(global_data, global);
  3875. /* clang-format off */
  3876. #if WASM_ENABLE_GC == 0
  3877. *(int32 *)global_addr = POP_I32();
  3878. #else
  3879. if (!wasm_is_type_reftype(global->type))
  3880. *(int32 *)global_addr = POP_I32();
  3881. else
  3882. PUT_REF_TO_ADDR((uint32 *)global_addr, POP_REF());
  3883. #endif
  3884. /* clang-format on */
  3885. HANDLE_OP_END();
  3886. }
  3887. HANDLE_OP(WASM_OP_SET_GLOBAL_AUX_STACK)
  3888. {
  3889. uint64 aux_stack_top;
  3890. read_leb_uint32(frame_ip, frame_ip_end, global_idx);
  3891. bh_assert(global_idx < module->e->global_count);
  3892. global = globals + global_idx;
  3893. global_addr = get_global_addr(global_data, global);
  3894. #if WASM_ENABLE_MEMORY64 != 0
  3895. if (is_memory64) {
  3896. aux_stack_top = *(uint64 *)(frame_sp - 2);
  3897. }
  3898. else
  3899. #endif
  3900. {
  3901. aux_stack_top = (uint64)(*(uint32 *)(frame_sp - 1));
  3902. }
  3903. if (aux_stack_top <= (uint64)exec_env->aux_stack_boundary) {
  3904. wasm_set_exception(module, "wasm auxiliary stack overflow");
  3905. goto got_exception;
  3906. }
  3907. if (aux_stack_top > (uint64)exec_env->aux_stack_bottom) {
  3908. wasm_set_exception(module,
  3909. "wasm auxiliary stack underflow");
  3910. goto got_exception;
  3911. }
  3912. #if WASM_ENABLE_MEMORY64 != 0
  3913. if (is_memory64) {
  3914. *(uint64 *)global_addr = aux_stack_top;
  3915. frame_sp -= 2;
  3916. }
  3917. else
  3918. #endif
  3919. {
  3920. *(uint32 *)global_addr = (uint32)aux_stack_top;
  3921. frame_sp--;
  3922. }
  3923. #if WASM_ENABLE_MEMORY_PROFILING != 0
  3924. if (module->module->aux_stack_top_global_index != (uint32)-1) {
  3925. uint32 aux_stack_used =
  3926. (uint32)(module->module->aux_stack_bottom
  3927. - *(uint32 *)global_addr);
  3928. if (aux_stack_used > module->e->max_aux_stack_used)
  3929. module->e->max_aux_stack_used = aux_stack_used;
  3930. }
  3931. #endif
  3932. HANDLE_OP_END();
  3933. }
  3934. HANDLE_OP(WASM_OP_SET_GLOBAL_64)
  3935. {
  3936. read_leb_uint32(frame_ip, frame_ip_end, global_idx);
  3937. bh_assert(global_idx < module->e->global_count);
  3938. global = globals + global_idx;
  3939. global_addr = get_global_addr(global_data, global);
  3940. PUT_I64_TO_ADDR((uint32 *)global_addr, POP_I64());
  3941. HANDLE_OP_END();
  3942. }
  3943. /* memory load instructions */
  3944. HANDLE_OP(WASM_OP_I32_LOAD)
  3945. HANDLE_OP(WASM_OP_F32_LOAD)
  3946. {
  3947. uint32 flags;
  3948. mem_offset_t offset, addr;
  3949. read_leb_memarg(frame_ip, frame_ip_end, flags);
  3950. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  3951. addr = POP_MEM_OFFSET();
  3952. CHECK_MEMORY_OVERFLOW(4);
  3953. PUSH_I32(LOAD_I32(maddr));
  3954. CHECK_READ_WATCHPOINT(addr, offset);
  3955. HANDLE_OP_END();
  3956. }
  3957. HANDLE_OP(WASM_OP_I64_LOAD)
  3958. HANDLE_OP(WASM_OP_F64_LOAD)
  3959. {
  3960. uint32 flags;
  3961. mem_offset_t offset, addr;
  3962. read_leb_memarg(frame_ip, frame_ip_end, flags);
  3963. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  3964. addr = POP_MEM_OFFSET();
  3965. CHECK_MEMORY_OVERFLOW(8);
  3966. PUSH_I64(LOAD_I64(maddr));
  3967. CHECK_READ_WATCHPOINT(addr, offset);
  3968. HANDLE_OP_END();
  3969. }
  3970. HANDLE_OP(WASM_OP_I32_LOAD8_S)
  3971. {
  3972. uint32 flags;
  3973. mem_offset_t offset, addr;
  3974. read_leb_memarg(frame_ip, frame_ip_end, flags);
  3975. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  3976. addr = POP_MEM_OFFSET();
  3977. CHECK_MEMORY_OVERFLOW(1);
  3978. PUSH_I32(sign_ext_8_32(*(int8 *)maddr));
  3979. CHECK_READ_WATCHPOINT(addr, offset);
  3980. HANDLE_OP_END();
  3981. }
  3982. HANDLE_OP(WASM_OP_I32_LOAD8_U)
  3983. {
  3984. uint32 flags;
  3985. mem_offset_t offset, addr;
  3986. read_leb_memarg(frame_ip, frame_ip_end, flags);
  3987. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  3988. addr = POP_MEM_OFFSET();
  3989. CHECK_MEMORY_OVERFLOW(1);
  3990. PUSH_I32((uint32)(*(uint8 *)maddr));
  3991. CHECK_READ_WATCHPOINT(addr, offset);
  3992. HANDLE_OP_END();
  3993. }
  3994. HANDLE_OP(WASM_OP_I32_LOAD16_S)
  3995. {
  3996. uint32 flags;
  3997. mem_offset_t offset, addr;
  3998. read_leb_memarg(frame_ip, frame_ip_end, flags);
  3999. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4000. addr = POP_MEM_OFFSET();
  4001. CHECK_MEMORY_OVERFLOW(2);
  4002. PUSH_I32(sign_ext_16_32(LOAD_I16(maddr)));
  4003. CHECK_READ_WATCHPOINT(addr, offset);
  4004. HANDLE_OP_END();
  4005. }
  4006. HANDLE_OP(WASM_OP_I32_LOAD16_U)
  4007. {
  4008. uint32 flags;
  4009. mem_offset_t offset, addr;
  4010. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4011. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4012. addr = POP_MEM_OFFSET();
  4013. CHECK_MEMORY_OVERFLOW(2);
  4014. PUSH_I32((uint32)(LOAD_U16(maddr)));
  4015. CHECK_READ_WATCHPOINT(addr, offset);
  4016. HANDLE_OP_END();
  4017. }
  4018. HANDLE_OP(WASM_OP_I64_LOAD8_S)
  4019. {
  4020. uint32 flags;
  4021. mem_offset_t offset, addr;
  4022. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4023. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4024. addr = POP_MEM_OFFSET();
  4025. CHECK_MEMORY_OVERFLOW(1);
  4026. PUSH_I64(sign_ext_8_64(*(int8 *)maddr));
  4027. CHECK_READ_WATCHPOINT(addr, offset);
  4028. HANDLE_OP_END();
  4029. }
  4030. HANDLE_OP(WASM_OP_I64_LOAD8_U)
  4031. {
  4032. uint32 flags;
  4033. mem_offset_t offset, addr;
  4034. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4035. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4036. addr = POP_MEM_OFFSET();
  4037. CHECK_MEMORY_OVERFLOW(1);
  4038. PUSH_I64((uint64)(*(uint8 *)maddr));
  4039. CHECK_READ_WATCHPOINT(addr, offset);
  4040. HANDLE_OP_END();
  4041. }
  4042. HANDLE_OP(WASM_OP_I64_LOAD16_S)
  4043. {
  4044. uint32 flags;
  4045. mem_offset_t offset, addr;
  4046. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4047. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4048. addr = POP_MEM_OFFSET();
  4049. CHECK_MEMORY_OVERFLOW(2);
  4050. PUSH_I64(sign_ext_16_64(LOAD_I16(maddr)));
  4051. CHECK_READ_WATCHPOINT(addr, offset);
  4052. HANDLE_OP_END();
  4053. }
  4054. HANDLE_OP(WASM_OP_I64_LOAD16_U)
  4055. {
  4056. uint32 flags;
  4057. mem_offset_t offset, addr;
  4058. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4059. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4060. addr = POP_MEM_OFFSET();
  4061. CHECK_MEMORY_OVERFLOW(2);
  4062. PUSH_I64((uint64)(LOAD_U16(maddr)));
  4063. CHECK_READ_WATCHPOINT(addr, offset);
  4064. HANDLE_OP_END();
  4065. }
  4066. HANDLE_OP(WASM_OP_I64_LOAD32_S)
  4067. {
  4068. uint32 flags;
  4069. mem_offset_t offset, addr;
  4070. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4071. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4072. addr = POP_MEM_OFFSET();
  4073. CHECK_MEMORY_OVERFLOW(4);
  4074. PUSH_I64(sign_ext_32_64(LOAD_I32(maddr)));
  4075. CHECK_READ_WATCHPOINT(addr, offset);
  4076. HANDLE_OP_END();
  4077. }
  4078. HANDLE_OP(WASM_OP_I64_LOAD32_U)
  4079. {
  4080. uint32 flags;
  4081. mem_offset_t offset, addr;
  4082. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4083. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4084. addr = POP_MEM_OFFSET();
  4085. CHECK_MEMORY_OVERFLOW(4);
  4086. PUSH_I64((uint64)(LOAD_U32(maddr)));
  4087. CHECK_READ_WATCHPOINT(addr, offset);
  4088. HANDLE_OP_END();
  4089. }
  4090. /* memory store instructions */
  4091. HANDLE_OP(WASM_OP_I32_STORE)
  4092. HANDLE_OP(WASM_OP_F32_STORE)
  4093. {
  4094. uint32 flags;
  4095. mem_offset_t offset, addr;
  4096. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4097. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4098. frame_sp--;
  4099. addr = POP_MEM_OFFSET();
  4100. CHECK_MEMORY_OVERFLOW(4);
  4101. #if WASM_ENABLE_MEMORY64 != 0
  4102. if (is_memory64) {
  4103. STORE_U32(maddr, frame_sp[2]);
  4104. }
  4105. else
  4106. #endif
  4107. {
  4108. STORE_U32(maddr, frame_sp[1]);
  4109. }
  4110. CHECK_WRITE_WATCHPOINT(addr, offset);
  4111. HANDLE_OP_END();
  4112. }
  4113. HANDLE_OP(WASM_OP_I64_STORE)
  4114. HANDLE_OP(WASM_OP_F64_STORE)
  4115. {
  4116. uint32 flags;
  4117. mem_offset_t offset, addr;
  4118. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4119. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4120. frame_sp -= 2;
  4121. addr = POP_MEM_OFFSET();
  4122. CHECK_MEMORY_OVERFLOW(8);
  4123. #if WASM_ENABLE_MEMORY64 != 0
  4124. if (is_memory64) {
  4125. PUT_I64_TO_ADDR((mem_offset_t *)maddr,
  4126. GET_I64_FROM_ADDR(frame_sp + 2));
  4127. }
  4128. else
  4129. #endif
  4130. {
  4131. PUT_I64_TO_ADDR((uint32 *)maddr,
  4132. GET_I64_FROM_ADDR(frame_sp + 1));
  4133. }
  4134. CHECK_WRITE_WATCHPOINT(addr, offset);
  4135. HANDLE_OP_END();
  4136. }
  4137. HANDLE_OP(WASM_OP_I32_STORE8)
  4138. HANDLE_OP(WASM_OP_I32_STORE16)
  4139. {
  4140. uint32 flags;
  4141. mem_offset_t offset, addr;
  4142. uint32 sval;
  4143. opcode = *(frame_ip - 1);
  4144. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4145. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4146. sval = (uint32)POP_I32();
  4147. addr = POP_MEM_OFFSET();
  4148. if (opcode == WASM_OP_I32_STORE8) {
  4149. CHECK_MEMORY_OVERFLOW(1);
  4150. *(uint8 *)maddr = (uint8)sval;
  4151. }
  4152. else {
  4153. CHECK_MEMORY_OVERFLOW(2);
  4154. STORE_U16(maddr, (uint16)sval);
  4155. }
  4156. CHECK_WRITE_WATCHPOINT(addr, offset);
  4157. HANDLE_OP_END();
  4158. }
  4159. HANDLE_OP(WASM_OP_I64_STORE8)
  4160. HANDLE_OP(WASM_OP_I64_STORE16)
  4161. HANDLE_OP(WASM_OP_I64_STORE32)
  4162. {
  4163. uint32 flags;
  4164. mem_offset_t offset, addr;
  4165. uint64 sval;
  4166. opcode = *(frame_ip - 1);
  4167. read_leb_memarg(frame_ip, frame_ip_end, flags);
  4168. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  4169. sval = (uint64)POP_I64();
  4170. addr = POP_MEM_OFFSET();
  4171. if (opcode == WASM_OP_I64_STORE8) {
  4172. CHECK_MEMORY_OVERFLOW(1);
  4173. *(uint8 *)maddr = (uint8)sval;
  4174. }
  4175. else if (opcode == WASM_OP_I64_STORE16) {
  4176. CHECK_MEMORY_OVERFLOW(2);
  4177. STORE_U16(maddr, (uint16)sval);
  4178. }
  4179. else {
  4180. CHECK_MEMORY_OVERFLOW(4);
  4181. STORE_U32(maddr, (uint32)sval);
  4182. }
  4183. CHECK_WRITE_WATCHPOINT(addr, offset);
  4184. HANDLE_OP_END();
  4185. }
  4186. /* memory size and memory grow instructions */
  4187. HANDLE_OP(WASM_OP_MEMORY_SIZE)
  4188. {
  4189. uint32 mem_idx;
  4190. read_leb_memidx(frame_ip, frame_ip_end, mem_idx);
  4191. PUSH_PAGE_COUNT(memory->cur_page_count);
  4192. HANDLE_OP_END();
  4193. }
  4194. HANDLE_OP(WASM_OP_MEMORY_GROW)
  4195. {
  4196. uint32 mem_idx, prev_page_count;
  4197. mem_offset_t delta;
  4198. read_leb_memidx(frame_ip, frame_ip_end, mem_idx);
  4199. prev_page_count = memory->cur_page_count;
  4200. delta = POP_PAGE_COUNT();
  4201. if (
  4202. #if WASM_ENABLE_MEMORY64 != 0
  4203. delta > UINT32_MAX ||
  4204. #endif
  4205. !wasm_enlarge_memory_with_idx(module, (uint32)delta,
  4206. mem_idx)) {
  4207. /* failed to memory.grow, return -1 */
  4208. PUSH_PAGE_COUNT(-1);
  4209. }
  4210. else {
  4211. /* success, return previous page count */
  4212. PUSH_PAGE_COUNT(prev_page_count);
  4213. /* update memory size, no need to update memory ptr as
  4214. it isn't changed in wasm_enlarge_memory */
  4215. #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
  4216. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
  4217. || WASM_ENABLE_BULK_MEMORY != 0
  4218. linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory);
  4219. #endif
  4220. }
  4221. HANDLE_OP_END();
  4222. }
  4223. /* constant instructions */
  4224. HANDLE_OP(WASM_OP_I32_CONST)
  4225. DEF_OP_I_CONST(int32, I32);
  4226. HANDLE_OP_END();
  4227. HANDLE_OP(WASM_OP_I64_CONST)
  4228. DEF_OP_I_CONST(int64, I64);
  4229. HANDLE_OP_END();
  4230. HANDLE_OP(WASM_OP_F32_CONST)
  4231. {
  4232. uint8 *p_float = (uint8 *)frame_sp++;
  4233. for (i = 0; i < sizeof(float32); i++)
  4234. *p_float++ = *frame_ip++;
  4235. HANDLE_OP_END();
  4236. }
  4237. HANDLE_OP(WASM_OP_F64_CONST)
  4238. {
  4239. uint8 *p_float = (uint8 *)frame_sp++;
  4240. frame_sp++;
  4241. for (i = 0; i < sizeof(float64); i++)
  4242. *p_float++ = *frame_ip++;
  4243. HANDLE_OP_END();
  4244. }
  4245. /* comparison instructions of i32 */
  4246. HANDLE_OP(WASM_OP_I32_EQZ)
  4247. {
  4248. DEF_OP_EQZ(I32);
  4249. HANDLE_OP_END();
  4250. }
  4251. HANDLE_OP(WASM_OP_I32_EQ)
  4252. {
  4253. DEF_OP_CMP(uint32, I32, ==);
  4254. HANDLE_OP_END();
  4255. }
  4256. HANDLE_OP(WASM_OP_I32_NE)
  4257. {
  4258. DEF_OP_CMP(uint32, I32, !=);
  4259. HANDLE_OP_END();
  4260. }
  4261. HANDLE_OP(WASM_OP_I32_LT_S)
  4262. {
  4263. DEF_OP_CMP(int32, I32, <);
  4264. HANDLE_OP_END();
  4265. }
  4266. HANDLE_OP(WASM_OP_I32_LT_U)
  4267. {
  4268. DEF_OP_CMP(uint32, I32, <);
  4269. HANDLE_OP_END();
  4270. }
  4271. HANDLE_OP(WASM_OP_I32_GT_S)
  4272. {
  4273. DEF_OP_CMP(int32, I32, >);
  4274. HANDLE_OP_END();
  4275. }
  4276. HANDLE_OP(WASM_OP_I32_GT_U)
  4277. {
  4278. DEF_OP_CMP(uint32, I32, >);
  4279. HANDLE_OP_END();
  4280. }
  4281. HANDLE_OP(WASM_OP_I32_LE_S)
  4282. {
  4283. DEF_OP_CMP(int32, I32, <=);
  4284. HANDLE_OP_END();
  4285. }
  4286. HANDLE_OP(WASM_OP_I32_LE_U)
  4287. {
  4288. DEF_OP_CMP(uint32, I32, <=);
  4289. HANDLE_OP_END();
  4290. }
  4291. HANDLE_OP(WASM_OP_I32_GE_S)
  4292. {
  4293. DEF_OP_CMP(int32, I32, >=);
  4294. HANDLE_OP_END();
  4295. }
  4296. HANDLE_OP(WASM_OP_I32_GE_U)
  4297. {
  4298. DEF_OP_CMP(uint32, I32, >=);
  4299. HANDLE_OP_END();
  4300. }
  4301. /* comparison instructions of i64 */
  4302. HANDLE_OP(WASM_OP_I64_EQZ)
  4303. {
  4304. DEF_OP_EQZ(I64);
  4305. HANDLE_OP_END();
  4306. }
  4307. HANDLE_OP(WASM_OP_I64_EQ)
  4308. {
  4309. DEF_OP_CMP(uint64, I64, ==);
  4310. HANDLE_OP_END();
  4311. }
  4312. HANDLE_OP(WASM_OP_I64_NE)
  4313. {
  4314. DEF_OP_CMP(uint64, I64, !=);
  4315. HANDLE_OP_END();
  4316. }
  4317. HANDLE_OP(WASM_OP_I64_LT_S)
  4318. {
  4319. DEF_OP_CMP(int64, I64, <);
  4320. HANDLE_OP_END();
  4321. }
  4322. HANDLE_OP(WASM_OP_I64_LT_U)
  4323. {
  4324. DEF_OP_CMP(uint64, I64, <);
  4325. HANDLE_OP_END();
  4326. }
  4327. HANDLE_OP(WASM_OP_I64_GT_S)
  4328. {
  4329. DEF_OP_CMP(int64, I64, >);
  4330. HANDLE_OP_END();
  4331. }
  4332. HANDLE_OP(WASM_OP_I64_GT_U)
  4333. {
  4334. DEF_OP_CMP(uint64, I64, >);
  4335. HANDLE_OP_END();
  4336. }
  4337. HANDLE_OP(WASM_OP_I64_LE_S)
  4338. {
  4339. DEF_OP_CMP(int64, I64, <=);
  4340. HANDLE_OP_END();
  4341. }
  4342. HANDLE_OP(WASM_OP_I64_LE_U)
  4343. {
  4344. DEF_OP_CMP(uint64, I64, <=);
  4345. HANDLE_OP_END();
  4346. }
  4347. HANDLE_OP(WASM_OP_I64_GE_S)
  4348. {
  4349. DEF_OP_CMP(int64, I64, >=);
  4350. HANDLE_OP_END();
  4351. }
  4352. HANDLE_OP(WASM_OP_I64_GE_U)
  4353. {
  4354. DEF_OP_CMP(uint64, I64, >=);
  4355. HANDLE_OP_END();
  4356. }
  4357. /* comparison instructions of f32 */
  4358. HANDLE_OP(WASM_OP_F32_EQ)
  4359. {
  4360. DEF_OP_CMP(float32, F32, ==);
  4361. HANDLE_OP_END();
  4362. }
  4363. HANDLE_OP(WASM_OP_F32_NE)
  4364. {
  4365. DEF_OP_CMP(float32, F32, !=);
  4366. HANDLE_OP_END();
  4367. }
  4368. HANDLE_OP(WASM_OP_F32_LT)
  4369. {
  4370. DEF_OP_CMP(float32, F32, <);
  4371. HANDLE_OP_END();
  4372. }
  4373. HANDLE_OP(WASM_OP_F32_GT)
  4374. {
  4375. DEF_OP_CMP(float32, F32, >);
  4376. HANDLE_OP_END();
  4377. }
  4378. HANDLE_OP(WASM_OP_F32_LE)
  4379. {
  4380. DEF_OP_CMP(float32, F32, <=);
  4381. HANDLE_OP_END();
  4382. }
  4383. HANDLE_OP(WASM_OP_F32_GE)
  4384. {
  4385. DEF_OP_CMP(float32, F32, >=);
  4386. HANDLE_OP_END();
  4387. }
  4388. /* comparison instructions of f64 */
  4389. HANDLE_OP(WASM_OP_F64_EQ)
  4390. {
  4391. DEF_OP_CMP(float64, F64, ==);
  4392. HANDLE_OP_END();
  4393. }
  4394. HANDLE_OP(WASM_OP_F64_NE)
  4395. {
  4396. DEF_OP_CMP(float64, F64, !=);
  4397. HANDLE_OP_END();
  4398. }
  4399. HANDLE_OP(WASM_OP_F64_LT)
  4400. {
  4401. DEF_OP_CMP(float64, F64, <);
  4402. HANDLE_OP_END();
  4403. }
  4404. HANDLE_OP(WASM_OP_F64_GT)
  4405. {
  4406. DEF_OP_CMP(float64, F64, >);
  4407. HANDLE_OP_END();
  4408. }
  4409. HANDLE_OP(WASM_OP_F64_LE)
  4410. {
  4411. DEF_OP_CMP(float64, F64, <=);
  4412. HANDLE_OP_END();
  4413. }
  4414. HANDLE_OP(WASM_OP_F64_GE)
  4415. {
  4416. DEF_OP_CMP(float64, F64, >=);
  4417. HANDLE_OP_END();
  4418. }
  4419. /* numeric instructions of i32 */
  4420. HANDLE_OP(WASM_OP_I32_CLZ)
  4421. {
  4422. DEF_OP_BIT_COUNT(uint32, I32, clz32);
  4423. HANDLE_OP_END();
  4424. }
  4425. HANDLE_OP(WASM_OP_I32_CTZ)
  4426. {
  4427. DEF_OP_BIT_COUNT(uint32, I32, ctz32);
  4428. HANDLE_OP_END();
  4429. }
  4430. HANDLE_OP(WASM_OP_I32_POPCNT)
  4431. {
  4432. DEF_OP_BIT_COUNT(uint32, I32, popcount32);
  4433. HANDLE_OP_END();
  4434. }
  4435. HANDLE_OP(WASM_OP_I32_ADD)
  4436. {
  4437. DEF_OP_NUMERIC(uint32, uint32, I32, +);
  4438. HANDLE_OP_END();
  4439. }
  4440. HANDLE_OP(WASM_OP_I32_SUB)
  4441. {
  4442. DEF_OP_NUMERIC(uint32, uint32, I32, -);
  4443. HANDLE_OP_END();
  4444. }
  4445. HANDLE_OP(WASM_OP_I32_MUL)
  4446. {
  4447. DEF_OP_NUMERIC(uint32, uint32, I32, *);
  4448. HANDLE_OP_END();
  4449. }
  4450. HANDLE_OP(WASM_OP_I32_DIV_S)
  4451. {
  4452. int32 a, b;
  4453. b = POP_I32();
  4454. a = POP_I32();
  4455. if (a == (int32)0x80000000 && b == -1) {
  4456. wasm_set_exception(module, "integer overflow");
  4457. goto got_exception;
  4458. }
  4459. if (b == 0) {
  4460. wasm_set_exception(module, "integer divide by zero");
  4461. goto got_exception;
  4462. }
  4463. PUSH_I32(a / b);
  4464. HANDLE_OP_END();
  4465. }
  4466. HANDLE_OP(WASM_OP_I32_DIV_U)
  4467. {
  4468. uint32 a, b;
  4469. b = (uint32)POP_I32();
  4470. a = (uint32)POP_I32();
  4471. if (b == 0) {
  4472. wasm_set_exception(module, "integer divide by zero");
  4473. goto got_exception;
  4474. }
  4475. PUSH_I32(a / b);
  4476. HANDLE_OP_END();
  4477. }
  4478. HANDLE_OP(WASM_OP_I32_REM_S)
  4479. {
  4480. int32 a, b;
  4481. b = POP_I32();
  4482. a = POP_I32();
  4483. if (a == (int32)0x80000000 && b == -1) {
  4484. PUSH_I32(0);
  4485. HANDLE_OP_END();
  4486. }
  4487. if (b == 0) {
  4488. wasm_set_exception(module, "integer divide by zero");
  4489. goto got_exception;
  4490. }
  4491. PUSH_I32(a % b);
  4492. HANDLE_OP_END();
  4493. }
  4494. HANDLE_OP(WASM_OP_I32_REM_U)
  4495. {
  4496. uint32 a, b;
  4497. b = (uint32)POP_I32();
  4498. a = (uint32)POP_I32();
  4499. if (b == 0) {
  4500. wasm_set_exception(module, "integer divide by zero");
  4501. goto got_exception;
  4502. }
  4503. PUSH_I32(a % b);
  4504. HANDLE_OP_END();
  4505. }
  4506. HANDLE_OP(WASM_OP_I32_AND)
  4507. {
  4508. DEF_OP_NUMERIC(uint32, uint32, I32, &);
  4509. HANDLE_OP_END();
  4510. }
  4511. HANDLE_OP(WASM_OP_I32_OR)
  4512. {
  4513. DEF_OP_NUMERIC(uint32, uint32, I32, |);
  4514. HANDLE_OP_END();
  4515. }
  4516. HANDLE_OP(WASM_OP_I32_XOR)
  4517. {
  4518. DEF_OP_NUMERIC(uint32, uint32, I32, ^);
  4519. HANDLE_OP_END();
  4520. }
  4521. HANDLE_OP(WASM_OP_I32_SHL)
  4522. {
  4523. DEF_OP_NUMERIC2(uint32, uint32, I32, <<);
  4524. HANDLE_OP_END();
  4525. }
  4526. HANDLE_OP(WASM_OP_I32_SHR_S)
  4527. {
  4528. DEF_OP_NUMERIC2(int32, uint32, I32, >>);
  4529. HANDLE_OP_END();
  4530. }
  4531. HANDLE_OP(WASM_OP_I32_SHR_U)
  4532. {
  4533. DEF_OP_NUMERIC2(uint32, uint32, I32, >>);
  4534. HANDLE_OP_END();
  4535. }
  4536. HANDLE_OP(WASM_OP_I32_ROTL)
  4537. {
  4538. uint32 a, b;
  4539. b = (uint32)POP_I32();
  4540. a = (uint32)POP_I32();
  4541. PUSH_I32(rotl32(a, b));
  4542. HANDLE_OP_END();
  4543. }
  4544. HANDLE_OP(WASM_OP_I32_ROTR)
  4545. {
  4546. uint32 a, b;
  4547. b = (uint32)POP_I32();
  4548. a = (uint32)POP_I32();
  4549. PUSH_I32(rotr32(a, b));
  4550. HANDLE_OP_END();
  4551. }
  4552. /* numeric instructions of i64 */
  4553. HANDLE_OP(WASM_OP_I64_CLZ)
  4554. {
  4555. DEF_OP_BIT_COUNT(uint64, I64, clz64);
  4556. HANDLE_OP_END();
  4557. }
  4558. HANDLE_OP(WASM_OP_I64_CTZ)
  4559. {
  4560. DEF_OP_BIT_COUNT(uint64, I64, ctz64);
  4561. HANDLE_OP_END();
  4562. }
  4563. HANDLE_OP(WASM_OP_I64_POPCNT)
  4564. {
  4565. DEF_OP_BIT_COUNT(uint64, I64, popcount64);
  4566. HANDLE_OP_END();
  4567. }
  4568. HANDLE_OP(WASM_OP_I64_ADD)
  4569. {
  4570. DEF_OP_NUMERIC_64(uint64, uint64, I64, +);
  4571. HANDLE_OP_END();
  4572. }
  4573. HANDLE_OP(WASM_OP_I64_SUB)
  4574. {
  4575. DEF_OP_NUMERIC_64(uint64, uint64, I64, -);
  4576. HANDLE_OP_END();
  4577. }
  4578. HANDLE_OP(WASM_OP_I64_MUL)
  4579. {
  4580. DEF_OP_NUMERIC_64(uint64, uint64, I64, *);
  4581. HANDLE_OP_END();
  4582. }
  4583. HANDLE_OP(WASM_OP_I64_DIV_S)
  4584. {
  4585. int64 a, b;
  4586. b = POP_I64();
  4587. a = POP_I64();
  4588. if (a == (int64)0x8000000000000000LL && b == -1) {
  4589. wasm_set_exception(module, "integer overflow");
  4590. goto got_exception;
  4591. }
  4592. if (b == 0) {
  4593. wasm_set_exception(module, "integer divide by zero");
  4594. goto got_exception;
  4595. }
  4596. PUSH_I64(a / b);
  4597. HANDLE_OP_END();
  4598. }
  4599. HANDLE_OP(WASM_OP_I64_DIV_U)
  4600. {
  4601. uint64 a, b;
  4602. b = (uint64)POP_I64();
  4603. a = (uint64)POP_I64();
  4604. if (b == 0) {
  4605. wasm_set_exception(module, "integer divide by zero");
  4606. goto got_exception;
  4607. }
  4608. PUSH_I64(a / b);
  4609. HANDLE_OP_END();
  4610. }
  4611. HANDLE_OP(WASM_OP_I64_REM_S)
  4612. {
  4613. int64 a, b;
  4614. b = POP_I64();
  4615. a = POP_I64();
  4616. if (a == (int64)0x8000000000000000LL && b == -1) {
  4617. PUSH_I64(0);
  4618. HANDLE_OP_END();
  4619. }
  4620. if (b == 0) {
  4621. wasm_set_exception(module, "integer divide by zero");
  4622. goto got_exception;
  4623. }
  4624. PUSH_I64(a % b);
  4625. HANDLE_OP_END();
  4626. }
  4627. HANDLE_OP(WASM_OP_I64_REM_U)
  4628. {
  4629. uint64 a, b;
  4630. b = (uint64)POP_I64();
  4631. a = (uint64)POP_I64();
  4632. if (b == 0) {
  4633. wasm_set_exception(module, "integer divide by zero");
  4634. goto got_exception;
  4635. }
  4636. PUSH_I64(a % b);
  4637. HANDLE_OP_END();
  4638. }
  4639. HANDLE_OP(WASM_OP_I64_AND)
  4640. {
  4641. DEF_OP_NUMERIC_64(uint64, uint64, I64, &);
  4642. HANDLE_OP_END();
  4643. }
  4644. HANDLE_OP(WASM_OP_I64_OR)
  4645. {
  4646. DEF_OP_NUMERIC_64(uint64, uint64, I64, |);
  4647. HANDLE_OP_END();
  4648. }
  4649. HANDLE_OP(WASM_OP_I64_XOR)
  4650. {
  4651. DEF_OP_NUMERIC_64(uint64, uint64, I64, ^);
  4652. HANDLE_OP_END();
  4653. }
  4654. HANDLE_OP(WASM_OP_I64_SHL)
  4655. {
  4656. DEF_OP_NUMERIC2_64(uint64, uint64, I64, <<);
  4657. HANDLE_OP_END();
  4658. }
  4659. HANDLE_OP(WASM_OP_I64_SHR_S)
  4660. {
  4661. DEF_OP_NUMERIC2_64(int64, uint64, I64, >>);
  4662. HANDLE_OP_END();
  4663. }
  4664. HANDLE_OP(WASM_OP_I64_SHR_U)
  4665. {
  4666. DEF_OP_NUMERIC2_64(uint64, uint64, I64, >>);
  4667. HANDLE_OP_END();
  4668. }
  4669. HANDLE_OP(WASM_OP_I64_ROTL)
  4670. {
  4671. uint64 a, b;
  4672. b = (uint64)POP_I64();
  4673. a = (uint64)POP_I64();
  4674. PUSH_I64(rotl64(a, b));
  4675. HANDLE_OP_END();
  4676. }
  4677. HANDLE_OP(WASM_OP_I64_ROTR)
  4678. {
  4679. uint64 a, b;
  4680. b = (uint64)POP_I64();
  4681. a = (uint64)POP_I64();
  4682. PUSH_I64(rotr64(a, b));
  4683. HANDLE_OP_END();
  4684. }
  4685. /* numeric instructions of f32 */
  4686. HANDLE_OP(WASM_OP_F32_ABS)
  4687. {
  4688. DEF_OP_MATH(float32, F32, fabsf);
  4689. HANDLE_OP_END();
  4690. }
  4691. HANDLE_OP(WASM_OP_F32_NEG)
  4692. {
  4693. uint32 u32 = frame_sp[-1];
  4694. uint32 sign_bit = u32 & ((uint32)1 << 31);
  4695. if (sign_bit)
  4696. frame_sp[-1] = u32 & ~((uint32)1 << 31);
  4697. else
  4698. frame_sp[-1] = u32 | ((uint32)1 << 31);
  4699. HANDLE_OP_END();
  4700. }
  4701. HANDLE_OP(WASM_OP_F32_CEIL)
  4702. {
  4703. DEF_OP_MATH(float32, F32, ceilf);
  4704. HANDLE_OP_END();
  4705. }
  4706. HANDLE_OP(WASM_OP_F32_FLOOR)
  4707. {
  4708. DEF_OP_MATH(float32, F32, floorf);
  4709. HANDLE_OP_END();
  4710. }
  4711. HANDLE_OP(WASM_OP_F32_TRUNC)
  4712. {
  4713. DEF_OP_MATH(float32, F32, truncf);
  4714. HANDLE_OP_END();
  4715. }
  4716. HANDLE_OP(WASM_OP_F32_NEAREST)
  4717. {
  4718. DEF_OP_MATH(float32, F32, rintf);
  4719. HANDLE_OP_END();
  4720. }
  4721. HANDLE_OP(WASM_OP_F32_SQRT)
  4722. {
  4723. DEF_OP_MATH(float32, F32, sqrtf);
  4724. HANDLE_OP_END();
  4725. }
  4726. HANDLE_OP(WASM_OP_F32_ADD)
  4727. {
  4728. DEF_OP_NUMERIC(float32, float32, F32, +);
  4729. HANDLE_OP_END();
  4730. }
  4731. HANDLE_OP(WASM_OP_F32_SUB)
  4732. {
  4733. DEF_OP_NUMERIC(float32, float32, F32, -);
  4734. HANDLE_OP_END();
  4735. }
  4736. HANDLE_OP(WASM_OP_F32_MUL)
  4737. {
  4738. DEF_OP_NUMERIC(float32, float32, F32, *);
  4739. HANDLE_OP_END();
  4740. }
  4741. HANDLE_OP(WASM_OP_F32_DIV)
  4742. {
  4743. DEF_OP_NUMERIC(float32, float32, F32, /);
  4744. HANDLE_OP_END();
  4745. }
  4746. HANDLE_OP(WASM_OP_F32_MIN)
  4747. {
  4748. float32 a, b;
  4749. b = POP_F32();
  4750. a = POP_F32();
  4751. PUSH_F32(f32_min(a, b));
  4752. HANDLE_OP_END();
  4753. }
  4754. HANDLE_OP(WASM_OP_F32_MAX)
  4755. {
  4756. float32 a, b;
  4757. b = POP_F32();
  4758. a = POP_F32();
  4759. PUSH_F32(f32_max(a, b));
  4760. HANDLE_OP_END();
  4761. }
  4762. HANDLE_OP(WASM_OP_F32_COPYSIGN)
  4763. {
  4764. float32 a, b;
  4765. b = POP_F32();
  4766. a = POP_F32();
  4767. PUSH_F32(local_copysignf(a, b));
  4768. HANDLE_OP_END();
  4769. }
  4770. /* numeric instructions of f64 */
  4771. HANDLE_OP(WASM_OP_F64_ABS)
  4772. {
  4773. DEF_OP_MATH(float64, F64, fabs);
  4774. HANDLE_OP_END();
  4775. }
  4776. HANDLE_OP(WASM_OP_F64_NEG)
  4777. {
  4778. uint64 u64 = GET_I64_FROM_ADDR(frame_sp - 2);
  4779. uint64 sign_bit = u64 & (((uint64)1) << 63);
  4780. if (sign_bit)
  4781. PUT_I64_TO_ADDR(frame_sp - 2, (u64 & ~(((uint64)1) << 63)));
  4782. else
  4783. PUT_I64_TO_ADDR(frame_sp - 2, (u64 | (((uint64)1) << 63)));
  4784. HANDLE_OP_END();
  4785. }
  4786. HANDLE_OP(WASM_OP_F64_CEIL)
  4787. {
  4788. DEF_OP_MATH(float64, F64, ceil);
  4789. HANDLE_OP_END();
  4790. }
  4791. HANDLE_OP(WASM_OP_F64_FLOOR)
  4792. {
  4793. DEF_OP_MATH(float64, F64, floor);
  4794. HANDLE_OP_END();
  4795. }
  4796. HANDLE_OP(WASM_OP_F64_TRUNC)
  4797. {
  4798. DEF_OP_MATH(float64, F64, trunc);
  4799. HANDLE_OP_END();
  4800. }
  4801. HANDLE_OP(WASM_OP_F64_NEAREST)
  4802. {
  4803. DEF_OP_MATH(float64, F64, rint);
  4804. HANDLE_OP_END();
  4805. }
  4806. HANDLE_OP(WASM_OP_F64_SQRT)
  4807. {
  4808. DEF_OP_MATH(float64, F64, sqrt);
  4809. HANDLE_OP_END();
  4810. }
  4811. HANDLE_OP(WASM_OP_F64_ADD)
  4812. {
  4813. DEF_OP_NUMERIC_64(float64, float64, F64, +);
  4814. HANDLE_OP_END();
  4815. }
  4816. HANDLE_OP(WASM_OP_F64_SUB)
  4817. {
  4818. DEF_OP_NUMERIC_64(float64, float64, F64, -);
  4819. HANDLE_OP_END();
  4820. }
  4821. HANDLE_OP(WASM_OP_F64_MUL)
  4822. {
  4823. DEF_OP_NUMERIC_64(float64, float64, F64, *);
  4824. HANDLE_OP_END();
  4825. }
  4826. HANDLE_OP(WASM_OP_F64_DIV)
  4827. {
  4828. DEF_OP_NUMERIC_64(float64, float64, F64, /);
  4829. HANDLE_OP_END();
  4830. }
  4831. HANDLE_OP(WASM_OP_F64_MIN)
  4832. {
  4833. float64 a, b;
  4834. b = POP_F64();
  4835. a = POP_F64();
  4836. PUSH_F64(f64_min(a, b));
  4837. HANDLE_OP_END();
  4838. }
  4839. HANDLE_OP(WASM_OP_F64_MAX)
  4840. {
  4841. float64 a, b;
  4842. b = POP_F64();
  4843. a = POP_F64();
  4844. PUSH_F64(f64_max(a, b));
  4845. HANDLE_OP_END();
  4846. }
  4847. HANDLE_OP(WASM_OP_F64_COPYSIGN)
  4848. {
  4849. float64 a, b;
  4850. b = POP_F64();
  4851. a = POP_F64();
  4852. PUSH_F64(local_copysign(a, b));
  4853. HANDLE_OP_END();
  4854. }
  4855. /* conversions of i32 */
  4856. HANDLE_OP(WASM_OP_I32_WRAP_I64)
  4857. {
  4858. int32 value = (int32)(POP_I64() & 0xFFFFFFFFLL);
  4859. PUSH_I32(value);
  4860. HANDLE_OP_END();
  4861. }
  4862. HANDLE_OP(WASM_OP_I32_TRUNC_S_F32)
  4863. {
  4864. /* We don't use INT32_MIN/INT32_MAX/UINT32_MIN/UINT32_MAX,
  4865. since float/double values of ieee754 cannot precisely
  4866. represent all int32/uint32/int64/uint64 values, e.g.
  4867. UINT32_MAX is 4294967295, but (float32)4294967295 is
  4868. 4294967296.0f, but not 4294967295.0f. */
  4869. DEF_OP_TRUNC_F32(-2147483904.0f, 2147483648.0f, true, true);
  4870. HANDLE_OP_END();
  4871. }
  4872. HANDLE_OP(WASM_OP_I32_TRUNC_U_F32)
  4873. {
  4874. DEF_OP_TRUNC_F32(-1.0f, 4294967296.0f, true, false);
  4875. HANDLE_OP_END();
  4876. }
  4877. HANDLE_OP(WASM_OP_I32_TRUNC_S_F64)
  4878. {
  4879. DEF_OP_TRUNC_F64(-2147483649.0, 2147483648.0, true, true);
  4880. /* frame_sp can't be moved in trunc function, we need to
  4881. manually adjust it if src and dst op's cell num is
  4882. different */
  4883. frame_sp--;
  4884. HANDLE_OP_END();
  4885. }
  4886. HANDLE_OP(WASM_OP_I32_TRUNC_U_F64)
  4887. {
  4888. DEF_OP_TRUNC_F64(-1.0, 4294967296.0, true, false);
  4889. frame_sp--;
  4890. HANDLE_OP_END();
  4891. }
  4892. /* conversions of i64 */
  4893. HANDLE_OP(WASM_OP_I64_EXTEND_S_I32)
  4894. {
  4895. DEF_OP_CONVERT(int64, I64, int32, I32);
  4896. HANDLE_OP_END();
  4897. }
  4898. HANDLE_OP(WASM_OP_I64_EXTEND_U_I32)
  4899. {
  4900. DEF_OP_CONVERT(int64, I64, uint32, I32);
  4901. HANDLE_OP_END();
  4902. }
  4903. HANDLE_OP(WASM_OP_I64_TRUNC_S_F32)
  4904. {
  4905. DEF_OP_TRUNC_F32(-9223373136366403584.0f,
  4906. 9223372036854775808.0f, false, true);
  4907. frame_sp++;
  4908. HANDLE_OP_END();
  4909. }
  4910. HANDLE_OP(WASM_OP_I64_TRUNC_U_F32)
  4911. {
  4912. DEF_OP_TRUNC_F32(-1.0f, 18446744073709551616.0f, false, false);
  4913. frame_sp++;
  4914. HANDLE_OP_END();
  4915. }
  4916. HANDLE_OP(WASM_OP_I64_TRUNC_S_F64)
  4917. {
  4918. DEF_OP_TRUNC_F64(-9223372036854777856.0, 9223372036854775808.0,
  4919. false, true);
  4920. HANDLE_OP_END();
  4921. }
  4922. HANDLE_OP(WASM_OP_I64_TRUNC_U_F64)
  4923. {
  4924. DEF_OP_TRUNC_F64(-1.0, 18446744073709551616.0, false, false);
  4925. HANDLE_OP_END();
  4926. }
  4927. /* conversions of f32 */
  4928. HANDLE_OP(WASM_OP_F32_CONVERT_S_I32)
  4929. {
  4930. DEF_OP_CONVERT(float32, F32, int32, I32);
  4931. HANDLE_OP_END();
  4932. }
  4933. HANDLE_OP(WASM_OP_F32_CONVERT_U_I32)
  4934. {
  4935. DEF_OP_CONVERT(float32, F32, uint32, I32);
  4936. HANDLE_OP_END();
  4937. }
  4938. HANDLE_OP(WASM_OP_F32_CONVERT_S_I64)
  4939. {
  4940. DEF_OP_CONVERT(float32, F32, int64, I64);
  4941. HANDLE_OP_END();
  4942. }
  4943. HANDLE_OP(WASM_OP_F32_CONVERT_U_I64)
  4944. {
  4945. DEF_OP_CONVERT(float32, F32, uint64, I64);
  4946. HANDLE_OP_END();
  4947. }
  4948. HANDLE_OP(WASM_OP_F32_DEMOTE_F64)
  4949. {
  4950. DEF_OP_CONVERT(float32, F32, float64, F64);
  4951. HANDLE_OP_END();
  4952. }
  4953. /* conversions of f64 */
  4954. HANDLE_OP(WASM_OP_F64_CONVERT_S_I32)
  4955. {
  4956. DEF_OP_CONVERT(float64, F64, int32, I32);
  4957. HANDLE_OP_END();
  4958. }
  4959. HANDLE_OP(WASM_OP_F64_CONVERT_U_I32)
  4960. {
  4961. DEF_OP_CONVERT(float64, F64, uint32, I32);
  4962. HANDLE_OP_END();
  4963. }
  4964. HANDLE_OP(WASM_OP_F64_CONVERT_S_I64)
  4965. {
  4966. DEF_OP_CONVERT(float64, F64, int64, I64);
  4967. HANDLE_OP_END();
  4968. }
  4969. HANDLE_OP(WASM_OP_F64_CONVERT_U_I64)
  4970. {
  4971. DEF_OP_CONVERT(float64, F64, uint64, I64);
  4972. HANDLE_OP_END();
  4973. }
  4974. HANDLE_OP(WASM_OP_F64_PROMOTE_F32)
  4975. {
  4976. DEF_OP_CONVERT(float64, F64, float32, F32);
  4977. HANDLE_OP_END();
  4978. }
  4979. /* reinterpretations */
  4980. HANDLE_OP(WASM_OP_I32_REINTERPRET_F32)
  4981. HANDLE_OP(WASM_OP_I64_REINTERPRET_F64)
  4982. HANDLE_OP(WASM_OP_F32_REINTERPRET_I32)
  4983. HANDLE_OP(WASM_OP_F64_REINTERPRET_I64)
  4984. {
  4985. HANDLE_OP_END();
  4986. }
  4987. HANDLE_OP(WASM_OP_I32_EXTEND8_S)
  4988. {
  4989. DEF_OP_CONVERT(int32, I32, int8, I32);
  4990. HANDLE_OP_END();
  4991. }
  4992. HANDLE_OP(WASM_OP_I32_EXTEND16_S)
  4993. {
  4994. DEF_OP_CONVERT(int32, I32, int16, I32);
  4995. HANDLE_OP_END();
  4996. }
  4997. HANDLE_OP(WASM_OP_I64_EXTEND8_S)
  4998. {
  4999. DEF_OP_CONVERT(int64, I64, int8, I64);
  5000. HANDLE_OP_END();
  5001. }
  5002. HANDLE_OP(WASM_OP_I64_EXTEND16_S)
  5003. {
  5004. DEF_OP_CONVERT(int64, I64, int16, I64);
  5005. HANDLE_OP_END();
  5006. }
  5007. HANDLE_OP(WASM_OP_I64_EXTEND32_S)
  5008. {
  5009. DEF_OP_CONVERT(int64, I64, int32, I64);
  5010. HANDLE_OP_END();
  5011. }
  5012. HANDLE_OP(WASM_OP_MISC_PREFIX)
  5013. {
  5014. uint32 opcode1;
  5015. read_leb_uint32(frame_ip, frame_ip_end, opcode1);
  5016. /* opcode1 was checked in loader and is no larger than
  5017. UINT8_MAX */
  5018. opcode = (uint8)opcode1;
  5019. switch (opcode) {
  5020. case WASM_OP_I32_TRUNC_SAT_S_F32:
  5021. DEF_OP_TRUNC_SAT_F32(-2147483904.0f, 2147483648.0f,
  5022. true, true);
  5023. break;
  5024. case WASM_OP_I32_TRUNC_SAT_U_F32:
  5025. DEF_OP_TRUNC_SAT_F32(-1.0f, 4294967296.0f, true, false);
  5026. break;
  5027. case WASM_OP_I32_TRUNC_SAT_S_F64:
  5028. DEF_OP_TRUNC_SAT_F64(-2147483649.0, 2147483648.0, true,
  5029. true);
  5030. frame_sp--;
  5031. break;
  5032. case WASM_OP_I32_TRUNC_SAT_U_F64:
  5033. DEF_OP_TRUNC_SAT_F64(-1.0, 4294967296.0, true, false);
  5034. frame_sp--;
  5035. break;
  5036. case WASM_OP_I64_TRUNC_SAT_S_F32:
  5037. DEF_OP_TRUNC_SAT_F32(-9223373136366403584.0f,
  5038. 9223372036854775808.0f, false,
  5039. true);
  5040. frame_sp++;
  5041. break;
  5042. case WASM_OP_I64_TRUNC_SAT_U_F32:
  5043. DEF_OP_TRUNC_SAT_F32(-1.0f, 18446744073709551616.0f,
  5044. false, false);
  5045. frame_sp++;
  5046. break;
  5047. case WASM_OP_I64_TRUNC_SAT_S_F64:
  5048. DEF_OP_TRUNC_SAT_F64(-9223372036854777856.0,
  5049. 9223372036854775808.0, false,
  5050. true);
  5051. break;
  5052. case WASM_OP_I64_TRUNC_SAT_U_F64:
  5053. DEF_OP_TRUNC_SAT_F64(-1.0, 18446744073709551616.0,
  5054. false, false);
  5055. break;
  5056. #if WASM_ENABLE_BULK_MEMORY != 0
  5057. case WASM_OP_MEMORY_INIT:
  5058. {
  5059. uint32 segment;
  5060. mem_offset_t addr;
  5061. uint64 bytes, offset, seg_len;
  5062. uint8 *data;
  5063. read_leb_uint32(frame_ip, frame_ip_end, segment);
  5064. #if WASM_ENABLE_MULTI_MEMORY != 0
  5065. read_leb_memidx(frame_ip, frame_ip_end, memidx);
  5066. #else
  5067. /* skip memory index */
  5068. frame_ip++;
  5069. #endif
  5070. bytes = (uint64)(uint32)POP_I32();
  5071. offset = (uint64)(uint32)POP_I32();
  5072. addr = (mem_offset_t)POP_MEM_OFFSET();
  5073. #if WASM_ENABLE_THREAD_MGR != 0
  5074. linear_mem_size = get_linear_mem_size();
  5075. #endif
  5076. #ifndef OS_ENABLE_HW_BOUND_CHECK
  5077. CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr);
  5078. #else
  5079. #if WASM_ENABLE_SHARED_HEAP != 0
  5080. if (app_addr_in_shared_heap((uint64)(uint32)addr,
  5081. bytes))
  5082. shared_heap_addr_app_to_native((uint64)(uint32)addr,
  5083. maddr);
  5084. else
  5085. #endif
  5086. {
  5087. if ((uint64)(uint32)addr + bytes > linear_mem_size)
  5088. goto out_of_bounds;
  5089. maddr = memory->memory_data + (uint32)addr;
  5090. }
  5091. #endif
  5092. if (bh_bitmap_get_bit(module->e->common.data_dropped,
  5093. segment)) {
  5094. seg_len = 0;
  5095. data = NULL;
  5096. }
  5097. else {
  5098. seg_len =
  5099. (uint64)module->module->data_segments[segment]
  5100. ->data_length;
  5101. data = module->module->data_segments[segment]->data;
  5102. }
  5103. if (offset + bytes > seg_len)
  5104. goto out_of_bounds;
  5105. bh_memcpy_s(maddr, (uint32)(linear_mem_size - addr),
  5106. data + offset, (uint32)bytes);
  5107. break;
  5108. }
  5109. case WASM_OP_DATA_DROP:
  5110. {
  5111. uint32 segment;
  5112. read_leb_uint32(frame_ip, frame_ip_end, segment);
  5113. bh_bitmap_set_bit(module->e->common.data_dropped,
  5114. segment);
  5115. break;
  5116. }
  5117. #endif /* WASM_ENABLE_BULK_MEMORY */
  5118. #if WASM_ENABLE_BULK_MEMORY_OPT != 0
  5119. case WASM_OP_MEMORY_COPY:
  5120. {
  5121. mem_offset_t dst, src, len;
  5122. uint8 *mdst, *msrc;
  5123. len = POP_MEM_OFFSET();
  5124. src = POP_MEM_OFFSET();
  5125. dst = POP_MEM_OFFSET();
  5126. #if WASM_ENABLE_MULTI_MEMORY != 0
  5127. /* dst memidx */
  5128. read_leb_memidx(frame_ip, frame_ip_end, memidx);
  5129. #else
  5130. /* skip dst memidx */
  5131. frame_ip += 1;
  5132. #endif
  5133. // TODO: apply memidx
  5134. #if WASM_ENABLE_THREAD_MGR != 0
  5135. linear_mem_size = get_linear_mem_size();
  5136. #endif
  5137. /* dst boundary check */
  5138. #ifndef OS_ENABLE_HW_BOUND_CHECK
  5139. CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
  5140. #else /* else of OS_ENABLE_HW_BOUND_CHECK */
  5141. #if WASM_ENABLE_SHARED_HEAP != 0
  5142. if (app_addr_in_shared_heap((uint64)dst, len)) {
  5143. shared_heap_addr_app_to_native((uint64)dst, mdst);
  5144. }
  5145. else
  5146. #endif
  5147. {
  5148. if ((uint64)dst + len > linear_mem_size)
  5149. goto out_of_bounds;
  5150. mdst = memory->memory_data + dst;
  5151. }
  5152. #endif /* end of OS_ENABLE_HW_BOUND_CHECK */
  5153. #if WASM_ENABLE_MULTI_MEMORY != 0
  5154. /* src memidx */
  5155. read_leb_memidx(frame_ip, frame_ip_end, memidx);
  5156. #else
  5157. /* skip src memidx */
  5158. frame_ip += 1;
  5159. #endif
  5160. // TODO: apply memidx
  5161. #if WASM_ENABLE_THREAD_MGR != 0
  5162. linear_mem_size = get_linear_mem_size();
  5163. #endif
  5164. /* src boundary check */
  5165. #ifndef OS_ENABLE_HW_BOUND_CHECK
  5166. CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc);
  5167. #else
  5168. #if WASM_ENABLE_SHARED_HEAP != 0
  5169. if (app_addr_in_shared_heap((uint64)src, len))
  5170. shared_heap_addr_app_to_native((uint64)src, msrc);
  5171. else
  5172. #endif
  5173. {
  5174. if ((uint64)src + len > linear_mem_size)
  5175. goto out_of_bounds;
  5176. msrc = memory->memory_data + src;
  5177. }
  5178. #endif
  5179. /*
  5180. * avoid unnecessary operations
  5181. *
  5182. * since dst and src both are valid indexes in the
  5183. * linear memory, mdst and msrc can't be NULL
  5184. *
  5185. * The spec. converts memory.copy into i32.load8 and
  5186. * i32.store8; the following are runtime-specific
  5187. * optimizations.
  5188. *
  5189. */
  5190. if (len && mdst != msrc) {
  5191. /* allowing the destination and source to overlap */
  5192. memmove(mdst, msrc, len);
  5193. }
  5194. break;
  5195. }
  5196. case WASM_OP_MEMORY_FILL:
  5197. {
  5198. mem_offset_t dst, len;
  5199. uint8 fill_val, *mdst;
  5200. #if WASM_ENABLE_MULTI_MEMORY != 0
  5201. read_leb_memidx(frame_ip, frame_ip_end, memidx);
  5202. #else
  5203. /* skip memory index */
  5204. frame_ip++;
  5205. #endif
  5206. len = POP_MEM_OFFSET();
  5207. fill_val = POP_I32();
  5208. dst = POP_MEM_OFFSET();
  5209. #if WASM_ENABLE_THREAD_MGR != 0
  5210. linear_mem_size = get_linear_mem_size();
  5211. #endif
  5212. #ifndef OS_ENABLE_HW_BOUND_CHECK
  5213. CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
  5214. #else
  5215. #if WASM_ENABLE_SHARED_HEAP != 0
  5216. if (app_addr_in_shared_heap((uint64)(uint32)dst, len))
  5217. shared_heap_addr_app_to_native((uint64)(uint32)dst,
  5218. mdst);
  5219. else
  5220. #endif
  5221. {
  5222. if ((uint64)(uint32)dst + len > linear_mem_size)
  5223. goto out_of_bounds;
  5224. mdst = memory->memory_data + (uint32)dst;
  5225. }
  5226. #endif
  5227. memset(mdst, fill_val, len);
  5228. break;
  5229. }
  5230. #endif /* WASM_ENABLE_BULK_MEMORY_OPT */
  5231. #if WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0
  5232. case WASM_OP_TABLE_INIT:
  5233. {
  5234. uint32 tbl_idx;
  5235. tbl_elem_idx_t elem_idx, d;
  5236. uint32 n, s;
  5237. WASMTableInstance *tbl_inst;
  5238. table_elem_type_t *table_elems;
  5239. InitializerExpression *tbl_seg_init_values = NULL,
  5240. *init_values;
  5241. uint32 tbl_seg_len = 0;
  5242. read_leb_uint32(frame_ip, frame_ip_end, elem_idx);
  5243. bh_assert(elem_idx < module->module->table_seg_count);
  5244. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  5245. bh_assert(tbl_idx < module->module->table_count);
  5246. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  5247. #if WASM_ENABLE_MEMORY64 != 0
  5248. is_table64 = tbl_inst->is_table64;
  5249. #endif
  5250. n = (uint32)POP_I32();
  5251. s = (uint32)POP_I32();
  5252. d = (tbl_elem_idx_t)POP_TBL_ELEM_IDX();
  5253. if (!bh_bitmap_get_bit(module->e->common.elem_dropped,
  5254. elem_idx)) {
  5255. /* table segment isn't dropped */
  5256. tbl_seg_init_values =
  5257. module->module->table_segments[elem_idx]
  5258. .init_values;
  5259. tbl_seg_len =
  5260. module->module->table_segments[elem_idx]
  5261. .value_count;
  5262. }
  5263. /* TODO: memory64 current implementation of table64
  5264. * still assumes the max table size UINT32_MAX
  5265. */
  5266. if (
  5267. #if WASM_ENABLE_MEMORY64 != 0
  5268. d > UINT32_MAX ||
  5269. #endif
  5270. offset_len_out_of_bounds(s, n, tbl_seg_len)
  5271. || offset_len_out_of_bounds((uint32)d, n,
  5272. tbl_inst->cur_size)) {
  5273. wasm_set_exception(module,
  5274. "out of bounds table access");
  5275. goto got_exception;
  5276. }
  5277. if (!n) {
  5278. break;
  5279. }
  5280. table_elems = tbl_inst->elems + d;
  5281. init_values = tbl_seg_init_values + s;
  5282. #if WASM_ENABLE_GC != 0
  5283. SYNC_ALL_TO_FRAME();
  5284. #endif
  5285. for (i = 0; i < n; i++) {
  5286. /* UINT32_MAX indicates that it is a null ref */
  5287. bh_assert(init_values[i].init_expr_type
  5288. == INIT_EXPR_TYPE_REFNULL_CONST
  5289. || init_values[i].init_expr_type
  5290. == INIT_EXPR_TYPE_FUNCREF_CONST);
  5291. #if WASM_ENABLE_GC == 0
  5292. table_elems[i] = (table_elem_type_t)init_values[i]
  5293. .u.unary.v.ref_index;
  5294. #else
  5295. if (init_values[i].u.unary.v.ref_index
  5296. != UINT32_MAX) {
  5297. if (!(func_obj = wasm_create_func_obj(
  5298. module,
  5299. init_values[i].u.unary.v.ref_index,
  5300. true, NULL, 0))) {
  5301. goto got_exception;
  5302. }
  5303. table_elems[i] = func_obj;
  5304. }
  5305. else {
  5306. table_elems[i] = NULL_REF;
  5307. }
  5308. #endif
  5309. }
  5310. break;
  5311. }
  5312. case WASM_OP_ELEM_DROP:
  5313. {
  5314. uint32 elem_idx;
  5315. read_leb_uint32(frame_ip, frame_ip_end, elem_idx);
  5316. bh_assert(elem_idx < module->module->table_seg_count);
  5317. bh_bitmap_set_bit(module->e->common.elem_dropped,
  5318. elem_idx);
  5319. break;
  5320. }
  5321. case WASM_OP_TABLE_COPY:
  5322. {
  5323. uint32 src_tbl_idx, dst_tbl_idx;
  5324. tbl_elem_idx_t n, s, d;
  5325. WASMTableInstance *src_tbl_inst, *dst_tbl_inst;
  5326. read_leb_uint32(frame_ip, frame_ip_end, dst_tbl_idx);
  5327. bh_assert(dst_tbl_idx < module->table_count);
  5328. dst_tbl_inst = wasm_get_table_inst(module, dst_tbl_idx);
  5329. read_leb_uint32(frame_ip, frame_ip_end, src_tbl_idx);
  5330. bh_assert(src_tbl_idx < module->table_count);
  5331. src_tbl_inst = wasm_get_table_inst(module, src_tbl_idx);
  5332. #if WASM_ENABLE_MEMORY64 != 0
  5333. is_table64 = src_tbl_inst->is_table64
  5334. && dst_tbl_inst->is_table64;
  5335. #endif
  5336. n = (tbl_elem_idx_t)POP_TBL_ELEM_IDX();
  5337. #if WASM_ENABLE_MEMORY64 != 0
  5338. is_table64 = src_tbl_inst->is_table64;
  5339. #endif
  5340. s = (tbl_elem_idx_t)POP_TBL_ELEM_IDX();
  5341. #if WASM_ENABLE_MEMORY64 != 0
  5342. is_table64 = dst_tbl_inst->is_table64;
  5343. #endif
  5344. d = (tbl_elem_idx_t)POP_TBL_ELEM_IDX();
  5345. if (
  5346. #if WASM_ENABLE_MEMORY64 != 0
  5347. n > UINT32_MAX || s > UINT32_MAX || d > UINT32_MAX
  5348. ||
  5349. #endif
  5350. offset_len_out_of_bounds((uint32)d, (uint32)n,
  5351. dst_tbl_inst->cur_size)
  5352. || offset_len_out_of_bounds(
  5353. (uint32)s, (uint32)n, src_tbl_inst->cur_size)) {
  5354. wasm_set_exception(module,
  5355. "out of bounds table access");
  5356. goto got_exception;
  5357. }
  5358. /* if s >= d, copy from front to back */
  5359. /* if s < d, copy from back to front */
  5360. /* merge all together */
  5361. bh_memmove_s((uint8 *)dst_tbl_inst
  5362. + offsetof(WASMTableInstance, elems)
  5363. + d * sizeof(table_elem_type_t),
  5364. (uint32)((dst_tbl_inst->cur_size - d)
  5365. * sizeof(table_elem_type_t)),
  5366. (uint8 *)src_tbl_inst
  5367. + offsetof(WASMTableInstance, elems)
  5368. + s * sizeof(table_elem_type_t),
  5369. (uint32)(n * sizeof(table_elem_type_t)));
  5370. break;
  5371. }
  5372. case WASM_OP_TABLE_GROW:
  5373. {
  5374. WASMTableInstance *tbl_inst;
  5375. uint32 tbl_idx, orig_tbl_sz;
  5376. tbl_elem_idx_t n;
  5377. table_elem_type_t init_val;
  5378. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  5379. bh_assert(tbl_idx < module->table_count);
  5380. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  5381. #if WASM_ENABLE_MEMORY64 != 0
  5382. is_table64 = tbl_inst->is_table64;
  5383. #endif
  5384. orig_tbl_sz = tbl_inst->cur_size;
  5385. n = POP_TBL_ELEM_IDX();
  5386. #if WASM_ENABLE_GC == 0
  5387. init_val = POP_I32();
  5388. #else
  5389. init_val = POP_REF();
  5390. #endif
  5391. if (
  5392. #if WASM_ENABLE_MEMORY64 != 0
  5393. n > UINT32_MAX ||
  5394. #endif
  5395. !wasm_enlarge_table(module, tbl_idx, (uint32)n,
  5396. init_val)) {
  5397. PUSH_TBL_ELEM_IDX(-1);
  5398. }
  5399. else {
  5400. PUSH_TBL_ELEM_IDX(orig_tbl_sz);
  5401. }
  5402. break;
  5403. }
  5404. case WASM_OP_TABLE_SIZE:
  5405. {
  5406. uint32 tbl_idx;
  5407. WASMTableInstance *tbl_inst;
  5408. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  5409. bh_assert(tbl_idx < module->table_count);
  5410. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  5411. #if WASM_ENABLE_MEMORY64 != 0
  5412. is_table64 = tbl_inst->is_table64;
  5413. #endif
  5414. PUSH_TBL_ELEM_IDX(tbl_inst->cur_size);
  5415. break;
  5416. }
  5417. case WASM_OP_TABLE_FILL:
  5418. {
  5419. uint32 tbl_idx;
  5420. tbl_elem_idx_t n, elem_idx;
  5421. WASMTableInstance *tbl_inst;
  5422. table_elem_type_t fill_val;
  5423. read_leb_uint32(frame_ip, frame_ip_end, tbl_idx);
  5424. bh_assert(tbl_idx < module->table_count);
  5425. tbl_inst = wasm_get_table_inst(module, tbl_idx);
  5426. #if WASM_ENABLE_MEMORY64 != 0
  5427. is_table64 = tbl_inst->is_table64;
  5428. #endif
  5429. n = POP_TBL_ELEM_IDX();
  5430. #if WASM_ENABLE_GC == 0
  5431. fill_val = POP_I32();
  5432. #else
  5433. fill_val = POP_REF();
  5434. #endif
  5435. elem_idx = POP_TBL_ELEM_IDX();
  5436. if (
  5437. #if WASM_ENABLE_MEMORY64 != 0
  5438. n > UINT32_MAX || elem_idx > UINT32_MAX ||
  5439. #endif
  5440. offset_len_out_of_bounds((uint32)elem_idx,
  5441. (uint32)n,
  5442. tbl_inst->cur_size)) {
  5443. wasm_set_exception(module,
  5444. "out of bounds table access");
  5445. goto got_exception;
  5446. }
  5447. for (; n != 0; elem_idx++, n--) {
  5448. tbl_inst->elems[elem_idx] = fill_val;
  5449. }
  5450. break;
  5451. }
  5452. #endif /* end of WASM_ENABLE_REF_TYPES != 0 || WASM_ENABLE_GC != 0 */
  5453. default:
  5454. wasm_set_exception(module, "unsupported opcode");
  5455. goto got_exception;
  5456. }
  5457. HANDLE_OP_END();
  5458. }
  5459. #if WASM_ENABLE_SHARED_MEMORY != 0
  5460. HANDLE_OP(WASM_OP_ATOMIC_PREFIX)
  5461. {
  5462. mem_offset_t offset = 0, addr;
  5463. uint32 align = 0;
  5464. uint32 opcode1;
  5465. read_leb_uint32(frame_ip, frame_ip_end, opcode1);
  5466. /* opcode1 was checked in loader and is no larger than
  5467. UINT8_MAX */
  5468. opcode = (uint8)opcode1;
  5469. if (opcode != WASM_OP_ATOMIC_FENCE) {
  5470. read_leb_uint32(frame_ip, frame_ip_end, align);
  5471. read_leb_mem_offset(frame_ip, frame_ip_end, offset);
  5472. }
  5473. switch (opcode) {
  5474. case WASM_OP_ATOMIC_NOTIFY:
  5475. {
  5476. uint32 notify_count, ret;
  5477. notify_count = POP_I32();
  5478. addr = POP_MEM_OFFSET();
  5479. CHECK_MEMORY_OVERFLOW(4);
  5480. CHECK_ATOMIC_MEMORY_ACCESS();
  5481. ret = wasm_runtime_atomic_notify(
  5482. (WASMModuleInstanceCommon *)module, maddr,
  5483. notify_count);
  5484. if (ret == (uint32)-1)
  5485. goto got_exception;
  5486. PUSH_I32(ret);
  5487. break;
  5488. }
  5489. case WASM_OP_ATOMIC_WAIT32:
  5490. {
  5491. uint64 timeout;
  5492. uint32 expect, ret;
  5493. timeout = POP_I64();
  5494. expect = POP_I32();
  5495. addr = POP_MEM_OFFSET();
  5496. CHECK_MEMORY_OVERFLOW(4);
  5497. CHECK_ATOMIC_MEMORY_ACCESS();
  5498. ret = wasm_runtime_atomic_wait(
  5499. (WASMModuleInstanceCommon *)module, maddr,
  5500. (uint64)expect, timeout, false);
  5501. if (ret == (uint32)-1)
  5502. goto got_exception;
  5503. #if WASM_ENABLE_THREAD_MGR != 0
  5504. CHECK_SUSPEND_FLAGS();
  5505. #endif
  5506. PUSH_I32(ret);
  5507. break;
  5508. }
  5509. case WASM_OP_ATOMIC_WAIT64:
  5510. {
  5511. uint64 timeout, expect;
  5512. uint32 ret;
  5513. timeout = POP_I64();
  5514. expect = POP_I64();
  5515. addr = POP_MEM_OFFSET();
  5516. CHECK_MEMORY_OVERFLOW(8);
  5517. CHECK_ATOMIC_MEMORY_ACCESS();
  5518. ret = wasm_runtime_atomic_wait(
  5519. (WASMModuleInstanceCommon *)module, maddr, expect,
  5520. timeout, true);
  5521. if (ret == (uint32)-1)
  5522. goto got_exception;
  5523. #if WASM_ENABLE_THREAD_MGR != 0
  5524. CHECK_SUSPEND_FLAGS();
  5525. #endif
  5526. PUSH_I32(ret);
  5527. break;
  5528. }
  5529. case WASM_OP_ATOMIC_FENCE:
  5530. {
  5531. /* Skip the memory index */
  5532. frame_ip++;
  5533. os_atomic_thread_fence(os_memory_order_seq_cst);
  5534. break;
  5535. }
  5536. case WASM_OP_ATOMIC_I32_LOAD:
  5537. case WASM_OP_ATOMIC_I32_LOAD8_U:
  5538. case WASM_OP_ATOMIC_I32_LOAD16_U:
  5539. {
  5540. uint32 readv;
  5541. addr = POP_MEM_OFFSET();
  5542. if (opcode == WASM_OP_ATOMIC_I32_LOAD8_U) {
  5543. CHECK_MEMORY_OVERFLOW(1);
  5544. CHECK_ATOMIC_MEMORY_ACCESS();
  5545. shared_memory_lock(memory);
  5546. readv = (uint32)(*(uint8 *)maddr);
  5547. shared_memory_unlock(memory);
  5548. }
  5549. else if (opcode == WASM_OP_ATOMIC_I32_LOAD16_U) {
  5550. CHECK_MEMORY_OVERFLOW(2);
  5551. CHECK_ATOMIC_MEMORY_ACCESS();
  5552. shared_memory_lock(memory);
  5553. readv = (uint32)LOAD_U16(maddr);
  5554. shared_memory_unlock(memory);
  5555. }
  5556. else {
  5557. CHECK_MEMORY_OVERFLOW(4);
  5558. CHECK_ATOMIC_MEMORY_ACCESS();
  5559. shared_memory_lock(memory);
  5560. readv = LOAD_I32(maddr);
  5561. shared_memory_unlock(memory);
  5562. }
  5563. PUSH_I32(readv);
  5564. break;
  5565. }
  5566. case WASM_OP_ATOMIC_I64_LOAD:
  5567. case WASM_OP_ATOMIC_I64_LOAD8_U:
  5568. case WASM_OP_ATOMIC_I64_LOAD16_U:
  5569. case WASM_OP_ATOMIC_I64_LOAD32_U:
  5570. {
  5571. uint64 readv;
  5572. addr = POP_MEM_OFFSET();
  5573. if (opcode == WASM_OP_ATOMIC_I64_LOAD8_U) {
  5574. CHECK_MEMORY_OVERFLOW(1);
  5575. CHECK_ATOMIC_MEMORY_ACCESS();
  5576. shared_memory_lock(memory);
  5577. readv = (uint64)(*(uint8 *)maddr);
  5578. shared_memory_unlock(memory);
  5579. }
  5580. else if (opcode == WASM_OP_ATOMIC_I64_LOAD16_U) {
  5581. CHECK_MEMORY_OVERFLOW(2);
  5582. CHECK_ATOMIC_MEMORY_ACCESS();
  5583. shared_memory_lock(memory);
  5584. readv = (uint64)LOAD_U16(maddr);
  5585. shared_memory_unlock(memory);
  5586. }
  5587. else if (opcode == WASM_OP_ATOMIC_I64_LOAD32_U) {
  5588. CHECK_MEMORY_OVERFLOW(4);
  5589. CHECK_ATOMIC_MEMORY_ACCESS();
  5590. shared_memory_lock(memory);
  5591. readv = (uint64)LOAD_U32(maddr);
  5592. shared_memory_unlock(memory);
  5593. }
  5594. else {
  5595. CHECK_MEMORY_OVERFLOW(8);
  5596. CHECK_ATOMIC_MEMORY_ACCESS();
  5597. shared_memory_lock(memory);
  5598. readv = LOAD_I64(maddr);
  5599. shared_memory_unlock(memory);
  5600. }
  5601. PUSH_I64(readv);
  5602. break;
  5603. }
  5604. case WASM_OP_ATOMIC_I32_STORE:
  5605. case WASM_OP_ATOMIC_I32_STORE8:
  5606. case WASM_OP_ATOMIC_I32_STORE16:
  5607. {
  5608. uint32 sval;
  5609. sval = (uint32)POP_I32();
  5610. addr = POP_MEM_OFFSET();
  5611. if (opcode == WASM_OP_ATOMIC_I32_STORE8) {
  5612. CHECK_MEMORY_OVERFLOW(1);
  5613. CHECK_ATOMIC_MEMORY_ACCESS();
  5614. shared_memory_lock(memory);
  5615. *(uint8 *)maddr = (uint8)sval;
  5616. shared_memory_unlock(memory);
  5617. }
  5618. else if (opcode == WASM_OP_ATOMIC_I32_STORE16) {
  5619. CHECK_MEMORY_OVERFLOW(2);
  5620. CHECK_ATOMIC_MEMORY_ACCESS();
  5621. shared_memory_lock(memory);
  5622. STORE_U16(maddr, (uint16)sval);
  5623. shared_memory_unlock(memory);
  5624. }
  5625. else {
  5626. CHECK_MEMORY_OVERFLOW(4);
  5627. CHECK_ATOMIC_MEMORY_ACCESS();
  5628. shared_memory_lock(memory);
  5629. STORE_U32(maddr, sval);
  5630. shared_memory_unlock(memory);
  5631. }
  5632. break;
  5633. }
  5634. case WASM_OP_ATOMIC_I64_STORE:
  5635. case WASM_OP_ATOMIC_I64_STORE8:
  5636. case WASM_OP_ATOMIC_I64_STORE16:
  5637. case WASM_OP_ATOMIC_I64_STORE32:
  5638. {
  5639. uint64 sval;
  5640. sval = (uint64)POP_I64();
  5641. addr = POP_MEM_OFFSET();
  5642. if (opcode == WASM_OP_ATOMIC_I64_STORE8) {
  5643. CHECK_MEMORY_OVERFLOW(1);
  5644. CHECK_ATOMIC_MEMORY_ACCESS();
  5645. shared_memory_lock(memory);
  5646. *(uint8 *)maddr = (uint8)sval;
  5647. shared_memory_unlock(memory);
  5648. }
  5649. else if (opcode == WASM_OP_ATOMIC_I64_STORE16) {
  5650. CHECK_MEMORY_OVERFLOW(2);
  5651. CHECK_ATOMIC_MEMORY_ACCESS();
  5652. shared_memory_lock(memory);
  5653. STORE_U16(maddr, (uint16)sval);
  5654. shared_memory_unlock(memory);
  5655. }
  5656. else if (opcode == WASM_OP_ATOMIC_I64_STORE32) {
  5657. CHECK_MEMORY_OVERFLOW(4);
  5658. CHECK_ATOMIC_MEMORY_ACCESS();
  5659. shared_memory_lock(memory);
  5660. STORE_U32(maddr, (uint32)sval);
  5661. shared_memory_unlock(memory);
  5662. }
  5663. else {
  5664. CHECK_MEMORY_OVERFLOW(8);
  5665. CHECK_ATOMIC_MEMORY_ACCESS();
  5666. shared_memory_lock(memory);
  5667. STORE_I64(maddr, sval);
  5668. shared_memory_unlock(memory);
  5669. }
  5670. break;
  5671. }
  5672. case WASM_OP_ATOMIC_RMW_I32_CMPXCHG:
  5673. case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U:
  5674. case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U:
  5675. {
  5676. uint32 readv, sval, expect;
  5677. sval = POP_I32();
  5678. expect = POP_I32();
  5679. addr = POP_MEM_OFFSET();
  5680. if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U) {
  5681. CHECK_MEMORY_OVERFLOW(1);
  5682. CHECK_ATOMIC_MEMORY_ACCESS();
  5683. expect = (uint8)expect;
  5684. shared_memory_lock(memory);
  5685. readv = (uint32)(*(uint8 *)maddr);
  5686. if (readv == expect)
  5687. *(uint8 *)maddr = (uint8)(sval);
  5688. shared_memory_unlock(memory);
  5689. }
  5690. else if (opcode == WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U) {
  5691. CHECK_MEMORY_OVERFLOW(2);
  5692. CHECK_ATOMIC_MEMORY_ACCESS();
  5693. expect = (uint16)expect;
  5694. shared_memory_lock(memory);
  5695. readv = (uint32)LOAD_U16(maddr);
  5696. if (readv == expect)
  5697. STORE_U16(maddr, (uint16)(sval));
  5698. shared_memory_unlock(memory);
  5699. }
  5700. else {
  5701. CHECK_MEMORY_OVERFLOW(4);
  5702. CHECK_ATOMIC_MEMORY_ACCESS();
  5703. shared_memory_lock(memory);
  5704. readv = LOAD_I32(maddr);
  5705. if (readv == expect)
  5706. STORE_U32(maddr, sval);
  5707. shared_memory_unlock(memory);
  5708. }
  5709. PUSH_I32(readv);
  5710. break;
  5711. }
  5712. case WASM_OP_ATOMIC_RMW_I64_CMPXCHG:
  5713. case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U:
  5714. case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U:
  5715. case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U:
  5716. {
  5717. uint64 readv, sval, expect;
  5718. sval = (uint64)POP_I64();
  5719. expect = (uint64)POP_I64();
  5720. addr = POP_MEM_OFFSET();
  5721. if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U) {
  5722. CHECK_MEMORY_OVERFLOW(1);
  5723. CHECK_ATOMIC_MEMORY_ACCESS();
  5724. expect = (uint8)expect;
  5725. shared_memory_lock(memory);
  5726. readv = (uint64)(*(uint8 *)maddr);
  5727. if (readv == expect)
  5728. *(uint8 *)maddr = (uint8)(sval);
  5729. shared_memory_unlock(memory);
  5730. }
  5731. else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U) {
  5732. CHECK_MEMORY_OVERFLOW(2);
  5733. CHECK_ATOMIC_MEMORY_ACCESS();
  5734. expect = (uint16)expect;
  5735. shared_memory_lock(memory);
  5736. readv = (uint64)LOAD_U16(maddr);
  5737. if (readv == expect)
  5738. STORE_U16(maddr, (uint16)(sval));
  5739. shared_memory_unlock(memory);
  5740. }
  5741. else if (opcode == WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U) {
  5742. CHECK_MEMORY_OVERFLOW(4);
  5743. CHECK_ATOMIC_MEMORY_ACCESS();
  5744. expect = (uint32)expect;
  5745. shared_memory_lock(memory);
  5746. readv = (uint64)LOAD_U32(maddr);
  5747. if (readv == expect)
  5748. STORE_U32(maddr, (uint32)(sval));
  5749. shared_memory_unlock(memory);
  5750. }
  5751. else {
  5752. CHECK_MEMORY_OVERFLOW(8);
  5753. CHECK_ATOMIC_MEMORY_ACCESS();
  5754. shared_memory_lock(memory);
  5755. readv = (uint64)LOAD_I64(maddr);
  5756. if (readv == expect)
  5757. STORE_I64(maddr, sval);
  5758. shared_memory_unlock(memory);
  5759. }
  5760. PUSH_I64(readv);
  5761. break;
  5762. }
  5763. DEF_ATOMIC_RMW_OPCODE(ADD, +);
  5764. DEF_ATOMIC_RMW_OPCODE(SUB, -);
  5765. DEF_ATOMIC_RMW_OPCODE(AND, &);
  5766. DEF_ATOMIC_RMW_OPCODE(OR, |);
  5767. DEF_ATOMIC_RMW_OPCODE(XOR, ^);
  5768. /* xchg, ignore the read value, and store the given
  5769. value: readv * 0 + sval */
  5770. DEF_ATOMIC_RMW_OPCODE(XCHG, *0 +);
  5771. }
  5772. HANDLE_OP_END();
  5773. }
  5774. #endif
  5775. HANDLE_OP(WASM_OP_IMPDEP)
  5776. {
  5777. frame = prev_frame;
  5778. frame_ip = frame->ip;
  5779. frame_sp = frame->sp;
  5780. frame_csp = frame->csp;
  5781. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  5782. is_return_call = false;
  5783. #endif
  5784. goto call_func_from_entry;
  5785. }
  5786. #if WASM_ENABLE_DEBUG_INTERP != 0
  5787. HANDLE_OP(DEBUG_OP_BREAK)
  5788. {
  5789. wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP);
  5790. WASM_SUSPEND_FLAGS_FETCH_OR(exec_env->suspend_flags,
  5791. WASM_SUSPEND_FLAG_SUSPEND);
  5792. frame_ip--;
  5793. SYNC_ALL_TO_FRAME();
  5794. CHECK_SUSPEND_FLAGS();
  5795. HANDLE_OP_END();
  5796. }
  5797. #endif
  5798. #if WASM_ENABLE_LABELS_AS_VALUES == 0
  5799. default:
  5800. wasm_set_exception(module, "unsupported opcode");
  5801. goto got_exception;
  5802. }
  5803. #endif
  5804. #if WASM_ENABLE_LABELS_AS_VALUES != 0
  5805. HANDLE_OP(WASM_OP_UNUSED_0x0a)
  5806. #if WASM_ENABLE_TAIL_CALL == 0
  5807. HANDLE_OP(WASM_OP_RETURN_CALL)
  5808. HANDLE_OP(WASM_OP_RETURN_CALL_INDIRECT)
  5809. #endif
  5810. #if WASM_ENABLE_SHARED_MEMORY == 0
  5811. HANDLE_OP(WASM_OP_ATOMIC_PREFIX)
  5812. #endif
  5813. #if WASM_ENABLE_REF_TYPES == 0 && WASM_ENABLE_GC == 0
  5814. HANDLE_OP(WASM_OP_SELECT_T)
  5815. HANDLE_OP(WASM_OP_TABLE_GET)
  5816. HANDLE_OP(WASM_OP_TABLE_SET)
  5817. HANDLE_OP(WASM_OP_REF_NULL)
  5818. HANDLE_OP(WASM_OP_REF_IS_NULL)
  5819. HANDLE_OP(WASM_OP_REF_FUNC)
  5820. #endif
  5821. #if WASM_ENABLE_GC == 0
  5822. HANDLE_OP(WASM_OP_CALL_REF)
  5823. HANDLE_OP(WASM_OP_RETURN_CALL_REF)
  5824. HANDLE_OP(WASM_OP_REF_EQ)
  5825. HANDLE_OP(WASM_OP_REF_AS_NON_NULL)
  5826. HANDLE_OP(WASM_OP_BR_ON_NULL)
  5827. HANDLE_OP(WASM_OP_BR_ON_NON_NULL)
  5828. HANDLE_OP(WASM_OP_GC_PREFIX)
  5829. #endif
  5830. #if WASM_ENABLE_EXCE_HANDLING == 0
  5831. HANDLE_OP(WASM_OP_TRY)
  5832. HANDLE_OP(WASM_OP_CATCH)
  5833. HANDLE_OP(WASM_OP_THROW)
  5834. HANDLE_OP(WASM_OP_RETHROW)
  5835. HANDLE_OP(WASM_OP_DELEGATE)
  5836. HANDLE_OP(WASM_OP_CATCH_ALL)
  5837. HANDLE_OP(EXT_OP_TRY)
  5838. #endif
  5839. /* SIMD isn't supported by interpreter, but when JIT is
  5840. enabled, `iwasm --interp <wasm_file>` may be run to
  5841. trigger the SIMD opcode in interpreter */
  5842. HANDLE_OP(WASM_OP_SIMD_PREFIX)
  5843. HANDLE_OP(WASM_OP_UNUSED_0x16)
  5844. HANDLE_OP(WASM_OP_UNUSED_0x17)
  5845. HANDLE_OP(WASM_OP_UNUSED_0x27)
  5846. /* Used by fast interpreter */
  5847. HANDLE_OP(EXT_OP_SET_LOCAL_FAST_I64)
  5848. HANDLE_OP(EXT_OP_TEE_LOCAL_FAST_I64)
  5849. HANDLE_OP(EXT_OP_COPY_STACK_TOP)
  5850. HANDLE_OP(EXT_OP_COPY_STACK_TOP_I64)
  5851. HANDLE_OP(EXT_OP_COPY_STACK_VALUES)
  5852. {
  5853. wasm_set_exception(module, "unsupported opcode");
  5854. goto got_exception;
  5855. }
  5856. #endif /* end of WASM_ENABLE_LABELS_AS_VALUES != 0 */
  5857. #if WASM_ENABLE_LABELS_AS_VALUES == 0
  5858. continue;
  5859. #else
  5860. FETCH_OPCODE_AND_DISPATCH();
  5861. #endif
  5862. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  5863. call_func_from_return_call:
  5864. {
  5865. POP(cur_func->param_cell_num);
  5866. if (cur_func->param_cell_num > 0) {
  5867. word_copy(frame->lp, frame_sp, cur_func->param_cell_num);
  5868. }
  5869. FREE_FRAME(exec_env, frame);
  5870. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  5871. is_return_call = true;
  5872. goto call_func_from_entry;
  5873. }
  5874. #endif
  5875. call_func_from_interp:
  5876. {
  5877. /* Only do the copy when it's called from interpreter. */
  5878. WASMInterpFrame *outs_area = wasm_exec_env_wasm_stack_top(exec_env);
  5879. if (cur_func->param_cell_num > 0) {
  5880. POP(cur_func->param_cell_num);
  5881. word_copy(outs_area->lp, frame_sp, cur_func->param_cell_num);
  5882. }
  5883. SYNC_ALL_TO_FRAME();
  5884. prev_frame = frame;
  5885. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  5886. is_return_call = false;
  5887. #endif
  5888. }
  5889. call_func_from_entry:
  5890. {
  5891. if (cur_func->is_import_func) {
  5892. #if WASM_ENABLE_MULTI_MODULE != 0
  5893. if (cur_func->import_func_inst) {
  5894. wasm_interp_call_func_import(module, exec_env, cur_func,
  5895. prev_frame);
  5896. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  5897. if (is_return_call) {
  5898. /* the frame was freed before tail calling and
  5899. the prev_frame was set as exec_env's cur_frame,
  5900. so here we recover context from prev_frame */
  5901. RECOVER_CONTEXT(prev_frame);
  5902. }
  5903. else
  5904. #endif
  5905. {
  5906. prev_frame = frame->prev_frame;
  5907. cur_func = frame->function;
  5908. UPDATE_ALL_FROM_FRAME();
  5909. }
  5910. #if WASM_ENABLE_EXCE_HANDLING != 0
  5911. char uncaught_exception[128] = { 0 };
  5912. bool has_exception =
  5913. wasm_copy_exception(module, uncaught_exception);
  5914. if (has_exception
  5915. && strstr(uncaught_exception, "uncaught wasm exception")) {
  5916. uint32 import_exception;
  5917. /* initialize imported exception index to be invalid */
  5918. SET_INVALID_TAGINDEX(import_exception);
  5919. /* pull external exception */
  5920. uint32 ext_exception = POP_I32();
  5921. /* external function came back with an exception or trap */
  5922. /* lookup exception in import tags */
  5923. WASMTagInstance *tag = module->e->tags;
  5924. for (uint32 t = 0; t < module->module->import_tag_count;
  5925. tag++, t++) {
  5926. /* compare the module and the external index with the
  5927. * import tag data */
  5928. if ((cur_func->u.func_import->import_module
  5929. == tag->u.tag_import->import_module)
  5930. && (ext_exception
  5931. == tag->u.tag_import
  5932. ->import_tag_index_linked)) {
  5933. /* set the import_exception to the import tag */
  5934. import_exception = t;
  5935. break;
  5936. }
  5937. }
  5938. /*
  5939. * exchange the thrown exception (index valid in submodule)
  5940. * with the imported exception index (valid in this module)
  5941. * if the module did not import the exception,
  5942. * that results in a "INVALID_TAGINDEX", that triggers
  5943. * an CATCH_ALL block, if there is one.
  5944. */
  5945. PUSH_I32(import_exception);
  5946. }
  5947. #endif /* end of WASM_ENABLE_EXCE_HANDLING != 0 */
  5948. }
  5949. else
  5950. #endif /* end of WASM_ENABLE_MULTI_MODULE != 0 */
  5951. {
  5952. wasm_interp_call_func_native(module, exec_env, cur_func,
  5953. prev_frame);
  5954. #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
  5955. if (is_return_call) {
  5956. /* the frame was freed before tail calling and
  5957. the prev_frame was set as exec_env's cur_frame,
  5958. so here we recover context from prev_frame */
  5959. RECOVER_CONTEXT(prev_frame);
  5960. }
  5961. else
  5962. #endif
  5963. {
  5964. prev_frame = frame->prev_frame;
  5965. cur_func = frame->function;
  5966. UPDATE_ALL_FROM_FRAME();
  5967. }
  5968. }
  5969. /* update memory size, no need to update memory ptr as
  5970. it isn't changed in wasm_enlarge_memory */
  5971. #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
  5972. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
  5973. || WASM_ENABLE_BULK_MEMORY != 0
  5974. if (memory)
  5975. linear_mem_size = GET_LINEAR_MEMORY_SIZE(memory);
  5976. #endif
  5977. if (wasm_copy_exception(module, NULL)) {
  5978. #if WASM_ENABLE_EXCE_HANDLING != 0
  5979. /* the caller raised an exception */
  5980. char uncaught_exception[128] = { 0 };
  5981. bool has_exception =
  5982. wasm_copy_exception(module, uncaught_exception);
  5983. /* libc_builtin signaled a "exception thrown by stdc++" trap */
  5984. if (has_exception
  5985. && strstr(uncaught_exception,
  5986. "exception thrown by stdc++")) {
  5987. wasm_set_exception(module, NULL);
  5988. /* setup internal c++ rethrow */
  5989. exception_tag_index = 0;
  5990. goto find_a_catch_handler;
  5991. }
  5992. /* when throw hits the end of a function it signals with a
  5993. * "uncaught wasm exception" trap */
  5994. if (has_exception
  5995. && strstr(uncaught_exception, "uncaught wasm exception")) {
  5996. wasm_set_exception(module, NULL);
  5997. exception_tag_index = POP_I32();
  5998. /* rethrow the exception into that frame */
  5999. goto find_a_catch_handler;
  6000. }
  6001. #endif /* WASM_ENABLE_EXCE_HANDLING != 0 */
  6002. goto got_exception;
  6003. }
  6004. }
  6005. else {
  6006. WASMFunction *cur_wasm_func = cur_func->u.func;
  6007. WASMFuncType *func_type = cur_wasm_func->func_type;
  6008. uint32 max_stack_cell_num = cur_wasm_func->max_stack_cell_num;
  6009. uint32 cell_num_of_local_stack;
  6010. #if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0
  6011. uint32 local_cell_idx;
  6012. #endif
  6013. #if WASM_ENABLE_EXCE_HANDLING != 0
  6014. /* account for exception handlers, bundle them here */
  6015. uint32 eh_size =
  6016. cur_wasm_func->exception_handler_count * sizeof(uint8 *);
  6017. max_stack_cell_num += eh_size;
  6018. #endif
  6019. cell_num_of_local_stack = cur_func->param_cell_num
  6020. + cur_func->local_cell_num
  6021. + max_stack_cell_num;
  6022. all_cell_num = cell_num_of_local_stack
  6023. + cur_wasm_func->max_block_num
  6024. * (uint32)sizeof(WASMBranchBlock) / 4;
  6025. #if WASM_ENABLE_GC != 0
  6026. /* area of frame_ref */
  6027. all_cell_num += (cell_num_of_local_stack + 3) / 4;
  6028. #endif
  6029. /* param_cell_num, local_cell_num, max_stack_cell_num and
  6030. max_block_num are all no larger than UINT16_MAX (checked
  6031. in loader), all_cell_num must be smaller than 1MB */
  6032. bh_assert(all_cell_num < 1 * BH_MB);
  6033. frame_size = wasm_interp_interp_frame_size(all_cell_num);
  6034. if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame))) {
  6035. frame = prev_frame;
  6036. goto got_exception;
  6037. }
  6038. /* Initialize the interpreter context. */
  6039. frame->function = cur_func;
  6040. frame_ip = wasm_get_func_code(cur_func);
  6041. frame_ip_end = wasm_get_func_code_end(cur_func);
  6042. frame_lp = frame->lp;
  6043. frame_sp = frame->sp_bottom =
  6044. frame_lp + cur_func->param_cell_num + cur_func->local_cell_num;
  6045. frame->sp_boundary = frame->sp_bottom + max_stack_cell_num;
  6046. frame_csp = frame->csp_bottom =
  6047. (WASMBranchBlock *)frame->sp_boundary;
  6048. frame->csp_boundary =
  6049. frame->csp_bottom + cur_wasm_func->max_block_num;
  6050. #if WASM_ENABLE_GC != 0
  6051. /* frame->sp and frame->ip are used during GC root set enumeration,
  6052. * so we must initialized these fields here */
  6053. frame->sp = frame_sp;
  6054. frame->ip = frame_ip;
  6055. frame_ref = (uint8 *)frame->csp_boundary;
  6056. init_frame_refs(frame_ref, (uint32)cell_num_of_local_stack,
  6057. cur_func);
  6058. #endif
  6059. /* Initialize the local variables */
  6060. memset(frame_lp + cur_func->param_cell_num, 0,
  6061. (uint32)(cur_func->local_cell_num * 4));
  6062. #if WASM_ENABLE_REF_TYPES != 0 && WASM_ENABLE_GC == 0
  6063. /* externref/funcref should be NULL_REF rather than 0 */
  6064. local_cell_idx = cur_func->param_cell_num;
  6065. for (i = 0; i < cur_wasm_func->local_count; i++) {
  6066. if (cur_wasm_func->local_types[i] == VALUE_TYPE_EXTERNREF
  6067. || cur_wasm_func->local_types[i] == VALUE_TYPE_FUNCREF) {
  6068. *(frame_lp + local_cell_idx) = NULL_REF;
  6069. }
  6070. local_cell_idx +=
  6071. wasm_value_type_cell_num(cur_wasm_func->local_types[i]);
  6072. }
  6073. #endif
  6074. /* Push function block as first block */
  6075. cell_num = func_type->ret_cell_num;
  6076. PUSH_CSP(LABEL_TYPE_FUNCTION, 0, cell_num, frame_ip_end - 1);
  6077. wasm_exec_env_set_cur_frame(exec_env, frame);
  6078. }
  6079. #if WASM_ENABLE_THREAD_MGR != 0
  6080. CHECK_SUSPEND_FLAGS();
  6081. #endif
  6082. HANDLE_OP_END();
  6083. }
  6084. return_func:
  6085. {
  6086. FREE_FRAME(exec_env, frame);
  6087. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  6088. if (!prev_frame->ip) {
  6089. /* Called from native. */
  6090. return;
  6091. }
  6092. RECOVER_CONTEXT(prev_frame);
  6093. #if WASM_ENABLE_EXCE_HANDLING != 0
  6094. if (wasm_get_exception(module)) {
  6095. wasm_set_exception(module, NULL);
  6096. exception_tag_index = POP_I32();
  6097. goto find_a_catch_handler;
  6098. }
  6099. #endif
  6100. HANDLE_OP_END();
  6101. }
  6102. #if WASM_ENABLE_SHARED_MEMORY != 0
  6103. unaligned_atomic:
  6104. wasm_set_exception(module, "unaligned atomic");
  6105. goto got_exception;
  6106. #endif
  6107. #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
  6108. || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
  6109. || WASM_ENABLE_BULK_MEMORY_OPT != 0
  6110. out_of_bounds:
  6111. wasm_set_exception(module, "out of bounds memory access");
  6112. #endif
  6113. got_exception:
  6114. #if WASM_ENABLE_DEBUG_INTERP != 0
  6115. if (wasm_exec_env_get_instance(exec_env) != NULL) {
  6116. uint8 *frame_ip_temp = frame_ip;
  6117. frame_ip = frame_ip_orig;
  6118. wasm_cluster_thread_send_signal(exec_env, WAMR_SIG_TRAP);
  6119. CHECK_SUSPEND_FLAGS();
  6120. frame_ip = frame_ip_temp;
  6121. }
  6122. #endif
  6123. SYNC_ALL_TO_FRAME();
  6124. return;
  6125. #if WASM_ENABLE_LABELS_AS_VALUES == 0
  6126. }
  6127. #else
  6128. FETCH_OPCODE_AND_DISPATCH();
  6129. #endif
  6130. }
  6131. #if WASM_ENABLE_GC != 0
  6132. bool
  6133. wasm_interp_traverse_gc_rootset(WASMExecEnv *exec_env, void *heap)
  6134. {
  6135. WASMInterpFrame *frame;
  6136. WASMObjectRef gc_obj;
  6137. int i;
  6138. frame = wasm_exec_env_get_cur_frame(exec_env);
  6139. for (; frame; frame = frame->prev_frame) {
  6140. uint8 *frame_ref = get_frame_ref(frame);
  6141. for (i = 0; i < frame->sp - frame->lp; i++) {
  6142. if (frame_ref[i]) {
  6143. gc_obj = GET_REF_FROM_ADDR(frame->lp + i);
  6144. if (wasm_obj_is_created_from_heap(gc_obj)) {
  6145. if (mem_allocator_add_root((mem_allocator_t)heap, gc_obj)) {
  6146. return false;
  6147. }
  6148. }
  6149. #if UINTPTR_MAX == UINT64_MAX
  6150. bh_assert(frame_ref[i + 1]);
  6151. i++;
  6152. #endif
  6153. }
  6154. }
  6155. }
  6156. return true;
  6157. }
  6158. #endif
  6159. #if WASM_ENABLE_FAST_JIT != 0
  6160. /*
  6161. * ASAN is not designed to work with custom stack unwind or other low-level
  6162. * things. Ignore a function that does some low-level magic. (e.g. walking
  6163. * through the thread's stack bypassing the frame boundaries)
  6164. */
  6165. #if defined(__GNUC__) || defined(__clang__)
  6166. __attribute__((no_sanitize_address))
  6167. #endif
  6168. static void
  6169. fast_jit_call_func_bytecode(WASMModuleInstance *module_inst,
  6170. WASMExecEnv *exec_env,
  6171. WASMFunctionInstance *function,
  6172. WASMInterpFrame *frame)
  6173. {
  6174. JitGlobals *jit_globals = jit_compiler_get_jit_globals();
  6175. JitInterpSwitchInfo info;
  6176. WASMModule *module = module_inst->module;
  6177. WASMFuncType *func_type = function->u.func->func_type;
  6178. uint8 type = func_type->result_count
  6179. ? func_type->types[func_type->param_count]
  6180. : VALUE_TYPE_VOID;
  6181. uint32 func_idx = (uint32)(function - module_inst->e->functions);
  6182. uint32 func_idx_non_import = func_idx - module->import_function_count;
  6183. int32 action;
  6184. #if WASM_ENABLE_REF_TYPES != 0
  6185. if (type == VALUE_TYPE_EXTERNREF || type == VALUE_TYPE_FUNCREF)
  6186. type = VALUE_TYPE_I32;
  6187. #endif
  6188. #if WASM_ENABLE_LAZY_JIT != 0
  6189. if (!jit_compiler_compile(module, func_idx)) {
  6190. wasm_set_exception(module_inst, "failed to compile fast jit function");
  6191. return;
  6192. }
  6193. #endif
  6194. bh_assert(jit_compiler_is_compiled(module, func_idx));
  6195. /* Switch to jitted code to call the jit function */
  6196. info.out.ret.last_return_type = type;
  6197. info.frame = frame;
  6198. frame->jitted_return_addr =
  6199. (uint8 *)jit_globals->return_to_interp_from_jitted;
  6200. action = jit_interp_switch_to_jitted(
  6201. exec_env, &info, func_idx,
  6202. module_inst->fast_jit_func_ptrs[func_idx_non_import]);
  6203. bh_assert(action == JIT_INTERP_ACTION_NORMAL
  6204. || (action == JIT_INTERP_ACTION_THROWN
  6205. && wasm_copy_exception(
  6206. (WASMModuleInstance *)exec_env->module_inst, NULL)));
  6207. /* Get the return values form info.out.ret */
  6208. if (func_type->result_count) {
  6209. switch (type) {
  6210. case VALUE_TYPE_I32:
  6211. *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
  6212. break;
  6213. case VALUE_TYPE_I64:
  6214. *(frame->sp - function->ret_cell_num) = info.out.ret.ival[0];
  6215. *(frame->sp - function->ret_cell_num + 1) =
  6216. info.out.ret.ival[1];
  6217. break;
  6218. case VALUE_TYPE_F32:
  6219. *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
  6220. break;
  6221. case VALUE_TYPE_F64:
  6222. *(frame->sp - function->ret_cell_num) = info.out.ret.fval[0];
  6223. *(frame->sp - function->ret_cell_num + 1) =
  6224. info.out.ret.fval[1];
  6225. break;
  6226. default:
  6227. bh_assert(0);
  6228. break;
  6229. }
  6230. }
  6231. (void)action;
  6232. (void)func_idx;
  6233. }
  6234. #endif /* end of WASM_ENABLE_FAST_JIT != 0 */
  6235. #if WASM_ENABLE_JIT != 0
  6236. #if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 \
  6237. || WASM_ENABLE_AOT_STACK_FRAME != 0
  6238. #if WASM_ENABLE_GC == 0
  6239. bool
  6240. llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
  6241. {
  6242. WASMModuleInstance *module_inst =
  6243. (WASMModuleInstance *)exec_env->module_inst;
  6244. WASMInterpFrame *cur_frame, *frame;
  6245. uint32 size = (uint32)offsetof(WASMInterpFrame, lp);
  6246. bh_assert(module_inst->module_type == Wasm_Module_Bytecode);
  6247. cur_frame = exec_env->cur_frame;
  6248. if (!cur_frame)
  6249. frame = (WASMInterpFrame *)exec_env->wasm_stack.bottom;
  6250. else
  6251. frame = (WASMInterpFrame *)((uint8 *)cur_frame + size);
  6252. if ((uint8 *)frame + size > exec_env->wasm_stack.top_boundary) {
  6253. wasm_set_exception(module_inst, "wasm operand stack overflow");
  6254. return false;
  6255. }
  6256. frame->function = module_inst->e->functions + func_index;
  6257. /* No need to initialize ip, it will be committed in jitted code
  6258. when needed */
  6259. /* frame->ip = NULL; */
  6260. frame->prev_frame = cur_frame;
  6261. #if WASM_ENABLE_PERF_PROFILING != 0
  6262. frame->time_started = os_time_thread_cputime_us();
  6263. #endif
  6264. #if WASM_ENABLE_MEMORY_PROFILING != 0
  6265. {
  6266. uint32 wasm_stack_used =
  6267. (uint8 *)frame + size - exec_env->wasm_stack.bottom;
  6268. if (wasm_stack_used > exec_env->max_wasm_stack_used)
  6269. exec_env->max_wasm_stack_used = wasm_stack_used;
  6270. }
  6271. #endif
  6272. exec_env->cur_frame = frame;
  6273. return true;
  6274. }
  6275. static inline void
  6276. llvm_jit_free_frame_internal(WASMExecEnv *exec_env)
  6277. {
  6278. WASMInterpFrame *frame = exec_env->cur_frame;
  6279. WASMInterpFrame *prev_frame = frame->prev_frame;
  6280. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  6281. #if WASM_ENABLE_PERF_PROFILING != 0
  6282. if (frame->function) {
  6283. uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started;
  6284. frame->function->total_exec_time += time_elapsed;
  6285. frame->function->total_exec_cnt++;
  6286. /* parent function */
  6287. if (prev_frame)
  6288. prev_frame->function->children_exec_time += time_elapsed;
  6289. }
  6290. #endif
  6291. exec_env->cur_frame = prev_frame;
  6292. }
  6293. void
  6294. llvm_jit_free_frame(WASMExecEnv *exec_env)
  6295. {
  6296. llvm_jit_free_frame_internal(exec_env);
  6297. }
  6298. #else /* else of WASM_ENABLE_GC == 0 */
  6299. bool
  6300. llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index)
  6301. {
  6302. WASMModuleInstance *module_inst;
  6303. WASMModule *module;
  6304. WASMInterpFrame *frame;
  6305. uint32 size, max_local_cell_num, max_stack_cell_num;
  6306. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  6307. module_inst = (WASMModuleInstance *)exec_env->module_inst;
  6308. module = module_inst->module;
  6309. if (func_index >= func_index - module->import_function_count) {
  6310. WASMFunction *func =
  6311. module->functions[func_index - module->import_function_count];
  6312. max_local_cell_num = func->param_cell_num + func->local_cell_num;
  6313. max_stack_cell_num = func->max_stack_cell_num;
  6314. }
  6315. else {
  6316. WASMFunctionImport *func =
  6317. &((module->import_functions + func_index)->u.function);
  6318. max_local_cell_num = func->func_type->param_cell_num > 2
  6319. ? func->func_type->param_cell_num
  6320. : 2;
  6321. max_stack_cell_num = 0;
  6322. }
  6323. size =
  6324. wasm_interp_interp_frame_size(max_local_cell_num + max_stack_cell_num);
  6325. frame = wasm_exec_env_alloc_wasm_frame(exec_env, size);
  6326. if (!frame) {
  6327. wasm_set_exception(module_inst, "wasm operand stack overflow");
  6328. return false;
  6329. }
  6330. frame->function = module_inst->e->functions + func_index;
  6331. #if WASM_ENABLE_PERF_PROFILING != 0
  6332. frame->time_started = os_time_thread_cputime_us();
  6333. #endif
  6334. frame->prev_frame = wasm_exec_env_get_cur_frame(exec_env);
  6335. /* No need to initialize ip, it will be committed in jitted code
  6336. when needed */
  6337. /* frame->ip = NULL; */
  6338. #if WASM_ENABLE_GC != 0
  6339. frame->sp = frame->lp + max_local_cell_num;
  6340. /* Initialize frame ref flags for import function */
  6341. if (func_index < module->import_function_count) {
  6342. WASMFunctionImport *func =
  6343. &((module->import_functions + func_index)->u.function);
  6344. WASMFuncType *func_type = func->func_type;
  6345. /* native function doesn't have operand stack and label stack */
  6346. uint8 *frame_ref = (uint8 *)frame->sp;
  6347. uint32 i, j, k, value_type_cell_num;
  6348. for (i = 0, j = 0; i < func_type->param_count; i++) {
  6349. if (wasm_is_type_reftype(func_type->types[i])
  6350. && !wasm_is_reftype_i31ref(func_type->types[i])) {
  6351. frame_ref[j++] = 1;
  6352. #if UINTPTR_MAX == UINT64_MAX
  6353. frame_ref[j++] = 1;
  6354. #endif
  6355. }
  6356. else {
  6357. value_type_cell_num =
  6358. wasm_value_type_cell_num(func_type->types[i]);
  6359. for (k = 0; k < value_type_cell_num; k++)
  6360. frame_ref[j++] = 0;
  6361. }
  6362. }
  6363. }
  6364. #endif
  6365. wasm_exec_env_set_cur_frame(exec_env, frame);
  6366. return true;
  6367. }
  6368. static inline void
  6369. llvm_jit_free_frame_internal(WASMExecEnv *exec_env)
  6370. {
  6371. WASMInterpFrame *frame;
  6372. WASMInterpFrame *prev_frame;
  6373. bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode);
  6374. frame = wasm_exec_env_get_cur_frame(exec_env);
  6375. prev_frame = frame->prev_frame;
  6376. #if WASM_ENABLE_PERF_PROFILING != 0
  6377. if (frame->function) {
  6378. uint64 time_elapsed = os_time_thread_cputime_us() - frame->time_started;
  6379. frame->function->total_exec_time += time_elapsed;
  6380. frame->function->total_exec_cnt++;
  6381. /* parent function */
  6382. if (prev_frame)
  6383. prev_frame->function->children_exec_time += time_elapsed;
  6384. }
  6385. #endif
  6386. wasm_exec_env_free_wasm_frame(exec_env, frame);
  6387. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  6388. }
  6389. void
  6390. llvm_jit_free_frame(WASMExecEnv *exec_env)
  6391. {
  6392. llvm_jit_free_frame_internal(exec_env);
  6393. }
  6394. #endif /* end of WASM_ENABLE_GC == 0 */
  6395. void
  6396. llvm_jit_frame_update_profile_info(WASMExecEnv *exec_env, bool alloc_frame)
  6397. {
  6398. #if WASM_ENABLE_PERF_PROFILING != 0
  6399. WASMInterpFrame *cur_frame = exec_env->cur_frame;
  6400. if (alloc_frame) {
  6401. cur_frame->time_started = os_time_thread_cputime_us();
  6402. }
  6403. else {
  6404. if (cur_frame->function) {
  6405. WASMInterpFrame *prev_frame = cur_frame->prev_frame;
  6406. uint64 time_elapsed =
  6407. os_time_thread_cputime_us() - cur_frame->time_started;
  6408. cur_frame->function->total_exec_time += time_elapsed;
  6409. cur_frame->function->total_exec_cnt++;
  6410. /* parent function */
  6411. if (prev_frame)
  6412. prev_frame->function->children_exec_time += time_elapsed;
  6413. }
  6414. }
  6415. #endif
  6416. #if WASM_ENABLE_MEMORY_PROFILING != 0
  6417. if (alloc_frame) {
  6418. #if WASM_ENABLE_GC == 0
  6419. uint32 wasm_stack_used = (uint8 *)exec_env->cur_frame
  6420. + (uint32)offsetof(WASMInterpFrame, lp)
  6421. - exec_env->wasm_stack.bottom;
  6422. #else
  6423. uint32 wasm_stack_used =
  6424. exec_env->wasm_stack.top - exec_env->wasm_stack.bottom;
  6425. #endif
  6426. if (wasm_stack_used > exec_env->max_wasm_stack_used)
  6427. exec_env->max_wasm_stack_used = wasm_stack_used;
  6428. }
  6429. #endif
  6430. }
  6431. #endif /* end of WASM_ENABLE_DUMP_CALL_STACK != 0 \
  6432. || WASM_ENABLE_PERF_PROFILING != 0 \
  6433. || WASM_ENABLE_AOT_STACK_FRAME != 0 */
  6434. static bool
  6435. llvm_jit_call_func_bytecode(WASMModuleInstance *module_inst,
  6436. WASMExecEnv *exec_env,
  6437. WASMFunctionInstance *function, uint32 argc,
  6438. uint32 argv[])
  6439. {
  6440. WASMFuncType *func_type = function->u.func->func_type;
  6441. uint32 result_count = func_type->result_count;
  6442. uint32 ext_ret_count = result_count > 1 ? result_count - 1 : 0;
  6443. uint32 func_idx = (uint32)(function - module_inst->e->functions);
  6444. bool ret = false;
  6445. #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) \
  6446. || (WASM_ENABLE_AOT_STACK_FRAME != 0)
  6447. if (!llvm_jit_alloc_frame(exec_env, function - module_inst->e->functions)) {
  6448. /* wasm operand stack overflow has been thrown,
  6449. no need to throw again */
  6450. return false;
  6451. }
  6452. #endif
  6453. if (ext_ret_count > 0) {
  6454. uint32 cell_num = 0, i;
  6455. uint8 *ext_ret_types = func_type->types + func_type->param_count + 1;
  6456. uint32 argv1_buf[32], *argv1 = argv1_buf, *ext_rets = NULL;
  6457. uint32 *argv_ret = argv;
  6458. uint32 ext_ret_cell = wasm_get_cell_num(ext_ret_types, ext_ret_count);
  6459. uint64 size;
  6460. /* Allocate memory all arguments */
  6461. size =
  6462. sizeof(uint32) * (uint64)argc /* original arguments */
  6463. + sizeof(void *)
  6464. * (uint64)ext_ret_count /* extra result values' addr */
  6465. + sizeof(uint32) * (uint64)ext_ret_cell; /* extra result values */
  6466. if (size > sizeof(argv1_buf)) {
  6467. if (size > UINT32_MAX
  6468. || !(argv1 = wasm_runtime_malloc((uint32)size))) {
  6469. wasm_set_exception(module_inst, "allocate memory failed");
  6470. ret = false;
  6471. goto fail;
  6472. }
  6473. }
  6474. /* Copy original arguments */
  6475. bh_memcpy_s(argv1, (uint32)size, argv, sizeof(uint32) * argc);
  6476. /* Get the extra result value's address */
  6477. ext_rets =
  6478. argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
  6479. /* Append each extra result value's address to original arguments */
  6480. for (i = 0; i < ext_ret_count; i++) {
  6481. *(uintptr_t *)(argv1 + argc + sizeof(void *) / sizeof(uint32) * i) =
  6482. (uintptr_t)(ext_rets + cell_num);
  6483. cell_num += wasm_value_type_cell_num(ext_ret_types[i]);
  6484. }
  6485. ret = wasm_runtime_invoke_native(
  6486. exec_env, module_inst->func_ptrs[func_idx], func_type, NULL, NULL,
  6487. argv1, argc, argv);
  6488. if (!ret) {
  6489. if (argv1 != argv1_buf)
  6490. wasm_runtime_free(argv1);
  6491. goto fail;
  6492. }
  6493. /* Get extra result values */
  6494. switch (func_type->types[func_type->param_count]) {
  6495. case VALUE_TYPE_I32:
  6496. case VALUE_TYPE_F32:
  6497. #if WASM_ENABLE_REF_TYPES != 0
  6498. case VALUE_TYPE_FUNCREF:
  6499. case VALUE_TYPE_EXTERNREF:
  6500. #endif
  6501. argv_ret++;
  6502. break;
  6503. case VALUE_TYPE_I64:
  6504. case VALUE_TYPE_F64:
  6505. argv_ret += 2;
  6506. break;
  6507. #if WASM_ENABLE_SIMD != 0
  6508. case VALUE_TYPE_V128:
  6509. argv_ret += 4;
  6510. break;
  6511. #endif
  6512. default:
  6513. bh_assert(0);
  6514. break;
  6515. }
  6516. ext_rets =
  6517. argv1 + argc + sizeof(void *) / sizeof(uint32) * ext_ret_count;
  6518. bh_memcpy_s(argv_ret, sizeof(uint32) * cell_num, ext_rets,
  6519. sizeof(uint32) * cell_num);
  6520. if (argv1 != argv1_buf)
  6521. wasm_runtime_free(argv1);
  6522. ret = true;
  6523. }
  6524. else {
  6525. #if WASM_ENABLE_QUICK_AOT_ENTRY != 0
  6526. /* Quick call if the quick jit entry is registered */
  6527. if (func_type->quick_aot_entry) {
  6528. void (*invoke_native)(void *func_ptr, void *exec_env, uint32 *argv,
  6529. uint32 *argv_ret) =
  6530. func_type->quick_aot_entry;
  6531. invoke_native(module_inst->func_ptrs[func_idx], exec_env, argv,
  6532. argv);
  6533. ret = !wasm_copy_exception(module_inst, NULL);
  6534. }
  6535. else
  6536. #endif
  6537. {
  6538. ret = wasm_runtime_invoke_native(
  6539. exec_env, module_inst->func_ptrs[func_idx], func_type, NULL,
  6540. NULL, argv, argc, argv);
  6541. if (ret)
  6542. ret = !wasm_copy_exception(module_inst, NULL);
  6543. }
  6544. }
  6545. fail:
  6546. #if (WASM_ENABLE_DUMP_CALL_STACK != 0) || (WASM_ENABLE_PERF_PROFILING != 0) \
  6547. || (WASM_ENABLE_AOT_STACK_FRAME != 0)
  6548. llvm_jit_free_frame_internal(exec_env);
  6549. #endif
  6550. return ret;
  6551. }
  6552. #endif /* end of WASM_ENABLE_JIT != 0 */
  6553. void
  6554. wasm_interp_call_wasm(WASMModuleInstance *module_inst, WASMExecEnv *exec_env,
  6555. WASMFunctionInstance *function, uint32 argc,
  6556. uint32 argv[])
  6557. {
  6558. WASMRuntimeFrame *frame = NULL, *prev_frame, *outs_area;
  6559. RunningMode running_mode =
  6560. wasm_runtime_get_running_mode((WASMModuleInstanceCommon *)module_inst);
  6561. /* Allocate sufficient cells for all kinds of return values. */
  6562. bool alloc_frame = true;
  6563. if (argc < function->param_cell_num) {
  6564. char buf[128];
  6565. snprintf(buf, sizeof(buf),
  6566. "invalid argument count %" PRIu32
  6567. ", must be no smaller than %u",
  6568. argc, function->param_cell_num);
  6569. wasm_set_exception(module_inst, buf);
  6570. return;
  6571. }
  6572. argc = function->param_cell_num;
  6573. #if defined(OS_ENABLE_HW_BOUND_CHECK) && WASM_DISABLE_STACK_HW_BOUND_CHECK == 0
  6574. /*
  6575. * wasm_runtime_detect_native_stack_overflow is done by
  6576. * call_wasm_with_hw_bound_check.
  6577. */
  6578. #else
  6579. if (!wasm_runtime_detect_native_stack_overflow(exec_env)) {
  6580. return;
  6581. }
  6582. #endif
  6583. if (!function->is_import_func) {
  6584. /* No need to alloc frame when calling LLVM JIT function */
  6585. #if WASM_ENABLE_JIT != 0
  6586. if (running_mode == Mode_LLVM_JIT) {
  6587. alloc_frame = false;
  6588. }
  6589. #if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0
  6590. else if (running_mode == Mode_Multi_Tier_JIT) {
  6591. /* Tier-up from Fast JIT to LLVM JIT, call llvm jit function
  6592. if it is compiled, else call fast jit function */
  6593. uint32 func_idx = (uint32)(function - module_inst->e->functions);
  6594. if (module_inst->module->func_ptrs_compiled
  6595. [func_idx - module_inst->module->import_function_count]) {
  6596. alloc_frame = false;
  6597. }
  6598. }
  6599. #endif
  6600. #endif
  6601. }
  6602. if (alloc_frame) {
  6603. unsigned all_cell_num =
  6604. function->ret_cell_num > 2 ? function->ret_cell_num : 2;
  6605. unsigned frame_size;
  6606. prev_frame = wasm_exec_env_get_cur_frame(exec_env);
  6607. /* This frame won't be used by JITed code, so only allocate interp
  6608. frame here. */
  6609. frame_size = wasm_interp_interp_frame_size(all_cell_num);
  6610. if (!(frame = ALLOC_FRAME(exec_env, frame_size, prev_frame)))
  6611. return;
  6612. outs_area = wasm_exec_env_wasm_stack_top(exec_env);
  6613. frame->function = NULL;
  6614. frame->ip = NULL;
  6615. /* There is no local variable. */
  6616. frame->sp = frame->lp + 0;
  6617. if ((uint8 *)(outs_area->lp + function->param_cell_num)
  6618. > exec_env->wasm_stack.top_boundary) {
  6619. wasm_set_exception(module_inst, "wasm operand stack overflow");
  6620. return;
  6621. }
  6622. if (argc > 0)
  6623. word_copy(outs_area->lp, argv, argc);
  6624. wasm_exec_env_set_cur_frame(exec_env, frame);
  6625. }
  6626. #if defined(os_writegsbase)
  6627. {
  6628. WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst);
  6629. if (memory_inst)
  6630. /* write base addr of linear memory to GS segment register */
  6631. os_writegsbase(memory_inst->memory_data);
  6632. }
  6633. #endif
  6634. if (function->is_import_func) {
  6635. #if WASM_ENABLE_MULTI_MODULE != 0
  6636. if (function->import_module_inst) {
  6637. wasm_interp_call_func_import(module_inst, exec_env, function,
  6638. frame);
  6639. }
  6640. else
  6641. #endif
  6642. {
  6643. /* it is a native function */
  6644. wasm_interp_call_func_native(module_inst, exec_env, function,
  6645. frame);
  6646. }
  6647. }
  6648. else {
  6649. if (running_mode == Mode_Interp) {
  6650. wasm_interp_call_func_bytecode(module_inst, exec_env, function,
  6651. frame);
  6652. }
  6653. #if WASM_ENABLE_FAST_JIT != 0
  6654. else if (running_mode == Mode_Fast_JIT) {
  6655. fast_jit_call_func_bytecode(module_inst, exec_env, function, frame);
  6656. }
  6657. #endif
  6658. #if WASM_ENABLE_JIT != 0
  6659. else if (running_mode == Mode_LLVM_JIT) {
  6660. llvm_jit_call_func_bytecode(module_inst, exec_env, function, argc,
  6661. argv);
  6662. }
  6663. #endif
  6664. #if WASM_ENABLE_LAZY_JIT != 0 && WASM_ENABLE_FAST_JIT != 0 \
  6665. && WASM_ENABLE_JIT != 0
  6666. else if (running_mode == Mode_Multi_Tier_JIT) {
  6667. /* Tier-up from Fast JIT to LLVM JIT, call llvm jit function
  6668. if it is compiled, else call fast jit function */
  6669. uint32 func_idx = (uint32)(function - module_inst->e->functions);
  6670. if (module_inst->module->func_ptrs_compiled
  6671. [func_idx - module_inst->module->import_function_count]) {
  6672. llvm_jit_call_func_bytecode(module_inst, exec_env, function,
  6673. argc, argv);
  6674. }
  6675. else {
  6676. fast_jit_call_func_bytecode(module_inst, exec_env, function,
  6677. frame);
  6678. }
  6679. }
  6680. #endif
  6681. else {
  6682. /* There should always be a supported running mode selected */
  6683. bh_assert(0);
  6684. }
  6685. (void)wasm_interp_call_func_bytecode;
  6686. #if WASM_ENABLE_FAST_JIT != 0
  6687. (void)fast_jit_call_func_bytecode;
  6688. #endif
  6689. }
  6690. /* Output the return value to the caller */
  6691. if (!wasm_copy_exception(module_inst, NULL)) {
  6692. if (alloc_frame) {
  6693. uint32 i;
  6694. for (i = 0; i < function->ret_cell_num; i++) {
  6695. argv[i] = *(frame->sp + i - function->ret_cell_num);
  6696. }
  6697. }
  6698. }
  6699. else {
  6700. #if WASM_ENABLE_DUMP_CALL_STACK != 0
  6701. if (wasm_interp_create_call_stack(exec_env)) {
  6702. wasm_interp_dump_call_stack(exec_env, true, NULL, 0);
  6703. }
  6704. #endif
  6705. }
  6706. if (alloc_frame) {
  6707. wasm_exec_env_set_cur_frame(exec_env, prev_frame);
  6708. FREE_FRAME(exec_env, frame);
  6709. }
  6710. }