aot_emit_aot_file.c 155 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603
  1. /*
  2. * Copyright (C) 2019 Intel Corporation. All rights reserved.
  3. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  4. */
  5. #include "aot_emit_aot_file.h"
  6. #include "../aot/aot_runtime.h"
  7. #define PUT_U64_TO_ADDR(addr, value) \
  8. do { \
  9. union { \
  10. uint64 val; \
  11. uint32 parts[2]; \
  12. } u; \
  13. u.val = (value); \
  14. ((uint32 *)(addr))[0] = u.parts[0]; \
  15. ((uint32 *)(addr))[1] = u.parts[1]; \
  16. } while (0)
  17. #define CHECK_SIZE(size) \
  18. do { \
  19. if (size == (uint32)-1) { \
  20. aot_set_last_error("get symbol size failed."); \
  21. return (uint32)-1; \
  22. } \
  23. } while (0)
  24. /* Internal function in object file */
  25. typedef struct AOTObjectFunc {
  26. char *func_name;
  27. /* text offset of aot_func#n */
  28. uint64 text_offset;
  29. /* text offset of aot_func_internal#n */
  30. uint64 text_offset_of_aot_func_internal;
  31. } AOTObjectFunc;
  32. /* Symbol table list node */
  33. typedef struct AOTSymbolNode {
  34. struct AOTSymbolNode *next;
  35. uint32 str_len;
  36. char *symbol;
  37. } AOTSymbolNode;
  38. typedef struct AOTSymbolList {
  39. AOTSymbolNode *head;
  40. AOTSymbolNode *end;
  41. uint32 len;
  42. } AOTSymbolList;
  43. /* AOT object data */
  44. typedef struct AOTObjectData {
  45. AOTCompContext *comp_ctx;
  46. LLVMMemoryBufferRef mem_buf;
  47. LLVMBinaryRef binary;
  48. AOTTargetInfo target_info;
  49. void *text;
  50. uint32 text_size;
  51. void *text_unlikely;
  52. uint32 text_unlikely_size;
  53. void *text_hot;
  54. uint32 text_hot_size;
  55. /* literal data and size */
  56. void *literal;
  57. uint32 literal_size;
  58. AOTObjectDataSection *data_sections;
  59. uint32 data_sections_count;
  60. AOTObjectFunc *funcs;
  61. uint32 func_count;
  62. AOTSymbolList symbol_list;
  63. AOTRelocationGroup *relocation_groups;
  64. uint32 relocation_group_count;
  65. const char *stack_sizes_section_name;
  66. uint32 stack_sizes_offset;
  67. uint32 *stack_sizes;
  68. } AOTObjectData;
  69. #if 0
  70. static void dump_buf(uint8 *buf, uint32 size, char *title)
  71. {
  72. int i;
  73. printf("------ %s -------", title);
  74. for (i = 0; i < size; i++) {
  75. if ((i % 16) == 0)
  76. printf("\n");
  77. printf("%02x ", (unsigned char)buf[i]);
  78. }
  79. printf("\n\n");
  80. }
  81. #endif
  82. static bool
  83. is_32bit_binary(const AOTObjectData *obj_data)
  84. {
  85. /* bit 1: 0 is 32-bit, 1 is 64-bit */
  86. return obj_data->target_info.bin_type & 2 ? false : true;
  87. }
  88. static bool
  89. is_little_endian_binary(const AOTObjectData *obj_data)
  90. {
  91. /* bit 0: 0 is little-endian, 1 is big-endian */
  92. return obj_data->target_info.bin_type & 1 ? false : true;
  93. }
  94. static bool
  95. str_starts_with(const char *str, const char *prefix)
  96. {
  97. size_t len_pre = strlen(prefix), len_str = strlen(str);
  98. return (len_str >= len_pre) && !memcmp(str, prefix, len_pre);
  99. }
  100. static uint32
  101. get_file_header_size()
  102. {
  103. /* magic number (4 bytes) + version (4 bytes) */
  104. return sizeof(uint32) + sizeof(uint32);
  105. }
  106. static uint32
  107. get_string_size(AOTCompContext *comp_ctx, const char *s)
  108. {
  109. /* string size (2 bytes) + string content + '\0' */
  110. return (uint32)sizeof(uint16) + (uint32)strlen(s) + 1;
  111. }
  112. static uint32
  113. get_target_info_section_size()
  114. {
  115. return sizeof(AOTTargetInfo);
  116. }
  117. static uint32
  118. get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data,
  119. InitializerExpression *expr);
  120. static uint32
  121. get_mem_init_data_size(AOTCompContext *comp_ctx, AOTMemInitData *mem_init_data)
  122. {
  123. /* init expr type (4 bytes)
  124. * + init expr value (4 bytes, valid value can only be i32/get_global)
  125. * + byte count (4 bytes) + bytes */
  126. uint32 total_size =
  127. (uint32)(get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  128. &mem_init_data->offset)
  129. + sizeof(uint32) + mem_init_data->byte_count);
  130. /* bulk_memory enabled:
  131. is_passive (4 bytes) + memory_index (4 bytes)
  132. bulk memory disabled:
  133. placeholder (4 bytes) + placeholder (4 bytes)
  134. */
  135. total_size += (sizeof(uint32) + sizeof(uint32));
  136. return total_size;
  137. }
  138. static uint32
  139. get_mem_init_data_list_size(AOTCompContext *comp_ctx,
  140. AOTMemInitData **mem_init_data_list,
  141. uint32 mem_init_data_count)
  142. {
  143. AOTMemInitData **mem_init_data = mem_init_data_list;
  144. uint32 size = 0, i;
  145. for (i = 0; i < mem_init_data_count; i++, mem_init_data++) {
  146. size = align_uint(size, 4);
  147. size += get_mem_init_data_size(comp_ctx, *mem_init_data);
  148. }
  149. return size;
  150. }
  151. static uint32
  152. get_import_memory_size(AOTCompData *comp_data)
  153. {
  154. /* currently we only emit import_memory_count = 0 */
  155. return sizeof(uint32);
  156. }
  157. static uint32
  158. get_memory_size(AOTCompData *comp_data)
  159. {
  160. /* memory_count + count * (flags + num_bytes_per_page +
  161. init_page_count + max_page_count) */
  162. return (uint32)(sizeof(uint32)
  163. + comp_data->memory_count * sizeof(uint32) * 4);
  164. }
  165. static uint32
  166. get_mem_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  167. {
  168. /* import_memory_size + memory_size
  169. + init_data_count + init_data_list */
  170. return get_import_memory_size(comp_data) + get_memory_size(comp_data)
  171. + (uint32)sizeof(uint32)
  172. + get_mem_init_data_list_size(comp_ctx,
  173. comp_data->mem_init_data_list,
  174. comp_data->mem_init_data_count);
  175. }
  176. static uint32
  177. get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data,
  178. InitializerExpression *expr)
  179. {
  180. /* init_expr_type */
  181. uint32 size = sizeof(uint32);
  182. #if WASM_ENABLE_GC != 0
  183. WASMModule *module = comp_data->wasm_module;
  184. #endif
  185. /* + init value size */
  186. switch (expr->init_expr_type) {
  187. case INIT_EXPR_NONE:
  188. /* no init value, used in table initializer */
  189. break;
  190. case INIT_EXPR_TYPE_I32_CONST:
  191. case INIT_EXPR_TYPE_F32_CONST:
  192. case INIT_EXPR_TYPE_GET_GLOBAL:
  193. size += sizeof(uint32);
  194. break;
  195. case INIT_EXPR_TYPE_I64_CONST:
  196. case INIT_EXPR_TYPE_F64_CONST:
  197. size += sizeof(uint64);
  198. break;
  199. case INIT_EXPR_TYPE_V128_CONST:
  200. size += sizeof(uint64) * 2;
  201. break;
  202. case INIT_EXPR_TYPE_FUNCREF_CONST:
  203. case INIT_EXPR_TYPE_REFNULL_CONST:
  204. /* ref_index */
  205. size += sizeof(uint32);
  206. break;
  207. #if WASM_ENABLE_GC != 0
  208. case INIT_EXPR_TYPE_I31_NEW:
  209. /* i32 */
  210. size += sizeof(uint32);
  211. break;
  212. case INIT_EXPR_TYPE_STRUCT_NEW:
  213. {
  214. uint32 i;
  215. WASMStructNewInitValues *struct_new_init_values =
  216. (WASMStructNewInitValues *)expr->u.data;
  217. /* type_index + field_count + fields */
  218. size += sizeof(uint32) + sizeof(uint32);
  219. bh_assert(struct_new_init_values->type_idx < module->type_count);
  220. for (i = 0; i < struct_new_init_values->count; i++) {
  221. WASMStructType *struct_type =
  222. (WASMStructType *)
  223. module->types[struct_new_init_values->type_idx];
  224. uint32 field_size;
  225. bh_assert(struct_type);
  226. bh_assert(struct_type->field_count
  227. == struct_new_init_values->count);
  228. field_size = wasm_value_type_size_internal(
  229. struct_type->fields[i].field_type, comp_ctx->pointer_size);
  230. if (field_size < sizeof(uint32))
  231. field_size = sizeof(uint32);
  232. size += field_size;
  233. }
  234. break;
  235. }
  236. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  237. /* type_index */
  238. size += sizeof(uint32);
  239. break;
  240. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  241. /* array_elem_type + type_index + len */
  242. size += sizeof(uint32) * 3;
  243. break;
  244. case INIT_EXPR_TYPE_ARRAY_NEW:
  245. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  246. {
  247. WASMArrayNewInitValues *array_new_init_values =
  248. (WASMArrayNewInitValues *)expr->u.data;
  249. WASMArrayType *array_type = NULL;
  250. uint32 value_count;
  251. array_type =
  252. (WASMArrayType *)module->types[array_new_init_values->type_idx];
  253. bh_assert(array_type);
  254. bh_assert(array_new_init_values->type_idx < module->type_count);
  255. value_count =
  256. (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED)
  257. ? array_new_init_values->length
  258. : 1;
  259. /* array_elem_type + type_index + len + elems */
  260. size += sizeof(uint32) * 3
  261. + wasm_value_type_size_internal(array_type->elem_type,
  262. comp_ctx->pointer_size)
  263. * value_count;
  264. break;
  265. }
  266. #endif /* end of WASM_ENABLE_GC != 0 */
  267. default:
  268. bh_assert(0);
  269. }
  270. return size;
  271. }
  272. static uint32
  273. get_table_init_data_size(AOTCompContext *comp_ctx,
  274. AOTTableInitData *table_init_data)
  275. {
  276. uint32 size, i;
  277. /*
  278. * mode (4 bytes), elem_type (4 bytes)
  279. *
  280. * table_index(4 bytes) + init expr type (4 bytes) + init expr value (8
  281. * bytes)
  282. */
  283. size = (uint32)(sizeof(uint32) * 2 + sizeof(uint32) + sizeof(uint32)
  284. + sizeof(uint64))
  285. /* Size of WasmRefType - inner padding (ref type + nullable +
  286. heap_type) */
  287. + 8;
  288. /* + value count/func index count (4 bytes) + init_values */
  289. size += sizeof(uint32);
  290. for (i = 0; i < table_init_data->value_count; i++) {
  291. size += get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  292. &table_init_data->init_values[i]);
  293. }
  294. return size;
  295. }
  296. static uint32
  297. get_table_init_data_list_size(AOTCompContext *comp_ctx,
  298. AOTTableInitData **table_init_data_list,
  299. uint32 table_init_data_count)
  300. {
  301. /*
  302. * ------------------------------
  303. * | table_init_data_count
  304. * ------------------------------
  305. * | | U32 mode
  306. * | AOTTableInitData[N] | U32 elem_type
  307. * | | U32 table_index
  308. * | | U32 offset.init_expr_type
  309. * | | U64 offset.u.i64
  310. * | | U32 func_index_count / elem_count
  311. * | | UINTPTR [func_index_count] / [elem_count]
  312. * ------------------------------
  313. */
  314. AOTTableInitData **table_init_data = table_init_data_list;
  315. uint32 size = 0, i;
  316. /* table_init_data_count(4 bytes) */
  317. size = (uint32)sizeof(uint32);
  318. for (i = 0; i < table_init_data_count; i++, table_init_data++) {
  319. size = align_uint(size, 4);
  320. size += get_table_init_data_size(comp_ctx, *table_init_data);
  321. }
  322. return size;
  323. }
  324. static uint32
  325. get_import_table_size(const AOTCompContext *comp_ctx,
  326. const AOTCompData *comp_data)
  327. {
  328. /*
  329. * ------------------------------
  330. * | import_table_count
  331. * ------------------------------
  332. * | | U8 elem_type
  333. * | | U8 flags
  334. * | | U8 possible_grow
  335. * | AOTImportTable[N] | U8 elem_ref_type.nullable (for GC only)
  336. * | | U32 init_size
  337. * | | U32 max_size
  338. * | | U32 elem_ref_type.heap_type (for GC only)
  339. * ------------------------------
  340. */
  341. uint32 size = 0, i;
  342. size = (uint32)sizeof(uint32);
  343. for (i = 0; i < comp_data->import_table_count; i++) {
  344. size += sizeof(uint32) * 3;
  345. #if WASM_ENABLE_GC != 0
  346. if (comp_ctx->enable_gc
  347. && comp_data->import_tables[i].table_type.elem_ref_type)
  348. size += sizeof(uint32);
  349. #endif
  350. }
  351. return size;
  352. }
  353. static uint32
  354. get_table_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data)
  355. {
  356. /*
  357. * ------------------------------
  358. * | table_count
  359. * ------------------------------
  360. * | | U8 elem_type
  361. * | | U8 flags
  362. * | | U8 possible_grow
  363. * | AOTTable[N] | U8 elem_ref_type.nullable (for GC only)
  364. * | | U32 init_size
  365. * | | U32 max_size
  366. * | | U32 elem_ref_type.heap_type (for GC only)
  367. * | | N init_expr (for GC only)
  368. * ------------------------------
  369. */
  370. uint32 size = 0, i;
  371. size = (uint32)sizeof(uint32);
  372. for (i = 0; i < comp_data->table_count; i++) {
  373. size += sizeof(uint32) * 3;
  374. #if WASM_ENABLE_GC != 0
  375. if (comp_ctx->enable_gc) {
  376. if (comp_data->tables[i].table_type.elem_ref_type) {
  377. size += sizeof(uint32);
  378. }
  379. size += get_init_expr_size(comp_ctx, comp_data,
  380. &comp_data->tables[i].init_expr);
  381. }
  382. #endif
  383. }
  384. return size;
  385. }
  386. static uint32
  387. get_table_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  388. {
  389. /*
  390. * ------------------------------
  391. * | import_table_count
  392. * ------------------------------
  393. * |
  394. * | AOTImportTable[import_table_count]
  395. * |
  396. * ------------------------------
  397. * | table_count
  398. * ------------------------------
  399. * |
  400. * | AOTTable[table_count]
  401. * |
  402. * ------------------------------
  403. * | table_init_data_count
  404. * ------------------------------
  405. * |
  406. * | AOTTableInitData*[table_init_data_count]
  407. * |
  408. * ------------------------------
  409. */
  410. return get_import_table_size(comp_ctx, comp_data)
  411. + get_table_size(comp_ctx, comp_data)
  412. + get_table_init_data_list_size(comp_ctx,
  413. comp_data->table_init_data_list,
  414. comp_data->table_init_data_count);
  415. }
  416. static uint32
  417. get_func_type_size(AOTCompContext *comp_ctx, AOTFuncType *func_type)
  418. {
  419. #if WASM_ENABLE_GC != 0
  420. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  421. + rec_count + rec_idx + param count + result count
  422. + ref_type_map_count + types + context of ref_type_map */
  423. if (comp_ctx->enable_gc) {
  424. uint32 size = 0;
  425. /* type flag */
  426. size += sizeof(func_type->base_type.type_flag);
  427. /* equivalence type flag + is_sub_final */
  428. size += sizeof(uint16);
  429. /* parent_type_idx */
  430. size += sizeof(func_type->base_type.parent_type_idx);
  431. /* rec_count */
  432. size += sizeof(func_type->base_type.rec_count);
  433. /* rec_idx */
  434. size += sizeof(func_type->base_type.rec_idx);
  435. /* param count */
  436. size += sizeof(func_type->param_count);
  437. /* result count */
  438. size += sizeof(func_type->result_count);
  439. /* ref_type_map_count */
  440. size += sizeof(func_type->ref_type_map_count);
  441. /* param and result types */
  442. size += func_type->param_count + func_type->result_count;
  443. /* align size */
  444. size = align_uint(size, 4);
  445. /* ref_type_map */
  446. size += func_type->ref_type_map_count * 8;
  447. return size;
  448. }
  449. else
  450. #endif
  451. {
  452. /* type flag + param count + result count + types */
  453. return (uint32)sizeof(uint16) * 3 + func_type->param_count
  454. + func_type->result_count;
  455. }
  456. }
  457. #if WASM_ENABLE_GC != 0
  458. static uint32
  459. get_struct_type_size(AOTCompContext *comp_ctx, AOTStructType *struct_type)
  460. {
  461. uint32 size = 0;
  462. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  463. + rec_count + rec_idx + field count + fields */
  464. /* type flag */
  465. size += sizeof(struct_type->base_type.type_flag);
  466. /* equivalence type flag + is_sub_final */
  467. size += sizeof(uint16);
  468. /* parent_type_idx */
  469. size += sizeof(struct_type->base_type.parent_type_idx);
  470. /* rec_count */
  471. size += sizeof(struct_type->base_type.rec_count);
  472. /* rec_idx */
  473. size += sizeof(struct_type->base_type.rec_idx);
  474. /* field count */
  475. size += sizeof(struct_type->field_count);
  476. /* field types */
  477. size += struct_type->field_count * 2;
  478. /* ref_type_map_count */
  479. size += sizeof(struct_type->ref_type_map_count);
  480. size = align_uint(size, 4);
  481. /* ref_type_map */
  482. size += struct_type->ref_type_map_count * 8;
  483. return size;
  484. }
  485. static uint32
  486. get_array_type_size(AOTCompContext *comp_ctx, AOTArrayType *array_type)
  487. {
  488. uint32 size = 0;
  489. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  490. + rec_count + rec_idx + elem_flags + elem_type + elem_ref_type */
  491. /* type flag */
  492. size += sizeof(array_type->base_type.type_flag);
  493. /* equivalence type flag + is_sub_final */
  494. size += sizeof(uint16);
  495. /* parent_type_idx (u32) */
  496. size += sizeof(array_type->base_type.parent_type_idx);
  497. /* rec_count */
  498. size += sizeof(array_type->base_type.rec_count);
  499. /* rec_idx */
  500. size += sizeof(array_type->base_type.rec_idx);
  501. /* elem_flags (u16) */
  502. size += sizeof(array_type->elem_flags);
  503. /* elem_type (u8) */
  504. size += sizeof(array_type->elem_type);
  505. /* elem_ref_type */
  506. if (array_type->elem_ref_type) {
  507. /* nullable (u8) */
  508. size += sizeof(uint8);
  509. /* heap type (u32) */
  510. size += sizeof(uint32);
  511. }
  512. return size;
  513. }
  514. #endif
  515. static uint32
  516. get_type_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  517. {
  518. /* Initial size with size of type count */
  519. uint32 size = 4;
  520. uint32 i;
  521. #if WASM_ENABLE_GC != 0
  522. if (comp_ctx->enable_gc) {
  523. for (i = 0; i < comp_data->type_count; i++) {
  524. uint32 j;
  525. size = align_uint(size, 4);
  526. /* Emit simple info if there is an equivalence type */
  527. for (j = 0; j < i; j++) {
  528. if (comp_data->types[j] == comp_data->types[i]) {
  529. /* type_flag (2 bytes) + equivalence type flag (1 byte)
  530. + padding (1 byte) + equivalence type index */
  531. size += 8;
  532. break;
  533. }
  534. }
  535. if (j < i)
  536. continue;
  537. if (comp_data->types[i]->type_flag == WASM_TYPE_FUNC)
  538. size += get_func_type_size(comp_ctx,
  539. (AOTFuncType *)comp_data->types[i]);
  540. else if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT)
  541. size += get_struct_type_size(
  542. comp_ctx, (AOTStructType *)comp_data->types[i]);
  543. else if (comp_data->types[i]->type_flag == WASM_TYPE_ARRAY)
  544. size += get_array_type_size(
  545. comp_ctx, (AOTArrayType *)comp_data->types[i]);
  546. else
  547. bh_assert(0);
  548. }
  549. }
  550. else
  551. #endif
  552. {
  553. for (i = 0; i < comp_data->type_count; i++) {
  554. size = align_uint(size, 4);
  555. size += get_func_type_size(comp_ctx,
  556. (AOTFuncType *)comp_data->types[i]);
  557. }
  558. }
  559. return size;
  560. }
  561. static uint32
  562. get_import_global_size(AOTCompContext *comp_ctx, AOTImportGlobal *import_global)
  563. {
  564. /* type (1 byte) + is_mutable (1 byte) + module_name + global_name */
  565. uint32 size = (uint32)sizeof(uint8) * 2
  566. + get_string_size(comp_ctx, import_global->module_name);
  567. size = align_uint(size, 2);
  568. size += get_string_size(comp_ctx, import_global->global_name);
  569. return size;
  570. }
  571. static uint32
  572. get_import_globals_size(AOTCompContext *comp_ctx,
  573. AOTImportGlobal *import_globals,
  574. uint32 import_global_count)
  575. {
  576. AOTImportGlobal *import_global = import_globals;
  577. uint32 size = 0, i;
  578. for (i = 0; i < import_global_count; i++, import_global++) {
  579. size = align_uint(size, 2);
  580. size += get_import_global_size(comp_ctx, import_global);
  581. }
  582. return size;
  583. }
  584. static uint32
  585. get_import_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  586. {
  587. /* import global count + import globals */
  588. return (uint32)sizeof(uint32)
  589. + get_import_globals_size(comp_ctx, comp_data->import_globals,
  590. comp_data->import_global_count);
  591. }
  592. static uint32
  593. get_global_size(AOTCompContext *comp_ctx, AOTGlobal *global)
  594. {
  595. /* type (1 byte) + is_mutable (1 byte) + padding (2 bytes)
  596. + init expr value (include init expr type) */
  597. return sizeof(uint8) * 2 + sizeof(uint8) * 2
  598. + get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  599. &global->init_expr);
  600. }
  601. static uint32
  602. get_globals_size(AOTCompContext *comp_ctx, AOTGlobal *globals,
  603. uint32 global_count)
  604. {
  605. AOTGlobal *global = globals;
  606. uint32 size = 0, i;
  607. for (i = 0; i < global_count; i++, global++) {
  608. size = align_uint(size, 4);
  609. size += get_global_size(comp_ctx, global);
  610. }
  611. return size;
  612. }
  613. static uint32
  614. get_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  615. {
  616. /* global count + globals */
  617. return (uint32)sizeof(uint32)
  618. + get_globals_size(comp_ctx, comp_data->globals,
  619. comp_data->global_count);
  620. }
  621. static uint32
  622. get_import_func_size(AOTCompContext *comp_ctx, AOTImportFunc *import_func)
  623. {
  624. /* type index (2 bytes) + module_name + func_name */
  625. uint32 size = (uint32)sizeof(uint16)
  626. + get_string_size(comp_ctx, import_func->module_name);
  627. size = align_uint(size, 2);
  628. size += get_string_size(comp_ctx, import_func->func_name);
  629. return size;
  630. }
  631. static uint32
  632. get_import_funcs_size(AOTCompContext *comp_ctx, AOTImportFunc *import_funcs,
  633. uint32 import_func_count)
  634. {
  635. AOTImportFunc *import_func = import_funcs;
  636. uint32 size = 0, i;
  637. for (i = 0; i < import_func_count; i++, import_func++) {
  638. size = align_uint(size, 2);
  639. size += get_import_func_size(comp_ctx, import_func);
  640. }
  641. return size;
  642. }
  643. static uint32
  644. get_import_func_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  645. {
  646. /* import func count + import funcs */
  647. return (uint32)sizeof(uint32)
  648. + get_import_funcs_size(comp_ctx, comp_data->import_funcs,
  649. comp_data->import_func_count);
  650. }
  651. static uint32
  652. get_object_data_sections_size(AOTCompContext *comp_ctx,
  653. AOTObjectDataSection *data_sections,
  654. uint32 data_sections_count)
  655. {
  656. AOTObjectDataSection *data_section = data_sections;
  657. uint32 size = 0, i;
  658. for (i = 0; i < data_sections_count; i++, data_section++) {
  659. /* name + size + data */
  660. size = align_uint(size, 2);
  661. size += get_string_size(comp_ctx, data_section->name);
  662. size = align_uint(size, 4);
  663. size += (uint32)sizeof(uint32);
  664. size += data_section->size;
  665. }
  666. return size;
  667. }
  668. static uint32
  669. get_object_data_section_info_size(AOTCompContext *comp_ctx,
  670. AOTObjectData *obj_data)
  671. {
  672. /* data sections count + data sections */
  673. return (uint32)sizeof(uint32)
  674. + get_object_data_sections_size(comp_ctx, obj_data->data_sections,
  675. obj_data->data_sections_count);
  676. }
  677. static uint32
  678. get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  679. AOTObjectData *obj_data)
  680. {
  681. uint32 size = 0;
  682. size += get_mem_info_size(comp_ctx, comp_data);
  683. size = align_uint(size, 4);
  684. size += get_table_info_size(comp_ctx, comp_data);
  685. size = align_uint(size, 4);
  686. size += get_type_info_size(comp_ctx, comp_data);
  687. size = align_uint(size, 4);
  688. size += get_import_global_info_size(comp_ctx, comp_data);
  689. size = align_uint(size, 4);
  690. size += get_global_info_size(comp_ctx, comp_data);
  691. size = align_uint(size, 4);
  692. size += get_import_func_info_size(comp_ctx, comp_data);
  693. /* func count + start func index */
  694. size = align_uint(size, 4);
  695. size += (uint32)sizeof(uint32) * 2;
  696. /* aux data/heap/stack data */
  697. size += sizeof(uint32) * 10;
  698. size += get_object_data_section_info_size(comp_ctx, obj_data);
  699. return size;
  700. }
  701. static uint32
  702. get_text_section_size(AOTObjectData *obj_data)
  703. {
  704. return sizeof(uint32) + align_uint(obj_data->literal_size, 4)
  705. + align_uint(obj_data->text_size, 4)
  706. + align_uint(obj_data->text_unlikely_size, 4)
  707. + align_uint(obj_data->text_hot_size, 4);
  708. }
  709. static uint32
  710. get_func_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  711. AOTObjectData *obj_data)
  712. {
  713. uint32 size = 0;
  714. /* text offsets */
  715. if (is_32bit_binary(obj_data))
  716. size = (uint32)sizeof(uint32) * comp_data->func_count;
  717. else
  718. size = (uint32)sizeof(uint64) * comp_data->func_count;
  719. /* function type indexes */
  720. size += (uint32)sizeof(uint32) * comp_data->func_count;
  721. /* max_local_cell_nums */
  722. size += (uint32)sizeof(uint32) * comp_data->func_count;
  723. /* max_stack_cell_nums */
  724. size += (uint32)sizeof(uint32) * comp_data->func_count;
  725. #if WASM_ENABLE_GC != 0
  726. /* func_local_ref_flags */
  727. if (comp_ctx->enable_gc) {
  728. AOTFuncType *func_type;
  729. uint32 i, j, local_ref_flags_cell_num;
  730. for (i = 0; i < comp_data->import_func_count; i++) {
  731. func_type = comp_data->import_funcs[i].func_type;
  732. /* recalculate cell_num based on target pointer size */
  733. local_ref_flags_cell_num = 0;
  734. for (j = 0; j < func_type->param_count; j++) {
  735. local_ref_flags_cell_num += wasm_value_type_cell_num_internal(
  736. func_type->types[j], comp_ctx->pointer_size);
  737. }
  738. local_ref_flags_cell_num =
  739. local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2;
  740. size = align_uint(size, 4);
  741. size += (uint32)sizeof(uint32);
  742. size += (uint32)sizeof(uint8) * local_ref_flags_cell_num;
  743. }
  744. for (i = 0; i < comp_data->func_count; i++) {
  745. func_type = comp_data->funcs[i]->func_type;
  746. local_ref_flags_cell_num = comp_data->funcs[i]->param_cell_num
  747. + comp_data->funcs[i]->local_cell_num;
  748. size = align_uint(size, 4);
  749. size += (uint32)sizeof(uint32);
  750. size += (uint32)sizeof(uint8) * local_ref_flags_cell_num;
  751. }
  752. }
  753. #endif
  754. return size;
  755. }
  756. static uint32
  757. get_export_size(AOTCompContext *comp_ctx, AOTExport *export)
  758. {
  759. /* export index + export kind + 1 byte padding + export name */
  760. return (uint32)sizeof(uint32) + sizeof(uint8) + 1
  761. + get_string_size(comp_ctx, export->name);
  762. }
  763. static uint32
  764. get_exports_size(AOTCompContext *comp_ctx, AOTExport *exports,
  765. uint32 export_count)
  766. {
  767. AOTExport *export = exports;
  768. uint32 size = 0, i;
  769. for (i = 0; i < export_count; i++, export ++) {
  770. size = align_uint(size, 4);
  771. size += get_export_size(comp_ctx, export);
  772. }
  773. return size;
  774. }
  775. static uint32
  776. get_export_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  777. {
  778. /* export count + exports */
  779. return (uint32)sizeof(uint32)
  780. + get_exports_size(comp_ctx, comp_data->wasm_module->exports,
  781. comp_data->wasm_module->export_count);
  782. }
  783. static uint32
  784. get_relocation_size(AOTRelocation *relocation, bool is_32bin)
  785. {
  786. /* offset + addend + relocation type + symbol name */
  787. uint32 size = 0;
  788. if (is_32bin)
  789. size = sizeof(uint32) * 2; /* offset and addend */
  790. else
  791. size = sizeof(uint64) * 2; /* offset and addend */
  792. size += (uint32)sizeof(uint32); /* relocation type */
  793. size += (uint32)sizeof(uint32); /* symbol name index */
  794. return size;
  795. }
  796. static uint32
  797. get_relocations_size(AOTObjectData *obj_data,
  798. AOTRelocationGroup *relocation_group,
  799. AOTRelocation *relocations, uint32 relocation_count,
  800. bool is_32bin)
  801. {
  802. AOTRelocation *relocation = relocations;
  803. uint32 size = 0, i;
  804. for (i = 0; i < relocation_count; i++, relocation++) {
  805. /* ignore the relocations to aot_func_internal#n in text section
  806. for windows platform since they will be applied in
  807. aot_emit_text_section */
  808. const char *name = relocation->symbol_name;
  809. if ((!strcmp(relocation_group->section_name, ".text")
  810. || !strcmp(relocation_group->section_name, ".ltext"))
  811. && !strncmp(name, AOT_FUNC_INTERNAL_PREFIX,
  812. strlen(AOT_FUNC_INTERNAL_PREFIX))
  813. && ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6)
  814. /* Windows AOT_COFF64_BIN_TYPE */
  815. && obj_data->target_info.bin_type == 6
  816. /* IMAGE_REL_AMD64_REL32 in windows x86_64 */
  817. && relocation->relocation_type == 4)
  818. || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4)
  819. /* Windows AOT_COFF32_BIN_TYPE */
  820. && obj_data->target_info.bin_type == 4
  821. /* IMAGE_REL_I386_REL32 in windows x86_32 */
  822. && relocation->relocation_type == 20))) {
  823. continue;
  824. }
  825. size = align_uint(size, 4);
  826. size += get_relocation_size(relocation, is_32bin);
  827. }
  828. return size;
  829. }
  830. static uint32
  831. get_relocation_group_size(AOTObjectData *obj_data,
  832. AOTRelocationGroup *relocation_group, bool is_32bin)
  833. {
  834. uint32 size = 0;
  835. /* section name index + relocation count + relocations */
  836. size += (uint32)sizeof(uint32);
  837. size += (uint32)sizeof(uint32);
  838. size += get_relocations_size(obj_data, relocation_group,
  839. relocation_group->relocations,
  840. relocation_group->relocation_count, is_32bin);
  841. return size;
  842. }
  843. static uint32
  844. get_relocation_groups_size(AOTObjectData *obj_data,
  845. AOTRelocationGroup *relocation_groups,
  846. uint32 relocation_group_count, bool is_32bin)
  847. {
  848. AOTRelocationGroup *relocation_group = relocation_groups;
  849. uint32 size = 0, i;
  850. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  851. size = align_uint(size, 4);
  852. size += get_relocation_group_size(obj_data, relocation_group, is_32bin);
  853. }
  854. return size;
  855. }
  856. /* return the index (in order of insertion) of the symbol,
  857. create if not exits, -1 if failed */
  858. static uint32
  859. get_relocation_symbol_index(const char *symbol_name, bool *is_new,
  860. AOTSymbolList *symbol_list)
  861. {
  862. AOTSymbolNode *sym;
  863. uint32 index = 0;
  864. sym = symbol_list->head;
  865. while (sym) {
  866. if (!strcmp(sym->symbol, symbol_name)) {
  867. if (is_new)
  868. *is_new = false;
  869. return index;
  870. }
  871. sym = sym->next;
  872. index++;
  873. }
  874. /* Not found in symbol_list, add it */
  875. sym = wasm_runtime_malloc(sizeof(AOTSymbolNode));
  876. if (!sym) {
  877. return (uint32)-1;
  878. }
  879. memset(sym, 0, sizeof(AOTSymbolNode));
  880. sym->symbol = (char *)symbol_name;
  881. sym->str_len = (uint32)strlen(symbol_name);
  882. if (!symbol_list->head) {
  883. symbol_list->head = symbol_list->end = sym;
  884. }
  885. else {
  886. symbol_list->end->next = sym;
  887. symbol_list->end = sym;
  888. }
  889. symbol_list->len++;
  890. if (is_new)
  891. *is_new = true;
  892. return index;
  893. }
  894. static uint32
  895. get_relocation_symbol_size(AOTCompContext *comp_ctx, AOTRelocation *relocation,
  896. AOTSymbolList *symbol_list)
  897. {
  898. uint32 size = 0, index = 0;
  899. bool is_new = false;
  900. index = get_relocation_symbol_index(relocation->symbol_name, &is_new,
  901. symbol_list);
  902. CHECK_SIZE(index);
  903. if (is_new) {
  904. size += get_string_size(comp_ctx, relocation->symbol_name);
  905. size = align_uint(size, 2);
  906. }
  907. relocation->symbol_index = index;
  908. return size;
  909. }
  910. static uint32
  911. get_relocations_symbol_size(AOTCompContext *comp_ctx,
  912. AOTRelocation *relocations, uint32 relocation_count,
  913. AOTSymbolList *symbol_list)
  914. {
  915. AOTRelocation *relocation = relocations;
  916. uint32 size = 0, curr_size, i;
  917. for (i = 0; i < relocation_count; i++, relocation++) {
  918. curr_size =
  919. get_relocation_symbol_size(comp_ctx, relocation, symbol_list);
  920. CHECK_SIZE(curr_size);
  921. size += curr_size;
  922. }
  923. return size;
  924. }
  925. static uint32
  926. get_relocation_group_symbol_size(AOTCompContext *comp_ctx,
  927. AOTRelocationGroup *relocation_group,
  928. AOTSymbolList *symbol_list)
  929. {
  930. uint32 size = 0, index = 0, curr_size;
  931. bool is_new = false;
  932. index = get_relocation_symbol_index(relocation_group->section_name, &is_new,
  933. symbol_list);
  934. CHECK_SIZE(index);
  935. if (is_new) {
  936. size += get_string_size(comp_ctx, relocation_group->section_name);
  937. size = align_uint(size, 2);
  938. }
  939. relocation_group->name_index = index;
  940. curr_size = get_relocations_symbol_size(
  941. comp_ctx, relocation_group->relocations,
  942. relocation_group->relocation_count, symbol_list);
  943. CHECK_SIZE(curr_size);
  944. size += curr_size;
  945. return size;
  946. }
  947. static uint32
  948. get_relocation_groups_symbol_size(AOTCompContext *comp_ctx,
  949. AOTRelocationGroup *relocation_groups,
  950. uint32 relocation_group_count,
  951. AOTSymbolList *symbol_list)
  952. {
  953. AOTRelocationGroup *relocation_group = relocation_groups;
  954. uint32 size = 0, curr_size, i;
  955. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  956. curr_size = get_relocation_group_symbol_size(comp_ctx, relocation_group,
  957. symbol_list);
  958. CHECK_SIZE(curr_size);
  959. size += curr_size;
  960. }
  961. return size;
  962. }
  963. static uint32
  964. get_symbol_size_from_symbol_list(AOTCompContext *comp_ctx,
  965. AOTSymbolList *symbol_list)
  966. {
  967. AOTSymbolNode *sym;
  968. uint32 size = 0;
  969. sym = symbol_list->head;
  970. while (sym) {
  971. /* (uint16)str_len + str */
  972. size += get_string_size(comp_ctx, sym->symbol);
  973. size = align_uint(size, 2);
  974. sym = sym->next;
  975. }
  976. return size;
  977. }
  978. static uint32
  979. get_relocation_section_symbol_size(AOTCompContext *comp_ctx,
  980. AOTObjectData *obj_data)
  981. {
  982. AOTRelocationGroup *relocation_groups = obj_data->relocation_groups;
  983. uint32 relocation_group_count = obj_data->relocation_group_count;
  984. uint32 string_count = 0, symbol_table_size = 0;
  985. /* section size will be calculated twice,
  986. get symbol size from symbol list directly in the second calculation */
  987. if (obj_data->symbol_list.len > 0) {
  988. symbol_table_size =
  989. get_symbol_size_from_symbol_list(comp_ctx, &obj_data->symbol_list);
  990. }
  991. else {
  992. symbol_table_size = get_relocation_groups_symbol_size(
  993. comp_ctx, relocation_groups, relocation_group_count,
  994. &obj_data->symbol_list);
  995. }
  996. CHECK_SIZE(symbol_table_size);
  997. string_count = obj_data->symbol_list.len;
  998. /* string_count + string_offsets + total_string_len
  999. + [str (string_len + str)] */
  1000. return (uint32)(sizeof(uint32) + sizeof(uint32) * string_count
  1001. + sizeof(uint32) + symbol_table_size);
  1002. }
  1003. static uint32
  1004. get_relocation_section_size(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  1005. {
  1006. AOTRelocationGroup *relocation_groups = obj_data->relocation_groups;
  1007. uint32 relocation_group_count = obj_data->relocation_group_count;
  1008. uint32 symbol_table_size = 0;
  1009. symbol_table_size = get_relocation_section_symbol_size(comp_ctx, obj_data);
  1010. CHECK_SIZE(symbol_table_size);
  1011. symbol_table_size = align_uint(symbol_table_size, 4);
  1012. /* relocation group count + symbol_table + relocation groups */
  1013. return (uint32)sizeof(uint32) + symbol_table_size
  1014. + get_relocation_groups_size(obj_data, relocation_groups,
  1015. relocation_group_count,
  1016. is_32bit_binary(obj_data));
  1017. }
  1018. static uint32
  1019. get_native_symbol_list_size(AOTCompContext *comp_ctx)
  1020. {
  1021. uint32 len = 0;
  1022. AOTNativeSymbol *sym = NULL;
  1023. sym = bh_list_first_elem(&comp_ctx->native_symbols);
  1024. while (sym) {
  1025. len = align_uint(len, 2);
  1026. len += get_string_size(comp_ctx, sym->symbol);
  1027. sym = bh_list_elem_next(sym);
  1028. }
  1029. return len;
  1030. }
  1031. #if WASM_ENABLE_STRINGREF != 0
  1032. static uint32
  1033. get_string_literal_section_size(AOTCompContext *comp_ctx,
  1034. AOTCompData *comp_data);
  1035. #endif
  1036. static uint32
  1037. get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data);
  1038. uint32
  1039. aot_get_aot_file_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1040. AOTObjectData *obj_data)
  1041. {
  1042. uint32 size = 0;
  1043. uint32 size_custom_section = 0;
  1044. #if WASM_ENABLE_STRINGREF != 0
  1045. uint32 size_string_literal_section = 0;
  1046. #endif
  1047. /* aot file header */
  1048. size += get_file_header_size();
  1049. /* target info section */
  1050. size = align_uint(size, 4);
  1051. /* section id + section size */
  1052. size += (uint32)sizeof(uint32) * 2;
  1053. size += get_target_info_section_size();
  1054. /* init data section */
  1055. size = align_uint(size, 4);
  1056. /* section id + section size */
  1057. size += (uint32)sizeof(uint32) * 2;
  1058. size += get_init_data_section_size(comp_ctx, comp_data, obj_data);
  1059. /* text section */
  1060. size = align_uint(size, 4);
  1061. /* section id + section size */
  1062. size += (uint32)sizeof(uint32) * 2;
  1063. size += get_text_section_size(obj_data);
  1064. /* function section */
  1065. size = align_uint(size, 4);
  1066. /* section id + section size */
  1067. size += (uint32)sizeof(uint32) * 2;
  1068. size += get_func_section_size(comp_ctx, comp_data, obj_data);
  1069. /* export section */
  1070. size = align_uint(size, 4);
  1071. /* section id + section size */
  1072. size += (uint32)sizeof(uint32) * 2;
  1073. size += get_export_section_size(comp_ctx, comp_data);
  1074. /* relocation section */
  1075. size = align_uint(size, 4);
  1076. /* section id + section size */
  1077. size += (uint32)sizeof(uint32) * 2;
  1078. size += get_relocation_section_size(comp_ctx, obj_data);
  1079. if (get_native_symbol_list_size(comp_ctx) > 0) {
  1080. /* emit only when there are native symbols */
  1081. size = align_uint(size, 4);
  1082. /* section id + section size + sub section id + symbol count */
  1083. size += (uint32)sizeof(uint32) * 4;
  1084. size += get_native_symbol_list_size(comp_ctx);
  1085. }
  1086. size_custom_section = get_custom_sections_size(comp_ctx, comp_data);
  1087. if (size_custom_section > 0) {
  1088. size = align_uint(size, 4);
  1089. size += size_custom_section;
  1090. }
  1091. #if WASM_ENABLE_STRINGREF != 0
  1092. /* string literal section */
  1093. size_string_literal_section =
  1094. get_string_literal_section_size(comp_ctx, comp_data);
  1095. if (size_string_literal_section > 0) {
  1096. size = align_uint(size, 4);
  1097. /* section id + section size + sub section id */
  1098. size += (uint32)sizeof(uint32) * 3;
  1099. size += size_string_literal_section;
  1100. }
  1101. #endif
  1102. return size;
  1103. }
  1104. #define exchange_uint8(p_data) (void)0
  1105. static void
  1106. exchange_uint16(uint8 *p_data)
  1107. {
  1108. uint8 value = *p_data;
  1109. *p_data = *(p_data + 1);
  1110. *(p_data + 1) = value;
  1111. }
  1112. static void
  1113. exchange_uint32(uint8 *p_data)
  1114. {
  1115. uint8 value = *p_data;
  1116. *p_data = *(p_data + 3);
  1117. *(p_data + 3) = value;
  1118. value = *(p_data + 1);
  1119. *(p_data + 1) = *(p_data + 2);
  1120. *(p_data + 2) = value;
  1121. }
  1122. static void
  1123. exchange_uint64(uint8 *p_data)
  1124. {
  1125. uint32 value;
  1126. value = *(uint32 *)p_data;
  1127. *(uint32 *)p_data = *(uint32 *)(p_data + 4);
  1128. *(uint32 *)(p_data + 4) = value;
  1129. exchange_uint32(p_data);
  1130. exchange_uint32(p_data + 4);
  1131. }
  1132. static void
  1133. exchange_uint128(uint8 *p_data)
  1134. {
  1135. /* swap high 64bit and low 64bit */
  1136. uint64 value = *(uint64 *)p_data;
  1137. *(uint64 *)p_data = *(uint64 *)(p_data + 8);
  1138. *(uint64 *)(p_data + 8) = value;
  1139. /* exchange high 64bit */
  1140. exchange_uint64(p_data);
  1141. /* exchange low 64bit */
  1142. exchange_uint64(p_data + 8);
  1143. }
  1144. static union {
  1145. int a;
  1146. char b;
  1147. } __ue = { .a = 1 };
  1148. #define is_little_endian() (__ue.b == 1)
  1149. #define CHECK_BUF(length) \
  1150. do { \
  1151. if (buf + offset + length > buf_end) { \
  1152. aot_set_last_error("buf overflow"); \
  1153. return false; \
  1154. } \
  1155. } while (0)
  1156. #define EMIT_U8(v) \
  1157. do { \
  1158. CHECK_BUF(1); \
  1159. *(uint8 *)(buf + offset) = (uint8)v; \
  1160. offset++; \
  1161. } while (0)
  1162. #define EMIT_U16(v) \
  1163. do { \
  1164. uint16 t = (uint16)v; \
  1165. CHECK_BUF(2); \
  1166. if (!is_little_endian()) \
  1167. exchange_uint16((uint8 *)&t); \
  1168. *(uint16 *)(buf + offset) = t; \
  1169. offset += (uint32)sizeof(uint16); \
  1170. } while (0)
  1171. #define EMIT_U32(v) \
  1172. do { \
  1173. uint32 t = (uint32)v; \
  1174. CHECK_BUF(4); \
  1175. if (!is_little_endian()) \
  1176. exchange_uint32((uint8 *)&t); \
  1177. *(uint32 *)(buf + offset) = t; \
  1178. offset += (uint32)sizeof(uint32); \
  1179. } while (0)
  1180. #define EMIT_U64(v) \
  1181. do { \
  1182. uint64 t = (uint64)v; \
  1183. CHECK_BUF(8); \
  1184. if (!is_little_endian()) \
  1185. exchange_uint64((uint8 *)&t); \
  1186. PUT_U64_TO_ADDR(buf + offset, t); \
  1187. offset += (uint32)sizeof(uint64); \
  1188. } while (0)
  1189. #define EMIT_V128(v) \
  1190. do { \
  1191. uint64 *t = (uint64 *)v.i64x2; \
  1192. CHECK_BUF(16); \
  1193. if (!is_little_endian()) \
  1194. exchange_uint128((uint8 *)t); \
  1195. PUT_U64_TO_ADDR(buf + offset, t[0]); \
  1196. offset += (uint32)sizeof(uint64); \
  1197. PUT_U64_TO_ADDR(buf + offset, t[1]); \
  1198. offset += (uint32)sizeof(uint64); \
  1199. } while (0)
  1200. #define EMIT_BUF(v, len) \
  1201. do { \
  1202. CHECK_BUF(len); \
  1203. memcpy(buf + offset, v, len); \
  1204. offset += len; \
  1205. } while (0)
  1206. /* Emit string with '\0'
  1207. */
  1208. #define EMIT_STR(s) \
  1209. do { \
  1210. uint32 str_len = (uint32)strlen(s) + 1; \
  1211. if (str_len > INT16_MAX) { \
  1212. aot_set_last_error("emit string failed: " \
  1213. "string too long"); \
  1214. return false; \
  1215. } \
  1216. EMIT_U16(str_len); \
  1217. EMIT_BUF(s, str_len); \
  1218. } while (0)
  1219. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  1220. static bool
  1221. read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
  1222. uint64 *p_result)
  1223. {
  1224. const uint8 *buf = *p_buf;
  1225. uint64 result = 0;
  1226. uint32 shift = 0;
  1227. uint32 offset = 0, bcnt = 0;
  1228. uint64 byte;
  1229. while (true) {
  1230. /* uN or SN must not exceed ceil(N/7) bytes */
  1231. if (bcnt + 1 > (maxbits + 6) / 7) {
  1232. aot_set_last_error("integer representation too long");
  1233. return false;
  1234. }
  1235. if (buf + offset + 1 > buf_end) {
  1236. aot_set_last_error("unexpected end of section or function");
  1237. return false;
  1238. }
  1239. byte = buf[offset];
  1240. offset += 1;
  1241. result |= ((byte & 0x7f) << shift);
  1242. shift += 7;
  1243. bcnt += 1;
  1244. if ((byte & 0x80) == 0) {
  1245. break;
  1246. }
  1247. }
  1248. if (!sign && maxbits == 32 && shift >= maxbits) {
  1249. /* The top bits set represent values > 32 bits */
  1250. if (((uint8)byte) & 0xf0)
  1251. goto fail_integer_too_large;
  1252. }
  1253. else if (sign && maxbits == 32) {
  1254. if (shift < maxbits) {
  1255. /* Sign extend, second highest bit is the sign bit */
  1256. if ((uint8)byte & 0x40)
  1257. result |= (~((uint64)0)) << shift;
  1258. }
  1259. else {
  1260. /* The top bits should be a sign-extension of the sign bit */
  1261. bool sign_bit_set = ((uint8)byte) & 0x8;
  1262. int top_bits = ((uint8)byte) & 0xf0;
  1263. if ((sign_bit_set && top_bits != 0x70)
  1264. || (!sign_bit_set && top_bits != 0))
  1265. goto fail_integer_too_large;
  1266. }
  1267. }
  1268. else if (sign && maxbits == 64) {
  1269. if (shift < maxbits) {
  1270. /* Sign extend, second highest bit is the sign bit */
  1271. if ((uint8)byte & 0x40)
  1272. result |= (~((uint64)0)) << shift;
  1273. }
  1274. else {
  1275. /* The top bits should be a sign-extension of the sign bit */
  1276. bool sign_bit_set = ((uint8)byte) & 0x1;
  1277. int top_bits = ((uint8)byte) & 0xfe;
  1278. if ((sign_bit_set && top_bits != 0x7e)
  1279. || (!sign_bit_set && top_bits != 0))
  1280. goto fail_integer_too_large;
  1281. }
  1282. }
  1283. *p_buf += offset;
  1284. *p_result = result;
  1285. return true;
  1286. fail_integer_too_large:
  1287. aot_set_last_error("integer too large");
  1288. return false;
  1289. }
  1290. /* NOLINTNEXTLINE */
  1291. #define read_leb_uint32(p, p_end, res) \
  1292. do { \
  1293. uint64 res64; \
  1294. if (!read_leb((uint8 **)&p, p_end, 32, false, &res64)) \
  1295. goto fail; \
  1296. res = (uint32)res64; \
  1297. } while (0)
  1298. /*
  1299. * - transfer .name section in .wasm (comp_data->name_section_buf) to
  1300. * aot buf (comp_data->aot_name_section_buf)
  1301. * - leb128 to u32
  1302. * - add `\0` at the end of every name, and adjust length(+1)
  1303. */
  1304. static uint32
  1305. get_name_section_size(AOTCompData *comp_data)
  1306. {
  1307. /* original name section content in .wasm */
  1308. const uint8 *p = comp_data->name_section_buf,
  1309. *p_end = comp_data->name_section_buf_end;
  1310. uint8 *buf, *buf_end;
  1311. uint32 name_type, subsection_size;
  1312. uint32 previous_name_type = 0;
  1313. uint32 num_func_name;
  1314. uint32 func_index;
  1315. uint32 previous_func_index = ~0U;
  1316. uint32 func_name_len;
  1317. uint32 name_index;
  1318. int i = 0;
  1319. uint32 name_len;
  1320. uint32 offset = 0;
  1321. uint32 max_aot_buf_size = 0;
  1322. if (p >= p_end) {
  1323. aot_set_last_error("unexpected end");
  1324. return 0;
  1325. }
  1326. max_aot_buf_size = 4 * (uint32)(p_end - p);
  1327. if (!(buf = comp_data->aot_name_section_buf =
  1328. wasm_runtime_malloc(max_aot_buf_size))) {
  1329. aot_set_last_error("allocate memory for custom name section failed.");
  1330. return 0;
  1331. }
  1332. memset(buf, 0, (uint32)max_aot_buf_size);
  1333. buf_end = buf + max_aot_buf_size;
  1334. /* the size of "name". it should be 4 */
  1335. read_leb_uint32(p, p_end, name_len);
  1336. offset = align_uint(offset, 4);
  1337. EMIT_U32(name_len);
  1338. if (name_len != 4 || p + name_len > p_end) {
  1339. aot_set_last_error("unexpected end");
  1340. return 0;
  1341. }
  1342. /* "name" */
  1343. if (memcmp(p, "name", 4) != 0) {
  1344. aot_set_last_error("invalid custom name section");
  1345. return 0;
  1346. }
  1347. EMIT_BUF(p, name_len);
  1348. p += name_len;
  1349. while (p < p_end) {
  1350. read_leb_uint32(p, p_end, name_type);
  1351. if (i != 0) {
  1352. if (name_type == previous_name_type) {
  1353. aot_set_last_error("duplicate sub-section");
  1354. return 0;
  1355. }
  1356. if (name_type < previous_name_type) {
  1357. aot_set_last_error("out-of-order sub-section");
  1358. return 0;
  1359. }
  1360. }
  1361. previous_name_type = name_type;
  1362. read_leb_uint32(p, p_end, subsection_size);
  1363. switch (name_type) {
  1364. case SUB_SECTION_TYPE_FUNC:
  1365. if (subsection_size) {
  1366. offset = align_uint(offset, 4);
  1367. EMIT_U32(name_type);
  1368. EMIT_U32(subsection_size);
  1369. read_leb_uint32(p, p_end, num_func_name);
  1370. EMIT_U32(num_func_name);
  1371. for (name_index = 0; name_index < num_func_name;
  1372. name_index++) {
  1373. read_leb_uint32(p, p_end, func_index);
  1374. offset = align_uint(offset, 4);
  1375. EMIT_U32(func_index);
  1376. if (func_index == previous_func_index) {
  1377. aot_set_last_error("duplicate function name");
  1378. return 0;
  1379. }
  1380. if (func_index < previous_func_index
  1381. && previous_func_index != ~0U) {
  1382. aot_set_last_error("out-of-order function index ");
  1383. return 0;
  1384. }
  1385. previous_func_index = func_index;
  1386. read_leb_uint32(p, p_end, func_name_len);
  1387. offset = align_uint(offset, 2);
  1388. /* emit a string ends with `\0` */
  1389. if (func_name_len + 1 > UINT16_MAX) {
  1390. aot_set_last_error(
  1391. "emit string failed: string too long");
  1392. goto fail;
  1393. }
  1394. /* extra 1 byte for \0 */
  1395. EMIT_U16(func_name_len + 1);
  1396. EMIT_BUF(p, func_name_len);
  1397. p += func_name_len;
  1398. EMIT_U8(0);
  1399. }
  1400. }
  1401. break;
  1402. case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection
  1403. */
  1404. case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */
  1405. default:
  1406. p = p + subsection_size;
  1407. break;
  1408. }
  1409. i++;
  1410. }
  1411. return offset;
  1412. fail:
  1413. return 0;
  1414. }
  1415. #endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */
  1416. #if WASM_ENABLE_STRINGREF != 0
  1417. static uint32
  1418. get_string_literal_section_size(AOTCompContext *comp_ctx,
  1419. AOTCompData *comp_data)
  1420. {
  1421. uint32 i;
  1422. uint32 size = 0;
  1423. uint32 string_count = comp_data->string_literal_count;
  1424. if (string_count == 0) {
  1425. return 0;
  1426. }
  1427. /* reserved slot + string count + string_lengths */
  1428. size += sizeof(uint32) * (2 + string_count);
  1429. for (i = 0; i < string_count; i++) {
  1430. size += comp_data->string_literal_lengths_wp[i];
  1431. }
  1432. return size;
  1433. }
  1434. #endif /* end of WASM_ENABLE_STRINGREF != 0 */
  1435. static uint32
  1436. get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  1437. {
  1438. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  1439. uint32 size = 0, i;
  1440. for (i = 0; i < comp_ctx->custom_sections_count; i++) {
  1441. const char *section_name = comp_ctx->custom_sections_wp[i];
  1442. const uint8 *content = NULL;
  1443. uint32 length = 0;
  1444. if (strcmp(section_name, "name") == 0) {
  1445. /* custom name section */
  1446. comp_data->aot_name_section_size = get_name_section_size(comp_data);
  1447. if (comp_data->aot_name_section_size == 0) {
  1448. LOG_WARNING("Can't find custom section [name], ignore it");
  1449. continue;
  1450. }
  1451. size = align_uint(size, 4);
  1452. /* section id + section size + sub section id */
  1453. size += (uint32)sizeof(uint32) * 3;
  1454. size += comp_data->aot_name_section_size;
  1455. continue;
  1456. }
  1457. content = wasm_loader_get_custom_section(comp_data->wasm_module,
  1458. section_name, &length);
  1459. if (!content) {
  1460. LOG_WARNING("Can't find custom section [%s], ignore it",
  1461. section_name);
  1462. continue;
  1463. }
  1464. size = align_uint(size, 4);
  1465. /* section id + section size + sub section id */
  1466. size += (uint32)sizeof(uint32) * 3;
  1467. /* section name and len */
  1468. size += get_string_size(comp_ctx, section_name);
  1469. /* section content */
  1470. size += length;
  1471. }
  1472. return size;
  1473. #else
  1474. return 0;
  1475. #endif
  1476. }
  1477. static bool
  1478. aot_emit_file_header(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1479. AOTCompData *comp_data, AOTObjectData *obj_data)
  1480. {
  1481. uint32 offset = *p_offset;
  1482. uint32 aot_curr_version = AOT_CURRENT_VERSION;
  1483. EMIT_U8('\0');
  1484. EMIT_U8('a');
  1485. EMIT_U8('o');
  1486. EMIT_U8('t');
  1487. EMIT_U32(aot_curr_version);
  1488. *p_offset = offset;
  1489. return true;
  1490. }
  1491. static bool
  1492. aot_emit_target_info_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1493. AOTCompData *comp_data, AOTObjectData *obj_data)
  1494. {
  1495. uint32 offset = *p_offset;
  1496. uint32 section_size = get_target_info_section_size();
  1497. AOTTargetInfo *target_info = &obj_data->target_info;
  1498. *p_offset = offset = align_uint(offset, 4);
  1499. EMIT_U32(AOT_SECTION_TYPE_TARGET_INFO);
  1500. EMIT_U32(section_size);
  1501. EMIT_U16(target_info->bin_type);
  1502. EMIT_U16(target_info->abi_type);
  1503. EMIT_U16(target_info->e_type);
  1504. EMIT_U16(target_info->e_machine);
  1505. EMIT_U32(target_info->e_version);
  1506. EMIT_U32(target_info->e_flags);
  1507. EMIT_U64(target_info->feature_flags);
  1508. EMIT_U64(target_info->reserved);
  1509. EMIT_BUF(target_info->arch, sizeof(target_info->arch));
  1510. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  1511. aot_set_last_error("emit target info failed.");
  1512. return false;
  1513. }
  1514. *p_offset = offset;
  1515. return true;
  1516. }
  1517. static bool
  1518. aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1519. AOTCompContext *comp_ctx, InitializerExpression *expr);
  1520. static bool
  1521. aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1522. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1523. AOTObjectData *obj_data)
  1524. {
  1525. uint32 offset = *p_offset, i;
  1526. AOTMemInitData **init_datas = comp_data->mem_init_data_list;
  1527. *p_offset = offset = align_uint(offset, 4);
  1528. /* Emit import memory count, only emit 0 currently.
  1529. TODO: emit the actual import memory count and
  1530. the full import memory info. */
  1531. EMIT_U32(0);
  1532. /* Emit memory count */
  1533. EMIT_U32(comp_data->memory_count);
  1534. /* Emit memory items */
  1535. for (i = 0; i < comp_data->memory_count; i++) {
  1536. EMIT_U32(comp_data->memories[i].flags);
  1537. EMIT_U32(comp_data->memories[i].num_bytes_per_page);
  1538. EMIT_U32(comp_data->memories[i].init_page_count);
  1539. EMIT_U32(comp_data->memories[i].max_page_count);
  1540. }
  1541. /* Emit mem init data count */
  1542. EMIT_U32(comp_data->mem_init_data_count);
  1543. /* Emit mem init data items */
  1544. for (i = 0; i < comp_data->mem_init_data_count; i++) {
  1545. offset = align_uint(offset, 4);
  1546. #if WASM_ENABLE_BULK_MEMORY != 0
  1547. if (comp_ctx->enable_bulk_memory) {
  1548. EMIT_U32(init_datas[i]->is_passive);
  1549. EMIT_U32(init_datas[i]->memory_index);
  1550. }
  1551. else
  1552. #endif
  1553. {
  1554. /* emit two placeholder to keep the same size */
  1555. EMIT_U32(0);
  1556. EMIT_U32(0);
  1557. }
  1558. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1559. &init_datas[i]->offset))
  1560. return false;
  1561. EMIT_U32(init_datas[i]->byte_count);
  1562. if (init_datas[i]->byte_count) {
  1563. EMIT_BUF(init_datas[i]->bytes, init_datas[i]->byte_count);
  1564. }
  1565. }
  1566. if (offset - *p_offset != get_mem_info_size(comp_ctx, comp_data)) {
  1567. aot_set_last_error("emit memory info failed.");
  1568. return false;
  1569. }
  1570. *p_offset = offset;
  1571. return true;
  1572. }
  1573. static bool
  1574. aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1575. AOTCompContext *comp_ctx, InitializerExpression *expr)
  1576. {
  1577. uint32 offset = *p_offset;
  1578. #if WASM_ENABLE_GC != 0
  1579. WASMModule *module = comp_ctx->comp_data->wasm_module;
  1580. #endif
  1581. *p_offset = offset = align_uint(offset, 4);
  1582. EMIT_U32(expr->init_expr_type);
  1583. switch (expr->init_expr_type) {
  1584. case INIT_EXPR_NONE:
  1585. break;
  1586. case INIT_EXPR_TYPE_I32_CONST:
  1587. case INIT_EXPR_TYPE_F32_CONST:
  1588. EMIT_U32(expr->u.i32);
  1589. break;
  1590. case INIT_EXPR_TYPE_I64_CONST:
  1591. case INIT_EXPR_TYPE_F64_CONST:
  1592. EMIT_U64(expr->u.i64);
  1593. break;
  1594. case INIT_EXPR_TYPE_V128_CONST:
  1595. EMIT_V128(expr->u.v128);
  1596. break;
  1597. case INIT_EXPR_TYPE_GET_GLOBAL:
  1598. EMIT_U32(expr->u.global_index);
  1599. break;
  1600. case INIT_EXPR_TYPE_FUNCREF_CONST:
  1601. case INIT_EXPR_TYPE_REFNULL_CONST:
  1602. EMIT_U32(expr->u.ref_index);
  1603. break;
  1604. #if WASM_ENABLE_GC != 0
  1605. case INIT_EXPR_TYPE_I31_NEW:
  1606. EMIT_U32(expr->u.i32);
  1607. break;
  1608. case INIT_EXPR_TYPE_STRUCT_NEW:
  1609. {
  1610. uint32 i;
  1611. WASMStructNewInitValues *init_values =
  1612. (WASMStructNewInitValues *)expr->u.data;
  1613. WASMStructType *struct_type = NULL;
  1614. EMIT_U32(init_values->type_idx);
  1615. EMIT_U32(init_values->count);
  1616. bh_assert(init_values->type_idx < module->type_count);
  1617. struct_type =
  1618. (WASMStructType *)module->types[init_values->type_idx];
  1619. bh_assert(struct_type);
  1620. bh_assert(struct_type->field_count == init_values->count);
  1621. for (i = 0; i < init_values->count; i++) {
  1622. uint32 field_size = wasm_value_type_size_internal(
  1623. struct_type->fields[i].field_type, comp_ctx->pointer_size);
  1624. if (field_size <= sizeof(uint32))
  1625. EMIT_U32(init_values->fields[i].u32);
  1626. else if (field_size == sizeof(uint64))
  1627. EMIT_U64(init_values->fields[i].u64);
  1628. else if (field_size == sizeof(uint64) * 2)
  1629. EMIT_V128(init_values->fields[i].v128);
  1630. else {
  1631. bh_assert(0);
  1632. }
  1633. }
  1634. break;
  1635. }
  1636. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  1637. EMIT_U32(expr->u.type_index);
  1638. break;
  1639. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  1640. {
  1641. WASMArrayType *array_type = NULL;
  1642. bh_assert(expr->u.array_new_default.type_index
  1643. < module->type_count);
  1644. array_type =
  1645. (WASMArrayType *)
  1646. module->types[expr->u.array_new_default.type_index];
  1647. EMIT_U32(array_type->elem_type);
  1648. EMIT_U32(expr->u.array_new_default.type_index);
  1649. EMIT_U32(expr->u.array_new_default.length);
  1650. break;
  1651. }
  1652. case INIT_EXPR_TYPE_ARRAY_NEW:
  1653. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  1654. {
  1655. uint32 value_count, i, field_size;
  1656. WASMArrayNewInitValues *init_values =
  1657. (WASMArrayNewInitValues *)expr->u.data;
  1658. WASMArrayType *array_type = NULL;
  1659. bh_assert(init_values->type_idx < module->type_count);
  1660. array_type = (WASMArrayType *)module->types[init_values->type_idx];
  1661. EMIT_U32(array_type->elem_type);
  1662. EMIT_U32(init_values->type_idx);
  1663. EMIT_U32(init_values->length);
  1664. value_count =
  1665. (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED)
  1666. ? init_values->length
  1667. : 1;
  1668. field_size = wasm_value_type_size_internal(array_type->elem_type,
  1669. comp_ctx->pointer_size);
  1670. for (i = 0; i < value_count; i++) {
  1671. if (field_size <= sizeof(uint32))
  1672. EMIT_U32(init_values->elem_data[i].u32);
  1673. else if (field_size == sizeof(uint64))
  1674. EMIT_U64(init_values->elem_data[i].u64);
  1675. else if (field_size == sizeof(uint64) * 2)
  1676. EMIT_V128(init_values->elem_data[i].v128);
  1677. else {
  1678. bh_assert(0);
  1679. }
  1680. }
  1681. break;
  1682. }
  1683. #endif /* end of WASM_ENABLE_GC != 0 */
  1684. default:
  1685. aot_set_last_error("invalid init expr type.");
  1686. return false;
  1687. }
  1688. *p_offset = offset;
  1689. return true;
  1690. }
  1691. static bool
  1692. aot_emit_table_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1693. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1694. AOTObjectData *obj_data)
  1695. {
  1696. uint32 offset = *p_offset, i, j;
  1697. AOTTableInitData **init_datas = comp_data->table_init_data_list;
  1698. *p_offset = offset = align_uint(offset, 4);
  1699. /* Emit import table count */
  1700. EMIT_U32(comp_data->import_table_count);
  1701. /* Emit table items */
  1702. for (i = 0; i < comp_data->import_table_count; i++) {
  1703. /* TODO:
  1704. * EMIT_STR(comp_data->import_tables[i].module_name );
  1705. * EMIT_STR(comp_data->import_tables[i].table_name);
  1706. */
  1707. EMIT_U8(comp_data->import_tables[i].table_type.elem_type);
  1708. EMIT_U8(comp_data->import_tables[i].table_type.flags);
  1709. EMIT_U8(comp_data->import_tables[i].table_type.possible_grow);
  1710. #if WASM_ENABLE_GC != 0
  1711. if (comp_ctx->enable_gc
  1712. && comp_data->import_tables[i].table_type.elem_ref_type) {
  1713. EMIT_U8(comp_data->import_tables[i]
  1714. .table_type.elem_ref_type->ref_ht_common.nullable);
  1715. }
  1716. else
  1717. #endif
  1718. {
  1719. /* emit one placeholder to keep the same size */
  1720. EMIT_U8(0);
  1721. }
  1722. EMIT_U32(comp_data->import_tables[i].table_type.init_size);
  1723. EMIT_U32(comp_data->import_tables[i].table_type.max_size);
  1724. #if WASM_ENABLE_GC != 0
  1725. if (comp_ctx->enable_gc
  1726. && comp_data->import_tables[i].table_type.elem_ref_type) {
  1727. bh_assert(wasm_is_type_multi_byte_type(
  1728. comp_data->import_tables[i].table_type.elem_type));
  1729. EMIT_U32(comp_data->import_tables[i]
  1730. .table_type.elem_ref_type->ref_ht_common.heap_type);
  1731. }
  1732. #endif
  1733. }
  1734. /* Emit table count */
  1735. EMIT_U32(comp_data->table_count);
  1736. /* Emit table items */
  1737. for (i = 0; i < comp_data->table_count; i++) {
  1738. EMIT_U8(comp_data->tables[i].table_type.elem_type);
  1739. EMIT_U8(comp_data->tables[i].table_type.flags);
  1740. EMIT_U8(comp_data->tables[i].table_type.possible_grow);
  1741. #if WASM_ENABLE_GC != 0
  1742. if (comp_ctx->enable_gc
  1743. && comp_data->tables[i].table_type.elem_ref_type) {
  1744. EMIT_U8(comp_data->tables[i]
  1745. .table_type.elem_ref_type->ref_ht_common.nullable);
  1746. }
  1747. else
  1748. #endif
  1749. {
  1750. /* emit one placeholder to keep the same size */
  1751. EMIT_U8(0);
  1752. }
  1753. EMIT_U32(comp_data->tables[i].table_type.init_size);
  1754. EMIT_U32(comp_data->tables[i].table_type.max_size);
  1755. #if WASM_ENABLE_GC != 0
  1756. if (comp_ctx->enable_gc) {
  1757. if (comp_data->tables[i].table_type.elem_ref_type) {
  1758. bh_assert(wasm_is_type_multi_byte_type(
  1759. comp_data->tables[i].table_type.elem_type));
  1760. EMIT_U32(
  1761. comp_data->tables[i]
  1762. .table_type.elem_ref_type->ref_ht_common.heap_type);
  1763. }
  1764. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1765. &comp_data->tables[i].init_expr)) {
  1766. return false;
  1767. }
  1768. }
  1769. #endif
  1770. }
  1771. /* Emit table init data count */
  1772. EMIT_U32(comp_data->table_init_data_count);
  1773. /* Emit table init data items */
  1774. for (i = 0; i < comp_data->table_init_data_count; i++) {
  1775. offset = align_uint(offset, 4);
  1776. EMIT_U32(init_datas[i]->mode);
  1777. EMIT_U32(init_datas[i]->elem_type);
  1778. EMIT_U32(init_datas[i]->table_index);
  1779. EMIT_U32(init_datas[i]->offset.init_expr_type);
  1780. EMIT_U64(init_datas[i]->offset.u.i64);
  1781. #if WASM_ENABLE_GC != 0
  1782. if (comp_ctx->enable_gc && init_datas[i]->elem_ref_type) {
  1783. EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.ref_type);
  1784. EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.nullable);
  1785. EMIT_U32(init_datas[i]->elem_ref_type->ref_ht_common.heap_type);
  1786. }
  1787. else
  1788. #endif
  1789. {
  1790. EMIT_U16(init_datas[i]->elem_type);
  1791. EMIT_U16(0);
  1792. EMIT_U32(0);
  1793. }
  1794. EMIT_U32(init_datas[i]->value_count);
  1795. for (j = 0; j < init_datas[i]->value_count; j++) {
  1796. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1797. &init_datas[i]->init_values[j]))
  1798. return false;
  1799. }
  1800. }
  1801. if (offset - *p_offset != get_table_info_size(comp_ctx, comp_data)) {
  1802. aot_set_last_error("emit table info failed.");
  1803. return false;
  1804. }
  1805. *p_offset = offset;
  1806. return true;
  1807. }
  1808. #if WASM_ENABLE_GC != 0
  1809. static bool
  1810. aot_emit_reftype_map(uint8 *buf, uint8 *buf_end, uint32 *p_offset, uint32 count,
  1811. WASMRefTypeMap *refmap)
  1812. {
  1813. uint32 offset = *p_offset, i;
  1814. for (i = 0; i < count; i++) {
  1815. EMIT_U16(refmap->index);
  1816. WASMRefType *ref_type = refmap->ref_type;
  1817. /* Note: WASMRefType is a union type */
  1818. EMIT_U8(ref_type->ref_ht_common.ref_type);
  1819. EMIT_U8(ref_type->ref_ht_common.nullable);
  1820. EMIT_U32(ref_type->ref_ht_common.heap_type);
  1821. refmap++;
  1822. }
  1823. *p_offset = offset;
  1824. return true;
  1825. }
  1826. #endif
  1827. static bool
  1828. aot_emit_type_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1829. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1830. AOTObjectData *obj_data)
  1831. {
  1832. uint32 offset = *p_offset, i;
  1833. *p_offset = offset = align_uint(offset, 4);
  1834. EMIT_U32(comp_data->type_count);
  1835. #if WASM_ENABLE_GC != 0
  1836. if (comp_ctx->enable_gc) {
  1837. AOTType **types = comp_data->types;
  1838. int32 idx;
  1839. uint32 j;
  1840. for (i = 0; i < comp_data->type_count; i++) {
  1841. offset = align_uint(offset, 4);
  1842. /* Emit simple info if there is an equivalence type */
  1843. for (j = 0; j < i; j++) {
  1844. if (types[j] == types[i]) {
  1845. EMIT_U16(types[i]->type_flag);
  1846. /* equivalence type flag is true */
  1847. EMIT_U8(1);
  1848. EMIT_U8(0);
  1849. /* equivalence type index */
  1850. EMIT_U32(j);
  1851. break;
  1852. }
  1853. }
  1854. if (j < i)
  1855. continue;
  1856. EMIT_U16(types[i]->type_flag);
  1857. /* equivalence type flag is false */
  1858. EMIT_U8(0);
  1859. EMIT_U8(types[i]->is_sub_final);
  1860. EMIT_U32(types[i]->parent_type_idx);
  1861. EMIT_U16(types[i]->rec_count);
  1862. EMIT_U16(types[i]->rec_idx);
  1863. /* Emit WASM_TYPE_FUNC */
  1864. if (types[i]->type_flag == WASM_TYPE_FUNC) {
  1865. AOTFuncType *func_type = (AOTFuncType *)types[i];
  1866. EMIT_U16(func_type->param_count);
  1867. EMIT_U16(func_type->result_count);
  1868. EMIT_U16(func_type->ref_type_map_count);
  1869. EMIT_BUF(func_type->types,
  1870. func_type->param_count + func_type->result_count);
  1871. offset = align_uint(offset, 4);
  1872. aot_emit_reftype_map(buf, buf_end, &offset,
  1873. func_type->ref_type_map_count,
  1874. func_type->ref_type_maps);
  1875. }
  1876. /* Emit WASM_TYPE_STRUCT */
  1877. else if (types[i]->type_flag == WASM_TYPE_STRUCT) {
  1878. AOTStructType *struct_type = (AOTStructType *)types[i];
  1879. EMIT_U16(struct_type->field_count);
  1880. EMIT_U16(struct_type->ref_type_map_count);
  1881. for (idx = 0; idx < struct_type->field_count; idx++) {
  1882. EMIT_U8(struct_type->fields[idx].field_flags);
  1883. EMIT_U8(struct_type->fields[idx].field_type);
  1884. }
  1885. offset = align_uint(offset, 4);
  1886. aot_emit_reftype_map(buf, buf_end, &offset,
  1887. struct_type->ref_type_map_count,
  1888. struct_type->ref_type_maps);
  1889. }
  1890. /* Emit WASM_TYPE_ARRAY */
  1891. else if (types[i]->type_flag == WASM_TYPE_ARRAY) {
  1892. AOTArrayType *array_type = (AOTArrayType *)types[i];
  1893. EMIT_U16(array_type->elem_flags);
  1894. EMIT_U8(array_type->elem_type);
  1895. if (array_type->elem_ref_type) {
  1896. bh_assert(
  1897. wasm_is_type_multi_byte_type(array_type->elem_type));
  1898. EMIT_U8(array_type->elem_ref_type->ref_ht_common.nullable);
  1899. EMIT_U32(
  1900. array_type->elem_ref_type->ref_ht_common.heap_type);
  1901. }
  1902. }
  1903. else {
  1904. aot_set_last_error("invalid type flag.");
  1905. return false;
  1906. }
  1907. }
  1908. if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) {
  1909. aot_set_last_error("emit function type info failed.");
  1910. return false;
  1911. }
  1912. *p_offset = offset;
  1913. }
  1914. else
  1915. #endif
  1916. {
  1917. AOTFuncType **func_types = (AOTFuncType **)comp_data->types;
  1918. for (i = 0; i < comp_data->type_count; i++) {
  1919. offset = align_uint(offset, 4);
  1920. /* If GC is disabled, only emit function type info */
  1921. EMIT_U16(WASM_TYPE_FUNC);
  1922. /* Omit to emit dummy padding for is_sub_final,
  1923. * parent_type_index, rec_count, rec_idx, 10 bytes in total */
  1924. EMIT_U16(func_types[i]->param_count);
  1925. EMIT_U16(func_types[i]->result_count);
  1926. /* Omit to emit dummy padding for ref_type_map_count, 2 bytes in
  1927. * total */
  1928. EMIT_BUF(func_types[i]->types,
  1929. func_types[i]->param_count + func_types[i]->result_count);
  1930. }
  1931. if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) {
  1932. aot_set_last_error("emit function type info failed.");
  1933. return false;
  1934. }
  1935. *p_offset = offset;
  1936. }
  1937. return true;
  1938. }
  1939. static bool
  1940. aot_emit_import_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1941. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1942. AOTObjectData *obj_data)
  1943. {
  1944. uint32 offset = *p_offset, i;
  1945. AOTImportGlobal *import_global = comp_data->import_globals;
  1946. *p_offset = offset = align_uint(offset, 4);
  1947. EMIT_U32(comp_data->import_global_count);
  1948. for (i = 0; i < comp_data->import_global_count; i++, import_global++) {
  1949. offset = align_uint(offset, 2);
  1950. EMIT_U8(import_global->type.val_type);
  1951. EMIT_U8(import_global->type.is_mutable);
  1952. EMIT_STR(import_global->module_name);
  1953. offset = align_uint(offset, 2);
  1954. EMIT_STR(import_global->global_name);
  1955. }
  1956. if (offset - *p_offset
  1957. != get_import_global_info_size(comp_ctx, comp_data)) {
  1958. aot_set_last_error("emit import global info failed.");
  1959. return false;
  1960. }
  1961. *p_offset = offset;
  1962. return true;
  1963. }
  1964. static bool
  1965. aot_emit_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1966. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1967. AOTObjectData *obj_data)
  1968. {
  1969. uint32 offset = *p_offset, i;
  1970. AOTGlobal *global = comp_data->globals;
  1971. *p_offset = offset = align_uint(offset, 4);
  1972. EMIT_U32(comp_data->global_count);
  1973. for (i = 0; i < comp_data->global_count; i++, global++) {
  1974. offset = align_uint(offset, 4);
  1975. EMIT_U8(global->type.val_type);
  1976. EMIT_U8(global->type.is_mutable);
  1977. offset = align_uint(offset, 4);
  1978. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1979. &global->init_expr))
  1980. return false;
  1981. }
  1982. if (offset - *p_offset != get_global_info_size(comp_ctx, comp_data)) {
  1983. aot_set_last_error("emit global info failed.");
  1984. return false;
  1985. }
  1986. *p_offset = offset;
  1987. return true;
  1988. }
  1989. static bool
  1990. aot_emit_import_func_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1991. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1992. AOTObjectData *obj_data)
  1993. {
  1994. uint32 offset = *p_offset, i;
  1995. AOTImportFunc *import_func = comp_data->import_funcs;
  1996. *p_offset = offset = align_uint(offset, 4);
  1997. EMIT_U32(comp_data->import_func_count);
  1998. for (i = 0; i < comp_data->import_func_count; i++, import_func++) {
  1999. offset = align_uint(offset, 2);
  2000. EMIT_U16(import_func->func_type_index);
  2001. EMIT_STR(import_func->module_name);
  2002. offset = align_uint(offset, 2);
  2003. EMIT_STR(import_func->func_name);
  2004. }
  2005. if (offset - *p_offset != get_import_func_info_size(comp_ctx, comp_data)) {
  2006. aot_set_last_error("emit import function info failed.");
  2007. return false;
  2008. }
  2009. *p_offset = offset;
  2010. return true;
  2011. }
  2012. static bool
  2013. aot_emit_object_data_section_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2014. AOTCompContext *comp_ctx,
  2015. AOTObjectData *obj_data)
  2016. {
  2017. uint32 offset = *p_offset, i;
  2018. AOTObjectDataSection *data_section = obj_data->data_sections;
  2019. *p_offset = offset = align_uint(offset, 4);
  2020. EMIT_U32(obj_data->data_sections_count);
  2021. for (i = 0; i < obj_data->data_sections_count; i++, data_section++) {
  2022. offset = align_uint(offset, 2);
  2023. EMIT_STR(data_section->name);
  2024. offset = align_uint(offset, 4);
  2025. EMIT_U32(data_section->size);
  2026. if (obj_data->stack_sizes_section_name != NULL
  2027. && !strcmp(obj_data->stack_sizes_section_name,
  2028. data_section->name)) {
  2029. uint32 ss_offset = obj_data->stack_sizes_offset;
  2030. uint32 ss_size =
  2031. obj_data->func_count * sizeof(*obj_data->stack_sizes);
  2032. LOG_VERBOSE("Replacing stack_sizes in %s section, offset %" PRIu32
  2033. ", size %" PRIu32,
  2034. obj_data->stack_sizes_section_name, ss_offset, ss_size);
  2035. bh_assert(ss_offset + ss_size <= data_section->size);
  2036. /* 0 .. ss_offset */
  2037. if (ss_offset > 0) {
  2038. EMIT_BUF(data_section->data, ss_offset);
  2039. }
  2040. /* ss_offset .. ss_offset+ss_size */
  2041. EMIT_BUF(obj_data->stack_sizes, ss_size);
  2042. /* ss_offset+ss_size .. data_section->size */
  2043. if (data_section->size > ss_offset + ss_size) {
  2044. EMIT_BUF(data_section->data + ss_offset + ss_size,
  2045. data_section->size - (ss_offset + ss_size));
  2046. }
  2047. }
  2048. else {
  2049. EMIT_BUF(data_section->data, data_section->size);
  2050. }
  2051. }
  2052. if (offset - *p_offset
  2053. != get_object_data_section_info_size(comp_ctx, obj_data)) {
  2054. aot_set_last_error("emit object data section info failed.");
  2055. return false;
  2056. }
  2057. *p_offset = offset;
  2058. return true;
  2059. }
  2060. static bool
  2061. aot_emit_init_data_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2062. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2063. AOTObjectData *obj_data)
  2064. {
  2065. uint32 section_size =
  2066. get_init_data_section_size(comp_ctx, comp_data, obj_data);
  2067. uint32 offset = *p_offset;
  2068. *p_offset = offset = align_uint(offset, 4);
  2069. EMIT_U32(AOT_SECTION_TYPE_INIT_DATA);
  2070. EMIT_U32(section_size);
  2071. if (!aot_emit_mem_info(buf, buf_end, &offset, comp_ctx, comp_data, obj_data)
  2072. || !aot_emit_table_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2073. obj_data)
  2074. || !aot_emit_type_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2075. obj_data)
  2076. || !aot_emit_import_global_info(buf, buf_end, &offset, comp_ctx,
  2077. comp_data, obj_data)
  2078. || !aot_emit_global_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2079. obj_data)
  2080. || !aot_emit_import_func_info(buf, buf_end, &offset, comp_ctx,
  2081. comp_data, obj_data))
  2082. return false;
  2083. offset = align_uint(offset, 4);
  2084. EMIT_U32(comp_data->func_count);
  2085. EMIT_U32(comp_data->start_func_index);
  2086. EMIT_U32(comp_data->aux_data_end_global_index);
  2087. EMIT_U64(comp_data->aux_data_end);
  2088. EMIT_U32(comp_data->aux_heap_base_global_index);
  2089. EMIT_U64(comp_data->aux_heap_base);
  2090. EMIT_U32(comp_data->aux_stack_top_global_index);
  2091. EMIT_U64(comp_data->aux_stack_bottom);
  2092. EMIT_U32(comp_data->aux_stack_size);
  2093. if (!aot_emit_object_data_section_info(buf, buf_end, &offset, comp_ctx,
  2094. obj_data))
  2095. return false;
  2096. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2097. aot_set_last_error("emit init data section failed.");
  2098. return false;
  2099. }
  2100. *p_offset = offset;
  2101. return true;
  2102. }
  2103. static bool
  2104. aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2105. AOTCompData *comp_data, AOTObjectData *obj_data)
  2106. {
  2107. uint32 section_size = get_text_section_size(obj_data);
  2108. uint32 offset = *p_offset;
  2109. uint8 placeholder = 0;
  2110. AOTRelocationGroup *relocation_group;
  2111. AOTRelocation *relocation;
  2112. uint32 i, j, relocation_count;
  2113. uint8 *text;
  2114. *p_offset = offset = align_uint(offset, 4);
  2115. EMIT_U32(AOT_SECTION_TYPE_TEXT);
  2116. EMIT_U32(section_size);
  2117. EMIT_U32(obj_data->literal_size);
  2118. if (obj_data->literal_size > 0) {
  2119. EMIT_BUF(obj_data->literal, obj_data->literal_size);
  2120. while (offset & 3)
  2121. EMIT_BUF(&placeholder, 1);
  2122. }
  2123. text = buf + offset;
  2124. if (obj_data->text_size > 0) {
  2125. EMIT_BUF(obj_data->text, obj_data->text_size);
  2126. while (offset & 3)
  2127. EMIT_BUF(&placeholder, 1);
  2128. }
  2129. if (obj_data->text_unlikely_size > 0) {
  2130. EMIT_BUF(obj_data->text_unlikely, obj_data->text_unlikely_size);
  2131. while (offset & 3)
  2132. EMIT_BUF(&placeholder, 1);
  2133. }
  2134. if (obj_data->text_hot_size > 0) {
  2135. EMIT_BUF(obj_data->text_hot, obj_data->text_hot_size);
  2136. while (offset & 3)
  2137. EMIT_BUF(&placeholder, 1);
  2138. }
  2139. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2140. aot_set_last_error("emit text section failed.");
  2141. return false;
  2142. }
  2143. /* apply relocations to aot_func_internal#n in text section for
  2144. windows platform */
  2145. if ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6)
  2146. /* Windows AOT_COFF64_BIN_TYPE */
  2147. && obj_data->target_info.bin_type == 6)
  2148. || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4)
  2149. /* Windows AOT_COFF32_BIN_TYPE */
  2150. && obj_data->target_info.bin_type == 4)) {
  2151. relocation_group = obj_data->relocation_groups;
  2152. for (i = 0; i < obj_data->relocation_group_count;
  2153. i++, relocation_group++) {
  2154. /* relocation in text section */
  2155. if ((!strcmp(relocation_group->section_name, ".text")
  2156. || !strcmp(relocation_group->section_name, ".ltext"))) {
  2157. relocation = relocation_group->relocations;
  2158. relocation_count = relocation_group->relocation_count;
  2159. for (j = 0; j < relocation_count; j++) {
  2160. /* relocation to aot_func_internal#n */
  2161. const char *name = relocation->symbol_name;
  2162. if (str_starts_with(name, AOT_FUNC_INTERNAL_PREFIX)
  2163. && ((obj_data->target_info.bin_type
  2164. == 6 /* AOT_COFF64_BIN_TYPE */
  2165. && relocation->relocation_type
  2166. == 4 /* IMAGE_REL_AMD64_REL32 */)
  2167. || (obj_data->target_info.bin_type
  2168. == 4 /* AOT_COFF32_BIN_TYPE */
  2169. && relocation->relocation_type
  2170. == 20 /* IMAGE_REL_I386_REL32 */))) {
  2171. uint32 func_idx =
  2172. atoi(name + strlen(AOT_FUNC_INTERNAL_PREFIX));
  2173. uint64 text_offset, reloc_offset, reloc_addend;
  2174. bh_assert(func_idx < obj_data->func_count);
  2175. text_offset = obj_data->funcs[func_idx]
  2176. .text_offset_of_aot_func_internal;
  2177. reloc_offset = relocation->relocation_offset;
  2178. reloc_addend = relocation->relocation_addend;
  2179. /* S + A - P */
  2180. *(uint32 *)(text + reloc_offset) =
  2181. (uint32)(text_offset + reloc_addend - reloc_offset
  2182. - 4);
  2183. /* remove current relocation as it has been applied */
  2184. if (j < relocation_count - 1) {
  2185. uint32 move_size =
  2186. (uint32)(sizeof(AOTRelocation)
  2187. * (relocation_count - 1 - j));
  2188. bh_memmove_s(relocation, move_size, relocation + 1,
  2189. move_size);
  2190. }
  2191. relocation_group->relocation_count--;
  2192. }
  2193. else {
  2194. relocation++;
  2195. }
  2196. }
  2197. }
  2198. }
  2199. }
  2200. *p_offset = offset;
  2201. return true;
  2202. }
  2203. #if WASM_ENABLE_GC != 0
  2204. static bool
  2205. aot_emit_ref_flag(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2206. uint8 pointer_size, int8 type)
  2207. {
  2208. uint32 j, offset = *p_offset;
  2209. uint16 value_type_cell_num;
  2210. if (wasm_is_type_reftype(type) && !wasm_is_reftype_i31ref(type)) {
  2211. EMIT_U8(1);
  2212. if (pointer_size == sizeof(uint64))
  2213. EMIT_U8(1);
  2214. }
  2215. else {
  2216. value_type_cell_num = wasm_value_type_cell_num(type);
  2217. for (j = 0; j < value_type_cell_num; j++)
  2218. EMIT_U8(0);
  2219. }
  2220. *p_offset = offset;
  2221. return true;
  2222. }
  2223. #endif
  2224. static bool
  2225. aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2226. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2227. AOTObjectData *obj_data)
  2228. {
  2229. uint32 section_size = get_func_section_size(comp_ctx, comp_data, obj_data);
  2230. uint32 i, offset = *p_offset;
  2231. AOTObjectFunc *func = obj_data->funcs;
  2232. AOTFunc **funcs = comp_data->funcs;
  2233. *p_offset = offset = align_uint(offset, 4);
  2234. EMIT_U32(AOT_SECTION_TYPE_FUNCTION);
  2235. EMIT_U32(section_size);
  2236. for (i = 0; i < obj_data->func_count; i++, func++) {
  2237. if (is_32bit_binary(obj_data))
  2238. EMIT_U32(func->text_offset);
  2239. else
  2240. EMIT_U64(func->text_offset);
  2241. }
  2242. for (i = 0; i < comp_data->func_count; i++)
  2243. EMIT_U32(funcs[i]->func_type_index);
  2244. for (i = 0; i < comp_data->func_count; i++) {
  2245. uint32 max_local_cell_num =
  2246. funcs[i]->param_cell_num + funcs[i]->local_cell_num;
  2247. EMIT_U32(max_local_cell_num);
  2248. }
  2249. for (i = 0; i < comp_data->func_count; i++)
  2250. EMIT_U32(funcs[i]->max_stack_cell_num);
  2251. #if WASM_ENABLE_GC != 0
  2252. if (comp_ctx->enable_gc) {
  2253. /* emit func_local_ref_flag arrays for both import and AOTed funcs */
  2254. AOTFuncType *func_type;
  2255. uint32 j, local_ref_flags_cell_num, paddings;
  2256. for (i = 0; i < comp_data->import_func_count; i++) {
  2257. func_type = comp_data->import_funcs[i].func_type;
  2258. /* recalculate cell_num based on target pointer size */
  2259. local_ref_flags_cell_num = 0;
  2260. for (j = 0; j < func_type->param_count; j++) {
  2261. local_ref_flags_cell_num += wasm_value_type_cell_num_internal(
  2262. func_type->types[j], comp_ctx->pointer_size);
  2263. }
  2264. paddings =
  2265. local_ref_flags_cell_num < 2 ? 2 - local_ref_flags_cell_num : 0;
  2266. local_ref_flags_cell_num =
  2267. local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2;
  2268. offset = align_uint(offset, 4);
  2269. EMIT_U32(local_ref_flags_cell_num);
  2270. for (j = 0; j < func_type->param_count; j++) {
  2271. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2272. comp_ctx->pointer_size,
  2273. func_type->types[j]))
  2274. return false;
  2275. }
  2276. for (j = 0; j < paddings; j++)
  2277. EMIT_U8(0);
  2278. }
  2279. for (i = 0; i < comp_data->func_count; i++) {
  2280. func_type = funcs[i]->func_type;
  2281. local_ref_flags_cell_num =
  2282. funcs[i]->param_cell_num + funcs[i]->local_cell_num;
  2283. offset = align_uint(offset, 4);
  2284. EMIT_U32(local_ref_flags_cell_num);
  2285. /* emit local_ref_flag for param variables */
  2286. for (j = 0; j < func_type->param_count; j++) {
  2287. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2288. comp_ctx->pointer_size,
  2289. func_type->types[j]))
  2290. return false;
  2291. }
  2292. /* emit local_ref_flag for local variables */
  2293. for (j = 0; j < funcs[i]->local_count; j++) {
  2294. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2295. comp_ctx->pointer_size,
  2296. funcs[i]->local_types_wp[j]))
  2297. return false;
  2298. }
  2299. }
  2300. }
  2301. #endif /* end of WASM_ENABLE_GC != 0 */
  2302. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2303. aot_set_last_error("emit function section failed.");
  2304. return false;
  2305. }
  2306. *p_offset = offset;
  2307. return true;
  2308. }
  2309. static bool
  2310. aot_emit_export_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2311. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2312. AOTObjectData *obj_data)
  2313. {
  2314. uint32 section_size = get_export_section_size(comp_ctx, comp_data);
  2315. AOTExport *export = comp_data->wasm_module->exports;
  2316. uint32 export_count = comp_data->wasm_module->export_count;
  2317. uint32 i, offset = *p_offset;
  2318. *p_offset = offset = align_uint(offset, 4);
  2319. EMIT_U32(AOT_SECTION_TYPE_EXPORT);
  2320. EMIT_U32(section_size);
  2321. EMIT_U32(export_count);
  2322. for (i = 0; i < export_count; i++, export ++) {
  2323. offset = align_uint(offset, 4);
  2324. EMIT_U32(export->index);
  2325. EMIT_U8(export->kind);
  2326. EMIT_U8(0);
  2327. EMIT_STR(export->name);
  2328. }
  2329. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2330. aot_set_last_error("emit export section failed.");
  2331. return false;
  2332. }
  2333. *p_offset = offset;
  2334. return true;
  2335. }
  2336. static bool
  2337. aot_emit_relocation_symbol_table(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2338. AOTCompContext *comp_ctx,
  2339. AOTCompData *comp_data,
  2340. AOTObjectData *obj_data)
  2341. {
  2342. uint32 symbol_offset = 0, total_string_len = 0;
  2343. uint32 offset = *p_offset;
  2344. AOTSymbolNode *sym;
  2345. EMIT_U32(obj_data->symbol_list.len);
  2346. /* emit symbol offsets */
  2347. sym = (AOTSymbolNode *)(obj_data->symbol_list.head);
  2348. while (sym) {
  2349. EMIT_U32(symbol_offset);
  2350. /* string_len + str[0 .. string_len - 1] */
  2351. symbol_offset += get_string_size(comp_ctx, sym->symbol);
  2352. symbol_offset = align_uint(symbol_offset, 2);
  2353. sym = sym->next;
  2354. }
  2355. /* emit total string len */
  2356. total_string_len = symbol_offset;
  2357. EMIT_U32(total_string_len);
  2358. /* emit symbols */
  2359. sym = (AOTSymbolNode *)(obj_data->symbol_list.head);
  2360. while (sym) {
  2361. EMIT_STR(sym->symbol);
  2362. offset = align_uint(offset, 2);
  2363. sym = sym->next;
  2364. }
  2365. *p_offset = offset;
  2366. return true;
  2367. }
  2368. static bool
  2369. aot_emit_relocation_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2370. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2371. AOTObjectData *obj_data)
  2372. {
  2373. uint32 section_size = get_relocation_section_size(comp_ctx, obj_data);
  2374. uint32 i, offset = *p_offset;
  2375. AOTRelocationGroup *relocation_group = obj_data->relocation_groups;
  2376. if (section_size == (uint32)-1)
  2377. return false;
  2378. *p_offset = offset = align_uint(offset, 4);
  2379. EMIT_U32(AOT_SECTION_TYPE_RELOCATION);
  2380. EMIT_U32(section_size);
  2381. aot_emit_relocation_symbol_table(buf, buf_end, &offset, comp_ctx, comp_data,
  2382. obj_data);
  2383. offset = align_uint(offset, 4);
  2384. EMIT_U32(obj_data->relocation_group_count);
  2385. /* emit each relocation group */
  2386. for (i = 0; i < obj_data->relocation_group_count; i++, relocation_group++) {
  2387. AOTRelocation *relocation = relocation_group->relocations;
  2388. uint32 j;
  2389. offset = align_uint(offset, 4);
  2390. EMIT_U32(relocation_group->name_index);
  2391. offset = align_uint(offset, 4);
  2392. EMIT_U32(relocation_group->relocation_count);
  2393. /* emit each relocation */
  2394. for (j = 0; j < relocation_group->relocation_count; j++, relocation++) {
  2395. offset = align_uint(offset, 4);
  2396. if (is_32bit_binary(obj_data)) {
  2397. EMIT_U32(relocation->relocation_offset);
  2398. EMIT_U32(relocation->relocation_addend);
  2399. }
  2400. else {
  2401. EMIT_U64(relocation->relocation_offset);
  2402. EMIT_U64(relocation->relocation_addend);
  2403. }
  2404. EMIT_U32(relocation->relocation_type);
  2405. EMIT_U32(relocation->symbol_index);
  2406. }
  2407. }
  2408. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2409. aot_set_last_error("emit relocation section failed.");
  2410. return false;
  2411. }
  2412. *p_offset = offset;
  2413. return true;
  2414. }
  2415. static bool
  2416. aot_emit_native_symbol(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2417. AOTCompContext *comp_ctx)
  2418. {
  2419. uint32 offset = *p_offset;
  2420. AOTNativeSymbol *sym = NULL;
  2421. if (bh_list_length(&comp_ctx->native_symbols) == 0)
  2422. /* emit only when there are native symbols */
  2423. return true;
  2424. *p_offset = offset = align_uint(offset, 4);
  2425. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2426. /* sub section id + symbol count + symbol list */
  2427. EMIT_U32(sizeof(uint32) * 2 + get_native_symbol_list_size(comp_ctx));
  2428. EMIT_U32(AOT_CUSTOM_SECTION_NATIVE_SYMBOL);
  2429. EMIT_U32(bh_list_length(&comp_ctx->native_symbols));
  2430. sym = bh_list_first_elem(&comp_ctx->native_symbols);
  2431. while (sym) {
  2432. offset = align_uint(offset, 2);
  2433. EMIT_STR(sym->symbol);
  2434. sym = bh_list_elem_next(sym);
  2435. }
  2436. *p_offset = offset;
  2437. return true;
  2438. }
  2439. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  2440. static bool
  2441. aot_emit_name_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2442. AOTCompData *comp_data, AOTCompContext *comp_ctx)
  2443. {
  2444. uint32 offset = *p_offset;
  2445. if (comp_data->aot_name_section_size == 0)
  2446. return true;
  2447. offset = align_uint(offset, 4);
  2448. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2449. /* sub section id + name section size */
  2450. EMIT_U32(sizeof(uint32) * 1 + comp_data->aot_name_section_size);
  2451. EMIT_U32(AOT_CUSTOM_SECTION_NAME);
  2452. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf),
  2453. comp_data->aot_name_section_buf,
  2454. (uint32)comp_data->aot_name_section_size);
  2455. offset += comp_data->aot_name_section_size;
  2456. *p_offset = offset;
  2457. LOG_DEBUG("emit name section");
  2458. return true;
  2459. }
  2460. #endif
  2461. #if WASM_ENABLE_STRINGREF != 0
  2462. static bool
  2463. aot_emit_string_literal_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2464. AOTCompData *comp_data,
  2465. AOTCompContext *comp_ctx)
  2466. {
  2467. uint32 string_count = comp_data->string_literal_count;
  2468. if (string_count > 0) {
  2469. uint32 offset = *p_offset;
  2470. uint32 i;
  2471. *p_offset = offset = align_uint(offset, 4);
  2472. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2473. /* sub section id + string literal section size */
  2474. EMIT_U32(sizeof(uint32) * 1
  2475. + get_string_literal_section_size(comp_ctx, comp_data));
  2476. EMIT_U32(AOT_CUSTOM_SECTION_STRING_LITERAL);
  2477. /* reserved */
  2478. EMIT_U32(0);
  2479. /* string literal count */
  2480. EMIT_U32(string_count);
  2481. for (i = 0; i < string_count; i++) {
  2482. EMIT_U32(comp_data->string_literal_lengths_wp[i]);
  2483. }
  2484. for (i = 0; i < string_count; i++) {
  2485. uint32 string_length = comp_data->string_literal_lengths_wp[i];
  2486. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf),
  2487. comp_data->string_literal_ptrs_wp[i], string_length);
  2488. offset += string_length;
  2489. }
  2490. *p_offset = offset;
  2491. }
  2492. return true;
  2493. }
  2494. #endif /* end of WASM_ENABLE_STRINGREF != 0 */
  2495. static bool
  2496. aot_emit_custom_sections(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2497. AOTCompData *comp_data, AOTCompContext *comp_ctx)
  2498. {
  2499. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  2500. uint32 offset = *p_offset, i;
  2501. for (i = 0; i < comp_ctx->custom_sections_count; i++) {
  2502. const char *section_name = comp_ctx->custom_sections_wp[i];
  2503. const uint8 *content = NULL;
  2504. uint32 length = 0;
  2505. if (strcmp(section_name, "name") == 0) {
  2506. *p_offset = offset;
  2507. if (!aot_emit_name_section(buf, buf_end, p_offset, comp_data,
  2508. comp_ctx))
  2509. return false;
  2510. offset = *p_offset;
  2511. continue;
  2512. }
  2513. content = wasm_loader_get_custom_section(comp_data->wasm_module,
  2514. section_name, &length);
  2515. if (!content) {
  2516. /* Warning has been reported during calculating size */
  2517. continue;
  2518. }
  2519. offset = align_uint(offset, 4);
  2520. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2521. /* sub section id + content */
  2522. EMIT_U32(sizeof(uint32) * 1 + get_string_size(comp_ctx, section_name)
  2523. + length);
  2524. EMIT_U32(AOT_CUSTOM_SECTION_RAW);
  2525. EMIT_STR(section_name);
  2526. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf), content,
  2527. length);
  2528. offset += length;
  2529. }
  2530. *p_offset = offset;
  2531. #endif
  2532. return true;
  2533. }
  2534. typedef uint32 U32;
  2535. typedef int32 I32;
  2536. typedef uint16 U16;
  2537. typedef uint8 U8;
  2538. struct coff_hdr {
  2539. U16 u16Machine;
  2540. U16 u16NumSections;
  2541. U32 u32DateTimeStamp;
  2542. U32 u32SymTblPtr;
  2543. U32 u32NumSymbols;
  2544. U16 u16PeHdrSize;
  2545. U16 u16Characs;
  2546. };
  2547. #define E_TYPE_REL 1
  2548. #define E_TYPE_XIP 4
  2549. #define IMAGE_FILE_MACHINE_AMD64 0x8664
  2550. #define IMAGE_FILE_MACHINE_I386 0x014c
  2551. #define IMAGE_FILE_MACHINE_IA64 0x0200
  2552. #define AOT_COFF32_BIN_TYPE 4 /* 32-bit little endian */
  2553. #define AOT_COFF64_BIN_TYPE 6 /* 64-bit little endian */
  2554. #define EI_NIDENT 16
  2555. typedef uint32 elf32_word;
  2556. typedef int32 elf32_sword;
  2557. typedef uint16 elf32_half;
  2558. typedef uint32 elf32_off;
  2559. typedef uint32 elf32_addr;
  2560. struct elf32_ehdr {
  2561. unsigned char e_ident[EI_NIDENT]; /* ident bytes */
  2562. elf32_half e_type; /* file type */
  2563. elf32_half e_machine; /* target machine */
  2564. elf32_word e_version; /* file version */
  2565. elf32_addr e_entry; /* start address */
  2566. elf32_off e_phoff; /* phdr file offset */
  2567. elf32_off e_shoff; /* shdr file offset */
  2568. elf32_word e_flags; /* file flags */
  2569. elf32_half e_ehsize; /* sizeof ehdr */
  2570. elf32_half e_phentsize; /* sizeof phdr */
  2571. elf32_half e_phnum; /* number phdrs */
  2572. elf32_half e_shentsize; /* sizeof shdr */
  2573. elf32_half e_shnum; /* number shdrs */
  2574. elf32_half e_shstrndx; /* shdr string index */
  2575. };
  2576. struct elf32_rel {
  2577. elf32_addr r_offset;
  2578. elf32_word r_info;
  2579. } elf32_rel;
  2580. struct elf32_rela {
  2581. elf32_addr r_offset;
  2582. elf32_word r_info;
  2583. elf32_sword r_addend;
  2584. } elf32_rela;
  2585. typedef uint32 elf64_word;
  2586. typedef int32 elf64_sword;
  2587. typedef uint64 elf64_xword;
  2588. typedef int64 elf64_sxword;
  2589. typedef uint16 elf64_half;
  2590. typedef uint64 elf64_off;
  2591. typedef uint64 elf64_addr;
  2592. struct elf64_ehdr {
  2593. unsigned char e_ident[EI_NIDENT]; /* ident bytes */
  2594. elf64_half e_type; /* file type */
  2595. elf64_half e_machine; /* target machine */
  2596. elf64_word e_version; /* file version */
  2597. elf64_addr e_entry; /* start address */
  2598. elf64_off e_phoff; /* phdr file offset */
  2599. elf64_off e_shoff; /* shdr file offset */
  2600. elf64_word e_flags; /* file flags */
  2601. elf64_half e_ehsize; /* sizeof ehdr */
  2602. elf64_half e_phentsize; /* sizeof phdr */
  2603. elf64_half e_phnum; /* number phdrs */
  2604. elf64_half e_shentsize; /* sizeof shdr */
  2605. elf64_half e_shnum; /* number shdrs */
  2606. elf64_half e_shstrndx; /* shdr string index */
  2607. };
  2608. typedef struct elf64_rel {
  2609. elf64_addr r_offset;
  2610. elf64_xword r_info;
  2611. } elf64_rel;
  2612. typedef struct elf64_rela {
  2613. elf64_addr r_offset;
  2614. elf64_xword r_info;
  2615. elf64_sxword r_addend;
  2616. } elf64_rela;
  2617. #define SET_TARGET_INFO_VALUE(f, val, type, little) \
  2618. do { \
  2619. type tmp = val; \
  2620. if ((little && !is_little_endian()) \
  2621. || (!little && is_little_endian())) \
  2622. exchange_##type((uint8 *)&tmp); \
  2623. obj_data->target_info.f = tmp; \
  2624. } while (0)
  2625. #define SET_TARGET_INFO_FIELD(f, v, type, little) \
  2626. SET_TARGET_INFO_VALUE(f, elf_header->v, type, little)
  2627. /* in windows 32, the symbol name may start with '_' */
  2628. static char *
  2629. LLVMGetSymbolNameAndUnDecorate(LLVMSymbolIteratorRef si,
  2630. AOTTargetInfo target_info)
  2631. {
  2632. char *original_name = (char *)LLVMGetSymbolName(si);
  2633. if (!original_name) {
  2634. return NULL;
  2635. }
  2636. if (target_info.bin_type != AOT_COFF32_BIN_TYPE) {
  2637. return original_name;
  2638. }
  2639. if (*original_name == '_') {
  2640. return ++original_name;
  2641. }
  2642. return original_name;
  2643. }
  2644. static bool
  2645. aot_resolve_target_info(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  2646. {
  2647. LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary);
  2648. const uint8 *elf_buf = (uint8 *)LLVMGetBufferStart(obj_data->mem_buf);
  2649. uint32 elf_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf);
  2650. if (bin_type != LLVMBinaryTypeCOFF && bin_type != LLVMBinaryTypeELF32L
  2651. && bin_type != LLVMBinaryTypeELF32B && bin_type != LLVMBinaryTypeELF64L
  2652. && bin_type != LLVMBinaryTypeELF64B
  2653. && bin_type != LLVMBinaryTypeMachO32L
  2654. && bin_type != LLVMBinaryTypeMachO32B
  2655. && bin_type != LLVMBinaryTypeMachO64L
  2656. && bin_type != LLVMBinaryTypeMachO64B) {
  2657. aot_set_last_error("invalid llvm binary bin_type.");
  2658. return false;
  2659. }
  2660. obj_data->target_info.bin_type = bin_type - LLVMBinaryTypeELF32L;
  2661. if (bin_type == LLVMBinaryTypeCOFF) {
  2662. struct coff_hdr *coff_header;
  2663. if (!elf_buf || elf_size < sizeof(struct coff_hdr)) {
  2664. aot_set_last_error("invalid coff_hdr buffer.");
  2665. return false;
  2666. }
  2667. coff_header = (struct coff_hdr *)elf_buf;
  2668. /* Emit eXecute In Place file type while in indirect mode */
  2669. if (comp_ctx->is_indirect_mode)
  2670. obj_data->target_info.e_type = E_TYPE_XIP;
  2671. else
  2672. obj_data->target_info.e_type = E_TYPE_REL;
  2673. obj_data->target_info.e_machine = coff_header->u16Machine;
  2674. obj_data->target_info.e_version = 1;
  2675. obj_data->target_info.e_flags = 0;
  2676. if (coff_header->u16Machine == IMAGE_FILE_MACHINE_AMD64
  2677. || coff_header->u16Machine == IMAGE_FILE_MACHINE_IA64)
  2678. obj_data->target_info.bin_type = AOT_COFF64_BIN_TYPE;
  2679. else if (coff_header->u16Machine == IMAGE_FILE_MACHINE_I386)
  2680. obj_data->target_info.bin_type = AOT_COFF32_BIN_TYPE;
  2681. }
  2682. else if (bin_type == LLVMBinaryTypeELF32L
  2683. || bin_type == LLVMBinaryTypeELF32B) {
  2684. struct elf32_ehdr *elf_header;
  2685. bool is_little_bin = bin_type == LLVMBinaryTypeELF32L;
  2686. uint16 e_type;
  2687. if (!elf_buf || elf_size < sizeof(struct elf32_ehdr)) {
  2688. aot_set_last_error("invalid elf32 buffer.");
  2689. return false;
  2690. }
  2691. elf_header = (struct elf32_ehdr *)elf_buf;
  2692. e_type = elf_header->e_type;
  2693. /* Emit eXecute In Place file type while in indirect mode */
  2694. if (comp_ctx->is_indirect_mode)
  2695. e_type = E_TYPE_XIP;
  2696. SET_TARGET_INFO_VALUE(e_type, e_type, uint16, is_little_bin);
  2697. SET_TARGET_INFO_FIELD(e_machine, e_machine, uint16, is_little_bin);
  2698. SET_TARGET_INFO_FIELD(e_version, e_version, uint32, is_little_bin);
  2699. SET_TARGET_INFO_FIELD(e_flags, e_flags, uint32, is_little_bin);
  2700. }
  2701. else if (bin_type == LLVMBinaryTypeELF64L
  2702. || bin_type == LLVMBinaryTypeELF64B) {
  2703. struct elf64_ehdr *elf_header;
  2704. bool is_little_bin = bin_type == LLVMBinaryTypeELF64L;
  2705. uint16 e_type;
  2706. if (!elf_buf || elf_size < sizeof(struct elf64_ehdr)) {
  2707. aot_set_last_error("invalid elf64 buffer.");
  2708. return false;
  2709. }
  2710. elf_header = (struct elf64_ehdr *)elf_buf;
  2711. e_type = elf_header->e_type;
  2712. /* Emit eXecute In Place file type while in indirect mode */
  2713. if (comp_ctx->is_indirect_mode)
  2714. e_type = E_TYPE_XIP;
  2715. SET_TARGET_INFO_VALUE(e_type, e_type, uint16, is_little_bin);
  2716. SET_TARGET_INFO_FIELD(e_machine, e_machine, uint16, is_little_bin);
  2717. SET_TARGET_INFO_FIELD(e_version, e_version, uint32, is_little_bin);
  2718. SET_TARGET_INFO_FIELD(e_flags, e_flags, uint32, is_little_bin);
  2719. }
  2720. else if (bin_type == LLVMBinaryTypeMachO32L
  2721. || bin_type == LLVMBinaryTypeMachO32B) {
  2722. /* TODO: parse file type of Mach-O 32 */
  2723. aot_set_last_error("invalid llvm binary bin_type.");
  2724. return false;
  2725. }
  2726. else if (bin_type == LLVMBinaryTypeMachO64L
  2727. || bin_type == LLVMBinaryTypeMachO64B) {
  2728. /* TODO: parse file type of Mach-O 64 */
  2729. aot_set_last_error("invalid llvm binary bin_type.");
  2730. return false;
  2731. }
  2732. bh_assert(sizeof(obj_data->target_info.arch)
  2733. == sizeof(comp_ctx->target_arch));
  2734. bh_memcpy_s(obj_data->target_info.arch, sizeof(obj_data->target_info.arch),
  2735. comp_ctx->target_arch, sizeof(comp_ctx->target_arch));
  2736. return true;
  2737. }
  2738. static bool
  2739. aot_resolve_text(AOTObjectData *obj_data)
  2740. {
  2741. #if WASM_ENABLE_DEBUG_AOT != 0
  2742. LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary);
  2743. if (bin_type == LLVMBinaryTypeELF32L || bin_type == LLVMBinaryTypeELF64L) {
  2744. obj_data->text = (char *)LLVMGetBufferStart(obj_data->mem_buf);
  2745. obj_data->text_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf);
  2746. }
  2747. else
  2748. #endif
  2749. {
  2750. LLVMSectionIteratorRef sec_itr;
  2751. char *name;
  2752. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2753. aot_set_last_error("llvm get section iterator failed.");
  2754. return false;
  2755. }
  2756. while (
  2757. !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2758. if ((name = (char *)LLVMGetSectionName(sec_itr))) {
  2759. if (!strcmp(name, ".text") || !strcmp(name, ".ltext")) {
  2760. obj_data->text = (char *)LLVMGetSectionContents(sec_itr);
  2761. obj_data->text_size = (uint32)LLVMGetSectionSize(sec_itr);
  2762. }
  2763. else if (!strcmp(name, ".text.unlikely.")
  2764. || !strcmp(name, ".ltext.unlikely.")) {
  2765. obj_data->text_unlikely =
  2766. (char *)LLVMGetSectionContents(sec_itr);
  2767. obj_data->text_unlikely_size =
  2768. (uint32)LLVMGetSectionSize(sec_itr);
  2769. }
  2770. else if (!strcmp(name, ".text.hot.")
  2771. || !strcmp(name, ".ltext.hot.")) {
  2772. obj_data->text_hot =
  2773. (char *)LLVMGetSectionContents(sec_itr);
  2774. obj_data->text_hot_size =
  2775. (uint32)LLVMGetSectionSize(sec_itr);
  2776. }
  2777. }
  2778. LLVMMoveToNextSection(sec_itr);
  2779. }
  2780. LLVMDisposeSectionIterator(sec_itr);
  2781. }
  2782. return true;
  2783. }
  2784. static bool
  2785. aot_resolve_literal(AOTObjectData *obj_data)
  2786. {
  2787. LLVMSectionIteratorRef sec_itr;
  2788. char *name;
  2789. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2790. aot_set_last_error("llvm get section iterator failed.");
  2791. return false;
  2792. }
  2793. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2794. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2795. && !strcmp(name, ".literal")) {
  2796. obj_data->literal = (char *)LLVMGetSectionContents(sec_itr);
  2797. obj_data->literal_size = (uint32)LLVMGetSectionSize(sec_itr);
  2798. break;
  2799. }
  2800. LLVMMoveToNextSection(sec_itr);
  2801. }
  2802. LLVMDisposeSectionIterator(sec_itr);
  2803. return true;
  2804. }
  2805. static bool
  2806. get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count);
  2807. static bool
  2808. is_data_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr,
  2809. char *section_name)
  2810. {
  2811. uint32 relocation_count = 0;
  2812. return (!strcmp(section_name, ".data") || !strcmp(section_name, ".sdata")
  2813. || !strcmp(section_name, ".rodata")
  2814. #if LLVM_VERSION_MAJOR >= 19
  2815. /* https://github.com/llvm/llvm-project/pull/82214 */
  2816. || !strcmp(section_name, ".srodata")
  2817. #endif
  2818. /* ".rodata.cst4/8/16/.." */
  2819. || !strncmp(section_name, ".rodata.cst", strlen(".rodata.cst"))
  2820. #if LLVM_VERSION_MAJOR >= 19
  2821. /* https://github.com/llvm/llvm-project/pull/82214
  2822. * ".srodata.cst4/8/16/.." */
  2823. || !strncmp(section_name, ".srodata.cst", strlen(".srodata.cst"))
  2824. #endif
  2825. /* ".rodata.strn.m" */
  2826. || !strncmp(section_name, ".rodata.str", strlen(".rodata.str"))
  2827. || (!strcmp(section_name, ".rdata")
  2828. && get_relocations_count(sec_itr, &relocation_count)
  2829. && relocation_count > 0)
  2830. || !strcmp(section_name, aot_stack_sizes_section_name)
  2831. || (obj_data->comp_ctx->enable_llvm_pgo
  2832. && (!strncmp(section_name, "__llvm_prf_cnts", 15)
  2833. || !strncmp(section_name, "__llvm_prf_data", 15)
  2834. || !strncmp(section_name, "__llvm_prf_names", 16))));
  2835. }
  2836. static bool
  2837. get_object_data_sections_count(AOTObjectData *obj_data, uint32 *p_count)
  2838. {
  2839. LLVMSectionIteratorRef sec_itr;
  2840. char *name;
  2841. uint32 count = 0;
  2842. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2843. aot_set_last_error("llvm get section iterator failed.");
  2844. return false;
  2845. }
  2846. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2847. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2848. && (is_data_section(obj_data, sec_itr, name))) {
  2849. count++;
  2850. }
  2851. LLVMMoveToNextSection(sec_itr);
  2852. }
  2853. LLVMDisposeSectionIterator(sec_itr);
  2854. *p_count = count;
  2855. return true;
  2856. }
  2857. static bool
  2858. aot_resolve_object_data_sections(AOTObjectData *obj_data)
  2859. {
  2860. LLVMSectionIteratorRef sec_itr;
  2861. char *name;
  2862. AOTObjectDataSection *data_section;
  2863. uint32 sections_count;
  2864. uint32 size;
  2865. if (!get_object_data_sections_count(obj_data, &sections_count)) {
  2866. return false;
  2867. }
  2868. if (sections_count > 0) {
  2869. uint32 llvm_prf_cnts_idx = 0, llvm_prf_data_idx = 0;
  2870. char buf[32];
  2871. size = (uint32)sizeof(AOTObjectDataSection) * sections_count;
  2872. if (!(data_section = obj_data->data_sections =
  2873. wasm_runtime_malloc(size))) {
  2874. aot_set_last_error("allocate memory for data sections failed.");
  2875. return false;
  2876. }
  2877. memset(obj_data->data_sections, 0, size);
  2878. obj_data->data_sections_count = sections_count;
  2879. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2880. aot_set_last_error("llvm get section iterator failed.");
  2881. return false;
  2882. }
  2883. while (
  2884. !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2885. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2886. && (is_data_section(obj_data, sec_itr, name))) {
  2887. data_section->name = name;
  2888. if (obj_data->comp_ctx->enable_llvm_pgo
  2889. && !strcmp(name, "__llvm_prf_cnts")) {
  2890. snprintf(buf, sizeof(buf), "%s%u", name,
  2891. llvm_prf_cnts_idx++);
  2892. size = (uint32)(strlen(buf) + 1);
  2893. if (!(data_section->name = wasm_runtime_malloc(size))) {
  2894. aot_set_last_error(
  2895. "allocate memory for data section name failed.");
  2896. return false;
  2897. }
  2898. bh_memcpy_s(data_section->name, size, buf, size);
  2899. data_section->is_name_allocated = true;
  2900. }
  2901. else if (obj_data->comp_ctx->enable_llvm_pgo
  2902. && !strcmp(name, "__llvm_prf_data")) {
  2903. snprintf(buf, sizeof(buf), "%s%u", name,
  2904. llvm_prf_data_idx++);
  2905. size = (uint32)(strlen(buf) + 1);
  2906. if (!(data_section->name = wasm_runtime_malloc(size))) {
  2907. aot_set_last_error(
  2908. "allocate memory for data section name failed.");
  2909. return false;
  2910. }
  2911. bh_memcpy_s(data_section->name, size, buf, size);
  2912. data_section->is_name_allocated = true;
  2913. }
  2914. if (obj_data->comp_ctx->enable_llvm_pgo
  2915. && !strcmp(name, "__llvm_prf_names")) {
  2916. data_section->data = (uint8 *)aot_compress_aot_func_names(
  2917. obj_data->comp_ctx, &data_section->size);
  2918. data_section->is_data_allocated = true;
  2919. }
  2920. else {
  2921. data_section->data =
  2922. (uint8 *)LLVMGetSectionContents(sec_itr);
  2923. data_section->size = (uint32)LLVMGetSectionSize(sec_itr);
  2924. }
  2925. data_section++;
  2926. }
  2927. LLVMMoveToNextSection(sec_itr);
  2928. }
  2929. LLVMDisposeSectionIterator(sec_itr);
  2930. }
  2931. return true;
  2932. }
  2933. static bool
  2934. read_stack_usage_file(const AOTCompContext *comp_ctx, const char *filename,
  2935. uint32 *sizes, uint32 count)
  2936. {
  2937. FILE *fp = NULL;
  2938. if (filename == NULL) {
  2939. aot_set_last_error("no stack usage file is specified.");
  2940. return false;
  2941. }
  2942. fp = fopen(filename, "r");
  2943. if (fp == NULL) {
  2944. LOG_ERROR("failed to open stack usage file: %s", filename);
  2945. goto fail;
  2946. }
  2947. /*
  2948. * the file consists of lines like:
  2949. *
  2950. * WASM Module:aot_func#9 72 static
  2951. */
  2952. const char *aot_func_prefix = AOT_FUNC_PREFIX;
  2953. const char *aot_func_internal_prefix = AOT_FUNC_INTERNAL_PREFIX;
  2954. uint32 precheck_found = 0;
  2955. uint32 precheck_stack_size_max = 0;
  2956. uint32 precheck_stack_size_min = UINT32_MAX;
  2957. uint32 found = 0;
  2958. while (true) {
  2959. const char *prefix;
  2960. char line[100];
  2961. char *cp = fgets(line, sizeof(line), fp);
  2962. char *fn;
  2963. char *colon;
  2964. uintmax_t func_idx;
  2965. uintmax_t sz;
  2966. int ret;
  2967. if (cp == NULL) {
  2968. break;
  2969. }
  2970. /*
  2971. * Note: strrchr (not strchr) because a module name can contain
  2972. * colons.
  2973. */
  2974. colon = strrchr(cp, ':');
  2975. if (colon == NULL) {
  2976. goto fail;
  2977. }
  2978. fn = strstr(colon, aot_func_prefix);
  2979. if (fn != NULL) {
  2980. prefix = aot_func_prefix;
  2981. }
  2982. else {
  2983. fn = strstr(colon, aot_func_internal_prefix);
  2984. if (fn == NULL) {
  2985. LOG_ERROR("failed to parse stack usage line: %s", cp);
  2986. goto fail;
  2987. }
  2988. prefix = aot_func_internal_prefix;
  2989. }
  2990. ret = sscanf(fn + strlen(prefix), "%ju %ju static", &func_idx, &sz);
  2991. if (ret != 2) {
  2992. goto fail;
  2993. }
  2994. if (sz > UINT32_MAX) {
  2995. goto fail;
  2996. }
  2997. if (func_idx > UINT32_MAX) {
  2998. goto fail;
  2999. }
  3000. if (func_idx >= count) {
  3001. goto fail;
  3002. }
  3003. if (prefix == aot_func_prefix) {
  3004. if (sz < precheck_stack_size_min) {
  3005. precheck_stack_size_min = (uint32)sz;
  3006. }
  3007. if (sz > precheck_stack_size_max) {
  3008. precheck_stack_size_max = (uint32)sz;
  3009. }
  3010. precheck_found++;
  3011. continue;
  3012. }
  3013. sizes[func_idx] = (uint32)sz;
  3014. found++;
  3015. }
  3016. fclose(fp);
  3017. if (precheck_found != count) {
  3018. LOG_ERROR("%" PRIu32 " precheck entries found while %" PRIu32
  3019. " entries are expected",
  3020. precheck_found, count);
  3021. return false;
  3022. }
  3023. if (found != count) {
  3024. /*
  3025. * LLVM seems to eliminate calls to an empty function
  3026. * (and eliminate the function) even if it's marked noinline.
  3027. */
  3028. LOG_VERBOSE("%" PRIu32 " entries found while %" PRIu32
  3029. " entries are expected. Maybe LLVM optimization eliminated "
  3030. "some functions.",
  3031. found, count);
  3032. }
  3033. if (precheck_stack_size_min != precheck_stack_size_max) {
  3034. /*
  3035. * Note: this is too strict.
  3036. *
  3037. * actually, the stack consumption of the precheck functions
  3038. * can depend on the type of them.
  3039. * that is, depending on various factors including
  3040. * calling conventions and compilers, a function with many
  3041. * parameters can consume more stack, even if it merely does
  3042. * a tail-call to another function.
  3043. */
  3044. bool musttail = aot_target_precheck_can_use_musttail(comp_ctx);
  3045. if (musttail) {
  3046. LOG_WARNING(
  3047. "precheck functions use variable amount of stack. (%" PRIu32
  3048. " - %" PRIu32 ")",
  3049. precheck_stack_size_min, precheck_stack_size_max);
  3050. }
  3051. else {
  3052. LOG_VERBOSE("precheck functions use %" PRIu32 " - %" PRIu32
  3053. " bytes of stack.",
  3054. precheck_stack_size_min, precheck_stack_size_max);
  3055. }
  3056. }
  3057. else {
  3058. LOG_VERBOSE("precheck functions use %" PRIu32 " bytes of stack.",
  3059. precheck_stack_size_max);
  3060. }
  3061. if (precheck_stack_size_max >= 1024) {
  3062. LOG_WARNING("precheck functions themselves consume relatively large "
  3063. "amount of stack (%" PRIu32
  3064. "). Please ensure the runtime has large enough "
  3065. "WASM_STACK_GUARD_SIZE.",
  3066. precheck_stack_size_max);
  3067. }
  3068. return true;
  3069. fail:
  3070. if (fp != NULL)
  3071. fclose(fp);
  3072. aot_set_last_error("failed to read stack usage file.");
  3073. return false;
  3074. }
  3075. static bool
  3076. aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  3077. {
  3078. LLVMSectionIteratorRef sec_itr = NULL;
  3079. LLVMSymbolIteratorRef sym_itr;
  3080. const char *name;
  3081. if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) {
  3082. aot_set_last_error("llvm get symbol iterator failed.");
  3083. return false;
  3084. }
  3085. while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) {
  3086. if ((name =
  3087. LLVMGetSymbolNameAndUnDecorate(sym_itr, obj_data->target_info))
  3088. && (!strcmp(name, aot_stack_sizes_alias_name))) {
  3089. #if 0 /* cf. https://github.com/llvm/llvm-project/issues/67765 */
  3090. uint64 sz = LLVMGetSymbolSize(sym_itr);
  3091. if (sz != sizeof(uint32) * obj_data->func_count
  3092. /* sz of COFF64/COFF32 is 0, ignore the check */
  3093. && obj_data->target_info.bin_type != AOT_COFF64_BIN_TYPE
  3094. && obj_data->target_info.bin_type != AOT_COFF32_BIN_TYPE) {
  3095. aot_set_last_error("stack_sizes had unexpected size.");
  3096. goto fail;
  3097. }
  3098. #endif
  3099. uint64 addr = LLVMGetSymbolAddress(sym_itr);
  3100. if (!(sec_itr =
  3101. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3102. aot_set_last_error("llvm get section iterator failed.");
  3103. goto fail;
  3104. }
  3105. LLVMMoveToContainingSection(sec_itr, sym_itr);
  3106. const char *sec_name = LLVMGetSectionName(sec_itr);
  3107. LOG_VERBOSE("stack_sizes found in section %s offset %" PRIu64 ".",
  3108. sec_name, addr);
  3109. if (strcmp(sec_name, aot_stack_sizes_section_name) || addr != 0) {
  3110. aot_set_last_error(
  3111. "stack_sizes found at an unexpected location.");
  3112. goto fail;
  3113. }
  3114. /*
  3115. * Note: We can't always modify stack_sizes in-place.
  3116. * E.g. When WAMRC_LLC_COMPILER is used, LLVM sometimes uses
  3117. * read-only mmap of the temporary file to back
  3118. * LLVMGetSectionContents.
  3119. */
  3120. const uint32 *ro_stack_sizes =
  3121. (const uint32 *)(LLVMGetSectionContents(sec_itr) + addr);
  3122. uint32 i;
  3123. for (i = 0; i < obj_data->func_count; i++) {
  3124. /* Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes */
  3125. if (ro_stack_sizes[i] != (uint32)-1) {
  3126. aot_set_last_error("unexpected data in stack_sizes.");
  3127. goto fail;
  3128. }
  3129. }
  3130. /*
  3131. * Record section/offset and construct a copy of stack_sizes.
  3132. * aot_emit_object_data_section_info will emit this copy.
  3133. */
  3134. obj_data->stack_sizes_section_name = sec_name;
  3135. obj_data->stack_sizes_offset = (uint32)addr;
  3136. obj_data->stack_sizes = wasm_runtime_malloc(
  3137. obj_data->func_count * sizeof(*obj_data->stack_sizes));
  3138. if (obj_data->stack_sizes == NULL) {
  3139. aot_set_last_error("failed to allocate memory.");
  3140. goto fail;
  3141. }
  3142. uint32 *stack_sizes = obj_data->stack_sizes;
  3143. for (i = 0; i < obj_data->func_count; i++) {
  3144. stack_sizes[i] = (uint32)-1;
  3145. }
  3146. if (!read_stack_usage_file(comp_ctx, comp_ctx->stack_usage_file,
  3147. stack_sizes, obj_data->func_count)) {
  3148. goto fail;
  3149. }
  3150. for (i = 0; i < obj_data->func_count; i++) {
  3151. const AOTFuncContext *func_ctx = comp_ctx->func_ctxes[i];
  3152. bool musttail = aot_target_precheck_can_use_musttail(comp_ctx);
  3153. unsigned int stack_consumption_to_call_wrapped_func =
  3154. musttail ? 0
  3155. : aot_estimate_stack_usage_for_function_call(
  3156. comp_ctx, func_ctx->aot_func->func_type);
  3157. /*
  3158. * LLVM seems to eliminate calls to an empty function
  3159. * (and eliminate the function) even if it's marked noinline.
  3160. *
  3161. * Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes
  3162. */
  3163. if (stack_sizes[i] == (uint32)-1) {
  3164. if (func_ctx->stack_consumption_for_func_call != 0) {
  3165. /*
  3166. * This happens if a function calling another
  3167. * function has been optimized out.
  3168. *
  3169. * for example,
  3170. *
  3171. * (func $func
  3172. * (local i32)
  3173. * local.get 0
  3174. * if
  3175. * call $another
  3176. * end
  3177. * )
  3178. */
  3179. LOG_VERBOSE("AOT func#%" PRIu32
  3180. " had call(s) but eliminated?",
  3181. i);
  3182. }
  3183. else {
  3184. LOG_VERBOSE("AOT func#%" PRIu32 " eliminated?", i);
  3185. }
  3186. stack_sizes[i] = 0;
  3187. }
  3188. else {
  3189. LOG_VERBOSE("AOT func#%" PRIu32 " stack_size %u + %" PRIu32
  3190. " + %u",
  3191. i, stack_consumption_to_call_wrapped_func,
  3192. stack_sizes[i],
  3193. func_ctx->stack_consumption_for_func_call);
  3194. if (UINT32_MAX - stack_sizes[i]
  3195. < func_ctx->stack_consumption_for_func_call) {
  3196. aot_set_last_error("stack size overflow.");
  3197. goto fail;
  3198. }
  3199. stack_sizes[i] += func_ctx->stack_consumption_for_func_call;
  3200. if (UINT32_MAX - stack_sizes[i]
  3201. < stack_consumption_to_call_wrapped_func) {
  3202. aot_set_last_error("stack size overflow.");
  3203. goto fail;
  3204. }
  3205. stack_sizes[i] += stack_consumption_to_call_wrapped_func;
  3206. }
  3207. }
  3208. LLVMDisposeSectionIterator(sec_itr);
  3209. LLVMDisposeSymbolIterator(sym_itr);
  3210. return true;
  3211. }
  3212. LLVMMoveToNextSymbol(sym_itr);
  3213. }
  3214. aot_set_last_error("stack_sizes not found.");
  3215. fail:
  3216. if (sec_itr)
  3217. LLVMDisposeSectionIterator(sec_itr);
  3218. LLVMDisposeSymbolIterator(sym_itr);
  3219. return false;
  3220. }
  3221. static bool
  3222. aot_resolve_functions(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  3223. {
  3224. AOTObjectFunc *func;
  3225. LLVMSymbolIteratorRef sym_itr;
  3226. char *name, *prefix = AOT_FUNC_PREFIX;
  3227. uint32 func_index, total_size;
  3228. /* allocate memory for aot function */
  3229. obj_data->func_count = comp_ctx->comp_data->func_count;
  3230. if (obj_data->func_count) {
  3231. if ((comp_ctx->enable_stack_bound_check
  3232. || comp_ctx->enable_stack_estimation)
  3233. && !aot_resolve_stack_sizes(comp_ctx, obj_data))
  3234. return false;
  3235. total_size = (uint32)sizeof(AOTObjectFunc) * obj_data->func_count;
  3236. if (!(obj_data->funcs = wasm_runtime_malloc(total_size))) {
  3237. aot_set_last_error("allocate memory for functions failed.");
  3238. return false;
  3239. }
  3240. memset(obj_data->funcs, 0, total_size);
  3241. }
  3242. if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) {
  3243. aot_set_last_error("llvm get symbol iterator failed.");
  3244. return false;
  3245. }
  3246. while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) {
  3247. name = LLVMGetSymbolNameAndUnDecorate(sym_itr, obj_data->target_info);
  3248. if (name && str_starts_with(name, prefix)) {
  3249. /* symbol aot_func#n */
  3250. func_index = (uint32)atoi(name + strlen(prefix));
  3251. if (func_index < obj_data->func_count) {
  3252. LLVMSectionIteratorRef contain_section;
  3253. char *contain_section_name;
  3254. func = obj_data->funcs + func_index;
  3255. func->func_name = name;
  3256. if (!(contain_section = LLVMObjectFileCopySectionIterator(
  3257. obj_data->binary))) {
  3258. aot_set_last_error("llvm get section iterator failed.");
  3259. LLVMDisposeSymbolIterator(sym_itr);
  3260. return false;
  3261. }
  3262. LLVMMoveToContainingSection(contain_section, sym_itr);
  3263. contain_section_name =
  3264. (char *)LLVMGetSectionName(contain_section);
  3265. LLVMDisposeSectionIterator(contain_section);
  3266. if (!strcmp(contain_section_name, ".text.unlikely.")
  3267. || !strcmp(contain_section_name, ".ltext.unlikely.")) {
  3268. func->text_offset = align_uint(obj_data->text_size, 4)
  3269. + LLVMGetSymbolAddress(sym_itr);
  3270. }
  3271. else if (!strcmp(contain_section_name, ".text.hot.")
  3272. || !strcmp(contain_section_name, ".ltext.hot.")) {
  3273. func->text_offset =
  3274. align_uint(obj_data->text_size, 4)
  3275. + align_uint(obj_data->text_unlikely_size, 4)
  3276. + LLVMGetSymbolAddress(sym_itr);
  3277. }
  3278. else {
  3279. func->text_offset = LLVMGetSymbolAddress(sym_itr);
  3280. }
  3281. }
  3282. }
  3283. else if (name && str_starts_with(name, AOT_FUNC_INTERNAL_PREFIX)) {
  3284. /* symbol aot_func_internal#n */
  3285. func_index = (uint32)atoi(name + strlen(AOT_FUNC_INTERNAL_PREFIX));
  3286. if (func_index < obj_data->func_count) {
  3287. LLVMSectionIteratorRef contain_section;
  3288. char *contain_section_name;
  3289. func = obj_data->funcs + func_index;
  3290. if (!(contain_section = LLVMObjectFileCopySectionIterator(
  3291. obj_data->binary))) {
  3292. aot_set_last_error("llvm get section iterator failed.");
  3293. LLVMDisposeSymbolIterator(sym_itr);
  3294. return false;
  3295. }
  3296. LLVMMoveToContainingSection(contain_section, sym_itr);
  3297. contain_section_name =
  3298. (char *)LLVMGetSectionName(contain_section);
  3299. LLVMDisposeSectionIterator(contain_section);
  3300. if (!strcmp(contain_section_name, ".text.unlikely.")
  3301. || !strcmp(contain_section_name, ".ltext.unlikely.")) {
  3302. func->text_offset_of_aot_func_internal =
  3303. align_uint(obj_data->text_size, 4)
  3304. + LLVMGetSymbolAddress(sym_itr);
  3305. }
  3306. else if (!strcmp(contain_section_name, ".text.hot.")
  3307. || !strcmp(contain_section_name, ".ltext.hot.")) {
  3308. func->text_offset_of_aot_func_internal =
  3309. align_uint(obj_data->text_size, 4)
  3310. + align_uint(obj_data->text_unlikely_size, 4)
  3311. + LLVMGetSymbolAddress(sym_itr);
  3312. }
  3313. else {
  3314. func->text_offset_of_aot_func_internal =
  3315. LLVMGetSymbolAddress(sym_itr);
  3316. }
  3317. }
  3318. }
  3319. LLVMMoveToNextSymbol(sym_itr);
  3320. }
  3321. LLVMDisposeSymbolIterator(sym_itr);
  3322. return true;
  3323. }
  3324. static bool
  3325. get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count)
  3326. {
  3327. uint32 relocation_count = 0;
  3328. LLVMRelocationIteratorRef rel_itr;
  3329. if (!(rel_itr = LLVMGetRelocations(sec_itr))) {
  3330. aot_set_last_error("llvm get relocations failed.");
  3331. LLVMDisposeSectionIterator(sec_itr);
  3332. return false;
  3333. }
  3334. while (!LLVMIsRelocationIteratorAtEnd(sec_itr, rel_itr)) {
  3335. relocation_count++;
  3336. LLVMMoveToNextRelocation(rel_itr);
  3337. }
  3338. LLVMDisposeRelocationIterator(rel_itr);
  3339. *p_count = relocation_count;
  3340. return true;
  3341. }
  3342. static bool
  3343. aot_resolve_object_relocation_group(AOTObjectData *obj_data,
  3344. AOTRelocationGroup *group,
  3345. LLVMSectionIteratorRef rel_sec)
  3346. {
  3347. LLVMRelocationIteratorRef rel_itr;
  3348. AOTRelocation *relocation = group->relocations;
  3349. uint32 size;
  3350. bool is_binary_32bit = is_32bit_binary(obj_data);
  3351. bool is_binary_little_endian = is_little_endian_binary(obj_data);
  3352. bool has_addend = str_starts_with(group->section_name, ".rela");
  3353. uint8 *rela_content = NULL;
  3354. /* calculate relocations count and allocate memory */
  3355. if (!get_relocations_count(rel_sec, &group->relocation_count))
  3356. return false;
  3357. if (group->relocation_count == 0) {
  3358. aot_set_last_error("invalid relocations count");
  3359. return false;
  3360. }
  3361. size = (uint32)sizeof(AOTRelocation) * group->relocation_count;
  3362. if (!(relocation = group->relocations = wasm_runtime_malloc(size))) {
  3363. aot_set_last_error("allocate memory for relocations failed.");
  3364. return false;
  3365. }
  3366. memset(group->relocations, 0, size);
  3367. if (has_addend) {
  3368. uint64 rela_content_size;
  3369. /* LLVM doesn't provide C API to get relocation addend. So we have to
  3370. * parse it manually. */
  3371. rela_content = (uint8 *)LLVMGetSectionContents(rel_sec);
  3372. rela_content_size = LLVMGetSectionSize(rel_sec);
  3373. if (is_binary_32bit)
  3374. size = (uint32)sizeof(struct elf32_rela) * group->relocation_count;
  3375. else
  3376. size = (uint32)sizeof(struct elf64_rela) * group->relocation_count;
  3377. if (rela_content_size != (uint64)size) {
  3378. aot_set_last_error("invalid relocation section content.");
  3379. return false;
  3380. }
  3381. }
  3382. /* parse each relocation */
  3383. if (!(rel_itr = LLVMGetRelocations(rel_sec))) {
  3384. aot_set_last_error("llvm get relocations failed.");
  3385. return false;
  3386. }
  3387. while (!LLVMIsRelocationIteratorAtEnd(rel_sec, rel_itr)) {
  3388. uint64 offset = LLVMGetRelocationOffset(rel_itr);
  3389. uint64 type = LLVMGetRelocationType(rel_itr);
  3390. LLVMSymbolIteratorRef rel_sym = LLVMGetRelocationSymbol(rel_itr);
  3391. if (!rel_sym) {
  3392. aot_set_last_error("llvm get relocation symbol failed.");
  3393. goto fail;
  3394. }
  3395. /* parse relocation addend from relocation content */
  3396. if (has_addend) {
  3397. if (is_binary_32bit) {
  3398. int32 addend =
  3399. (int32)(((struct elf32_rela *)rela_content)->r_addend);
  3400. if (is_binary_little_endian != is_little_endian())
  3401. exchange_uint32((uint8 *)&addend);
  3402. relocation->relocation_addend = (int64)addend;
  3403. rela_content += sizeof(struct elf32_rela);
  3404. }
  3405. else {
  3406. int64 addend =
  3407. (int64)(((struct elf64_rela *)rela_content)->r_addend);
  3408. if (is_binary_little_endian != is_little_endian())
  3409. exchange_uint64((uint8 *)&addend);
  3410. relocation->relocation_addend = addend;
  3411. rela_content += sizeof(struct elf64_rela);
  3412. }
  3413. }
  3414. /* set relocation fields */
  3415. relocation->relocation_type = (uint32)type;
  3416. relocation->symbol_name =
  3417. LLVMGetSymbolNameAndUnDecorate(rel_sym, obj_data->target_info);
  3418. relocation->relocation_offset = offset;
  3419. if (!strcmp(group->section_name, ".rela.text.unlikely.")
  3420. || !strcmp(group->section_name, ".rel.text.unlikely.")) {
  3421. relocation->relocation_offset += align_uint(obj_data->text_size, 4);
  3422. }
  3423. else if (!strcmp(group->section_name, ".rela.text.hot.")
  3424. || !strcmp(group->section_name, ".rel.text.hot.")) {
  3425. relocation->relocation_offset +=
  3426. align_uint(obj_data->text_size, 4)
  3427. + align_uint(obj_data->text_unlikely_size, 4);
  3428. }
  3429. if (!strcmp(relocation->symbol_name, ".text.unlikely.")) {
  3430. relocation->symbol_name = ".text";
  3431. relocation->relocation_addend += align_uint(obj_data->text_size, 4);
  3432. }
  3433. if (!strcmp(relocation->symbol_name, ".text.hot.")) {
  3434. relocation->symbol_name = ".text";
  3435. relocation->relocation_addend +=
  3436. align_uint(obj_data->text_size, 4)
  3437. + align_uint(obj_data->text_unlikely_size, 4);
  3438. }
  3439. /*
  3440. * Note: aot_stack_sizes_section_name section only contains
  3441. * stack_sizes table.
  3442. */
  3443. if (!strcmp(relocation->symbol_name, aot_stack_sizes_name)) {
  3444. /* discard const */
  3445. relocation->symbol_name = (char *)aot_stack_sizes_section_name;
  3446. }
  3447. if (obj_data->comp_ctx->enable_llvm_pgo
  3448. && (!strcmp(relocation->symbol_name, "__llvm_prf_cnts")
  3449. || !strcmp(relocation->symbol_name, "__llvm_prf_data"))) {
  3450. LLVMSectionIteratorRef sec_itr;
  3451. char buf[32], *section_name;
  3452. uint32 prof_section_idx = 0;
  3453. if (!(sec_itr =
  3454. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3455. aot_set_last_error("llvm get section iterator failed.");
  3456. LLVMDisposeSymbolIterator(rel_sym);
  3457. goto fail;
  3458. }
  3459. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary,
  3460. sec_itr)) {
  3461. section_name = (char *)LLVMGetSectionName(sec_itr);
  3462. if (section_name
  3463. && !strcmp(section_name, relocation->symbol_name)) {
  3464. if (LLVMGetSectionContainsSymbol(sec_itr, rel_sym))
  3465. break;
  3466. prof_section_idx++;
  3467. }
  3468. LLVMMoveToNextSection(sec_itr);
  3469. }
  3470. LLVMDisposeSectionIterator(sec_itr);
  3471. if (!strcmp(group->section_name, ".rela.text")
  3472. || !strcmp(group->section_name, ".rel.text")) {
  3473. snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name,
  3474. prof_section_idx);
  3475. size = (uint32)(strlen(buf) + 1);
  3476. if (!(relocation->symbol_name = wasm_runtime_malloc(size))) {
  3477. aot_set_last_error(
  3478. "allocate memory for relocation symbol name failed.");
  3479. LLVMDisposeSymbolIterator(rel_sym);
  3480. goto fail;
  3481. }
  3482. bh_memcpy_s(relocation->symbol_name, size, buf, size);
  3483. relocation->is_symbol_name_allocated = true;
  3484. }
  3485. else if (!strncmp(group->section_name, ".rela__llvm_prf_data", 20)
  3486. || !strncmp(group->section_name, ".rel__llvm_prf_data",
  3487. 19)) {
  3488. snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name,
  3489. prof_section_idx);
  3490. size = (uint32)(strlen(buf) + 1);
  3491. if (!(relocation->symbol_name = wasm_runtime_malloc(size))) {
  3492. aot_set_last_error(
  3493. "allocate memory for relocation symbol name failed.");
  3494. LLVMDisposeSymbolIterator(rel_sym);
  3495. goto fail;
  3496. }
  3497. bh_memcpy_s(relocation->symbol_name, size, buf, size);
  3498. relocation->is_symbol_name_allocated = true;
  3499. }
  3500. }
  3501. /* for ".LCPIxxx", ".LJTIxxx", ".LBBxxx" and switch lookup table
  3502. * relocation, transform the symbol name to real section name and set
  3503. * addend to the offset of the symbol in the real section */
  3504. if (relocation->symbol_name
  3505. && (str_starts_with(relocation->symbol_name, ".LCPI")
  3506. || str_starts_with(relocation->symbol_name, ".LJTI")
  3507. || str_starts_with(relocation->symbol_name, ".LBB")
  3508. || str_starts_with(relocation->symbol_name, ".Lswitch.table.")
  3509. #if LLVM_VERSION_MAJOR >= 16
  3510. /* cf. https://reviews.llvm.org/D123264 */
  3511. || str_starts_with(relocation->symbol_name, ".Lpcrel_hi")
  3512. #endif
  3513. #if LLVM_VERSION_MAJOR >= 19
  3514. /* cf.
  3515. * https://github.com/llvm/llvm-project/pull/95031
  3516. * https://github.com/llvm/llvm-project/pull/89693
  3517. *
  3518. * note: the trailing space in ".L0 " is intentional. */
  3519. || !strcmp(relocation->symbol_name, "")
  3520. || !strcmp(relocation->symbol_name, ".L0 ")
  3521. #endif
  3522. )) {
  3523. /* change relocation->relocation_addend and
  3524. relocation->symbol_name */
  3525. LLVMSectionIteratorRef contain_section;
  3526. if (!(contain_section =
  3527. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3528. aot_set_last_error("llvm get section iterator failed.");
  3529. goto fail;
  3530. }
  3531. LLVMMoveToContainingSection(contain_section, rel_sym);
  3532. if (LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary,
  3533. contain_section)) {
  3534. LLVMDisposeSectionIterator(contain_section);
  3535. aot_set_last_error("llvm get containing section failed.");
  3536. goto fail;
  3537. }
  3538. relocation->relocation_addend += LLVMGetSymbolAddress(rel_sym);
  3539. relocation->symbol_name =
  3540. (char *)LLVMGetSectionName(contain_section);
  3541. LLVMDisposeSectionIterator(contain_section);
  3542. }
  3543. LLVMDisposeSymbolIterator(rel_sym);
  3544. LLVMMoveToNextRelocation(rel_itr);
  3545. relocation++;
  3546. }
  3547. LLVMDisposeRelocationIterator(rel_itr);
  3548. return true;
  3549. fail:
  3550. LLVMDisposeRelocationIterator(rel_itr);
  3551. return false;
  3552. }
  3553. static bool
  3554. is_relocation_section_name(AOTObjectData *obj_data, char *section_name)
  3555. {
  3556. return (!strcmp(section_name, ".rela.text")
  3557. || !strcmp(section_name, ".rel.text")
  3558. || !strcmp(section_name, ".rela.text.unlikely.")
  3559. || !strcmp(section_name, ".rel.text.unlikely.")
  3560. || !strcmp(section_name, ".rela.text.hot.")
  3561. || !strcmp(section_name, ".rel.text.hot.")
  3562. || !strcmp(section_name, ".rela.ltext")
  3563. || !strcmp(section_name, ".rel.ltext")
  3564. || !strcmp(section_name, ".rela.ltext.unlikely.")
  3565. || !strcmp(section_name, ".rel.ltext.unlikely.")
  3566. || !strcmp(section_name, ".rela.ltext.hot.")
  3567. || !strcmp(section_name, ".rel.ltext.hot.")
  3568. || !strcmp(section_name, ".rela.literal")
  3569. || !strcmp(section_name, ".rela.data")
  3570. || !strcmp(section_name, ".rel.data")
  3571. || !strcmp(section_name, ".rela.sdata")
  3572. || !strcmp(section_name, ".rel.sdata")
  3573. || !strcmp(section_name, ".rela.rodata")
  3574. || !strcmp(section_name, ".rel.rodata")
  3575. || (obj_data->comp_ctx->enable_llvm_pgo
  3576. && (!strcmp(section_name, ".rela__llvm_prf_data")
  3577. || !strcmp(section_name, ".rel__llvm_prf_data")))
  3578. /* ".rela.rodata.cst4/8/16/.." */
  3579. || !strncmp(section_name, ".rela.rodata.cst",
  3580. strlen(".rela.rodata.cst"))
  3581. /* ".rel.rodata.cst4/8/16/.." */
  3582. || !strncmp(section_name, ".rel.rodata.cst",
  3583. strlen(".rel.rodata.cst")));
  3584. }
  3585. static bool
  3586. is_relocation_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr)
  3587. {
  3588. uint32 count = 0;
  3589. char *name = (char *)LLVMGetSectionName(sec_itr);
  3590. if (name) {
  3591. if (is_relocation_section_name(obj_data, name))
  3592. return true;
  3593. else if ((!strcmp(name, ".text") || !strcmp(name, ".text.unlikely.")
  3594. || !strcmp(name, ".text.hot.") || !strcmp(name, ".rdata"))
  3595. && get_relocations_count(sec_itr, &count) && count > 0)
  3596. return true;
  3597. }
  3598. return false;
  3599. }
  3600. static bool
  3601. is_readonly_section(const char *name)
  3602. {
  3603. return !strcmp(name, ".rel.text") || !strcmp(name, ".rela.text")
  3604. || !strcmp(name, ".rel.ltext") || !strcmp(name, ".rela.ltext")
  3605. || !strcmp(name, ".rela.literal") || !strcmp(name, ".text")
  3606. || !strcmp(name, ".ltext");
  3607. }
  3608. static bool
  3609. get_relocation_groups_count(AOTObjectData *obj_data, uint32 *p_count)
  3610. {
  3611. uint32 count = 0;
  3612. LLVMSectionIteratorRef sec_itr;
  3613. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3614. aot_set_last_error("llvm get section iterator failed.");
  3615. return false;
  3616. }
  3617. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  3618. if (is_relocation_section(obj_data, sec_itr)) {
  3619. count++;
  3620. }
  3621. LLVMMoveToNextSection(sec_itr);
  3622. }
  3623. LLVMDisposeSectionIterator(sec_itr);
  3624. *p_count = count;
  3625. return true;
  3626. }
  3627. static bool
  3628. aot_resolve_object_relocation_groups(AOTObjectData *obj_data)
  3629. {
  3630. LLVMSectionIteratorRef sec_itr;
  3631. AOTRelocationGroup *relocation_group;
  3632. uint32 group_count, llvm_prf_data_idx = 0;
  3633. char *name;
  3634. uint32 size;
  3635. /* calculate relocation groups count and allocate memory */
  3636. if (!get_relocation_groups_count(obj_data, &group_count))
  3637. return false;
  3638. if (0 == (obj_data->relocation_group_count = group_count))
  3639. return true;
  3640. size = (uint32)sizeof(AOTRelocationGroup) * group_count;
  3641. if (!(relocation_group = obj_data->relocation_groups =
  3642. wasm_runtime_malloc(size))) {
  3643. aot_set_last_error("allocate memory for relocation groups failed.");
  3644. return false;
  3645. }
  3646. memset(obj_data->relocation_groups, 0, size);
  3647. /* resolve each relocation group */
  3648. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3649. aot_set_last_error("llvm get section iterator failed.");
  3650. return false;
  3651. }
  3652. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  3653. if (is_relocation_section(obj_data, sec_itr)) {
  3654. name = (char *)LLVMGetSectionName(sec_itr);
  3655. relocation_group->section_name = name;
  3656. if (obj_data->comp_ctx->enable_llvm_pgo
  3657. && (!strcmp(name, ".rela__llvm_prf_data")
  3658. || !strcmp(name, ".rel__llvm_prf_data"))) {
  3659. char buf[32];
  3660. snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_data_idx);
  3661. size = (uint32)(strlen(buf) + 1);
  3662. if (!(relocation_group->section_name =
  3663. wasm_runtime_malloc(size))) {
  3664. aot_set_last_error(
  3665. "allocate memory for section name failed.");
  3666. LLVMDisposeSectionIterator(sec_itr);
  3667. return false;
  3668. }
  3669. bh_memcpy_s(relocation_group->section_name, size, buf, size);
  3670. relocation_group->is_section_name_allocated = true;
  3671. }
  3672. if (!aot_resolve_object_relocation_group(obj_data, relocation_group,
  3673. sec_itr)) {
  3674. LLVMDisposeSectionIterator(sec_itr);
  3675. return false;
  3676. }
  3677. if (obj_data->comp_ctx->enable_llvm_pgo
  3678. && (!strcmp(name, ".rela__llvm_prf_data")
  3679. || !strcmp(name, ".rel__llvm_prf_data"))) {
  3680. llvm_prf_data_idx++;
  3681. }
  3682. if (!strcmp(relocation_group->section_name, ".rela.text.unlikely.")
  3683. || !strcmp(relocation_group->section_name, ".rela.text.hot.")) {
  3684. relocation_group->section_name = ".rela.text";
  3685. }
  3686. else if (!strcmp(relocation_group->section_name,
  3687. ".rela.ltext.unlikely.")
  3688. || !strcmp(relocation_group->section_name,
  3689. ".rela.ltext.hot.")) {
  3690. relocation_group->section_name = ".rela.ltext";
  3691. }
  3692. else if (!strcmp(relocation_group->section_name,
  3693. ".rel.text.unlikely.")
  3694. || !strcmp(relocation_group->section_name,
  3695. ".rel.text.hot.")) {
  3696. relocation_group->section_name = ".rel.text";
  3697. }
  3698. else if (!strcmp(relocation_group->section_name,
  3699. ".rel.ltext.unlikely.")
  3700. || !strcmp(relocation_group->section_name,
  3701. ".rel.ltext.hot.")) {
  3702. relocation_group->section_name = ".rel.ltext";
  3703. }
  3704. /*
  3705. * Relocations in read-only sections are problematic,
  3706. * especially for XIP on platforms which don't have
  3707. * copy-on-write mappings.
  3708. */
  3709. if (obj_data->comp_ctx->is_indirect_mode
  3710. && is_readonly_section(relocation_group->section_name)) {
  3711. LOG_WARNING("%" PRIu32
  3712. " text relocations in %s section for indirect mode",
  3713. relocation_group->relocation_count,
  3714. relocation_group->section_name);
  3715. }
  3716. relocation_group++;
  3717. }
  3718. LLVMMoveToNextSection(sec_itr);
  3719. }
  3720. LLVMDisposeSectionIterator(sec_itr);
  3721. return true;
  3722. }
  3723. static void
  3724. destroy_relocation_groups(AOTRelocationGroup *relocation_groups,
  3725. uint32 relocation_group_count)
  3726. {
  3727. uint32 i, j;
  3728. AOTRelocationGroup *relocation_group = relocation_groups;
  3729. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  3730. if (relocation_group->relocations) {
  3731. for (j = 0; j < relocation_group->relocation_count; j++) {
  3732. if (relocation_group->relocations[j].is_symbol_name_allocated)
  3733. wasm_runtime_free(
  3734. relocation_group->relocations[j].symbol_name);
  3735. }
  3736. wasm_runtime_free(relocation_group->relocations);
  3737. }
  3738. if (relocation_group->is_section_name_allocated)
  3739. wasm_runtime_free(relocation_group->section_name);
  3740. }
  3741. wasm_runtime_free(relocation_groups);
  3742. }
  3743. static void
  3744. destroy_relocation_symbol_list(AOTSymbolList *symbol_list)
  3745. {
  3746. AOTSymbolNode *elem;
  3747. elem = symbol_list->head;
  3748. while (elem) {
  3749. AOTSymbolNode *next = elem->next;
  3750. wasm_runtime_free(elem);
  3751. elem = next;
  3752. }
  3753. }
  3754. void
  3755. aot_obj_data_destroy(AOTObjectData *obj_data)
  3756. {
  3757. if (obj_data->binary)
  3758. LLVMDisposeBinary(obj_data->binary);
  3759. if (obj_data->mem_buf)
  3760. LLVMDisposeMemoryBuffer(obj_data->mem_buf);
  3761. if (obj_data->funcs)
  3762. wasm_runtime_free(obj_data->funcs);
  3763. if (obj_data->data_sections) {
  3764. uint32 i;
  3765. for (i = 0; i < obj_data->data_sections_count; i++) {
  3766. if (obj_data->data_sections[i].name
  3767. && obj_data->data_sections[i].is_name_allocated) {
  3768. wasm_runtime_free(obj_data->data_sections[i].name);
  3769. }
  3770. if (obj_data->data_sections[i].data
  3771. && obj_data->data_sections[i].is_data_allocated) {
  3772. wasm_runtime_free(obj_data->data_sections[i].data);
  3773. }
  3774. }
  3775. wasm_runtime_free(obj_data->data_sections);
  3776. }
  3777. if (obj_data->relocation_groups)
  3778. destroy_relocation_groups(obj_data->relocation_groups,
  3779. obj_data->relocation_group_count);
  3780. if (obj_data->symbol_list.len)
  3781. destroy_relocation_symbol_list(&obj_data->symbol_list);
  3782. if (obj_data->stack_sizes)
  3783. wasm_runtime_free(obj_data->stack_sizes);
  3784. wasm_runtime_free(obj_data);
  3785. }
  3786. AOTObjectData *
  3787. aot_obj_data_create(AOTCompContext *comp_ctx)
  3788. {
  3789. char *err = NULL;
  3790. AOTObjectData *obj_data;
  3791. LLVMTargetRef target = LLVMGetTargetMachineTarget(comp_ctx->target_machine);
  3792. bh_print_time("Begin to emit object file to buffer");
  3793. if (!(obj_data = wasm_runtime_malloc(sizeof(AOTObjectData)))) {
  3794. aot_set_last_error("allocate memory failed.");
  3795. return false;
  3796. }
  3797. memset(obj_data, 0, sizeof(AOTObjectData));
  3798. obj_data->comp_ctx = comp_ctx;
  3799. bh_print_time("Begin to emit object file");
  3800. if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
  3801. /* Generate a temp file name */
  3802. int ret;
  3803. char obj_file_name[64];
  3804. if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name,
  3805. sizeof(obj_file_name))) {
  3806. goto fail;
  3807. }
  3808. if (!aot_emit_object_file(comp_ctx, obj_file_name)) {
  3809. goto fail;
  3810. }
  3811. /* create memory buffer from object file */
  3812. ret = LLVMCreateMemoryBufferWithContentsOfFile(
  3813. obj_file_name, &obj_data->mem_buf, &err);
  3814. /* remove temp object file */
  3815. unlink(obj_file_name);
  3816. if (ret != 0) {
  3817. if (err) {
  3818. LLVMDisposeMessage(err);
  3819. err = NULL;
  3820. }
  3821. aot_set_last_error("create mem buffer with file failed.");
  3822. goto fail;
  3823. }
  3824. }
  3825. else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
  3826. /* Emit to assembly file instead for arc target
  3827. as it cannot emit to object file */
  3828. char file_name[] = "wasm-XXXXXX", buf[128];
  3829. int ret;
  3830. if (!bh_mkstemp(file_name, sizeof(file_name))) {
  3831. aot_set_last_error("make temp file failed.");
  3832. goto fail;
  3833. }
  3834. snprintf(buf, sizeof(buf), "%s%s", file_name, ".s");
  3835. if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine,
  3836. comp_ctx->module, buf, LLVMAssemblyFile,
  3837. &err)
  3838. != 0) {
  3839. if (err) {
  3840. LLVMDisposeMessage(err);
  3841. err = NULL;
  3842. }
  3843. aot_set_last_error("emit elf to object file failed.");
  3844. goto fail;
  3845. }
  3846. /* call arc gcc to compile assembly file to object file */
  3847. /* TODO: get arc gcc from environment variable firstly
  3848. and check whether the toolchain exists actually */
  3849. snprintf(buf, sizeof(buf), "%s%s%s%s%s%s",
  3850. "/opt/zephyr-sdk/arc-zephyr-elf/bin/arc-zephyr-elf-gcc ",
  3851. "-mcpu=arcem -o ", file_name, ".o -c ", file_name, ".s");
  3852. /* TODO: use try..catch to handle possible exceptions */
  3853. ret = bh_system(buf);
  3854. /* remove temp assembly file */
  3855. snprintf(buf, sizeof(buf), "%s%s", file_name, ".s");
  3856. unlink(buf);
  3857. if (ret != 0) {
  3858. aot_set_last_error("failed to compile asm file to obj file "
  3859. "with arc gcc toolchain.");
  3860. goto fail;
  3861. }
  3862. /* create memory buffer from object file */
  3863. snprintf(buf, sizeof(buf), "%s%s", file_name, ".o");
  3864. ret = LLVMCreateMemoryBufferWithContentsOfFile(buf, &obj_data->mem_buf,
  3865. &err);
  3866. /* remove temp object file */
  3867. snprintf(buf, sizeof(buf), "%s%s", file_name, ".o");
  3868. unlink(buf);
  3869. if (ret != 0) {
  3870. if (err) {
  3871. LLVMDisposeMessage(err);
  3872. err = NULL;
  3873. }
  3874. aot_set_last_error("create mem buffer with file failed.");
  3875. goto fail;
  3876. }
  3877. }
  3878. else {
  3879. if (LLVMTargetMachineEmitToMemoryBuffer(
  3880. comp_ctx->target_machine, comp_ctx->module, LLVMObjectFile,
  3881. &err, &obj_data->mem_buf)
  3882. != 0) {
  3883. if (err) {
  3884. LLVMDisposeMessage(err);
  3885. err = NULL;
  3886. }
  3887. aot_set_last_error("llvm emit to memory buffer failed.");
  3888. goto fail;
  3889. }
  3890. }
  3891. if (!(obj_data->binary = LLVMCreateBinary(obj_data->mem_buf, NULL, &err))) {
  3892. if (err) {
  3893. LLVMDisposeMessage(err);
  3894. err = NULL;
  3895. }
  3896. aot_set_last_error("llvm create binary failed.");
  3897. goto fail;
  3898. }
  3899. /* Create wasm feature flags form compile options */
  3900. obj_data->target_info.feature_flags = 0;
  3901. if (comp_ctx->enable_simd) {
  3902. obj_data->target_info.feature_flags |= WASM_FEATURE_SIMD_128BIT;
  3903. }
  3904. if (comp_ctx->enable_bulk_memory) {
  3905. obj_data->target_info.feature_flags |= WASM_FEATURE_BULK_MEMORY;
  3906. }
  3907. if (comp_ctx->enable_thread_mgr) {
  3908. obj_data->target_info.feature_flags |= WASM_FEATURE_MULTI_THREAD;
  3909. }
  3910. if (comp_ctx->enable_ref_types) {
  3911. obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES;
  3912. }
  3913. if (comp_ctx->enable_gc) {
  3914. obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION;
  3915. }
  3916. if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_TINY) {
  3917. obj_data->target_info.feature_flags |= WASM_FEATURE_TINY_STACK_FRAME;
  3918. }
  3919. if (comp_ctx->call_stack_features.frame_per_function) {
  3920. obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_PER_FUNCTION;
  3921. }
  3922. if (!comp_ctx->call_stack_features.func_idx) {
  3923. obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_NO_FUNC_IDX;
  3924. }
  3925. bh_print_time("Begin to resolve object file info");
  3926. /* resolve target info/text/relocations/functions */
  3927. if (!aot_resolve_target_info(comp_ctx, obj_data)
  3928. || !aot_resolve_text(obj_data) || !aot_resolve_literal(obj_data)
  3929. || !aot_resolve_object_data_sections(obj_data)
  3930. || !aot_resolve_functions(comp_ctx, obj_data)
  3931. || !aot_resolve_object_relocation_groups(obj_data))
  3932. goto fail;
  3933. return obj_data;
  3934. fail:
  3935. aot_obj_data_destroy(obj_data);
  3936. return NULL;
  3937. }
  3938. uint8 *
  3939. aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  3940. uint32 *p_aot_file_size)
  3941. {
  3942. AOTObjectData *obj_data = aot_obj_data_create(comp_ctx);
  3943. uint8 *aot_file_buf;
  3944. uint32 aot_file_size;
  3945. if (!obj_data)
  3946. return NULL;
  3947. aot_file_size = aot_get_aot_file_size(comp_ctx, comp_data, obj_data);
  3948. if (aot_file_size == 0) {
  3949. aot_set_last_error("get aot file size failed");
  3950. goto fail1;
  3951. }
  3952. if (!(aot_file_buf = wasm_runtime_malloc(aot_file_size))) {
  3953. aot_set_last_error("allocate memory failed.");
  3954. goto fail1;
  3955. }
  3956. memset(aot_file_buf, 0, aot_file_size);
  3957. if (!aot_emit_aot_file_buf_ex(comp_ctx, comp_data, obj_data, aot_file_buf,
  3958. aot_file_size))
  3959. goto fail2;
  3960. *p_aot_file_size = aot_file_size;
  3961. aot_obj_data_destroy(obj_data);
  3962. return aot_file_buf;
  3963. fail2:
  3964. wasm_runtime_free(aot_file_buf);
  3965. fail1:
  3966. aot_obj_data_destroy(obj_data);
  3967. return NULL;
  3968. }
  3969. bool
  3970. aot_emit_aot_file_buf_ex(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  3971. AOTObjectData *obj_data, uint8 *buf,
  3972. uint32 aot_file_size)
  3973. {
  3974. uint8 *buf_end = buf + aot_file_size;
  3975. uint32 offset = 0;
  3976. if (!aot_emit_file_header(buf, buf_end, &offset, comp_data, obj_data)
  3977. || !aot_emit_target_info_section(buf, buf_end, &offset, comp_data,
  3978. obj_data)
  3979. || !aot_emit_init_data_section(buf, buf_end, &offset, comp_ctx,
  3980. comp_data, obj_data)
  3981. || !aot_emit_text_section(buf, buf_end, &offset, comp_data, obj_data)
  3982. || !aot_emit_func_section(buf, buf_end, &offset, comp_ctx, comp_data,
  3983. obj_data)
  3984. || !aot_emit_export_section(buf, buf_end, &offset, comp_ctx, comp_data,
  3985. obj_data)
  3986. || !aot_emit_relocation_section(buf, buf_end, &offset, comp_ctx,
  3987. comp_data, obj_data)
  3988. || !aot_emit_native_symbol(buf, buf_end, &offset, comp_ctx)
  3989. || !aot_emit_custom_sections(buf, buf_end, &offset, comp_data, comp_ctx)
  3990. #if WASM_ENABLE_STRINGREF != 0
  3991. || !aot_emit_string_literal_section(buf, buf_end, &offset, comp_data,
  3992. comp_ctx)
  3993. #endif
  3994. )
  3995. return false;
  3996. #if 0
  3997. dump_buf(buf, offset, "sections");
  3998. #endif
  3999. if (offset != aot_file_size) {
  4000. aot_set_last_error("emit aot file failed.");
  4001. return false;
  4002. }
  4003. return true;
  4004. }
  4005. bool
  4006. aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  4007. const char *file_name)
  4008. {
  4009. uint8 *aot_file_buf;
  4010. uint32 aot_file_size;
  4011. bool ret = false;
  4012. FILE *file;
  4013. bh_print_time("Begin to emit AOT file");
  4014. if (!(aot_file_buf =
  4015. aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size))) {
  4016. return false;
  4017. }
  4018. /* write buffer to file */
  4019. if (!(file = fopen(file_name, "wb"))) {
  4020. aot_set_last_error("open or create aot file failed.");
  4021. goto fail1;
  4022. }
  4023. if (!fwrite(aot_file_buf, aot_file_size, 1, file)) {
  4024. aot_set_last_error("write to aot file failed.");
  4025. goto fail2;
  4026. }
  4027. ret = true;
  4028. fail2:
  4029. fclose(file);
  4030. fail1:
  4031. wasm_runtime_free(aot_file_buf);
  4032. return ret;
  4033. }