aot_emit_aot_file.c 155 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579
  1. /*
  2. * Copyright (C) 2019 Intel Corporation. All rights reserved.
  3. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  4. */
  5. #include "aot_emit_aot_file.h"
  6. #include "../aot/aot_runtime.h"
  7. #define PUT_U64_TO_ADDR(addr, value) \
  8. do { \
  9. union { \
  10. uint64 val; \
  11. uint32 parts[2]; \
  12. } u; \
  13. u.val = (value); \
  14. ((uint32 *)(addr))[0] = u.parts[0]; \
  15. ((uint32 *)(addr))[1] = u.parts[1]; \
  16. } while (0)
  17. #define CHECK_SIZE(size) \
  18. do { \
  19. if (size == (uint32)-1) { \
  20. aot_set_last_error("get symbol size failed."); \
  21. return (uint32)-1; \
  22. } \
  23. } while (0)
  24. /* Internal function in object file */
  25. typedef struct AOTObjectFunc {
  26. char *func_name;
  27. /* text offset of aot_func#n */
  28. uint64 text_offset;
  29. /* text offset of aot_func_internal#n */
  30. uint64 text_offset_of_aot_func_internal;
  31. } AOTObjectFunc;
  32. /* Symbol table list node */
  33. typedef struct AOTSymbolNode {
  34. struct AOTSymbolNode *next;
  35. uint32 str_len;
  36. char *symbol;
  37. } AOTSymbolNode;
  38. typedef struct AOTSymbolList {
  39. AOTSymbolNode *head;
  40. AOTSymbolNode *end;
  41. uint32 len;
  42. } AOTSymbolList;
  43. /* AOT object data */
  44. typedef struct AOTObjectData {
  45. AOTCompContext *comp_ctx;
  46. LLVMMemoryBufferRef mem_buf;
  47. LLVMBinaryRef binary;
  48. AOTTargetInfo target_info;
  49. void *text;
  50. uint32 text_size;
  51. void *text_unlikely;
  52. uint32 text_unlikely_size;
  53. void *text_hot;
  54. uint32 text_hot_size;
  55. /* literal data and size */
  56. void *literal;
  57. uint32 literal_size;
  58. AOTObjectDataSection *data_sections;
  59. uint32 data_sections_count;
  60. AOTObjectFunc *funcs;
  61. uint32 func_count;
  62. AOTSymbolList symbol_list;
  63. AOTRelocationGroup *relocation_groups;
  64. uint32 relocation_group_count;
  65. const char *stack_sizes_section_name;
  66. uint32 stack_sizes_offset;
  67. uint32 *stack_sizes;
  68. } AOTObjectData;
  69. #if 0
  70. static void dump_buf(uint8 *buf, uint32 size, char *title)
  71. {
  72. int i;
  73. printf("------ %s -------", title);
  74. for (i = 0; i < size; i++) {
  75. if ((i % 16) == 0)
  76. printf("\n");
  77. printf("%02x ", (unsigned char)buf[i]);
  78. }
  79. printf("\n\n");
  80. }
  81. #endif
  82. static bool
  83. is_32bit_binary(const AOTObjectData *obj_data)
  84. {
  85. /* bit 1: 0 is 32-bit, 1 is 64-bit */
  86. return obj_data->target_info.bin_type & 2 ? false : true;
  87. }
  88. static bool
  89. is_little_endian_binary(const AOTObjectData *obj_data)
  90. {
  91. /* bit 0: 0 is little-endian, 1 is big-endian */
  92. return obj_data->target_info.bin_type & 1 ? false : true;
  93. }
  94. static bool
  95. str_starts_with(const char *str, const char *prefix)
  96. {
  97. size_t len_pre = strlen(prefix), len_str = strlen(str);
  98. return (len_str >= len_pre) && !memcmp(str, prefix, len_pre);
  99. }
  100. static uint32
  101. get_file_header_size()
  102. {
  103. /* magic number (4 bytes) + version (4 bytes) */
  104. return sizeof(uint32) + sizeof(uint32);
  105. }
  106. static uint32
  107. get_string_size(AOTCompContext *comp_ctx, const char *s)
  108. {
  109. /* string size (2 bytes) + string content + '\0' */
  110. return (uint32)sizeof(uint16) + (uint32)strlen(s) + 1;
  111. }
  112. static uint32
  113. get_target_info_section_size()
  114. {
  115. return sizeof(AOTTargetInfo);
  116. }
  117. static uint32
  118. get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data,
  119. InitializerExpression *expr);
  120. static uint32
  121. get_mem_init_data_size(AOTCompContext *comp_ctx, AOTMemInitData *mem_init_data)
  122. {
  123. /* init expr type (4 bytes)
  124. * + init expr value (4 bytes, valid value can only be i32/get_global)
  125. * + byte count (4 bytes) + bytes */
  126. uint32 total_size =
  127. (uint32)(get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  128. &mem_init_data->offset)
  129. + sizeof(uint32) + mem_init_data->byte_count);
  130. /* bulk_memory enabled:
  131. is_passive (4 bytes) + memory_index (4 bytes)
  132. bulk memory disabled:
  133. placeholder (4 bytes) + placeholder (4 bytes)
  134. */
  135. total_size += (sizeof(uint32) + sizeof(uint32));
  136. return total_size;
  137. }
  138. static uint32
  139. get_mem_init_data_list_size(AOTCompContext *comp_ctx,
  140. AOTMemInitData **mem_init_data_list,
  141. uint32 mem_init_data_count)
  142. {
  143. AOTMemInitData **mem_init_data = mem_init_data_list;
  144. uint32 size = 0, i;
  145. for (i = 0; i < mem_init_data_count; i++, mem_init_data++) {
  146. size = align_uint(size, 4);
  147. size += get_mem_init_data_size(comp_ctx, *mem_init_data);
  148. }
  149. return size;
  150. }
  151. static uint32
  152. get_import_memory_size(AOTCompData *comp_data)
  153. {
  154. /* currently we only emit import_memory_count = 0 */
  155. return sizeof(uint32);
  156. }
  157. static uint32
  158. get_memory_size(AOTCompData *comp_data)
  159. {
  160. /* memory_count + count * (flags + num_bytes_per_page +
  161. init_page_count + max_page_count) */
  162. return (uint32)(sizeof(uint32)
  163. + comp_data->memory_count * sizeof(uint32) * 4);
  164. }
  165. static uint32
  166. get_mem_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  167. {
  168. /* import_memory_size + memory_size
  169. + init_data_count + init_data_list */
  170. return get_import_memory_size(comp_data) + get_memory_size(comp_data)
  171. + (uint32)sizeof(uint32)
  172. + get_mem_init_data_list_size(comp_ctx,
  173. comp_data->mem_init_data_list,
  174. comp_data->mem_init_data_count);
  175. }
  176. static uint32
  177. get_init_expr_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data,
  178. InitializerExpression *expr)
  179. {
  180. /* init_expr_type */
  181. uint32 size = sizeof(uint32);
  182. #if WASM_ENABLE_GC != 0
  183. WASMModule *module = comp_data->wasm_module;
  184. #endif
  185. /* + init value size */
  186. switch (expr->init_expr_type) {
  187. case INIT_EXPR_NONE:
  188. /* no init value, used in table initializer */
  189. break;
  190. case INIT_EXPR_TYPE_I32_CONST:
  191. case INIT_EXPR_TYPE_F32_CONST:
  192. case INIT_EXPR_TYPE_GET_GLOBAL:
  193. size += sizeof(uint32);
  194. break;
  195. case INIT_EXPR_TYPE_I64_CONST:
  196. case INIT_EXPR_TYPE_F64_CONST:
  197. size += sizeof(uint64);
  198. break;
  199. case INIT_EXPR_TYPE_V128_CONST:
  200. size += sizeof(uint64) * 2;
  201. break;
  202. case INIT_EXPR_TYPE_FUNCREF_CONST:
  203. case INIT_EXPR_TYPE_REFNULL_CONST:
  204. /* ref_index */
  205. size += sizeof(uint32);
  206. break;
  207. #if WASM_ENABLE_GC != 0
  208. case INIT_EXPR_TYPE_I31_NEW:
  209. /* i32 */
  210. size += sizeof(uint32);
  211. break;
  212. case INIT_EXPR_TYPE_STRUCT_NEW:
  213. {
  214. uint32 i;
  215. WASMStructNewInitValues *struct_new_init_values =
  216. (WASMStructNewInitValues *)expr->u.data;
  217. /* type_index + field_count + fields */
  218. size += sizeof(uint32) + sizeof(uint32);
  219. bh_assert(struct_new_init_values->type_idx < module->type_count);
  220. for (i = 0; i < struct_new_init_values->count; i++) {
  221. WASMStructType *struct_type =
  222. (WASMStructType *)
  223. module->types[struct_new_init_values->type_idx];
  224. uint32 field_size;
  225. bh_assert(struct_type);
  226. bh_assert(struct_type->field_count
  227. == struct_new_init_values->count);
  228. field_size = wasm_value_type_size_internal(
  229. struct_type->fields[i].field_type, comp_ctx->pointer_size);
  230. if (field_size < sizeof(uint32))
  231. field_size = sizeof(uint32);
  232. size += field_size;
  233. }
  234. break;
  235. }
  236. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  237. /* type_index */
  238. size += sizeof(uint32);
  239. break;
  240. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  241. /* array_elem_type + type_index + len */
  242. size += sizeof(uint32) * 3;
  243. break;
  244. case INIT_EXPR_TYPE_ARRAY_NEW:
  245. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  246. {
  247. WASMArrayNewInitValues *array_new_init_values =
  248. (WASMArrayNewInitValues *)expr->u.data;
  249. WASMArrayType *array_type = NULL;
  250. uint32 value_count;
  251. array_type =
  252. (WASMArrayType *)module->types[array_new_init_values->type_idx];
  253. bh_assert(array_type);
  254. bh_assert(array_new_init_values->type_idx < module->type_count);
  255. value_count =
  256. (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED)
  257. ? array_new_init_values->length
  258. : 1;
  259. /* array_elem_type + type_index + len + elems */
  260. size += sizeof(uint32) * 3
  261. + wasm_value_type_size_internal(array_type->elem_type,
  262. comp_ctx->pointer_size)
  263. * value_count;
  264. break;
  265. }
  266. #endif /* end of WASM_ENABLE_GC != 0 */
  267. default:
  268. bh_assert(0);
  269. }
  270. return size;
  271. }
  272. static uint32
  273. get_table_init_data_size(AOTCompContext *comp_ctx,
  274. AOTTableInitData *table_init_data)
  275. {
  276. uint32 size, i;
  277. /*
  278. * mode (4 bytes), elem_type (4 bytes)
  279. *
  280. * table_index(4 bytes) + init expr type (4 bytes) + init expr value (8
  281. * bytes)
  282. */
  283. size = (uint32)(sizeof(uint32) * 2 + sizeof(uint32) + sizeof(uint32)
  284. + sizeof(uint64))
  285. /* Size of WasmRefType - inner padding (ref type + nullable +
  286. heap_type) */
  287. + 8;
  288. /* + value count/func index count (4 bytes) + init_values */
  289. size += sizeof(uint32);
  290. for (i = 0; i < table_init_data->value_count; i++) {
  291. size += get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  292. &table_init_data->init_values[i]);
  293. }
  294. return size;
  295. }
  296. static uint32
  297. get_table_init_data_list_size(AOTCompContext *comp_ctx,
  298. AOTTableInitData **table_init_data_list,
  299. uint32 table_init_data_count)
  300. {
  301. /*
  302. * ------------------------------
  303. * | table_init_data_count
  304. * ------------------------------
  305. * | | U32 mode
  306. * | AOTTableInitData[N] | U32 elem_type
  307. * | | U32 table_index
  308. * | | U32 offset.init_expr_type
  309. * | | U64 offset.u.i64
  310. * | | U32 func_index_count / elem_count
  311. * | | UINTPTR [func_index_count] / [elem_count]
  312. * ------------------------------
  313. */
  314. AOTTableInitData **table_init_data = table_init_data_list;
  315. uint32 size = 0, i;
  316. /* table_init_data_count(4 bytes) */
  317. size = (uint32)sizeof(uint32);
  318. for (i = 0; i < table_init_data_count; i++, table_init_data++) {
  319. size = align_uint(size, 4);
  320. size += get_table_init_data_size(comp_ctx, *table_init_data);
  321. }
  322. return size;
  323. }
  324. static uint32
  325. get_import_table_size(const AOTCompContext *comp_ctx,
  326. const AOTCompData *comp_data)
  327. {
  328. /*
  329. * ------------------------------
  330. * | import_table_count
  331. * ------------------------------
  332. * | | U8 elem_type
  333. * | | U8 flags
  334. * | | U8 possible_grow
  335. * | AOTImportTable[N] | U8 elem_ref_type.nullable (for GC only)
  336. * | | U32 init_size
  337. * | | U32 max_size
  338. * | | U32 elem_ref_type.heap_type (for GC only)
  339. * ------------------------------
  340. */
  341. uint32 size = 0, i;
  342. size = (uint32)sizeof(uint32);
  343. for (i = 0; i < comp_data->import_table_count; i++) {
  344. size += sizeof(uint32) * 3;
  345. #if WASM_ENABLE_GC != 0
  346. if (comp_ctx->enable_gc
  347. && comp_data->import_tables[i].table_type.elem_ref_type)
  348. size += sizeof(uint32);
  349. #endif
  350. }
  351. return size;
  352. }
  353. static uint32
  354. get_table_size(const AOTCompContext *comp_ctx, const AOTCompData *comp_data)
  355. {
  356. /*
  357. * ------------------------------
  358. * | table_count
  359. * ------------------------------
  360. * | | U8 elem_type
  361. * | | U8 flags
  362. * | | U8 possible_grow
  363. * | AOTTable[N] | U8 elem_ref_type.nullable (for GC only)
  364. * | | U32 init_size
  365. * | | U32 max_size
  366. * | | U32 elem_ref_type.heap_type (for GC only)
  367. * | | N init_expr (for GC only)
  368. * ------------------------------
  369. */
  370. uint32 size = 0, i;
  371. size = (uint32)sizeof(uint32);
  372. for (i = 0; i < comp_data->table_count; i++) {
  373. size += sizeof(uint32) * 3;
  374. #if WASM_ENABLE_GC != 0
  375. if (comp_ctx->enable_gc) {
  376. if (comp_data->tables[i].table_type.elem_ref_type) {
  377. size += sizeof(uint32);
  378. }
  379. size += get_init_expr_size(comp_ctx, comp_data,
  380. &comp_data->tables[i].init_expr);
  381. }
  382. #endif
  383. }
  384. return size;
  385. }
  386. static uint32
  387. get_table_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  388. {
  389. /*
  390. * ------------------------------
  391. * | import_table_count
  392. * ------------------------------
  393. * |
  394. * | AOTImportTable[import_table_count]
  395. * |
  396. * ------------------------------
  397. * | table_count
  398. * ------------------------------
  399. * |
  400. * | AOTTable[table_count]
  401. * |
  402. * ------------------------------
  403. * | table_init_data_count
  404. * ------------------------------
  405. * |
  406. * | AOTTableInitData*[table_init_data_count]
  407. * |
  408. * ------------------------------
  409. */
  410. return get_import_table_size(comp_ctx, comp_data)
  411. + get_table_size(comp_ctx, comp_data)
  412. + get_table_init_data_list_size(comp_ctx,
  413. comp_data->table_init_data_list,
  414. comp_data->table_init_data_count);
  415. }
  416. static uint32
  417. get_func_type_size(AOTCompContext *comp_ctx, AOTFuncType *func_type)
  418. {
  419. #if WASM_ENABLE_GC != 0
  420. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  421. + rec_count + rec_idx + param count + result count
  422. + ref_type_map_count + types + context of ref_type_map */
  423. if (comp_ctx->enable_gc) {
  424. uint32 size = 0;
  425. /* type flag */
  426. size += sizeof(func_type->base_type.type_flag);
  427. /* equivalence type flag + is_sub_final */
  428. size += sizeof(uint16);
  429. /* parent_type_idx */
  430. size += sizeof(func_type->base_type.parent_type_idx);
  431. /* rec_count */
  432. size += sizeof(func_type->base_type.rec_count);
  433. /* rec_idx */
  434. size += sizeof(func_type->base_type.rec_idx);
  435. /* param count */
  436. size += sizeof(func_type->param_count);
  437. /* result count */
  438. size += sizeof(func_type->result_count);
  439. /* ref_type_map_count */
  440. size += sizeof(func_type->ref_type_map_count);
  441. /* param and result types */
  442. size += func_type->param_count + func_type->result_count;
  443. /* align size */
  444. size = align_uint(size, 4);
  445. /* ref_type_map */
  446. size += func_type->ref_type_map_count * 8;
  447. return size;
  448. }
  449. else
  450. #endif
  451. {
  452. /* type flag + param count + result count + types */
  453. return (uint32)sizeof(uint16) * 3 + func_type->param_count
  454. + func_type->result_count;
  455. }
  456. }
  457. #if WASM_ENABLE_GC != 0
  458. static uint32
  459. get_struct_type_size(AOTCompContext *comp_ctx, AOTStructType *struct_type)
  460. {
  461. uint32 size = 0;
  462. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  463. + rec_count + rec_idx + field count + fields */
  464. /* type flag */
  465. size += sizeof(struct_type->base_type.type_flag);
  466. /* equivalence type flag + is_sub_final */
  467. size += sizeof(uint16);
  468. /* parent_type_idx */
  469. size += sizeof(struct_type->base_type.parent_type_idx);
  470. /* rec_count */
  471. size += sizeof(struct_type->base_type.rec_count);
  472. /* rec_idx */
  473. size += sizeof(struct_type->base_type.rec_idx);
  474. /* field count */
  475. size += sizeof(struct_type->field_count);
  476. /* field types */
  477. size += struct_type->field_count * 2;
  478. /* ref_type_map_count */
  479. size += sizeof(struct_type->ref_type_map_count);
  480. size = align_uint(size, 4);
  481. /* ref_type_map */
  482. size += struct_type->ref_type_map_count * 8;
  483. return size;
  484. }
  485. static uint32
  486. get_array_type_size(AOTCompContext *comp_ctx, AOTArrayType *array_type)
  487. {
  488. uint32 size = 0;
  489. /* type flag + equivalence type flag + is_sub_final + parent_type_idx
  490. + rec_count + rec_idx + elem_flags + elem_type + elem_ref_type */
  491. /* type flag */
  492. size += sizeof(array_type->base_type.type_flag);
  493. /* equivalence type flag + is_sub_final */
  494. size += sizeof(uint16);
  495. /* parent_type_idx (u32) */
  496. size += sizeof(array_type->base_type.parent_type_idx);
  497. /* rec_count */
  498. size += sizeof(array_type->base_type.rec_count);
  499. /* rec_idx */
  500. size += sizeof(array_type->base_type.rec_idx);
  501. /* elem_flags (u16) */
  502. size += sizeof(array_type->elem_flags);
  503. /* elem_type (u8) */
  504. size += sizeof(array_type->elem_type);
  505. /* elem_ref_type */
  506. if (array_type->elem_ref_type) {
  507. /* nullable (u8) */
  508. size += sizeof(uint8);
  509. /* heap type (u32) */
  510. size += sizeof(uint32);
  511. }
  512. return size;
  513. }
  514. #endif
  515. static uint32
  516. get_type_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  517. {
  518. /* Initial size with size of type count */
  519. uint32 size = 4;
  520. uint32 i;
  521. #if WASM_ENABLE_GC != 0
  522. if (comp_ctx->enable_gc) {
  523. for (i = 0; i < comp_data->type_count; i++) {
  524. uint32 j;
  525. size = align_uint(size, 4);
  526. /* Emit simple info if there is an equivalence type */
  527. for (j = 0; j < i; j++) {
  528. if (comp_data->types[j] == comp_data->types[i]) {
  529. /* type_flag (2 bytes) + equivalence type flag (1 byte)
  530. + padding (1 byte) + equivalence type index */
  531. size += 8;
  532. break;
  533. }
  534. }
  535. if (j < i)
  536. continue;
  537. if (comp_data->types[i]->type_flag == WASM_TYPE_FUNC)
  538. size += get_func_type_size(comp_ctx,
  539. (AOTFuncType *)comp_data->types[i]);
  540. else if (comp_data->types[i]->type_flag == WASM_TYPE_STRUCT)
  541. size += get_struct_type_size(
  542. comp_ctx, (AOTStructType *)comp_data->types[i]);
  543. else if (comp_data->types[i]->type_flag == WASM_TYPE_ARRAY)
  544. size += get_array_type_size(
  545. comp_ctx, (AOTArrayType *)comp_data->types[i]);
  546. else
  547. bh_assert(0);
  548. }
  549. }
  550. else
  551. #endif
  552. {
  553. for (i = 0; i < comp_data->type_count; i++) {
  554. size = align_uint(size, 4);
  555. size += get_func_type_size(comp_ctx,
  556. (AOTFuncType *)comp_data->types[i]);
  557. }
  558. }
  559. return size;
  560. }
  561. static uint32
  562. get_import_global_size(AOTCompContext *comp_ctx, AOTImportGlobal *import_global)
  563. {
  564. /* type (1 byte) + is_mutable (1 byte) + module_name + global_name */
  565. uint32 size = (uint32)sizeof(uint8) * 2
  566. + get_string_size(comp_ctx, import_global->module_name);
  567. size = align_uint(size, 2);
  568. size += get_string_size(comp_ctx, import_global->global_name);
  569. return size;
  570. }
  571. static uint32
  572. get_import_globals_size(AOTCompContext *comp_ctx,
  573. AOTImportGlobal *import_globals,
  574. uint32 import_global_count)
  575. {
  576. AOTImportGlobal *import_global = import_globals;
  577. uint32 size = 0, i;
  578. for (i = 0; i < import_global_count; i++, import_global++) {
  579. size = align_uint(size, 2);
  580. size += get_import_global_size(comp_ctx, import_global);
  581. }
  582. return size;
  583. }
  584. static uint32
  585. get_import_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  586. {
  587. /* import global count + import globals */
  588. return (uint32)sizeof(uint32)
  589. + get_import_globals_size(comp_ctx, comp_data->import_globals,
  590. comp_data->import_global_count);
  591. }
  592. static uint32
  593. get_global_size(AOTCompContext *comp_ctx, AOTGlobal *global)
  594. {
  595. /* type (1 byte) + is_mutable (1 byte) + padding (2 bytes)
  596. + init expr value (include init expr type) */
  597. return sizeof(uint8) * 2 + sizeof(uint8) * 2
  598. + get_init_expr_size(comp_ctx, comp_ctx->comp_data,
  599. &global->init_expr);
  600. }
  601. static uint32
  602. get_globals_size(AOTCompContext *comp_ctx, AOTGlobal *globals,
  603. uint32 global_count)
  604. {
  605. AOTGlobal *global = globals;
  606. uint32 size = 0, i;
  607. for (i = 0; i < global_count; i++, global++) {
  608. size = align_uint(size, 4);
  609. size += get_global_size(comp_ctx, global);
  610. }
  611. return size;
  612. }
  613. static uint32
  614. get_global_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  615. {
  616. /* global count + globals */
  617. return (uint32)sizeof(uint32)
  618. + get_globals_size(comp_ctx, comp_data->globals,
  619. comp_data->global_count);
  620. }
  621. static uint32
  622. get_import_func_size(AOTCompContext *comp_ctx, AOTImportFunc *import_func)
  623. {
  624. /* type index (2 bytes) + module_name + func_name */
  625. uint32 size = (uint32)sizeof(uint16)
  626. + get_string_size(comp_ctx, import_func->module_name);
  627. size = align_uint(size, 2);
  628. size += get_string_size(comp_ctx, import_func->func_name);
  629. return size;
  630. }
  631. static uint32
  632. get_import_funcs_size(AOTCompContext *comp_ctx, AOTImportFunc *import_funcs,
  633. uint32 import_func_count)
  634. {
  635. AOTImportFunc *import_func = import_funcs;
  636. uint32 size = 0, i;
  637. for (i = 0; i < import_func_count; i++, import_func++) {
  638. size = align_uint(size, 2);
  639. size += get_import_func_size(comp_ctx, import_func);
  640. }
  641. return size;
  642. }
  643. static uint32
  644. get_import_func_info_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  645. {
  646. /* import func count + import funcs */
  647. return (uint32)sizeof(uint32)
  648. + get_import_funcs_size(comp_ctx, comp_data->import_funcs,
  649. comp_data->import_func_count);
  650. }
  651. static uint32
  652. get_object_data_sections_size(AOTCompContext *comp_ctx,
  653. AOTObjectDataSection *data_sections,
  654. uint32 data_sections_count)
  655. {
  656. AOTObjectDataSection *data_section = data_sections;
  657. uint32 size = 0, i;
  658. for (i = 0; i < data_sections_count; i++, data_section++) {
  659. /* name + size + data */
  660. size = align_uint(size, 2);
  661. size += get_string_size(comp_ctx, data_section->name);
  662. size = align_uint(size, 4);
  663. size += (uint32)sizeof(uint32);
  664. size += data_section->size;
  665. }
  666. return size;
  667. }
  668. static uint32
  669. get_object_data_section_info_size(AOTCompContext *comp_ctx,
  670. AOTObjectData *obj_data)
  671. {
  672. /* data sections count + data sections */
  673. return (uint32)sizeof(uint32)
  674. + get_object_data_sections_size(comp_ctx, obj_data->data_sections,
  675. obj_data->data_sections_count);
  676. }
  677. static uint32
  678. get_init_data_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  679. AOTObjectData *obj_data)
  680. {
  681. uint32 size = 0;
  682. size += get_mem_info_size(comp_ctx, comp_data);
  683. size = align_uint(size, 4);
  684. size += get_table_info_size(comp_ctx, comp_data);
  685. size = align_uint(size, 4);
  686. size += get_type_info_size(comp_ctx, comp_data);
  687. size = align_uint(size, 4);
  688. size += get_import_global_info_size(comp_ctx, comp_data);
  689. size = align_uint(size, 4);
  690. size += get_global_info_size(comp_ctx, comp_data);
  691. size = align_uint(size, 4);
  692. size += get_import_func_info_size(comp_ctx, comp_data);
  693. /* func count + start func index */
  694. size = align_uint(size, 4);
  695. size += (uint32)sizeof(uint32) * 2;
  696. /* aux data/heap/stack data */
  697. size += sizeof(uint32) * 10;
  698. size += get_object_data_section_info_size(comp_ctx, obj_data);
  699. return size;
  700. }
  701. static uint32
  702. get_text_section_size(AOTObjectData *obj_data)
  703. {
  704. return sizeof(uint32) + align_uint(obj_data->literal_size, 4)
  705. + align_uint(obj_data->text_size, 4)
  706. + align_uint(obj_data->text_unlikely_size, 4)
  707. + align_uint(obj_data->text_hot_size, 4);
  708. }
  709. static uint32
  710. get_func_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  711. AOTObjectData *obj_data)
  712. {
  713. uint32 size = 0;
  714. /* text offsets */
  715. if (is_32bit_binary(obj_data))
  716. size = (uint32)sizeof(uint32) * comp_data->func_count;
  717. else
  718. size = (uint32)sizeof(uint64) * comp_data->func_count;
  719. /* function type indexes */
  720. size += (uint32)sizeof(uint32) * comp_data->func_count;
  721. /* max_local_cell_nums */
  722. size += (uint32)sizeof(uint32) * comp_data->func_count;
  723. /* max_stack_cell_nums */
  724. size += (uint32)sizeof(uint32) * comp_data->func_count;
  725. #if WASM_ENABLE_GC != 0
  726. /* func_local_ref_flags */
  727. if (comp_ctx->enable_gc) {
  728. AOTFuncType *func_type;
  729. uint32 i, j, local_ref_flags_cell_num;
  730. for (i = 0; i < comp_data->import_func_count; i++) {
  731. func_type = comp_data->import_funcs[i].func_type;
  732. /* recalculate cell_num based on target pointer size */
  733. local_ref_flags_cell_num = 0;
  734. for (j = 0; j < func_type->param_count; j++) {
  735. local_ref_flags_cell_num += wasm_value_type_cell_num_internal(
  736. func_type->types[j], comp_ctx->pointer_size);
  737. }
  738. local_ref_flags_cell_num =
  739. local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2;
  740. size = align_uint(size, 4);
  741. size += (uint32)sizeof(uint32);
  742. size += (uint32)sizeof(uint8) * local_ref_flags_cell_num;
  743. }
  744. for (i = 0; i < comp_data->func_count; i++) {
  745. func_type = comp_data->funcs[i]->func_type;
  746. local_ref_flags_cell_num = comp_data->funcs[i]->param_cell_num
  747. + comp_data->funcs[i]->local_cell_num;
  748. size = align_uint(size, 4);
  749. size += (uint32)sizeof(uint32);
  750. size += (uint32)sizeof(uint8) * local_ref_flags_cell_num;
  751. }
  752. }
  753. #endif
  754. return size;
  755. }
  756. static uint32
  757. get_export_size(AOTCompContext *comp_ctx, AOTExport *export)
  758. {
  759. /* export index + export kind + 1 byte padding + export name */
  760. return (uint32)sizeof(uint32) + sizeof(uint8) + 1
  761. + get_string_size(comp_ctx, export->name);
  762. }
  763. static uint32
  764. get_exports_size(AOTCompContext *comp_ctx, AOTExport *exports,
  765. uint32 export_count)
  766. {
  767. AOTExport *export = exports;
  768. uint32 size = 0, i;
  769. for (i = 0; i < export_count; i++, export ++) {
  770. size = align_uint(size, 4);
  771. size += get_export_size(comp_ctx, export);
  772. }
  773. return size;
  774. }
  775. static uint32
  776. get_export_section_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  777. {
  778. /* export count + exports */
  779. return (uint32)sizeof(uint32)
  780. + get_exports_size(comp_ctx, comp_data->wasm_module->exports,
  781. comp_data->wasm_module->export_count);
  782. }
  783. static uint32
  784. get_relocation_size(AOTRelocation *relocation, bool is_32bin)
  785. {
  786. /* offset + addend + relocation type + symbol name */
  787. uint32 size = 0;
  788. if (is_32bin)
  789. size = sizeof(uint32) * 2; /* offset and addend */
  790. else
  791. size = sizeof(uint64) * 2; /* offset and addend */
  792. size += (uint32)sizeof(uint32); /* relocation type */
  793. size += (uint32)sizeof(uint32); /* symbol name index */
  794. return size;
  795. }
  796. static uint32
  797. get_relocations_size(AOTObjectData *obj_data,
  798. AOTRelocationGroup *relocation_group,
  799. AOTRelocation *relocations, uint32 relocation_count,
  800. bool is_32bin)
  801. {
  802. AOTRelocation *relocation = relocations;
  803. uint32 size = 0, i;
  804. for (i = 0; i < relocation_count; i++, relocation++) {
  805. /* ignore the relocations to aot_func_internal#n in text section
  806. for windows platform since they will be applied in
  807. aot_emit_text_section */
  808. if ((!strcmp(relocation_group->section_name, ".text")
  809. || !strcmp(relocation_group->section_name, ".ltext"))
  810. && !strncmp(relocation->symbol_name, AOT_FUNC_INTERNAL_PREFIX,
  811. strlen(AOT_FUNC_INTERNAL_PREFIX))
  812. && ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6)
  813. /* Windows AOT_COFF64_BIN_TYPE */
  814. && obj_data->target_info.bin_type == 6
  815. /* IMAGE_REL_AMD64_REL32 in windows x86_64 */
  816. && relocation->relocation_type == 4)
  817. || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4)
  818. /* Windows AOT_COFF32_BIN_TYPE */
  819. && obj_data->target_info.bin_type == 4
  820. /* IMAGE_REL_I386_REL32 in windows x86_32 */
  821. && relocation->relocation_type == 20))) {
  822. continue;
  823. }
  824. size = align_uint(size, 4);
  825. size += get_relocation_size(relocation, is_32bin);
  826. }
  827. return size;
  828. }
  829. static uint32
  830. get_relocation_group_size(AOTObjectData *obj_data,
  831. AOTRelocationGroup *relocation_group, bool is_32bin)
  832. {
  833. uint32 size = 0;
  834. /* section name index + relocation count + relocations */
  835. size += (uint32)sizeof(uint32);
  836. size += (uint32)sizeof(uint32);
  837. size += get_relocations_size(obj_data, relocation_group,
  838. relocation_group->relocations,
  839. relocation_group->relocation_count, is_32bin);
  840. return size;
  841. }
  842. static uint32
  843. get_relocation_groups_size(AOTObjectData *obj_data,
  844. AOTRelocationGroup *relocation_groups,
  845. uint32 relocation_group_count, bool is_32bin)
  846. {
  847. AOTRelocationGroup *relocation_group = relocation_groups;
  848. uint32 size = 0, i;
  849. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  850. size = align_uint(size, 4);
  851. size += get_relocation_group_size(obj_data, relocation_group, is_32bin);
  852. }
  853. return size;
  854. }
  855. /* return the index (in order of insertion) of the symbol,
  856. create if not exits, -1 if failed */
  857. static uint32
  858. get_relocation_symbol_index(const char *symbol_name, bool *is_new,
  859. AOTSymbolList *symbol_list)
  860. {
  861. AOTSymbolNode *sym;
  862. uint32 index = 0;
  863. sym = symbol_list->head;
  864. while (sym) {
  865. if (!strcmp(sym->symbol, symbol_name)) {
  866. if (is_new)
  867. *is_new = false;
  868. return index;
  869. }
  870. sym = sym->next;
  871. index++;
  872. }
  873. /* Not found in symbol_list, add it */
  874. sym = wasm_runtime_malloc(sizeof(AOTSymbolNode));
  875. if (!sym) {
  876. return (uint32)-1;
  877. }
  878. memset(sym, 0, sizeof(AOTSymbolNode));
  879. sym->symbol = (char *)symbol_name;
  880. sym->str_len = (uint32)strlen(symbol_name);
  881. if (!symbol_list->head) {
  882. symbol_list->head = symbol_list->end = sym;
  883. }
  884. else {
  885. symbol_list->end->next = sym;
  886. symbol_list->end = sym;
  887. }
  888. symbol_list->len++;
  889. if (is_new)
  890. *is_new = true;
  891. return index;
  892. }
  893. static uint32
  894. get_relocation_symbol_size(AOTCompContext *comp_ctx, AOTRelocation *relocation,
  895. AOTSymbolList *symbol_list)
  896. {
  897. uint32 size = 0, index = 0;
  898. bool is_new = false;
  899. index = get_relocation_symbol_index(relocation->symbol_name, &is_new,
  900. symbol_list);
  901. CHECK_SIZE(index);
  902. if (is_new) {
  903. size += get_string_size(comp_ctx, relocation->symbol_name);
  904. size = align_uint(size, 2);
  905. }
  906. relocation->symbol_index = index;
  907. return size;
  908. }
  909. static uint32
  910. get_relocations_symbol_size(AOTCompContext *comp_ctx,
  911. AOTRelocation *relocations, uint32 relocation_count,
  912. AOTSymbolList *symbol_list)
  913. {
  914. AOTRelocation *relocation = relocations;
  915. uint32 size = 0, curr_size, i;
  916. for (i = 0; i < relocation_count; i++, relocation++) {
  917. curr_size =
  918. get_relocation_symbol_size(comp_ctx, relocation, symbol_list);
  919. CHECK_SIZE(curr_size);
  920. size += curr_size;
  921. }
  922. return size;
  923. }
  924. static uint32
  925. get_relocation_group_symbol_size(AOTCompContext *comp_ctx,
  926. AOTRelocationGroup *relocation_group,
  927. AOTSymbolList *symbol_list)
  928. {
  929. uint32 size = 0, index = 0, curr_size;
  930. bool is_new = false;
  931. index = get_relocation_symbol_index(relocation_group->section_name, &is_new,
  932. symbol_list);
  933. CHECK_SIZE(index);
  934. if (is_new) {
  935. size += get_string_size(comp_ctx, relocation_group->section_name);
  936. size = align_uint(size, 2);
  937. }
  938. relocation_group->name_index = index;
  939. curr_size = get_relocations_symbol_size(
  940. comp_ctx, relocation_group->relocations,
  941. relocation_group->relocation_count, symbol_list);
  942. CHECK_SIZE(curr_size);
  943. size += curr_size;
  944. return size;
  945. }
  946. static uint32
  947. get_relocation_groups_symbol_size(AOTCompContext *comp_ctx,
  948. AOTRelocationGroup *relocation_groups,
  949. uint32 relocation_group_count,
  950. AOTSymbolList *symbol_list)
  951. {
  952. AOTRelocationGroup *relocation_group = relocation_groups;
  953. uint32 size = 0, curr_size, i;
  954. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  955. curr_size = get_relocation_group_symbol_size(comp_ctx, relocation_group,
  956. symbol_list);
  957. CHECK_SIZE(curr_size);
  958. size += curr_size;
  959. }
  960. return size;
  961. }
  962. static uint32
  963. get_symbol_size_from_symbol_list(AOTCompContext *comp_ctx,
  964. AOTSymbolList *symbol_list)
  965. {
  966. AOTSymbolNode *sym;
  967. uint32 size = 0;
  968. sym = symbol_list->head;
  969. while (sym) {
  970. /* (uint16)str_len + str */
  971. size += get_string_size(comp_ctx, sym->symbol);
  972. size = align_uint(size, 2);
  973. sym = sym->next;
  974. }
  975. return size;
  976. }
  977. static uint32
  978. get_relocation_section_symbol_size(AOTCompContext *comp_ctx,
  979. AOTObjectData *obj_data)
  980. {
  981. AOTRelocationGroup *relocation_groups = obj_data->relocation_groups;
  982. uint32 relocation_group_count = obj_data->relocation_group_count;
  983. uint32 string_count = 0, symbol_table_size = 0;
  984. /* section size will be calculated twice,
  985. get symbol size from symbol list directly in the second calculation */
  986. if (obj_data->symbol_list.len > 0) {
  987. symbol_table_size =
  988. get_symbol_size_from_symbol_list(comp_ctx, &obj_data->symbol_list);
  989. }
  990. else {
  991. symbol_table_size = get_relocation_groups_symbol_size(
  992. comp_ctx, relocation_groups, relocation_group_count,
  993. &obj_data->symbol_list);
  994. }
  995. CHECK_SIZE(symbol_table_size);
  996. string_count = obj_data->symbol_list.len;
  997. /* string_count + string_offsets + total_string_len
  998. + [str (string_len + str)] */
  999. return (uint32)(sizeof(uint32) + sizeof(uint32) * string_count
  1000. + sizeof(uint32) + symbol_table_size);
  1001. }
  1002. static uint32
  1003. get_relocation_section_size(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  1004. {
  1005. AOTRelocationGroup *relocation_groups = obj_data->relocation_groups;
  1006. uint32 relocation_group_count = obj_data->relocation_group_count;
  1007. uint32 symbol_table_size = 0;
  1008. symbol_table_size = get_relocation_section_symbol_size(comp_ctx, obj_data);
  1009. CHECK_SIZE(symbol_table_size);
  1010. symbol_table_size = align_uint(symbol_table_size, 4);
  1011. /* relocation group count + symbol_table + relocation groups */
  1012. return (uint32)sizeof(uint32) + symbol_table_size
  1013. + get_relocation_groups_size(obj_data, relocation_groups,
  1014. relocation_group_count,
  1015. is_32bit_binary(obj_data));
  1016. }
  1017. static uint32
  1018. get_native_symbol_list_size(AOTCompContext *comp_ctx)
  1019. {
  1020. uint32 len = 0;
  1021. AOTNativeSymbol *sym = NULL;
  1022. sym = bh_list_first_elem(&comp_ctx->native_symbols);
  1023. while (sym) {
  1024. len = align_uint(len, 2);
  1025. len += get_string_size(comp_ctx, sym->symbol);
  1026. sym = bh_list_elem_next(sym);
  1027. }
  1028. return len;
  1029. }
  1030. #if WASM_ENABLE_STRINGREF != 0
  1031. static uint32
  1032. get_string_literal_section_size(AOTCompContext *comp_ctx,
  1033. AOTCompData *comp_data);
  1034. #endif
  1035. static uint32
  1036. get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data);
  1037. uint32
  1038. aot_get_aot_file_size(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1039. AOTObjectData *obj_data)
  1040. {
  1041. uint32 size = 0;
  1042. uint32 size_custom_section = 0;
  1043. #if WASM_ENABLE_STRINGREF != 0
  1044. uint32 size_string_literal_section = 0;
  1045. #endif
  1046. /* aot file header */
  1047. size += get_file_header_size();
  1048. /* target info section */
  1049. size = align_uint(size, 4);
  1050. /* section id + section size */
  1051. size += (uint32)sizeof(uint32) * 2;
  1052. size += get_target_info_section_size();
  1053. /* init data section */
  1054. size = align_uint(size, 4);
  1055. /* section id + section size */
  1056. size += (uint32)sizeof(uint32) * 2;
  1057. size += get_init_data_section_size(comp_ctx, comp_data, obj_data);
  1058. /* text section */
  1059. size = align_uint(size, 4);
  1060. /* section id + section size */
  1061. size += (uint32)sizeof(uint32) * 2;
  1062. size += get_text_section_size(obj_data);
  1063. /* function section */
  1064. size = align_uint(size, 4);
  1065. /* section id + section size */
  1066. size += (uint32)sizeof(uint32) * 2;
  1067. size += get_func_section_size(comp_ctx, comp_data, obj_data);
  1068. /* export section */
  1069. size = align_uint(size, 4);
  1070. /* section id + section size */
  1071. size += (uint32)sizeof(uint32) * 2;
  1072. size += get_export_section_size(comp_ctx, comp_data);
  1073. /* relocation section */
  1074. size = align_uint(size, 4);
  1075. /* section id + section size */
  1076. size += (uint32)sizeof(uint32) * 2;
  1077. size += get_relocation_section_size(comp_ctx, obj_data);
  1078. if (get_native_symbol_list_size(comp_ctx) > 0) {
  1079. /* emit only when there are native symbols */
  1080. size = align_uint(size, 4);
  1081. /* section id + section size + sub section id + symbol count */
  1082. size += (uint32)sizeof(uint32) * 4;
  1083. size += get_native_symbol_list_size(comp_ctx);
  1084. }
  1085. size_custom_section = get_custom_sections_size(comp_ctx, comp_data);
  1086. if (size_custom_section > 0) {
  1087. size = align_uint(size, 4);
  1088. size += size_custom_section;
  1089. }
  1090. #if WASM_ENABLE_STRINGREF != 0
  1091. /* string literal section */
  1092. size_string_literal_section =
  1093. get_string_literal_section_size(comp_ctx, comp_data);
  1094. if (size_string_literal_section > 0) {
  1095. size = align_uint(size, 4);
  1096. /* section id + section size + sub section id */
  1097. size += (uint32)sizeof(uint32) * 3;
  1098. size += size_string_literal_section;
  1099. }
  1100. #endif
  1101. return size;
  1102. }
  1103. #define exchange_uint8(p_data) (void)0
  1104. static void
  1105. exchange_uint16(uint8 *p_data)
  1106. {
  1107. uint8 value = *p_data;
  1108. *p_data = *(p_data + 1);
  1109. *(p_data + 1) = value;
  1110. }
  1111. static void
  1112. exchange_uint32(uint8 *p_data)
  1113. {
  1114. uint8 value = *p_data;
  1115. *p_data = *(p_data + 3);
  1116. *(p_data + 3) = value;
  1117. value = *(p_data + 1);
  1118. *(p_data + 1) = *(p_data + 2);
  1119. *(p_data + 2) = value;
  1120. }
  1121. static void
  1122. exchange_uint64(uint8 *p_data)
  1123. {
  1124. uint32 value;
  1125. value = *(uint32 *)p_data;
  1126. *(uint32 *)p_data = *(uint32 *)(p_data + 4);
  1127. *(uint32 *)(p_data + 4) = value;
  1128. exchange_uint32(p_data);
  1129. exchange_uint32(p_data + 4);
  1130. }
  1131. static void
  1132. exchange_uint128(uint8 *p_data)
  1133. {
  1134. /* swap high 64bit and low 64bit */
  1135. uint64 value = *(uint64 *)p_data;
  1136. *(uint64 *)p_data = *(uint64 *)(p_data + 8);
  1137. *(uint64 *)(p_data + 8) = value;
  1138. /* exchange high 64bit */
  1139. exchange_uint64(p_data);
  1140. /* exchange low 64bit */
  1141. exchange_uint64(p_data + 8);
  1142. }
  1143. static union {
  1144. int a;
  1145. char b;
  1146. } __ue = { .a = 1 };
  1147. #define is_little_endian() (__ue.b == 1)
  1148. #define CHECK_BUF(length) \
  1149. do { \
  1150. if (buf + offset + length > buf_end) { \
  1151. aot_set_last_error("buf overflow"); \
  1152. return false; \
  1153. } \
  1154. } while (0)
  1155. #define EMIT_U8(v) \
  1156. do { \
  1157. CHECK_BUF(1); \
  1158. *(uint8 *)(buf + offset) = (uint8)v; \
  1159. offset++; \
  1160. } while (0)
  1161. #define EMIT_U16(v) \
  1162. do { \
  1163. uint16 t = (uint16)v; \
  1164. CHECK_BUF(2); \
  1165. if (!is_little_endian()) \
  1166. exchange_uint16((uint8 *)&t); \
  1167. *(uint16 *)(buf + offset) = t; \
  1168. offset += (uint32)sizeof(uint16); \
  1169. } while (0)
  1170. #define EMIT_U32(v) \
  1171. do { \
  1172. uint32 t = (uint32)v; \
  1173. CHECK_BUF(4); \
  1174. if (!is_little_endian()) \
  1175. exchange_uint32((uint8 *)&t); \
  1176. *(uint32 *)(buf + offset) = t; \
  1177. offset += (uint32)sizeof(uint32); \
  1178. } while (0)
  1179. #define EMIT_U64(v) \
  1180. do { \
  1181. uint64 t = (uint64)v; \
  1182. CHECK_BUF(8); \
  1183. if (!is_little_endian()) \
  1184. exchange_uint64((uint8 *)&t); \
  1185. PUT_U64_TO_ADDR(buf + offset, t); \
  1186. offset += (uint32)sizeof(uint64); \
  1187. } while (0)
  1188. #define EMIT_V128(v) \
  1189. do { \
  1190. uint64 *t = (uint64 *)v.i64x2; \
  1191. CHECK_BUF(16); \
  1192. if (!is_little_endian()) \
  1193. exchange_uint128((uint8 *)t); \
  1194. PUT_U64_TO_ADDR(buf + offset, t[0]); \
  1195. offset += (uint32)sizeof(uint64); \
  1196. PUT_U64_TO_ADDR(buf + offset, t[1]); \
  1197. offset += (uint32)sizeof(uint64); \
  1198. } while (0)
  1199. #define EMIT_BUF(v, len) \
  1200. do { \
  1201. CHECK_BUF(len); \
  1202. memcpy(buf + offset, v, len); \
  1203. offset += len; \
  1204. } while (0)
  1205. /* Emit string with '\0'
  1206. */
  1207. #define EMIT_STR(s) \
  1208. do { \
  1209. uint32 str_len = (uint32)strlen(s) + 1; \
  1210. if (str_len > INT16_MAX) { \
  1211. aot_set_last_error("emit string failed: " \
  1212. "string too long"); \
  1213. return false; \
  1214. } \
  1215. EMIT_U16(str_len); \
  1216. EMIT_BUF(s, str_len); \
  1217. } while (0)
  1218. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  1219. static bool
  1220. read_leb(uint8 **p_buf, const uint8 *buf_end, uint32 maxbits, bool sign,
  1221. uint64 *p_result)
  1222. {
  1223. const uint8 *buf = *p_buf;
  1224. uint64 result = 0;
  1225. uint32 shift = 0;
  1226. uint32 offset = 0, bcnt = 0;
  1227. uint64 byte;
  1228. while (true) {
  1229. /* uN or SN must not exceed ceil(N/7) bytes */
  1230. if (bcnt + 1 > (maxbits + 6) / 7) {
  1231. aot_set_last_error("integer representation too long");
  1232. return false;
  1233. }
  1234. if (buf + offset + 1 > buf_end) {
  1235. aot_set_last_error("unexpected end of section or function");
  1236. return false;
  1237. }
  1238. byte = buf[offset];
  1239. offset += 1;
  1240. result |= ((byte & 0x7f) << shift);
  1241. shift += 7;
  1242. bcnt += 1;
  1243. if ((byte & 0x80) == 0) {
  1244. break;
  1245. }
  1246. }
  1247. if (!sign && maxbits == 32 && shift >= maxbits) {
  1248. /* The top bits set represent values > 32 bits */
  1249. if (((uint8)byte) & 0xf0)
  1250. goto fail_integer_too_large;
  1251. }
  1252. else if (sign && maxbits == 32) {
  1253. if (shift < maxbits) {
  1254. /* Sign extend, second highest bit is the sign bit */
  1255. if ((uint8)byte & 0x40)
  1256. result |= (~((uint64)0)) << shift;
  1257. }
  1258. else {
  1259. /* The top bits should be a sign-extension of the sign bit */
  1260. bool sign_bit_set = ((uint8)byte) & 0x8;
  1261. int top_bits = ((uint8)byte) & 0xf0;
  1262. if ((sign_bit_set && top_bits != 0x70)
  1263. || (!sign_bit_set && top_bits != 0))
  1264. goto fail_integer_too_large;
  1265. }
  1266. }
  1267. else if (sign && maxbits == 64) {
  1268. if (shift < maxbits) {
  1269. /* Sign extend, second highest bit is the sign bit */
  1270. if ((uint8)byte & 0x40)
  1271. result |= (~((uint64)0)) << shift;
  1272. }
  1273. else {
  1274. /* The top bits should be a sign-extension of the sign bit */
  1275. bool sign_bit_set = ((uint8)byte) & 0x1;
  1276. int top_bits = ((uint8)byte) & 0xfe;
  1277. if ((sign_bit_set && top_bits != 0x7e)
  1278. || (!sign_bit_set && top_bits != 0))
  1279. goto fail_integer_too_large;
  1280. }
  1281. }
  1282. *p_buf += offset;
  1283. *p_result = result;
  1284. return true;
  1285. fail_integer_too_large:
  1286. aot_set_last_error("integer too large");
  1287. return false;
  1288. }
  1289. /* NOLINTNEXTLINE */
  1290. #define read_leb_uint32(p, p_end, res) \
  1291. do { \
  1292. uint64 res64; \
  1293. if (!read_leb((uint8 **)&p, p_end, 32, false, &res64)) \
  1294. goto fail; \
  1295. res = (uint32)res64; \
  1296. } while (0)
  1297. /*
  1298. * - transfer .name section in .wasm (comp_data->name_section_buf) to
  1299. * aot buf (comp_data->aot_name_section_buf)
  1300. * - leb128 to u32
  1301. * - add `\0` at the end of every name, and adjust length(+1)
  1302. */
  1303. static uint32
  1304. get_name_section_size(AOTCompData *comp_data)
  1305. {
  1306. /* original name section content in .wasm */
  1307. const uint8 *p = comp_data->name_section_buf,
  1308. *p_end = comp_data->name_section_buf_end;
  1309. uint8 *buf, *buf_end;
  1310. uint32 name_type, subsection_size;
  1311. uint32 previous_name_type = 0;
  1312. uint32 num_func_name;
  1313. uint32 func_index;
  1314. uint32 previous_func_index = ~0U;
  1315. uint32 func_name_len;
  1316. uint32 name_index;
  1317. int i = 0;
  1318. uint32 name_len;
  1319. uint32 offset = 0;
  1320. uint32 max_aot_buf_size = 0;
  1321. if (p >= p_end) {
  1322. aot_set_last_error("unexpected end");
  1323. return 0;
  1324. }
  1325. max_aot_buf_size = 4 * (uint32)(p_end - p);
  1326. if (!(buf = comp_data->aot_name_section_buf =
  1327. wasm_runtime_malloc(max_aot_buf_size))) {
  1328. aot_set_last_error("allocate memory for custom name section failed.");
  1329. return 0;
  1330. }
  1331. memset(buf, 0, (uint32)max_aot_buf_size);
  1332. buf_end = buf + max_aot_buf_size;
  1333. /* the size of "name". it should be 4 */
  1334. read_leb_uint32(p, p_end, name_len);
  1335. offset = align_uint(offset, 4);
  1336. EMIT_U32(name_len);
  1337. if (name_len != 4 || p + name_len > p_end) {
  1338. aot_set_last_error("unexpected end");
  1339. return 0;
  1340. }
  1341. /* "name" */
  1342. if (memcmp(p, "name", 4) != 0) {
  1343. aot_set_last_error("invalid custom name section");
  1344. return 0;
  1345. }
  1346. EMIT_BUF(p, name_len);
  1347. p += name_len;
  1348. while (p < p_end) {
  1349. read_leb_uint32(p, p_end, name_type);
  1350. if (i != 0) {
  1351. if (name_type == previous_name_type) {
  1352. aot_set_last_error("duplicate sub-section");
  1353. return 0;
  1354. }
  1355. if (name_type < previous_name_type) {
  1356. aot_set_last_error("out-of-order sub-section");
  1357. return 0;
  1358. }
  1359. }
  1360. previous_name_type = name_type;
  1361. read_leb_uint32(p, p_end, subsection_size);
  1362. switch (name_type) {
  1363. case SUB_SECTION_TYPE_FUNC:
  1364. if (subsection_size) {
  1365. offset = align_uint(offset, 4);
  1366. EMIT_U32(name_type);
  1367. EMIT_U32(subsection_size);
  1368. read_leb_uint32(p, p_end, num_func_name);
  1369. EMIT_U32(num_func_name);
  1370. for (name_index = 0; name_index < num_func_name;
  1371. name_index++) {
  1372. read_leb_uint32(p, p_end, func_index);
  1373. offset = align_uint(offset, 4);
  1374. EMIT_U32(func_index);
  1375. if (func_index == previous_func_index) {
  1376. aot_set_last_error("duplicate function name");
  1377. return 0;
  1378. }
  1379. if (func_index < previous_func_index
  1380. && previous_func_index != ~0U) {
  1381. aot_set_last_error("out-of-order function index ");
  1382. return 0;
  1383. }
  1384. previous_func_index = func_index;
  1385. read_leb_uint32(p, p_end, func_name_len);
  1386. offset = align_uint(offset, 2);
  1387. /* emit a string ends with `\0` */
  1388. if (func_name_len + 1 > UINT16_MAX) {
  1389. aot_set_last_error(
  1390. "emit string failed: string too long");
  1391. goto fail;
  1392. }
  1393. /* extra 1 byte for \0 */
  1394. EMIT_U16(func_name_len + 1);
  1395. EMIT_BUF(p, func_name_len);
  1396. p += func_name_len;
  1397. EMIT_U8(0);
  1398. }
  1399. }
  1400. break;
  1401. case SUB_SECTION_TYPE_MODULE: /* TODO: Parse for module subsection
  1402. */
  1403. case SUB_SECTION_TYPE_LOCAL: /* TODO: Parse for local subsection */
  1404. default:
  1405. p = p + subsection_size;
  1406. break;
  1407. }
  1408. i++;
  1409. }
  1410. return offset;
  1411. fail:
  1412. return 0;
  1413. }
  1414. #endif /* end of WASM_ENABLE_LOAD_CUSTOM_SECTION != 0 */
  1415. #if WASM_ENABLE_STRINGREF != 0
  1416. static uint32
  1417. get_string_literal_section_size(AOTCompContext *comp_ctx,
  1418. AOTCompData *comp_data)
  1419. {
  1420. uint32 i;
  1421. uint32 size = 0;
  1422. uint32 string_count = comp_data->string_literal_count;
  1423. if (string_count == 0) {
  1424. return 0;
  1425. }
  1426. /* reserved slot + string count + string_lengths */
  1427. size += sizeof(uint32) * (2 + string_count);
  1428. for (i = 0; i < string_count; i++) {
  1429. size += comp_data->string_literal_lengths_wp[i];
  1430. }
  1431. return size;
  1432. }
  1433. #endif /* end of WASM_ENABLE_STRINGREF != 0 */
  1434. static uint32
  1435. get_custom_sections_size(AOTCompContext *comp_ctx, AOTCompData *comp_data)
  1436. {
  1437. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  1438. uint32 size = 0, i;
  1439. for (i = 0; i < comp_ctx->custom_sections_count; i++) {
  1440. const char *section_name = comp_ctx->custom_sections_wp[i];
  1441. const uint8 *content = NULL;
  1442. uint32 length = 0;
  1443. if (strcmp(section_name, "name") == 0) {
  1444. /* custom name section */
  1445. comp_data->aot_name_section_size = get_name_section_size(comp_data);
  1446. if (comp_data->aot_name_section_size == 0) {
  1447. LOG_WARNING("Can't find custom section [name], ignore it");
  1448. continue;
  1449. }
  1450. size = align_uint(size, 4);
  1451. /* section id + section size + sub section id */
  1452. size += (uint32)sizeof(uint32) * 3;
  1453. size += comp_data->aot_name_section_size;
  1454. continue;
  1455. }
  1456. content = wasm_loader_get_custom_section(comp_data->wasm_module,
  1457. section_name, &length);
  1458. if (!content) {
  1459. LOG_WARNING("Can't find custom section [%s], ignore it",
  1460. section_name);
  1461. continue;
  1462. }
  1463. size = align_uint(size, 4);
  1464. /* section id + section size + sub section id */
  1465. size += (uint32)sizeof(uint32) * 3;
  1466. /* section name and len */
  1467. size += get_string_size(comp_ctx, section_name);
  1468. /* section content */
  1469. size += length;
  1470. }
  1471. return size;
  1472. #else
  1473. return 0;
  1474. #endif
  1475. }
  1476. static bool
  1477. aot_emit_file_header(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1478. AOTCompData *comp_data, AOTObjectData *obj_data)
  1479. {
  1480. uint32 offset = *p_offset;
  1481. uint32 aot_curr_version = AOT_CURRENT_VERSION;
  1482. EMIT_U8('\0');
  1483. EMIT_U8('a');
  1484. EMIT_U8('o');
  1485. EMIT_U8('t');
  1486. EMIT_U32(aot_curr_version);
  1487. *p_offset = offset;
  1488. return true;
  1489. }
  1490. static bool
  1491. aot_emit_target_info_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1492. AOTCompData *comp_data, AOTObjectData *obj_data)
  1493. {
  1494. uint32 offset = *p_offset;
  1495. uint32 section_size = get_target_info_section_size();
  1496. AOTTargetInfo *target_info = &obj_data->target_info;
  1497. *p_offset = offset = align_uint(offset, 4);
  1498. EMIT_U32(AOT_SECTION_TYPE_TARGET_INFO);
  1499. EMIT_U32(section_size);
  1500. EMIT_U16(target_info->bin_type);
  1501. EMIT_U16(target_info->abi_type);
  1502. EMIT_U16(target_info->e_type);
  1503. EMIT_U16(target_info->e_machine);
  1504. EMIT_U32(target_info->e_version);
  1505. EMIT_U32(target_info->e_flags);
  1506. EMIT_U64(target_info->feature_flags);
  1507. EMIT_U64(target_info->reserved);
  1508. EMIT_BUF(target_info->arch, sizeof(target_info->arch));
  1509. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  1510. aot_set_last_error("emit target info failed.");
  1511. return false;
  1512. }
  1513. *p_offset = offset;
  1514. return true;
  1515. }
  1516. static bool
  1517. aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1518. AOTCompContext *comp_ctx, InitializerExpression *expr);
  1519. static bool
  1520. aot_emit_mem_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1521. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1522. AOTObjectData *obj_data)
  1523. {
  1524. uint32 offset = *p_offset, i;
  1525. AOTMemInitData **init_datas = comp_data->mem_init_data_list;
  1526. *p_offset = offset = align_uint(offset, 4);
  1527. /* Emit import memory count, only emit 0 currently.
  1528. TODO: emit the actual import memory count and
  1529. the full import memory info. */
  1530. EMIT_U32(0);
  1531. /* Emit memory count */
  1532. EMIT_U32(comp_data->memory_count);
  1533. /* Emit memory items */
  1534. for (i = 0; i < comp_data->memory_count; i++) {
  1535. EMIT_U32(comp_data->memories[i].flags);
  1536. EMIT_U32(comp_data->memories[i].num_bytes_per_page);
  1537. EMIT_U32(comp_data->memories[i].init_page_count);
  1538. EMIT_U32(comp_data->memories[i].max_page_count);
  1539. }
  1540. /* Emit mem init data count */
  1541. EMIT_U32(comp_data->mem_init_data_count);
  1542. /* Emit mem init data items */
  1543. for (i = 0; i < comp_data->mem_init_data_count; i++) {
  1544. offset = align_uint(offset, 4);
  1545. #if WASM_ENABLE_BULK_MEMORY != 0
  1546. if (comp_ctx->enable_bulk_memory) {
  1547. EMIT_U32(init_datas[i]->is_passive);
  1548. EMIT_U32(init_datas[i]->memory_index);
  1549. }
  1550. else
  1551. #endif
  1552. {
  1553. /* emit two placeholder to keep the same size */
  1554. EMIT_U32(0);
  1555. EMIT_U32(0);
  1556. }
  1557. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1558. &init_datas[i]->offset))
  1559. return false;
  1560. EMIT_U32(init_datas[i]->byte_count);
  1561. EMIT_BUF(init_datas[i]->bytes, init_datas[i]->byte_count);
  1562. }
  1563. if (offset - *p_offset != get_mem_info_size(comp_ctx, comp_data)) {
  1564. aot_set_last_error("emit memory info failed.");
  1565. return false;
  1566. }
  1567. *p_offset = offset;
  1568. return true;
  1569. }
  1570. static bool
  1571. aot_emit_init_expr(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1572. AOTCompContext *comp_ctx, InitializerExpression *expr)
  1573. {
  1574. uint32 offset = *p_offset;
  1575. #if WASM_ENABLE_GC != 0
  1576. WASMModule *module = comp_ctx->comp_data->wasm_module;
  1577. #endif
  1578. *p_offset = offset = align_uint(offset, 4);
  1579. EMIT_U32(expr->init_expr_type);
  1580. switch (expr->init_expr_type) {
  1581. case INIT_EXPR_NONE:
  1582. break;
  1583. case INIT_EXPR_TYPE_I32_CONST:
  1584. case INIT_EXPR_TYPE_F32_CONST:
  1585. EMIT_U32(expr->u.i32);
  1586. break;
  1587. case INIT_EXPR_TYPE_I64_CONST:
  1588. case INIT_EXPR_TYPE_F64_CONST:
  1589. EMIT_U64(expr->u.i64);
  1590. break;
  1591. case INIT_EXPR_TYPE_V128_CONST:
  1592. EMIT_V128(expr->u.v128);
  1593. break;
  1594. case INIT_EXPR_TYPE_GET_GLOBAL:
  1595. EMIT_U32(expr->u.global_index);
  1596. break;
  1597. case INIT_EXPR_TYPE_FUNCREF_CONST:
  1598. case INIT_EXPR_TYPE_REFNULL_CONST:
  1599. EMIT_U32(expr->u.ref_index);
  1600. break;
  1601. #if WASM_ENABLE_GC != 0
  1602. case INIT_EXPR_TYPE_I31_NEW:
  1603. EMIT_U32(expr->u.i32);
  1604. break;
  1605. case INIT_EXPR_TYPE_STRUCT_NEW:
  1606. {
  1607. uint32 i;
  1608. WASMStructNewInitValues *init_values =
  1609. (WASMStructNewInitValues *)expr->u.data;
  1610. WASMStructType *struct_type = NULL;
  1611. EMIT_U32(init_values->type_idx);
  1612. EMIT_U32(init_values->count);
  1613. bh_assert(init_values->type_idx < module->type_count);
  1614. struct_type =
  1615. (WASMStructType *)module->types[init_values->type_idx];
  1616. bh_assert(struct_type);
  1617. bh_assert(struct_type->field_count == init_values->count);
  1618. for (i = 0; i < init_values->count; i++) {
  1619. uint32 field_size = wasm_value_type_size_internal(
  1620. struct_type->fields[i].field_type, comp_ctx->pointer_size);
  1621. if (field_size <= sizeof(uint32))
  1622. EMIT_U32(init_values->fields[i].u32);
  1623. else if (field_size == sizeof(uint64))
  1624. EMIT_U64(init_values->fields[i].u64);
  1625. else if (field_size == sizeof(uint64) * 2)
  1626. EMIT_V128(init_values->fields[i].v128);
  1627. else {
  1628. bh_assert(0);
  1629. }
  1630. }
  1631. break;
  1632. }
  1633. case INIT_EXPR_TYPE_STRUCT_NEW_DEFAULT:
  1634. EMIT_U32(expr->u.type_index);
  1635. break;
  1636. case INIT_EXPR_TYPE_ARRAY_NEW_DEFAULT:
  1637. {
  1638. WASMArrayType *array_type = NULL;
  1639. bh_assert(expr->u.array_new_default.type_index
  1640. < module->type_count);
  1641. array_type =
  1642. (WASMArrayType *)
  1643. module->types[expr->u.array_new_default.type_index];
  1644. EMIT_U32(array_type->elem_type);
  1645. EMIT_U32(expr->u.array_new_default.type_index);
  1646. EMIT_U32(expr->u.array_new_default.length);
  1647. break;
  1648. }
  1649. case INIT_EXPR_TYPE_ARRAY_NEW:
  1650. case INIT_EXPR_TYPE_ARRAY_NEW_FIXED:
  1651. {
  1652. uint32 value_count, i, field_size;
  1653. WASMArrayNewInitValues *init_values =
  1654. (WASMArrayNewInitValues *)expr->u.data;
  1655. WASMArrayType *array_type = NULL;
  1656. bh_assert(init_values->type_idx < module->type_count);
  1657. array_type = (WASMArrayType *)module->types[init_values->type_idx];
  1658. EMIT_U32(array_type->elem_type);
  1659. EMIT_U32(init_values->type_idx);
  1660. EMIT_U32(init_values->length);
  1661. value_count =
  1662. (expr->init_expr_type == INIT_EXPR_TYPE_ARRAY_NEW_FIXED)
  1663. ? init_values->length
  1664. : 1;
  1665. field_size = wasm_value_type_size_internal(array_type->elem_type,
  1666. comp_ctx->pointer_size);
  1667. for (i = 0; i < value_count; i++) {
  1668. if (field_size <= sizeof(uint32))
  1669. EMIT_U32(init_values->elem_data[i].u32);
  1670. else if (field_size == sizeof(uint64))
  1671. EMIT_U64(init_values->elem_data[i].u64);
  1672. else if (field_size == sizeof(uint64) * 2)
  1673. EMIT_V128(init_values->elem_data[i].v128);
  1674. else {
  1675. bh_assert(0);
  1676. }
  1677. }
  1678. break;
  1679. }
  1680. #endif /* end of WASM_ENABLE_GC != 0 */
  1681. default:
  1682. aot_set_last_error("invalid init expr type.");
  1683. return false;
  1684. }
  1685. *p_offset = offset;
  1686. return true;
  1687. }
  1688. static bool
  1689. aot_emit_table_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1690. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1691. AOTObjectData *obj_data)
  1692. {
  1693. uint32 offset = *p_offset, i, j;
  1694. AOTTableInitData **init_datas = comp_data->table_init_data_list;
  1695. *p_offset = offset = align_uint(offset, 4);
  1696. /* Emit import table count */
  1697. EMIT_U32(comp_data->import_table_count);
  1698. /* Emit table items */
  1699. for (i = 0; i < comp_data->import_table_count; i++) {
  1700. /* TODO:
  1701. * EMIT_STR(comp_data->import_tables[i].module_name );
  1702. * EMIT_STR(comp_data->import_tables[i].table_name);
  1703. */
  1704. EMIT_U8(comp_data->import_tables[i].table_type.elem_type);
  1705. EMIT_U8(comp_data->import_tables[i].table_type.flags);
  1706. EMIT_U8(comp_data->import_tables[i].table_type.possible_grow);
  1707. #if WASM_ENABLE_GC != 0
  1708. if (comp_ctx->enable_gc
  1709. && comp_data->import_tables[i].table_type.elem_ref_type) {
  1710. EMIT_U8(comp_data->import_tables[i]
  1711. .table_type.elem_ref_type->ref_ht_common.nullable);
  1712. }
  1713. else
  1714. #endif
  1715. {
  1716. /* emit one placeholder to keep the same size */
  1717. EMIT_U8(0);
  1718. }
  1719. EMIT_U32(comp_data->import_tables[i].table_type.init_size);
  1720. EMIT_U32(comp_data->import_tables[i].table_type.max_size);
  1721. #if WASM_ENABLE_GC != 0
  1722. if (comp_ctx->enable_gc
  1723. && comp_data->import_tables[i].table_type.elem_ref_type) {
  1724. bh_assert(wasm_is_type_multi_byte_type(
  1725. comp_data->import_tables[i].table_type.elem_type));
  1726. EMIT_U32(comp_data->import_tables[i]
  1727. .table_type.elem_ref_type->ref_ht_common.heap_type);
  1728. }
  1729. #endif
  1730. }
  1731. /* Emit table count */
  1732. EMIT_U32(comp_data->table_count);
  1733. /* Emit table items */
  1734. for (i = 0; i < comp_data->table_count; i++) {
  1735. EMIT_U8(comp_data->tables[i].table_type.elem_type);
  1736. EMIT_U8(comp_data->tables[i].table_type.flags);
  1737. EMIT_U8(comp_data->tables[i].table_type.possible_grow);
  1738. #if WASM_ENABLE_GC != 0
  1739. if (comp_ctx->enable_gc
  1740. && comp_data->tables[i].table_type.elem_ref_type) {
  1741. EMIT_U8(comp_data->tables[i]
  1742. .table_type.elem_ref_type->ref_ht_common.nullable);
  1743. }
  1744. else
  1745. #endif
  1746. {
  1747. /* emit one placeholder to keep the same size */
  1748. EMIT_U8(0);
  1749. }
  1750. EMIT_U32(comp_data->tables[i].table_type.init_size);
  1751. EMIT_U32(comp_data->tables[i].table_type.max_size);
  1752. #if WASM_ENABLE_GC != 0
  1753. if (comp_ctx->enable_gc) {
  1754. if (comp_data->tables[i].table_type.elem_ref_type) {
  1755. bh_assert(wasm_is_type_multi_byte_type(
  1756. comp_data->tables[i].table_type.elem_type));
  1757. EMIT_U32(
  1758. comp_data->tables[i]
  1759. .table_type.elem_ref_type->ref_ht_common.heap_type);
  1760. }
  1761. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1762. &comp_data->tables[i].init_expr)) {
  1763. return false;
  1764. }
  1765. }
  1766. #endif
  1767. }
  1768. /* Emit table init data count */
  1769. EMIT_U32(comp_data->table_init_data_count);
  1770. /* Emit table init data items */
  1771. for (i = 0; i < comp_data->table_init_data_count; i++) {
  1772. offset = align_uint(offset, 4);
  1773. EMIT_U32(init_datas[i]->mode);
  1774. EMIT_U32(init_datas[i]->elem_type);
  1775. EMIT_U32(init_datas[i]->table_index);
  1776. EMIT_U32(init_datas[i]->offset.init_expr_type);
  1777. EMIT_U64(init_datas[i]->offset.u.i64);
  1778. #if WASM_ENABLE_GC != 0
  1779. if (comp_ctx->enable_gc && init_datas[i]->elem_ref_type) {
  1780. EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.ref_type);
  1781. EMIT_U16(init_datas[i]->elem_ref_type->ref_ht_common.nullable);
  1782. EMIT_U32(init_datas[i]->elem_ref_type->ref_ht_common.heap_type);
  1783. }
  1784. else
  1785. #endif
  1786. {
  1787. EMIT_U16(init_datas[i]->elem_type);
  1788. EMIT_U16(0);
  1789. EMIT_U32(0);
  1790. }
  1791. EMIT_U32(init_datas[i]->value_count);
  1792. for (j = 0; j < init_datas[i]->value_count; j++) {
  1793. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1794. &init_datas[i]->init_values[j]))
  1795. return false;
  1796. }
  1797. }
  1798. if (offset - *p_offset != get_table_info_size(comp_ctx, comp_data)) {
  1799. aot_set_last_error("emit table info failed.");
  1800. return false;
  1801. }
  1802. *p_offset = offset;
  1803. return true;
  1804. }
  1805. #if WASM_ENABLE_GC != 0
  1806. static bool
  1807. aot_emit_reftype_map(uint8 *buf, uint8 *buf_end, uint32 *p_offset, uint32 count,
  1808. WASMRefTypeMap *refmap)
  1809. {
  1810. uint32 offset = *p_offset, i;
  1811. for (i = 0; i < count; i++) {
  1812. EMIT_U16(refmap->index);
  1813. WASMRefType *ref_type = refmap->ref_type;
  1814. /* Note: WASMRefType is a union type */
  1815. EMIT_U8(ref_type->ref_ht_common.ref_type);
  1816. EMIT_U8(ref_type->ref_ht_common.nullable);
  1817. EMIT_U32(ref_type->ref_ht_common.heap_type);
  1818. refmap++;
  1819. }
  1820. *p_offset = offset;
  1821. return true;
  1822. }
  1823. #endif
  1824. static bool
  1825. aot_emit_type_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1826. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1827. AOTObjectData *obj_data)
  1828. {
  1829. uint32 offset = *p_offset, i;
  1830. *p_offset = offset = align_uint(offset, 4);
  1831. EMIT_U32(comp_data->type_count);
  1832. #if WASM_ENABLE_GC != 0
  1833. if (comp_ctx->enable_gc) {
  1834. AOTType **types = comp_data->types;
  1835. int32 idx;
  1836. uint32 j;
  1837. for (i = 0; i < comp_data->type_count; i++) {
  1838. offset = align_uint(offset, 4);
  1839. /* Emit simple info if there is an equivalence type */
  1840. for (j = 0; j < i; j++) {
  1841. if (types[j] == types[i]) {
  1842. EMIT_U16(types[i]->type_flag);
  1843. /* equivalence type flag is true */
  1844. EMIT_U8(1);
  1845. EMIT_U8(0);
  1846. /* equivalence type index */
  1847. EMIT_U32(j);
  1848. break;
  1849. }
  1850. }
  1851. if (j < i)
  1852. continue;
  1853. EMIT_U16(types[i]->type_flag);
  1854. /* equivalence type flag is false */
  1855. EMIT_U8(0);
  1856. EMIT_U8(types[i]->is_sub_final);
  1857. EMIT_U32(types[i]->parent_type_idx);
  1858. EMIT_U16(types[i]->rec_count);
  1859. EMIT_U16(types[i]->rec_idx);
  1860. /* Emit WASM_TYPE_FUNC */
  1861. if (types[i]->type_flag == WASM_TYPE_FUNC) {
  1862. AOTFuncType *func_type = (AOTFuncType *)types[i];
  1863. EMIT_U16(func_type->param_count);
  1864. EMIT_U16(func_type->result_count);
  1865. EMIT_U16(func_type->ref_type_map_count);
  1866. EMIT_BUF(func_type->types,
  1867. func_type->param_count + func_type->result_count);
  1868. offset = align_uint(offset, 4);
  1869. aot_emit_reftype_map(buf, buf_end, &offset,
  1870. func_type->ref_type_map_count,
  1871. func_type->ref_type_maps);
  1872. }
  1873. /* Emit WASM_TYPE_STRUCT */
  1874. else if (types[i]->type_flag == WASM_TYPE_STRUCT) {
  1875. AOTStructType *struct_type = (AOTStructType *)types[i];
  1876. EMIT_U16(struct_type->field_count);
  1877. EMIT_U16(struct_type->ref_type_map_count);
  1878. for (idx = 0; idx < struct_type->field_count; idx++) {
  1879. EMIT_U8(struct_type->fields[idx].field_flags);
  1880. EMIT_U8(struct_type->fields[idx].field_type);
  1881. }
  1882. offset = align_uint(offset, 4);
  1883. aot_emit_reftype_map(buf, buf_end, &offset,
  1884. struct_type->ref_type_map_count,
  1885. struct_type->ref_type_maps);
  1886. }
  1887. /* Emit WASM_TYPE_ARRAY */
  1888. else if (types[i]->type_flag == WASM_TYPE_ARRAY) {
  1889. AOTArrayType *array_type = (AOTArrayType *)types[i];
  1890. EMIT_U16(array_type->elem_flags);
  1891. EMIT_U8(array_type->elem_type);
  1892. if (array_type->elem_ref_type) {
  1893. bh_assert(
  1894. wasm_is_type_multi_byte_type(array_type->elem_type));
  1895. EMIT_U8(array_type->elem_ref_type->ref_ht_common.nullable);
  1896. EMIT_U32(
  1897. array_type->elem_ref_type->ref_ht_common.heap_type);
  1898. }
  1899. }
  1900. else {
  1901. aot_set_last_error("invalid type flag.");
  1902. return false;
  1903. }
  1904. }
  1905. if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) {
  1906. aot_set_last_error("emit function type info failed.");
  1907. return false;
  1908. }
  1909. *p_offset = offset;
  1910. }
  1911. else
  1912. #endif
  1913. {
  1914. AOTFuncType **func_types = (AOTFuncType **)comp_data->types;
  1915. for (i = 0; i < comp_data->type_count; i++) {
  1916. offset = align_uint(offset, 4);
  1917. /* If GC is disabled, only emit function type info */
  1918. EMIT_U16(WASM_TYPE_FUNC);
  1919. /* Omit to emit dummy padding for is_sub_final,
  1920. * parent_type_index, rec_count, rec_idx, 10 bytes in total */
  1921. EMIT_U16(func_types[i]->param_count);
  1922. EMIT_U16(func_types[i]->result_count);
  1923. /* Omit to emit dummy padding for ref_type_map_count, 2 bytes in
  1924. * total */
  1925. EMIT_BUF(func_types[i]->types,
  1926. func_types[i]->param_count + func_types[i]->result_count);
  1927. }
  1928. if (offset - *p_offset != get_type_info_size(comp_ctx, comp_data)) {
  1929. aot_set_last_error("emit function type info failed.");
  1930. return false;
  1931. }
  1932. *p_offset = offset;
  1933. }
  1934. return true;
  1935. }
  1936. static bool
  1937. aot_emit_import_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1938. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1939. AOTObjectData *obj_data)
  1940. {
  1941. uint32 offset = *p_offset, i;
  1942. AOTImportGlobal *import_global = comp_data->import_globals;
  1943. *p_offset = offset = align_uint(offset, 4);
  1944. EMIT_U32(comp_data->import_global_count);
  1945. for (i = 0; i < comp_data->import_global_count; i++, import_global++) {
  1946. offset = align_uint(offset, 2);
  1947. EMIT_U8(import_global->type.val_type);
  1948. EMIT_U8(import_global->type.is_mutable);
  1949. EMIT_STR(import_global->module_name);
  1950. offset = align_uint(offset, 2);
  1951. EMIT_STR(import_global->global_name);
  1952. }
  1953. if (offset - *p_offset
  1954. != get_import_global_info_size(comp_ctx, comp_data)) {
  1955. aot_set_last_error("emit import global info failed.");
  1956. return false;
  1957. }
  1958. *p_offset = offset;
  1959. return true;
  1960. }
  1961. static bool
  1962. aot_emit_global_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1963. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1964. AOTObjectData *obj_data)
  1965. {
  1966. uint32 offset = *p_offset, i;
  1967. AOTGlobal *global = comp_data->globals;
  1968. *p_offset = offset = align_uint(offset, 4);
  1969. EMIT_U32(comp_data->global_count);
  1970. for (i = 0; i < comp_data->global_count; i++, global++) {
  1971. offset = align_uint(offset, 4);
  1972. EMIT_U8(global->type.val_type);
  1973. EMIT_U8(global->type.is_mutable);
  1974. offset = align_uint(offset, 4);
  1975. if (!aot_emit_init_expr(buf, buf_end, &offset, comp_ctx,
  1976. &global->init_expr))
  1977. return false;
  1978. }
  1979. if (offset - *p_offset != get_global_info_size(comp_ctx, comp_data)) {
  1980. aot_set_last_error("emit global info failed.");
  1981. return false;
  1982. }
  1983. *p_offset = offset;
  1984. return true;
  1985. }
  1986. static bool
  1987. aot_emit_import_func_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  1988. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  1989. AOTObjectData *obj_data)
  1990. {
  1991. uint32 offset = *p_offset, i;
  1992. AOTImportFunc *import_func = comp_data->import_funcs;
  1993. *p_offset = offset = align_uint(offset, 4);
  1994. EMIT_U32(comp_data->import_func_count);
  1995. for (i = 0; i < comp_data->import_func_count; i++, import_func++) {
  1996. offset = align_uint(offset, 2);
  1997. EMIT_U16(import_func->func_type_index);
  1998. EMIT_STR(import_func->module_name);
  1999. offset = align_uint(offset, 2);
  2000. EMIT_STR(import_func->func_name);
  2001. }
  2002. if (offset - *p_offset != get_import_func_info_size(comp_ctx, comp_data)) {
  2003. aot_set_last_error("emit import function info failed.");
  2004. return false;
  2005. }
  2006. *p_offset = offset;
  2007. return true;
  2008. }
  2009. static bool
  2010. aot_emit_object_data_section_info(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2011. AOTCompContext *comp_ctx,
  2012. AOTObjectData *obj_data)
  2013. {
  2014. uint32 offset = *p_offset, i;
  2015. AOTObjectDataSection *data_section = obj_data->data_sections;
  2016. *p_offset = offset = align_uint(offset, 4);
  2017. EMIT_U32(obj_data->data_sections_count);
  2018. for (i = 0; i < obj_data->data_sections_count; i++, data_section++) {
  2019. offset = align_uint(offset, 2);
  2020. EMIT_STR(data_section->name);
  2021. offset = align_uint(offset, 4);
  2022. EMIT_U32(data_section->size);
  2023. if (obj_data->stack_sizes_section_name != NULL
  2024. && !strcmp(obj_data->stack_sizes_section_name,
  2025. data_section->name)) {
  2026. uint32 ss_offset = obj_data->stack_sizes_offset;
  2027. uint32 ss_size =
  2028. obj_data->func_count * sizeof(*obj_data->stack_sizes);
  2029. LOG_VERBOSE("Replacing stack_sizes in %s section, offset %" PRIu32
  2030. ", size %" PRIu32,
  2031. obj_data->stack_sizes_section_name, ss_offset, ss_size);
  2032. bh_assert(ss_offset + ss_size <= data_section->size);
  2033. /* 0 .. ss_offset */
  2034. if (ss_offset > 0) {
  2035. EMIT_BUF(data_section->data, ss_offset);
  2036. }
  2037. /* ss_offset .. ss_offset+ss_size */
  2038. EMIT_BUF(obj_data->stack_sizes, ss_size);
  2039. /* ss_offset+ss_size .. data_section->size */
  2040. if (data_section->size > ss_offset + ss_size) {
  2041. EMIT_BUF(data_section->data + ss_offset + ss_size,
  2042. data_section->size - (ss_offset + ss_size));
  2043. }
  2044. }
  2045. else {
  2046. EMIT_BUF(data_section->data, data_section->size);
  2047. }
  2048. }
  2049. if (offset - *p_offset
  2050. != get_object_data_section_info_size(comp_ctx, obj_data)) {
  2051. aot_set_last_error("emit object data section info failed.");
  2052. return false;
  2053. }
  2054. *p_offset = offset;
  2055. return true;
  2056. }
  2057. static bool
  2058. aot_emit_init_data_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2059. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2060. AOTObjectData *obj_data)
  2061. {
  2062. uint32 section_size =
  2063. get_init_data_section_size(comp_ctx, comp_data, obj_data);
  2064. uint32 offset = *p_offset;
  2065. *p_offset = offset = align_uint(offset, 4);
  2066. EMIT_U32(AOT_SECTION_TYPE_INIT_DATA);
  2067. EMIT_U32(section_size);
  2068. if (!aot_emit_mem_info(buf, buf_end, &offset, comp_ctx, comp_data, obj_data)
  2069. || !aot_emit_table_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2070. obj_data)
  2071. || !aot_emit_type_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2072. obj_data)
  2073. || !aot_emit_import_global_info(buf, buf_end, &offset, comp_ctx,
  2074. comp_data, obj_data)
  2075. || !aot_emit_global_info(buf, buf_end, &offset, comp_ctx, comp_data,
  2076. obj_data)
  2077. || !aot_emit_import_func_info(buf, buf_end, &offset, comp_ctx,
  2078. comp_data, obj_data))
  2079. return false;
  2080. offset = align_uint(offset, 4);
  2081. EMIT_U32(comp_data->func_count);
  2082. EMIT_U32(comp_data->start_func_index);
  2083. EMIT_U32(comp_data->aux_data_end_global_index);
  2084. EMIT_U64(comp_data->aux_data_end);
  2085. EMIT_U32(comp_data->aux_heap_base_global_index);
  2086. EMIT_U64(comp_data->aux_heap_base);
  2087. EMIT_U32(comp_data->aux_stack_top_global_index);
  2088. EMIT_U64(comp_data->aux_stack_bottom);
  2089. EMIT_U32(comp_data->aux_stack_size);
  2090. if (!aot_emit_object_data_section_info(buf, buf_end, &offset, comp_ctx,
  2091. obj_data))
  2092. return false;
  2093. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2094. aot_set_last_error("emit init data section failed.");
  2095. return false;
  2096. }
  2097. *p_offset = offset;
  2098. return true;
  2099. }
  2100. static bool
  2101. aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2102. AOTCompData *comp_data, AOTObjectData *obj_data)
  2103. {
  2104. uint32 section_size = get_text_section_size(obj_data);
  2105. uint32 offset = *p_offset;
  2106. uint8 placeholder = 0;
  2107. AOTRelocationGroup *relocation_group;
  2108. AOTRelocation *relocation;
  2109. uint32 i, j, relocation_count;
  2110. uint8 *text;
  2111. *p_offset = offset = align_uint(offset, 4);
  2112. EMIT_U32(AOT_SECTION_TYPE_TEXT);
  2113. EMIT_U32(section_size);
  2114. EMIT_U32(obj_data->literal_size);
  2115. if (obj_data->literal_size > 0) {
  2116. EMIT_BUF(obj_data->literal, obj_data->literal_size);
  2117. while (offset & 3)
  2118. EMIT_BUF(&placeholder, 1);
  2119. }
  2120. text = buf + offset;
  2121. if (obj_data->text_size > 0) {
  2122. EMIT_BUF(obj_data->text, obj_data->text_size);
  2123. while (offset & 3)
  2124. EMIT_BUF(&placeholder, 1);
  2125. }
  2126. if (obj_data->text_unlikely_size > 0) {
  2127. EMIT_BUF(obj_data->text_unlikely, obj_data->text_unlikely_size);
  2128. while (offset & 3)
  2129. EMIT_BUF(&placeholder, 1);
  2130. }
  2131. if (obj_data->text_hot_size > 0) {
  2132. EMIT_BUF(obj_data->text_hot, obj_data->text_hot_size);
  2133. while (offset & 3)
  2134. EMIT_BUF(&placeholder, 1);
  2135. }
  2136. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2137. aot_set_last_error("emit text section failed.");
  2138. return false;
  2139. }
  2140. /* apply relocations to aot_func_internal#n in text section for
  2141. windows platform */
  2142. if ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6)
  2143. /* Windows AOT_COFF64_BIN_TYPE */
  2144. && obj_data->target_info.bin_type == 6)
  2145. || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4)
  2146. /* Windows AOT_COFF32_BIN_TYPE */
  2147. && obj_data->target_info.bin_type == 4)) {
  2148. relocation_group = obj_data->relocation_groups;
  2149. for (i = 0; i < obj_data->relocation_group_count;
  2150. i++, relocation_group++) {
  2151. /* relocation in text section */
  2152. if ((!strcmp(relocation_group->section_name, ".text")
  2153. || !strcmp(relocation_group->section_name, ".ltext"))) {
  2154. relocation = relocation_group->relocations;
  2155. relocation_count = relocation_group->relocation_count;
  2156. for (j = 0; j < relocation_count; j++) {
  2157. /* relocation to aot_func_internal#n */
  2158. if (str_starts_with(relocation->symbol_name,
  2159. AOT_FUNC_INTERNAL_PREFIX)
  2160. && ((obj_data->target_info.bin_type
  2161. == 6 /* AOT_COFF64_BIN_TYPE */
  2162. && relocation->relocation_type
  2163. == 4 /* IMAGE_REL_AMD64_REL32 */)
  2164. || (obj_data->target_info.bin_type
  2165. == 4 /* AOT_COFF32_BIN_TYPE */
  2166. && relocation->relocation_type
  2167. == 20 /* IMAGE_REL_I386_REL32 */))) {
  2168. uint32 func_idx =
  2169. atoi(relocation->symbol_name
  2170. + strlen(AOT_FUNC_INTERNAL_PREFIX));
  2171. uint64 text_offset, reloc_offset, reloc_addend;
  2172. bh_assert(func_idx < obj_data->func_count);
  2173. text_offset = obj_data->funcs[func_idx]
  2174. .text_offset_of_aot_func_internal;
  2175. reloc_offset = relocation->relocation_offset;
  2176. reloc_addend = relocation->relocation_addend;
  2177. /* S + A - P */
  2178. *(uint32 *)(text + reloc_offset) =
  2179. (uint32)(text_offset + reloc_addend - reloc_offset
  2180. - 4);
  2181. /* remove current relocation as it has been applied */
  2182. if (j < relocation_count - 1) {
  2183. uint32 move_size =
  2184. (uint32)(sizeof(AOTRelocation)
  2185. * (relocation_count - 1 - j));
  2186. bh_memmove_s(relocation, move_size, relocation + 1,
  2187. move_size);
  2188. }
  2189. relocation_group->relocation_count--;
  2190. }
  2191. else {
  2192. relocation++;
  2193. }
  2194. }
  2195. }
  2196. }
  2197. }
  2198. *p_offset = offset;
  2199. return true;
  2200. }
  2201. #if WASM_ENABLE_GC != 0
  2202. static bool
  2203. aot_emit_ref_flag(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2204. uint8 pointer_size, int8 type)
  2205. {
  2206. uint32 j, offset = *p_offset;
  2207. uint16 value_type_cell_num;
  2208. if (wasm_is_type_reftype(type) && !wasm_is_reftype_i31ref(type)) {
  2209. EMIT_U8(1);
  2210. if (pointer_size == sizeof(uint64))
  2211. EMIT_U8(1);
  2212. }
  2213. else {
  2214. value_type_cell_num = wasm_value_type_cell_num(type);
  2215. for (j = 0; j < value_type_cell_num; j++)
  2216. EMIT_U8(0);
  2217. }
  2218. *p_offset = offset;
  2219. return true;
  2220. }
  2221. #endif
  2222. static bool
  2223. aot_emit_func_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2224. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2225. AOTObjectData *obj_data)
  2226. {
  2227. uint32 section_size = get_func_section_size(comp_ctx, comp_data, obj_data);
  2228. uint32 i, offset = *p_offset;
  2229. AOTObjectFunc *func = obj_data->funcs;
  2230. AOTFunc **funcs = comp_data->funcs;
  2231. *p_offset = offset = align_uint(offset, 4);
  2232. EMIT_U32(AOT_SECTION_TYPE_FUNCTION);
  2233. EMIT_U32(section_size);
  2234. for (i = 0; i < obj_data->func_count; i++, func++) {
  2235. if (is_32bit_binary(obj_data))
  2236. EMIT_U32(func->text_offset);
  2237. else
  2238. EMIT_U64(func->text_offset);
  2239. }
  2240. for (i = 0; i < comp_data->func_count; i++)
  2241. EMIT_U32(funcs[i]->func_type_index);
  2242. for (i = 0; i < comp_data->func_count; i++) {
  2243. uint32 max_local_cell_num =
  2244. funcs[i]->param_cell_num + funcs[i]->local_cell_num;
  2245. EMIT_U32(max_local_cell_num);
  2246. }
  2247. for (i = 0; i < comp_data->func_count; i++)
  2248. EMIT_U32(funcs[i]->max_stack_cell_num);
  2249. #if WASM_ENABLE_GC != 0
  2250. if (comp_ctx->enable_gc) {
  2251. /* emit func_local_ref_flag arrays for both import and AOTed funcs */
  2252. AOTFuncType *func_type;
  2253. uint32 j, local_ref_flags_cell_num, paddings;
  2254. for (i = 0; i < comp_data->import_func_count; i++) {
  2255. func_type = comp_data->import_funcs[i].func_type;
  2256. /* recalculate cell_num based on target pointer size */
  2257. local_ref_flags_cell_num = 0;
  2258. for (j = 0; j < func_type->param_count; j++) {
  2259. local_ref_flags_cell_num += wasm_value_type_cell_num_internal(
  2260. func_type->types[j], comp_ctx->pointer_size);
  2261. }
  2262. paddings =
  2263. local_ref_flags_cell_num < 2 ? 2 - local_ref_flags_cell_num : 0;
  2264. local_ref_flags_cell_num =
  2265. local_ref_flags_cell_num > 2 ? local_ref_flags_cell_num : 2;
  2266. offset = align_uint(offset, 4);
  2267. EMIT_U32(local_ref_flags_cell_num);
  2268. for (j = 0; j < func_type->param_count; j++) {
  2269. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2270. comp_ctx->pointer_size,
  2271. func_type->types[j]))
  2272. return false;
  2273. }
  2274. for (j = 0; j < paddings; j++)
  2275. EMIT_U8(0);
  2276. }
  2277. for (i = 0; i < comp_data->func_count; i++) {
  2278. func_type = funcs[i]->func_type;
  2279. local_ref_flags_cell_num =
  2280. funcs[i]->param_cell_num + funcs[i]->local_cell_num;
  2281. offset = align_uint(offset, 4);
  2282. EMIT_U32(local_ref_flags_cell_num);
  2283. /* emit local_ref_flag for param variables */
  2284. for (j = 0; j < func_type->param_count; j++) {
  2285. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2286. comp_ctx->pointer_size,
  2287. func_type->types[j]))
  2288. return false;
  2289. }
  2290. /* emit local_ref_flag for local variables */
  2291. for (j = 0; j < funcs[i]->local_count; j++) {
  2292. if (!aot_emit_ref_flag(buf, buf_end, &offset,
  2293. comp_ctx->pointer_size,
  2294. funcs[i]->local_types_wp[j]))
  2295. return false;
  2296. }
  2297. }
  2298. }
  2299. #endif /* end of WASM_ENABLE_GC != 0 */
  2300. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2301. aot_set_last_error("emit function section failed.");
  2302. return false;
  2303. }
  2304. *p_offset = offset;
  2305. return true;
  2306. }
  2307. static bool
  2308. aot_emit_export_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2309. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2310. AOTObjectData *obj_data)
  2311. {
  2312. uint32 section_size = get_export_section_size(comp_ctx, comp_data);
  2313. AOTExport *export = comp_data->wasm_module->exports;
  2314. uint32 export_count = comp_data->wasm_module->export_count;
  2315. uint32 i, offset = *p_offset;
  2316. *p_offset = offset = align_uint(offset, 4);
  2317. EMIT_U32(AOT_SECTION_TYPE_EXPORT);
  2318. EMIT_U32(section_size);
  2319. EMIT_U32(export_count);
  2320. for (i = 0; i < export_count; i++, export ++) {
  2321. offset = align_uint(offset, 4);
  2322. EMIT_U32(export->index);
  2323. EMIT_U8(export->kind);
  2324. EMIT_U8(0);
  2325. EMIT_STR(export->name);
  2326. }
  2327. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2328. aot_set_last_error("emit export section failed.");
  2329. return false;
  2330. }
  2331. *p_offset = offset;
  2332. return true;
  2333. }
  2334. static bool
  2335. aot_emit_relocation_symbol_table(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2336. AOTCompContext *comp_ctx,
  2337. AOTCompData *comp_data,
  2338. AOTObjectData *obj_data)
  2339. {
  2340. uint32 symbol_offset = 0, total_string_len = 0;
  2341. uint32 offset = *p_offset;
  2342. AOTSymbolNode *sym;
  2343. EMIT_U32(obj_data->symbol_list.len);
  2344. /* emit symbol offsets */
  2345. sym = (AOTSymbolNode *)(obj_data->symbol_list.head);
  2346. while (sym) {
  2347. EMIT_U32(symbol_offset);
  2348. /* string_len + str[0 .. string_len - 1] */
  2349. symbol_offset += get_string_size(comp_ctx, sym->symbol);
  2350. symbol_offset = align_uint(symbol_offset, 2);
  2351. sym = sym->next;
  2352. }
  2353. /* emit total string len */
  2354. total_string_len = symbol_offset;
  2355. EMIT_U32(total_string_len);
  2356. /* emit symbols */
  2357. sym = (AOTSymbolNode *)(obj_data->symbol_list.head);
  2358. while (sym) {
  2359. EMIT_STR(sym->symbol);
  2360. offset = align_uint(offset, 2);
  2361. sym = sym->next;
  2362. }
  2363. *p_offset = offset;
  2364. return true;
  2365. }
  2366. static bool
  2367. aot_emit_relocation_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2368. AOTCompContext *comp_ctx, AOTCompData *comp_data,
  2369. AOTObjectData *obj_data)
  2370. {
  2371. uint32 section_size = get_relocation_section_size(comp_ctx, obj_data);
  2372. uint32 i, offset = *p_offset;
  2373. AOTRelocationGroup *relocation_group = obj_data->relocation_groups;
  2374. if (section_size == (uint32)-1)
  2375. return false;
  2376. *p_offset = offset = align_uint(offset, 4);
  2377. EMIT_U32(AOT_SECTION_TYPE_RELOCATION);
  2378. EMIT_U32(section_size);
  2379. aot_emit_relocation_symbol_table(buf, buf_end, &offset, comp_ctx, comp_data,
  2380. obj_data);
  2381. offset = align_uint(offset, 4);
  2382. EMIT_U32(obj_data->relocation_group_count);
  2383. /* emit each relocation group */
  2384. for (i = 0; i < obj_data->relocation_group_count; i++, relocation_group++) {
  2385. AOTRelocation *relocation = relocation_group->relocations;
  2386. uint32 j;
  2387. offset = align_uint(offset, 4);
  2388. EMIT_U32(relocation_group->name_index);
  2389. offset = align_uint(offset, 4);
  2390. EMIT_U32(relocation_group->relocation_count);
  2391. /* emit each relocation */
  2392. for (j = 0; j < relocation_group->relocation_count; j++, relocation++) {
  2393. offset = align_uint(offset, 4);
  2394. if (is_32bit_binary(obj_data)) {
  2395. EMIT_U32(relocation->relocation_offset);
  2396. EMIT_U32(relocation->relocation_addend);
  2397. }
  2398. else {
  2399. EMIT_U64(relocation->relocation_offset);
  2400. EMIT_U64(relocation->relocation_addend);
  2401. }
  2402. EMIT_U32(relocation->relocation_type);
  2403. EMIT_U32(relocation->symbol_index);
  2404. }
  2405. }
  2406. if (offset - *p_offset != section_size + sizeof(uint32) * 2) {
  2407. aot_set_last_error("emit relocation section failed.");
  2408. return false;
  2409. }
  2410. *p_offset = offset;
  2411. return true;
  2412. }
  2413. static bool
  2414. aot_emit_native_symbol(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2415. AOTCompContext *comp_ctx)
  2416. {
  2417. uint32 offset = *p_offset;
  2418. AOTNativeSymbol *sym = NULL;
  2419. if (bh_list_length(&comp_ctx->native_symbols) == 0)
  2420. /* emit only when there are native symbols */
  2421. return true;
  2422. *p_offset = offset = align_uint(offset, 4);
  2423. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2424. /* sub section id + symbol count + symbol list */
  2425. EMIT_U32(sizeof(uint32) * 2 + get_native_symbol_list_size(comp_ctx));
  2426. EMIT_U32(AOT_CUSTOM_SECTION_NATIVE_SYMBOL);
  2427. EMIT_U32(bh_list_length(&comp_ctx->native_symbols));
  2428. sym = bh_list_first_elem(&comp_ctx->native_symbols);
  2429. while (sym) {
  2430. offset = align_uint(offset, 2);
  2431. EMIT_STR(sym->symbol);
  2432. sym = bh_list_elem_next(sym);
  2433. }
  2434. *p_offset = offset;
  2435. return true;
  2436. }
  2437. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  2438. static bool
  2439. aot_emit_name_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2440. AOTCompData *comp_data, AOTCompContext *comp_ctx)
  2441. {
  2442. uint32 offset = *p_offset;
  2443. if (comp_data->aot_name_section_size == 0)
  2444. return true;
  2445. offset = align_uint(offset, 4);
  2446. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2447. /* sub section id + name section size */
  2448. EMIT_U32(sizeof(uint32) * 1 + comp_data->aot_name_section_size);
  2449. EMIT_U32(AOT_CUSTOM_SECTION_NAME);
  2450. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf),
  2451. comp_data->aot_name_section_buf,
  2452. (uint32)comp_data->aot_name_section_size);
  2453. offset += comp_data->aot_name_section_size;
  2454. *p_offset = offset;
  2455. LOG_DEBUG("emit name section");
  2456. return true;
  2457. }
  2458. #endif
  2459. #if WASM_ENABLE_STRINGREF != 0
  2460. static bool
  2461. aot_emit_string_literal_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2462. AOTCompData *comp_data,
  2463. AOTCompContext *comp_ctx)
  2464. {
  2465. uint32 string_count = comp_data->string_literal_count;
  2466. if (string_count > 0) {
  2467. uint32 offset = *p_offset;
  2468. uint32 i;
  2469. *p_offset = offset = align_uint(offset, 4);
  2470. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2471. /* sub section id + string literal section size */
  2472. EMIT_U32(sizeof(uint32) * 1
  2473. + get_string_literal_section_size(comp_ctx, comp_data));
  2474. EMIT_U32(AOT_CUSTOM_SECTION_STRING_LITERAL);
  2475. /* reserved */
  2476. EMIT_U32(0);
  2477. /* string literal count */
  2478. EMIT_U32(string_count);
  2479. for (i = 0; i < string_count; i++) {
  2480. EMIT_U32(comp_data->string_literal_lengths_wp[i]);
  2481. }
  2482. for (i = 0; i < string_count; i++) {
  2483. uint32 string_length = comp_data->string_literal_lengths_wp[i];
  2484. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf),
  2485. comp_data->string_literal_ptrs_wp[i], string_length);
  2486. offset += string_length;
  2487. }
  2488. *p_offset = offset;
  2489. }
  2490. return true;
  2491. }
  2492. #endif /* end of WASM_ENABLE_STRINGREF != 0 */
  2493. static bool
  2494. aot_emit_custom_sections(uint8 *buf, uint8 *buf_end, uint32 *p_offset,
  2495. AOTCompData *comp_data, AOTCompContext *comp_ctx)
  2496. {
  2497. #if WASM_ENABLE_LOAD_CUSTOM_SECTION != 0
  2498. uint32 offset = *p_offset, i;
  2499. for (i = 0; i < comp_ctx->custom_sections_count; i++) {
  2500. const char *section_name = comp_ctx->custom_sections_wp[i];
  2501. const uint8 *content = NULL;
  2502. uint32 length = 0;
  2503. if (strcmp(section_name, "name") == 0) {
  2504. *p_offset = offset;
  2505. if (!aot_emit_name_section(buf, buf_end, p_offset, comp_data,
  2506. comp_ctx))
  2507. return false;
  2508. offset = *p_offset;
  2509. continue;
  2510. }
  2511. content = wasm_loader_get_custom_section(comp_data->wasm_module,
  2512. section_name, &length);
  2513. if (!content) {
  2514. /* Warning has been reported during calculating size */
  2515. continue;
  2516. }
  2517. offset = align_uint(offset, 4);
  2518. EMIT_U32(AOT_SECTION_TYPE_CUSTOM);
  2519. /* sub section id + content */
  2520. EMIT_U32(sizeof(uint32) * 1 + get_string_size(comp_ctx, section_name)
  2521. + length);
  2522. EMIT_U32(AOT_CUSTOM_SECTION_RAW);
  2523. EMIT_STR(section_name);
  2524. bh_memcpy_s((uint8 *)(buf + offset), (uint32)(buf_end - buf), content,
  2525. length);
  2526. offset += length;
  2527. }
  2528. *p_offset = offset;
  2529. #endif
  2530. return true;
  2531. }
  2532. typedef uint32 U32;
  2533. typedef int32 I32;
  2534. typedef uint16 U16;
  2535. typedef uint8 U8;
  2536. struct coff_hdr {
  2537. U16 u16Machine;
  2538. U16 u16NumSections;
  2539. U32 u32DateTimeStamp;
  2540. U32 u32SymTblPtr;
  2541. U32 u32NumSymbols;
  2542. U16 u16PeHdrSize;
  2543. U16 u16Characs;
  2544. };
  2545. #define E_TYPE_REL 1
  2546. #define E_TYPE_XIP 4
  2547. #define IMAGE_FILE_MACHINE_AMD64 0x8664
  2548. #define IMAGE_FILE_MACHINE_I386 0x014c
  2549. #define IMAGE_FILE_MACHINE_IA64 0x0200
  2550. #define AOT_COFF32_BIN_TYPE 4 /* 32-bit little endian */
  2551. #define AOT_COFF64_BIN_TYPE 6 /* 64-bit little endian */
  2552. #define EI_NIDENT 16
  2553. typedef uint32 elf32_word;
  2554. typedef int32 elf32_sword;
  2555. typedef uint16 elf32_half;
  2556. typedef uint32 elf32_off;
  2557. typedef uint32 elf32_addr;
  2558. struct elf32_ehdr {
  2559. unsigned char e_ident[EI_NIDENT]; /* ident bytes */
  2560. elf32_half e_type; /* file type */
  2561. elf32_half e_machine; /* target machine */
  2562. elf32_word e_version; /* file version */
  2563. elf32_addr e_entry; /* start address */
  2564. elf32_off e_phoff; /* phdr file offset */
  2565. elf32_off e_shoff; /* shdr file offset */
  2566. elf32_word e_flags; /* file flags */
  2567. elf32_half e_ehsize; /* sizeof ehdr */
  2568. elf32_half e_phentsize; /* sizeof phdr */
  2569. elf32_half e_phnum; /* number phdrs */
  2570. elf32_half e_shentsize; /* sizeof shdr */
  2571. elf32_half e_shnum; /* number shdrs */
  2572. elf32_half e_shstrndx; /* shdr string index */
  2573. };
  2574. struct elf32_rel {
  2575. elf32_addr r_offset;
  2576. elf32_word r_info;
  2577. } elf32_rel;
  2578. struct elf32_rela {
  2579. elf32_addr r_offset;
  2580. elf32_word r_info;
  2581. elf32_sword r_addend;
  2582. } elf32_rela;
  2583. typedef uint32 elf64_word;
  2584. typedef int32 elf64_sword;
  2585. typedef uint64 elf64_xword;
  2586. typedef int64 elf64_sxword;
  2587. typedef uint16 elf64_half;
  2588. typedef uint64 elf64_off;
  2589. typedef uint64 elf64_addr;
  2590. struct elf64_ehdr {
  2591. unsigned char e_ident[EI_NIDENT]; /* ident bytes */
  2592. elf64_half e_type; /* file type */
  2593. elf64_half e_machine; /* target machine */
  2594. elf64_word e_version; /* file version */
  2595. elf64_addr e_entry; /* start address */
  2596. elf64_off e_phoff; /* phdr file offset */
  2597. elf64_off e_shoff; /* shdr file offset */
  2598. elf64_word e_flags; /* file flags */
  2599. elf64_half e_ehsize; /* sizeof ehdr */
  2600. elf64_half e_phentsize; /* sizeof phdr */
  2601. elf64_half e_phnum; /* number phdrs */
  2602. elf64_half e_shentsize; /* sizeof shdr */
  2603. elf64_half e_shnum; /* number shdrs */
  2604. elf64_half e_shstrndx; /* shdr string index */
  2605. };
  2606. typedef struct elf64_rel {
  2607. elf64_addr r_offset;
  2608. elf64_xword r_info;
  2609. } elf64_rel;
  2610. typedef struct elf64_rela {
  2611. elf64_addr r_offset;
  2612. elf64_xword r_info;
  2613. elf64_sxword r_addend;
  2614. } elf64_rela;
  2615. #define SET_TARGET_INFO_VALUE(f, val, type, little) \
  2616. do { \
  2617. type tmp = val; \
  2618. if ((little && !is_little_endian()) \
  2619. || (!little && is_little_endian())) \
  2620. exchange_##type((uint8 *)&tmp); \
  2621. obj_data->target_info.f = tmp; \
  2622. } while (0)
  2623. #define SET_TARGET_INFO_FIELD(f, v, type, little) \
  2624. SET_TARGET_INFO_VALUE(f, elf_header->v, type, little)
  2625. static bool
  2626. aot_resolve_target_info(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  2627. {
  2628. LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary);
  2629. const uint8 *elf_buf = (uint8 *)LLVMGetBufferStart(obj_data->mem_buf);
  2630. uint32 elf_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf);
  2631. if (bin_type != LLVMBinaryTypeCOFF && bin_type != LLVMBinaryTypeELF32L
  2632. && bin_type != LLVMBinaryTypeELF32B && bin_type != LLVMBinaryTypeELF64L
  2633. && bin_type != LLVMBinaryTypeELF64B
  2634. && bin_type != LLVMBinaryTypeMachO32L
  2635. && bin_type != LLVMBinaryTypeMachO32B
  2636. && bin_type != LLVMBinaryTypeMachO64L
  2637. && bin_type != LLVMBinaryTypeMachO64B) {
  2638. aot_set_last_error("invalid llvm binary bin_type.");
  2639. return false;
  2640. }
  2641. obj_data->target_info.bin_type = bin_type - LLVMBinaryTypeELF32L;
  2642. if (bin_type == LLVMBinaryTypeCOFF) {
  2643. struct coff_hdr *coff_header;
  2644. if (!elf_buf || elf_size < sizeof(struct coff_hdr)) {
  2645. aot_set_last_error("invalid coff_hdr buffer.");
  2646. return false;
  2647. }
  2648. coff_header = (struct coff_hdr *)elf_buf;
  2649. /* Emit eXecute In Place file type while in indirect mode */
  2650. if (comp_ctx->is_indirect_mode)
  2651. obj_data->target_info.e_type = E_TYPE_XIP;
  2652. else
  2653. obj_data->target_info.e_type = E_TYPE_REL;
  2654. obj_data->target_info.e_machine = coff_header->u16Machine;
  2655. obj_data->target_info.e_version = 1;
  2656. obj_data->target_info.e_flags = 0;
  2657. if (coff_header->u16Machine == IMAGE_FILE_MACHINE_AMD64
  2658. || coff_header->u16Machine == IMAGE_FILE_MACHINE_IA64)
  2659. obj_data->target_info.bin_type = AOT_COFF64_BIN_TYPE;
  2660. else if (coff_header->u16Machine == IMAGE_FILE_MACHINE_I386)
  2661. obj_data->target_info.bin_type = AOT_COFF32_BIN_TYPE;
  2662. }
  2663. else if (bin_type == LLVMBinaryTypeELF32L
  2664. || bin_type == LLVMBinaryTypeELF32B) {
  2665. struct elf32_ehdr *elf_header;
  2666. bool is_little_bin = bin_type == LLVMBinaryTypeELF32L;
  2667. uint16 e_type;
  2668. if (!elf_buf || elf_size < sizeof(struct elf32_ehdr)) {
  2669. aot_set_last_error("invalid elf32 buffer.");
  2670. return false;
  2671. }
  2672. elf_header = (struct elf32_ehdr *)elf_buf;
  2673. e_type = elf_header->e_type;
  2674. /* Emit eXecute In Place file type while in indirect mode */
  2675. if (comp_ctx->is_indirect_mode)
  2676. e_type = E_TYPE_XIP;
  2677. SET_TARGET_INFO_VALUE(e_type, e_type, uint16, is_little_bin);
  2678. SET_TARGET_INFO_FIELD(e_machine, e_machine, uint16, is_little_bin);
  2679. SET_TARGET_INFO_FIELD(e_version, e_version, uint32, is_little_bin);
  2680. SET_TARGET_INFO_FIELD(e_flags, e_flags, uint32, is_little_bin);
  2681. }
  2682. else if (bin_type == LLVMBinaryTypeELF64L
  2683. || bin_type == LLVMBinaryTypeELF64B) {
  2684. struct elf64_ehdr *elf_header;
  2685. bool is_little_bin = bin_type == LLVMBinaryTypeELF64L;
  2686. uint16 e_type;
  2687. if (!elf_buf || elf_size < sizeof(struct elf64_ehdr)) {
  2688. aot_set_last_error("invalid elf64 buffer.");
  2689. return false;
  2690. }
  2691. elf_header = (struct elf64_ehdr *)elf_buf;
  2692. e_type = elf_header->e_type;
  2693. /* Emit eXecute In Place file type while in indirect mode */
  2694. if (comp_ctx->is_indirect_mode)
  2695. e_type = E_TYPE_XIP;
  2696. SET_TARGET_INFO_VALUE(e_type, e_type, uint16, is_little_bin);
  2697. SET_TARGET_INFO_FIELD(e_machine, e_machine, uint16, is_little_bin);
  2698. SET_TARGET_INFO_FIELD(e_version, e_version, uint32, is_little_bin);
  2699. SET_TARGET_INFO_FIELD(e_flags, e_flags, uint32, is_little_bin);
  2700. }
  2701. else if (bin_type == LLVMBinaryTypeMachO32L
  2702. || bin_type == LLVMBinaryTypeMachO32B) {
  2703. /* TODO: parse file type of Mach-O 32 */
  2704. aot_set_last_error("invalid llvm binary bin_type.");
  2705. return false;
  2706. }
  2707. else if (bin_type == LLVMBinaryTypeMachO64L
  2708. || bin_type == LLVMBinaryTypeMachO64B) {
  2709. /* TODO: parse file type of Mach-O 64 */
  2710. aot_set_last_error("invalid llvm binary bin_type.");
  2711. return false;
  2712. }
  2713. bh_assert(sizeof(obj_data->target_info.arch)
  2714. == sizeof(comp_ctx->target_arch));
  2715. bh_memcpy_s(obj_data->target_info.arch, sizeof(obj_data->target_info.arch),
  2716. comp_ctx->target_arch, sizeof(comp_ctx->target_arch));
  2717. return true;
  2718. }
  2719. static bool
  2720. aot_resolve_text(AOTObjectData *obj_data)
  2721. {
  2722. #if WASM_ENABLE_DEBUG_AOT != 0
  2723. LLVMBinaryType bin_type = LLVMBinaryGetType(obj_data->binary);
  2724. if (bin_type == LLVMBinaryTypeELF32L || bin_type == LLVMBinaryTypeELF64L) {
  2725. obj_data->text = (char *)LLVMGetBufferStart(obj_data->mem_buf);
  2726. obj_data->text_size = (uint32)LLVMGetBufferSize(obj_data->mem_buf);
  2727. }
  2728. else
  2729. #endif
  2730. {
  2731. LLVMSectionIteratorRef sec_itr;
  2732. char *name;
  2733. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2734. aot_set_last_error("llvm get section iterator failed.");
  2735. return false;
  2736. }
  2737. while (
  2738. !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2739. if ((name = (char *)LLVMGetSectionName(sec_itr))) {
  2740. if (!strcmp(name, ".text") || !strcmp(name, ".ltext")) {
  2741. obj_data->text = (char *)LLVMGetSectionContents(sec_itr);
  2742. obj_data->text_size = (uint32)LLVMGetSectionSize(sec_itr);
  2743. }
  2744. else if (!strcmp(name, ".text.unlikely.")
  2745. || !strcmp(name, ".ltext.unlikely.")) {
  2746. obj_data->text_unlikely =
  2747. (char *)LLVMGetSectionContents(sec_itr);
  2748. obj_data->text_unlikely_size =
  2749. (uint32)LLVMGetSectionSize(sec_itr);
  2750. }
  2751. else if (!strcmp(name, ".text.hot.")
  2752. || !strcmp(name, ".ltext.hot.")) {
  2753. obj_data->text_hot =
  2754. (char *)LLVMGetSectionContents(sec_itr);
  2755. obj_data->text_hot_size =
  2756. (uint32)LLVMGetSectionSize(sec_itr);
  2757. }
  2758. }
  2759. LLVMMoveToNextSection(sec_itr);
  2760. }
  2761. LLVMDisposeSectionIterator(sec_itr);
  2762. }
  2763. return true;
  2764. }
  2765. static bool
  2766. aot_resolve_literal(AOTObjectData *obj_data)
  2767. {
  2768. LLVMSectionIteratorRef sec_itr;
  2769. char *name;
  2770. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2771. aot_set_last_error("llvm get section iterator failed.");
  2772. return false;
  2773. }
  2774. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2775. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2776. && !strcmp(name, ".literal")) {
  2777. obj_data->literal = (char *)LLVMGetSectionContents(sec_itr);
  2778. obj_data->literal_size = (uint32)LLVMGetSectionSize(sec_itr);
  2779. break;
  2780. }
  2781. LLVMMoveToNextSection(sec_itr);
  2782. }
  2783. LLVMDisposeSectionIterator(sec_itr);
  2784. return true;
  2785. }
  2786. static bool
  2787. get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count);
  2788. static bool
  2789. is_data_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr,
  2790. char *section_name)
  2791. {
  2792. uint32 relocation_count = 0;
  2793. return (!strcmp(section_name, ".data") || !strcmp(section_name, ".sdata")
  2794. || !strcmp(section_name, ".rodata")
  2795. /* ".rodata.cst4/8/16/.." */
  2796. || !strncmp(section_name, ".rodata.cst", strlen(".rodata.cst"))
  2797. /* ".rodata.strn.m" */
  2798. || !strncmp(section_name, ".rodata.str", strlen(".rodata.str"))
  2799. || (!strcmp(section_name, ".rdata")
  2800. && get_relocations_count(sec_itr, &relocation_count)
  2801. && relocation_count > 0)
  2802. || !strcmp(section_name, aot_stack_sizes_section_name)
  2803. || (obj_data->comp_ctx->enable_llvm_pgo
  2804. && (!strncmp(section_name, "__llvm_prf_cnts", 15)
  2805. || !strncmp(section_name, "__llvm_prf_data", 15)
  2806. || !strncmp(section_name, "__llvm_prf_names", 16))));
  2807. }
  2808. static bool
  2809. get_object_data_sections_count(AOTObjectData *obj_data, uint32 *p_count)
  2810. {
  2811. LLVMSectionIteratorRef sec_itr;
  2812. char *name;
  2813. uint32 count = 0;
  2814. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2815. aot_set_last_error("llvm get section iterator failed.");
  2816. return false;
  2817. }
  2818. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2819. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2820. && (is_data_section(obj_data, sec_itr, name))) {
  2821. count++;
  2822. }
  2823. LLVMMoveToNextSection(sec_itr);
  2824. }
  2825. LLVMDisposeSectionIterator(sec_itr);
  2826. *p_count = count;
  2827. return true;
  2828. }
  2829. static bool
  2830. aot_resolve_object_data_sections(AOTObjectData *obj_data)
  2831. {
  2832. LLVMSectionIteratorRef sec_itr;
  2833. char *name;
  2834. AOTObjectDataSection *data_section;
  2835. uint32 sections_count;
  2836. uint32 size;
  2837. if (!get_object_data_sections_count(obj_data, &sections_count)) {
  2838. return false;
  2839. }
  2840. if (sections_count > 0) {
  2841. uint32 llvm_prf_cnts_idx = 0, llvm_prf_data_idx = 0;
  2842. char buf[32];
  2843. size = (uint32)sizeof(AOTObjectDataSection) * sections_count;
  2844. if (!(data_section = obj_data->data_sections =
  2845. wasm_runtime_malloc(size))) {
  2846. aot_set_last_error("allocate memory for data sections failed.");
  2847. return false;
  2848. }
  2849. memset(obj_data->data_sections, 0, size);
  2850. obj_data->data_sections_count = sections_count;
  2851. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  2852. aot_set_last_error("llvm get section iterator failed.");
  2853. return false;
  2854. }
  2855. while (
  2856. !LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  2857. if ((name = (char *)LLVMGetSectionName(sec_itr))
  2858. && (is_data_section(obj_data, sec_itr, name))) {
  2859. data_section->name = name;
  2860. if (obj_data->comp_ctx->enable_llvm_pgo
  2861. && !strcmp(name, "__llvm_prf_cnts")) {
  2862. snprintf(buf, sizeof(buf), "%s%u", name,
  2863. llvm_prf_cnts_idx++);
  2864. size = (uint32)(strlen(buf) + 1);
  2865. if (!(data_section->name = wasm_runtime_malloc(size))) {
  2866. aot_set_last_error(
  2867. "allocate memory for data section name failed.");
  2868. return false;
  2869. }
  2870. bh_memcpy_s(data_section->name, size, buf, size);
  2871. data_section->is_name_allocated = true;
  2872. }
  2873. else if (obj_data->comp_ctx->enable_llvm_pgo
  2874. && !strcmp(name, "__llvm_prf_data")) {
  2875. snprintf(buf, sizeof(buf), "%s%u", name,
  2876. llvm_prf_data_idx++);
  2877. size = (uint32)(strlen(buf) + 1);
  2878. if (!(data_section->name = wasm_runtime_malloc(size))) {
  2879. aot_set_last_error(
  2880. "allocate memory for data section name failed.");
  2881. return false;
  2882. }
  2883. bh_memcpy_s(data_section->name, size, buf, size);
  2884. data_section->is_name_allocated = true;
  2885. }
  2886. if (obj_data->comp_ctx->enable_llvm_pgo
  2887. && !strcmp(name, "__llvm_prf_names")) {
  2888. data_section->data = (uint8 *)aot_compress_aot_func_names(
  2889. obj_data->comp_ctx, &data_section->size);
  2890. data_section->is_data_allocated = true;
  2891. }
  2892. else {
  2893. data_section->data =
  2894. (uint8 *)LLVMGetSectionContents(sec_itr);
  2895. data_section->size = (uint32)LLVMGetSectionSize(sec_itr);
  2896. }
  2897. data_section++;
  2898. }
  2899. LLVMMoveToNextSection(sec_itr);
  2900. }
  2901. LLVMDisposeSectionIterator(sec_itr);
  2902. }
  2903. return true;
  2904. }
  2905. static bool
  2906. read_stack_usage_file(const AOTCompContext *comp_ctx, const char *filename,
  2907. uint32 *sizes, uint32 count)
  2908. {
  2909. FILE *fp = NULL;
  2910. if (filename == NULL) {
  2911. aot_set_last_error("no stack usage file is specified.");
  2912. return false;
  2913. }
  2914. fp = fopen(filename, "r");
  2915. if (fp == NULL) {
  2916. LOG_ERROR("failed to open stack usage file: %s", filename);
  2917. goto fail;
  2918. }
  2919. /*
  2920. * the file consists of lines like:
  2921. *
  2922. * WASM Module:aot_func#9 72 static
  2923. */
  2924. const char *aot_func_prefix = AOT_FUNC_PREFIX;
  2925. const char *aot_func_internal_prefix = AOT_FUNC_INTERNAL_PREFIX;
  2926. uint32 precheck_found = 0;
  2927. uint32 precheck_stack_size_max = 0;
  2928. uint32 precheck_stack_size_min = UINT32_MAX;
  2929. uint32 found = 0;
  2930. while (true) {
  2931. const char *prefix;
  2932. char line[100];
  2933. char *cp = fgets(line, sizeof(line), fp);
  2934. char *fn;
  2935. char *colon;
  2936. uintmax_t func_idx;
  2937. uintmax_t sz;
  2938. int ret;
  2939. if (cp == NULL) {
  2940. break;
  2941. }
  2942. /*
  2943. * Note: strrchr (not strchr) because a module name can contain
  2944. * colons.
  2945. */
  2946. colon = strrchr(cp, ':');
  2947. if (colon == NULL) {
  2948. goto fail;
  2949. }
  2950. fn = strstr(colon, aot_func_prefix);
  2951. if (fn != NULL) {
  2952. prefix = aot_func_prefix;
  2953. }
  2954. else {
  2955. fn = strstr(colon, aot_func_internal_prefix);
  2956. if (fn == NULL) {
  2957. LOG_ERROR("failed to parse stack usage line: %s", cp);
  2958. goto fail;
  2959. }
  2960. prefix = aot_func_internal_prefix;
  2961. }
  2962. ret = sscanf(fn + strlen(prefix), "%ju %ju static", &func_idx, &sz);
  2963. if (ret != 2) {
  2964. goto fail;
  2965. }
  2966. if (sz > UINT32_MAX) {
  2967. goto fail;
  2968. }
  2969. if (func_idx > UINT32_MAX) {
  2970. goto fail;
  2971. }
  2972. if (func_idx >= count) {
  2973. goto fail;
  2974. }
  2975. if (prefix == aot_func_prefix) {
  2976. if (sz < precheck_stack_size_min) {
  2977. precheck_stack_size_min = (uint32)sz;
  2978. }
  2979. if (sz > precheck_stack_size_max) {
  2980. precheck_stack_size_max = (uint32)sz;
  2981. }
  2982. precheck_found++;
  2983. continue;
  2984. }
  2985. sizes[func_idx] = (uint32)sz;
  2986. found++;
  2987. }
  2988. fclose(fp);
  2989. if (precheck_found != count) {
  2990. LOG_ERROR("%" PRIu32 " precheck entries found while %" PRIu32
  2991. " entries are expected",
  2992. precheck_found, count);
  2993. return false;
  2994. }
  2995. if (found != count) {
  2996. /*
  2997. * LLVM seems to eliminate calls to an empty function
  2998. * (and eliminate the function) even if it's marked noinline.
  2999. */
  3000. LOG_VERBOSE("%" PRIu32 " entries found while %" PRIu32
  3001. " entries are expected. Maybe LLVM optimization eliminated "
  3002. "some functions.",
  3003. found, count);
  3004. }
  3005. if (precheck_stack_size_min != precheck_stack_size_max) {
  3006. /*
  3007. * Note: this is too strict.
  3008. *
  3009. * actually, the stack consumption of the precheck functions
  3010. * can depend on the type of them.
  3011. * that is, depending on various factors including
  3012. * calling conventions and compilers, a function with many
  3013. * parameters can consume more stack, even if it merely does
  3014. * a tail-call to another function.
  3015. */
  3016. bool musttail = aot_target_precheck_can_use_musttail(comp_ctx);
  3017. if (musttail) {
  3018. LOG_WARNING(
  3019. "precheck functions use variable amount of stack. (%" PRIu32
  3020. " - %" PRIu32 ")",
  3021. precheck_stack_size_min, precheck_stack_size_max);
  3022. }
  3023. else {
  3024. LOG_VERBOSE("precheck functions use %" PRIu32 " - %" PRIu32
  3025. " bytes of stack.",
  3026. precheck_stack_size_min, precheck_stack_size_max);
  3027. }
  3028. }
  3029. else {
  3030. LOG_VERBOSE("precheck functions use %" PRIu32 " bytes of stack.",
  3031. precheck_stack_size_max);
  3032. }
  3033. if (precheck_stack_size_max >= 1024) {
  3034. LOG_WARNING("precheck functions themselves consume relatively large "
  3035. "amount of stack (%" PRIu32
  3036. "). Please ensure the runtime has large enough "
  3037. "WASM_STACK_GUARD_SIZE.",
  3038. precheck_stack_size_max);
  3039. }
  3040. return true;
  3041. fail:
  3042. if (fp != NULL)
  3043. fclose(fp);
  3044. aot_set_last_error("failed to read stack usage file.");
  3045. return false;
  3046. }
  3047. static bool
  3048. aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  3049. {
  3050. LLVMSectionIteratorRef sec_itr = NULL;
  3051. LLVMSymbolIteratorRef sym_itr;
  3052. const char *name;
  3053. if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) {
  3054. aot_set_last_error("llvm get symbol iterator failed.");
  3055. return false;
  3056. }
  3057. while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) {
  3058. if ((name = LLVMGetSymbolName(sym_itr))
  3059. && (!strcmp(name, aot_stack_sizes_alias_name)
  3060. /* symbol of COFF32 starts with "_" */
  3061. || (obj_data->target_info.bin_type == AOT_COFF32_BIN_TYPE
  3062. && !strncmp(name, "_", 1)
  3063. && !strcmp(name + 1, aot_stack_sizes_alias_name)))) {
  3064. #if 0 /* cf. https://github.com/llvm/llvm-project/issues/67765 */
  3065. uint64 sz = LLVMGetSymbolSize(sym_itr);
  3066. if (sz != sizeof(uint32) * obj_data->func_count
  3067. /* sz of COFF64/COFF32 is 0, ignore the check */
  3068. && obj_data->target_info.bin_type != AOT_COFF64_BIN_TYPE
  3069. && obj_data->target_info.bin_type != AOT_COFF32_BIN_TYPE) {
  3070. aot_set_last_error("stack_sizes had unexpected size.");
  3071. goto fail;
  3072. }
  3073. #endif
  3074. uint64 addr = LLVMGetSymbolAddress(sym_itr);
  3075. if (!(sec_itr =
  3076. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3077. aot_set_last_error("llvm get section iterator failed.");
  3078. goto fail;
  3079. }
  3080. LLVMMoveToContainingSection(sec_itr, sym_itr);
  3081. const char *sec_name = LLVMGetSectionName(sec_itr);
  3082. LOG_VERBOSE("stack_sizes found in section %s offset %" PRIu64 ".",
  3083. sec_name, addr);
  3084. if (strcmp(sec_name, aot_stack_sizes_section_name) || addr != 0) {
  3085. aot_set_last_error(
  3086. "stack_sizes found at an unexpected location.");
  3087. goto fail;
  3088. }
  3089. /*
  3090. * Note: We can't always modify stack_sizes in-place.
  3091. * E.g. When WAMRC_LLC_COMPILER is used, LLVM sometimes uses
  3092. * read-only mmap of the temporary file to back
  3093. * LLVMGetSectionContents.
  3094. */
  3095. const uint32 *ro_stack_sizes =
  3096. (const uint32 *)(LLVMGetSectionContents(sec_itr) + addr);
  3097. uint32 i;
  3098. for (i = 0; i < obj_data->func_count; i++) {
  3099. /* Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes */
  3100. if (ro_stack_sizes[i] != (uint32)-1) {
  3101. aot_set_last_error("unexpected data in stack_sizes.");
  3102. goto fail;
  3103. }
  3104. }
  3105. /*
  3106. * Record section/offset and construct a copy of stack_sizes.
  3107. * aot_emit_object_data_section_info will emit this copy.
  3108. */
  3109. obj_data->stack_sizes_section_name = sec_name;
  3110. obj_data->stack_sizes_offset = (uint32)addr;
  3111. obj_data->stack_sizes = wasm_runtime_malloc(
  3112. obj_data->func_count * sizeof(*obj_data->stack_sizes));
  3113. if (obj_data->stack_sizes == NULL) {
  3114. aot_set_last_error("failed to allocate memory.");
  3115. goto fail;
  3116. }
  3117. uint32 *stack_sizes = obj_data->stack_sizes;
  3118. for (i = 0; i < obj_data->func_count; i++) {
  3119. stack_sizes[i] = (uint32)-1;
  3120. }
  3121. if (!read_stack_usage_file(comp_ctx, comp_ctx->stack_usage_file,
  3122. stack_sizes, obj_data->func_count)) {
  3123. goto fail;
  3124. }
  3125. for (i = 0; i < obj_data->func_count; i++) {
  3126. const AOTFuncContext *func_ctx = comp_ctx->func_ctxes[i];
  3127. bool musttail = aot_target_precheck_can_use_musttail(comp_ctx);
  3128. unsigned int stack_consumption_to_call_wrapped_func =
  3129. musttail ? 0
  3130. : aot_estimate_stack_usage_for_function_call(
  3131. comp_ctx, func_ctx->aot_func->func_type);
  3132. /*
  3133. * LLVM seems to eliminate calls to an empty function
  3134. * (and eliminate the function) even if it's marked noinline.
  3135. *
  3136. * Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes
  3137. */
  3138. if (stack_sizes[i] == (uint32)-1) {
  3139. if (func_ctx->stack_consumption_for_func_call != 0) {
  3140. /*
  3141. * This happens if a function calling another
  3142. * function has been optimized out.
  3143. *
  3144. * for example,
  3145. *
  3146. * (func $func
  3147. * (local i32)
  3148. * local.get 0
  3149. * if
  3150. * call $another
  3151. * end
  3152. * )
  3153. */
  3154. LOG_VERBOSE("AOT func#%" PRIu32
  3155. " had call(s) but eliminated?",
  3156. i);
  3157. }
  3158. else {
  3159. LOG_VERBOSE("AOT func#%" PRIu32 " eliminated?", i);
  3160. }
  3161. stack_sizes[i] = 0;
  3162. }
  3163. else {
  3164. LOG_VERBOSE("AOT func#%" PRIu32 " stack_size %u + %" PRIu32
  3165. " + %u",
  3166. i, stack_consumption_to_call_wrapped_func,
  3167. stack_sizes[i],
  3168. func_ctx->stack_consumption_for_func_call);
  3169. if (UINT32_MAX - stack_sizes[i]
  3170. < func_ctx->stack_consumption_for_func_call) {
  3171. aot_set_last_error("stack size overflow.");
  3172. goto fail;
  3173. }
  3174. stack_sizes[i] += func_ctx->stack_consumption_for_func_call;
  3175. if (UINT32_MAX - stack_sizes[i]
  3176. < stack_consumption_to_call_wrapped_func) {
  3177. aot_set_last_error("stack size overflow.");
  3178. goto fail;
  3179. }
  3180. stack_sizes[i] += stack_consumption_to_call_wrapped_func;
  3181. }
  3182. }
  3183. LLVMDisposeSectionIterator(sec_itr);
  3184. LLVMDisposeSymbolIterator(sym_itr);
  3185. return true;
  3186. }
  3187. LLVMMoveToNextSymbol(sym_itr);
  3188. }
  3189. aot_set_last_error("stack_sizes not found.");
  3190. fail:
  3191. if (sec_itr)
  3192. LLVMDisposeSectionIterator(sec_itr);
  3193. LLVMDisposeSymbolIterator(sym_itr);
  3194. return false;
  3195. }
  3196. static bool
  3197. aot_resolve_functions(AOTCompContext *comp_ctx, AOTObjectData *obj_data)
  3198. {
  3199. AOTObjectFunc *func;
  3200. LLVMSymbolIteratorRef sym_itr;
  3201. char *name, *prefix = AOT_FUNC_PREFIX;
  3202. uint32 func_index, total_size;
  3203. /* allocate memory for aot function */
  3204. obj_data->func_count = comp_ctx->comp_data->func_count;
  3205. if (obj_data->func_count) {
  3206. if ((comp_ctx->enable_stack_bound_check
  3207. || comp_ctx->enable_stack_estimation)
  3208. && !aot_resolve_stack_sizes(comp_ctx, obj_data))
  3209. return false;
  3210. total_size = (uint32)sizeof(AOTObjectFunc) * obj_data->func_count;
  3211. if (!(obj_data->funcs = wasm_runtime_malloc(total_size))) {
  3212. aot_set_last_error("allocate memory for functions failed.");
  3213. return false;
  3214. }
  3215. memset(obj_data->funcs, 0, total_size);
  3216. }
  3217. if (!(sym_itr = LLVMObjectFileCopySymbolIterator(obj_data->binary))) {
  3218. aot_set_last_error("llvm get symbol iterator failed.");
  3219. return false;
  3220. }
  3221. while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) {
  3222. if ((name = (char *)LLVMGetSymbolName(sym_itr))
  3223. && str_starts_with(name, prefix)) {
  3224. /* symbol aot_func#n */
  3225. func_index = (uint32)atoi(name + strlen(prefix));
  3226. if (func_index < obj_data->func_count) {
  3227. LLVMSectionIteratorRef contain_section;
  3228. char *contain_section_name;
  3229. func = obj_data->funcs + func_index;
  3230. func->func_name = name;
  3231. if (!(contain_section = LLVMObjectFileCopySectionIterator(
  3232. obj_data->binary))) {
  3233. aot_set_last_error("llvm get section iterator failed.");
  3234. LLVMDisposeSymbolIterator(sym_itr);
  3235. return false;
  3236. }
  3237. LLVMMoveToContainingSection(contain_section, sym_itr);
  3238. contain_section_name =
  3239. (char *)LLVMGetSectionName(contain_section);
  3240. LLVMDisposeSectionIterator(contain_section);
  3241. if (!strcmp(contain_section_name, ".text.unlikely.")
  3242. || !strcmp(contain_section_name, ".ltext.unlikely.")) {
  3243. func->text_offset = align_uint(obj_data->text_size, 4)
  3244. + LLVMGetSymbolAddress(sym_itr);
  3245. }
  3246. else if (!strcmp(contain_section_name, ".text.hot.")
  3247. || !strcmp(contain_section_name, ".ltext.hot.")) {
  3248. func->text_offset =
  3249. align_uint(obj_data->text_size, 4)
  3250. + align_uint(obj_data->text_unlikely_size, 4)
  3251. + LLVMGetSymbolAddress(sym_itr);
  3252. }
  3253. else {
  3254. func->text_offset = LLVMGetSymbolAddress(sym_itr);
  3255. }
  3256. }
  3257. }
  3258. else if ((name = (char *)LLVMGetSymbolName(sym_itr))
  3259. && str_starts_with(name, AOT_FUNC_INTERNAL_PREFIX)) {
  3260. /* symbol aot_func_internal#n */
  3261. func_index = (uint32)atoi(name + strlen(AOT_FUNC_INTERNAL_PREFIX));
  3262. if (func_index < obj_data->func_count) {
  3263. LLVMSectionIteratorRef contain_section;
  3264. char *contain_section_name;
  3265. func = obj_data->funcs + func_index;
  3266. if (!(contain_section = LLVMObjectFileCopySectionIterator(
  3267. obj_data->binary))) {
  3268. aot_set_last_error("llvm get section iterator failed.");
  3269. LLVMDisposeSymbolIterator(sym_itr);
  3270. return false;
  3271. }
  3272. LLVMMoveToContainingSection(contain_section, sym_itr);
  3273. contain_section_name =
  3274. (char *)LLVMGetSectionName(contain_section);
  3275. LLVMDisposeSectionIterator(contain_section);
  3276. if (!strcmp(contain_section_name, ".text.unlikely.")
  3277. || !strcmp(contain_section_name, ".ltext.unlikely.")) {
  3278. func->text_offset_of_aot_func_internal =
  3279. align_uint(obj_data->text_size, 4)
  3280. + LLVMGetSymbolAddress(sym_itr);
  3281. }
  3282. else if (!strcmp(contain_section_name, ".text.hot.")
  3283. || !strcmp(contain_section_name, ".ltext.hot.")) {
  3284. func->text_offset_of_aot_func_internal =
  3285. align_uint(obj_data->text_size, 4)
  3286. + align_uint(obj_data->text_unlikely_size, 4)
  3287. + LLVMGetSymbolAddress(sym_itr);
  3288. }
  3289. else {
  3290. func->text_offset_of_aot_func_internal =
  3291. LLVMGetSymbolAddress(sym_itr);
  3292. }
  3293. }
  3294. }
  3295. LLVMMoveToNextSymbol(sym_itr);
  3296. }
  3297. LLVMDisposeSymbolIterator(sym_itr);
  3298. return true;
  3299. }
  3300. static bool
  3301. get_relocations_count(LLVMSectionIteratorRef sec_itr, uint32 *p_count)
  3302. {
  3303. uint32 relocation_count = 0;
  3304. LLVMRelocationIteratorRef rel_itr;
  3305. if (!(rel_itr = LLVMGetRelocations(sec_itr))) {
  3306. aot_set_last_error("llvm get relocations failed.");
  3307. LLVMDisposeSectionIterator(sec_itr);
  3308. return false;
  3309. }
  3310. while (!LLVMIsRelocationIteratorAtEnd(sec_itr, rel_itr)) {
  3311. relocation_count++;
  3312. LLVMMoveToNextRelocation(rel_itr);
  3313. }
  3314. LLVMDisposeRelocationIterator(rel_itr);
  3315. *p_count = relocation_count;
  3316. return true;
  3317. }
  3318. static bool
  3319. aot_resolve_object_relocation_group(AOTObjectData *obj_data,
  3320. AOTRelocationGroup *group,
  3321. LLVMSectionIteratorRef rel_sec)
  3322. {
  3323. LLVMRelocationIteratorRef rel_itr;
  3324. AOTRelocation *relocation = group->relocations;
  3325. uint32 size;
  3326. bool is_binary_32bit = is_32bit_binary(obj_data);
  3327. bool is_binary_little_endian = is_little_endian_binary(obj_data);
  3328. bool has_addend = str_starts_with(group->section_name, ".rela");
  3329. uint8 *rela_content = NULL;
  3330. /* calculate relocations count and allocate memory */
  3331. if (!get_relocations_count(rel_sec, &group->relocation_count))
  3332. return false;
  3333. if (group->relocation_count == 0) {
  3334. aot_set_last_error("invalid relocations count");
  3335. return false;
  3336. }
  3337. size = (uint32)sizeof(AOTRelocation) * group->relocation_count;
  3338. if (!(relocation = group->relocations = wasm_runtime_malloc(size))) {
  3339. aot_set_last_error("allocate memory for relocations failed.");
  3340. return false;
  3341. }
  3342. memset(group->relocations, 0, size);
  3343. if (has_addend) {
  3344. uint64 rela_content_size;
  3345. /* LLVM doesn't provide C API to get relocation addend. So we have to
  3346. * parse it manually. */
  3347. rela_content = (uint8 *)LLVMGetSectionContents(rel_sec);
  3348. rela_content_size = LLVMGetSectionSize(rel_sec);
  3349. if (is_binary_32bit)
  3350. size = (uint32)sizeof(struct elf32_rela) * group->relocation_count;
  3351. else
  3352. size = (uint32)sizeof(struct elf64_rela) * group->relocation_count;
  3353. if (rela_content_size != (uint64)size) {
  3354. aot_set_last_error("invalid relocation section content.");
  3355. return false;
  3356. }
  3357. }
  3358. /* pares each relocation */
  3359. if (!(rel_itr = LLVMGetRelocations(rel_sec))) {
  3360. aot_set_last_error("llvm get relocations failed.");
  3361. return false;
  3362. }
  3363. while (!LLVMIsRelocationIteratorAtEnd(rel_sec, rel_itr)) {
  3364. uint64 offset = LLVMGetRelocationOffset(rel_itr);
  3365. uint64 type = LLVMGetRelocationType(rel_itr);
  3366. LLVMSymbolIteratorRef rel_sym = LLVMGetRelocationSymbol(rel_itr);
  3367. if (!rel_sym) {
  3368. aot_set_last_error("llvm get relocation symbol failed.");
  3369. goto fail;
  3370. }
  3371. /* parse relocation addend from relocation content */
  3372. if (has_addend) {
  3373. if (is_binary_32bit) {
  3374. int32 addend =
  3375. (int32)(((struct elf32_rela *)rela_content)->r_addend);
  3376. if (is_binary_little_endian != is_little_endian())
  3377. exchange_uint32((uint8 *)&addend);
  3378. relocation->relocation_addend = (int64)addend;
  3379. rela_content += sizeof(struct elf32_rela);
  3380. }
  3381. else {
  3382. int64 addend =
  3383. (int64)(((struct elf64_rela *)rela_content)->r_addend);
  3384. if (is_binary_little_endian != is_little_endian())
  3385. exchange_uint64((uint8 *)&addend);
  3386. relocation->relocation_addend = addend;
  3387. rela_content += sizeof(struct elf64_rela);
  3388. }
  3389. }
  3390. /* set relocation fields */
  3391. relocation->relocation_type = (uint32)type;
  3392. relocation->symbol_name = (char *)LLVMGetSymbolName(rel_sym);
  3393. relocation->relocation_offset = offset;
  3394. if (!strcmp(group->section_name, ".rela.text.unlikely.")
  3395. || !strcmp(group->section_name, ".rel.text.unlikely.")) {
  3396. relocation->relocation_offset += align_uint(obj_data->text_size, 4);
  3397. }
  3398. else if (!strcmp(group->section_name, ".rela.text.hot.")
  3399. || !strcmp(group->section_name, ".rel.text.hot.")) {
  3400. relocation->relocation_offset +=
  3401. align_uint(obj_data->text_size, 4)
  3402. + align_uint(obj_data->text_unlikely_size, 4);
  3403. }
  3404. if (!strcmp(relocation->symbol_name, ".text.unlikely.")) {
  3405. relocation->symbol_name = ".text";
  3406. relocation->relocation_addend += align_uint(obj_data->text_size, 4);
  3407. }
  3408. if (!strcmp(relocation->symbol_name, ".text.hot.")) {
  3409. relocation->symbol_name = ".text";
  3410. relocation->relocation_addend +=
  3411. align_uint(obj_data->text_size, 4)
  3412. + align_uint(obj_data->text_unlikely_size, 4);
  3413. }
  3414. /*
  3415. * Note: aot_stack_sizes_section_name section only contains
  3416. * stack_sizes table.
  3417. */
  3418. if (!strcmp(relocation->symbol_name, aot_stack_sizes_name)
  3419. /* in windows 32, the symbol name may start with '_' */
  3420. || (strlen(relocation->symbol_name) > 0
  3421. && relocation->symbol_name[0] == '_'
  3422. && !strcmp(relocation->symbol_name + 1,
  3423. aot_stack_sizes_name))) {
  3424. /* discard const */
  3425. relocation->symbol_name = (char *)aot_stack_sizes_section_name;
  3426. }
  3427. if (obj_data->comp_ctx->enable_llvm_pgo
  3428. && (!strcmp(relocation->symbol_name, "__llvm_prf_cnts")
  3429. || !strcmp(relocation->symbol_name, "__llvm_prf_data"))) {
  3430. LLVMSectionIteratorRef sec_itr;
  3431. char buf[32], *section_name;
  3432. uint32 prof_section_idx = 0;
  3433. if (!(sec_itr =
  3434. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3435. aot_set_last_error("llvm get section iterator failed.");
  3436. LLVMDisposeSymbolIterator(rel_sym);
  3437. goto fail;
  3438. }
  3439. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary,
  3440. sec_itr)) {
  3441. section_name = (char *)LLVMGetSectionName(sec_itr);
  3442. if (section_name
  3443. && !strcmp(section_name, relocation->symbol_name)) {
  3444. if (LLVMGetSectionContainsSymbol(sec_itr, rel_sym))
  3445. break;
  3446. prof_section_idx++;
  3447. }
  3448. LLVMMoveToNextSection(sec_itr);
  3449. }
  3450. LLVMDisposeSectionIterator(sec_itr);
  3451. if (!strcmp(group->section_name, ".rela.text")
  3452. || !strcmp(group->section_name, ".rel.text")) {
  3453. snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name,
  3454. prof_section_idx);
  3455. size = (uint32)(strlen(buf) + 1);
  3456. if (!(relocation->symbol_name = wasm_runtime_malloc(size))) {
  3457. aot_set_last_error(
  3458. "allocate memory for relocation symbol name failed.");
  3459. LLVMDisposeSymbolIterator(rel_sym);
  3460. goto fail;
  3461. }
  3462. bh_memcpy_s(relocation->symbol_name, size, buf, size);
  3463. relocation->is_symbol_name_allocated = true;
  3464. }
  3465. else if (!strncmp(group->section_name, ".rela__llvm_prf_data", 20)
  3466. || !strncmp(group->section_name, ".rel__llvm_prf_data",
  3467. 19)) {
  3468. snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name,
  3469. prof_section_idx);
  3470. size = (uint32)(strlen(buf) + 1);
  3471. if (!(relocation->symbol_name = wasm_runtime_malloc(size))) {
  3472. aot_set_last_error(
  3473. "allocate memory for relocation symbol name failed.");
  3474. LLVMDisposeSymbolIterator(rel_sym);
  3475. goto fail;
  3476. }
  3477. bh_memcpy_s(relocation->symbol_name, size, buf, size);
  3478. relocation->is_symbol_name_allocated = true;
  3479. }
  3480. }
  3481. /* for ".LCPIxxx", ".LJTIxxx", ".LBBxxx" and switch lookup table
  3482. * relocation, transform the symbol name to real section name and set
  3483. * addend to the offset of the symbol in the real section */
  3484. if (relocation->symbol_name
  3485. && (str_starts_with(relocation->symbol_name, ".LCPI")
  3486. || str_starts_with(relocation->symbol_name, ".LJTI")
  3487. || str_starts_with(relocation->symbol_name, ".LBB")
  3488. || str_starts_with(relocation->symbol_name,
  3489. ".Lswitch.table."))) {
  3490. /* change relocation->relocation_addend and
  3491. relocation->symbol_name */
  3492. LLVMSectionIteratorRef contain_section;
  3493. if (!(contain_section =
  3494. LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3495. aot_set_last_error("llvm get section iterator failed.");
  3496. goto fail;
  3497. }
  3498. LLVMMoveToContainingSection(contain_section, rel_sym);
  3499. if (LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary,
  3500. contain_section)) {
  3501. LLVMDisposeSectionIterator(contain_section);
  3502. aot_set_last_error("llvm get containing section failed.");
  3503. goto fail;
  3504. }
  3505. relocation->relocation_addend += LLVMGetSymbolAddress(rel_sym);
  3506. relocation->symbol_name =
  3507. (char *)LLVMGetSectionName(contain_section);
  3508. LLVMDisposeSectionIterator(contain_section);
  3509. }
  3510. LLVMDisposeSymbolIterator(rel_sym);
  3511. LLVMMoveToNextRelocation(rel_itr);
  3512. relocation++;
  3513. }
  3514. LLVMDisposeRelocationIterator(rel_itr);
  3515. return true;
  3516. fail:
  3517. LLVMDisposeRelocationIterator(rel_itr);
  3518. return false;
  3519. }
  3520. static bool
  3521. is_relocation_section_name(AOTObjectData *obj_data, char *section_name)
  3522. {
  3523. return (!strcmp(section_name, ".rela.text")
  3524. || !strcmp(section_name, ".rel.text")
  3525. || !strcmp(section_name, ".rela.text.unlikely.")
  3526. || !strcmp(section_name, ".rel.text.unlikely.")
  3527. || !strcmp(section_name, ".rela.text.hot.")
  3528. || !strcmp(section_name, ".rel.text.hot.")
  3529. || !strcmp(section_name, ".rela.ltext")
  3530. || !strcmp(section_name, ".rel.ltext")
  3531. || !strcmp(section_name, ".rela.ltext.unlikely.")
  3532. || !strcmp(section_name, ".rel.ltext.unlikely.")
  3533. || !strcmp(section_name, ".rela.ltext.hot.")
  3534. || !strcmp(section_name, ".rel.ltext.hot.")
  3535. || !strcmp(section_name, ".rela.literal")
  3536. || !strcmp(section_name, ".rela.data")
  3537. || !strcmp(section_name, ".rel.data")
  3538. || !strcmp(section_name, ".rela.sdata")
  3539. || !strcmp(section_name, ".rel.sdata")
  3540. || !strcmp(section_name, ".rela.rodata")
  3541. || !strcmp(section_name, ".rel.rodata")
  3542. || (obj_data->comp_ctx->enable_llvm_pgo
  3543. && (!strcmp(section_name, ".rela__llvm_prf_data")
  3544. || !strcmp(section_name, ".rel__llvm_prf_data")))
  3545. /* ".rela.rodata.cst4/8/16/.." */
  3546. || !strncmp(section_name, ".rela.rodata.cst",
  3547. strlen(".rela.rodata.cst"))
  3548. /* ".rel.rodata.cst4/8/16/.." */
  3549. || !strncmp(section_name, ".rel.rodata.cst",
  3550. strlen(".rel.rodata.cst")));
  3551. }
  3552. static bool
  3553. is_relocation_section(AOTObjectData *obj_data, LLVMSectionIteratorRef sec_itr)
  3554. {
  3555. uint32 count = 0;
  3556. char *name = (char *)LLVMGetSectionName(sec_itr);
  3557. if (name) {
  3558. if (is_relocation_section_name(obj_data, name))
  3559. return true;
  3560. else if ((!strcmp(name, ".text") || !strcmp(name, ".text.unlikely.")
  3561. || !strcmp(name, ".text.hot.") || !strcmp(name, ".rdata"))
  3562. && get_relocations_count(sec_itr, &count) && count > 0)
  3563. return true;
  3564. }
  3565. return false;
  3566. }
  3567. static bool
  3568. is_readonly_section(const char *name)
  3569. {
  3570. return !strcmp(name, ".rel.text") || !strcmp(name, ".rela.text")
  3571. || !strcmp(name, ".rel.ltext") || !strcmp(name, ".rela.ltext")
  3572. || !strcmp(name, ".rela.literal") || !strcmp(name, ".text")
  3573. || !strcmp(name, ".ltext");
  3574. }
  3575. static bool
  3576. get_relocation_groups_count(AOTObjectData *obj_data, uint32 *p_count)
  3577. {
  3578. uint32 count = 0;
  3579. LLVMSectionIteratorRef sec_itr;
  3580. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3581. aot_set_last_error("llvm get section iterator failed.");
  3582. return false;
  3583. }
  3584. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  3585. if (is_relocation_section(obj_data, sec_itr)) {
  3586. count++;
  3587. }
  3588. LLVMMoveToNextSection(sec_itr);
  3589. }
  3590. LLVMDisposeSectionIterator(sec_itr);
  3591. *p_count = count;
  3592. return true;
  3593. }
  3594. static bool
  3595. aot_resolve_object_relocation_groups(AOTObjectData *obj_data)
  3596. {
  3597. LLVMSectionIteratorRef sec_itr;
  3598. AOTRelocationGroup *relocation_group;
  3599. uint32 group_count, llvm_prf_data_idx = 0;
  3600. char *name;
  3601. uint32 size;
  3602. /* calculate relocation groups count and allocate memory */
  3603. if (!get_relocation_groups_count(obj_data, &group_count))
  3604. return false;
  3605. if (0 == (obj_data->relocation_group_count = group_count))
  3606. return true;
  3607. size = (uint32)sizeof(AOTRelocationGroup) * group_count;
  3608. if (!(relocation_group = obj_data->relocation_groups =
  3609. wasm_runtime_malloc(size))) {
  3610. aot_set_last_error("allocate memory for relocation groups failed.");
  3611. return false;
  3612. }
  3613. memset(obj_data->relocation_groups, 0, size);
  3614. /* resolve each relocation group */
  3615. if (!(sec_itr = LLVMObjectFileCopySectionIterator(obj_data->binary))) {
  3616. aot_set_last_error("llvm get section iterator failed.");
  3617. return false;
  3618. }
  3619. while (!LLVMObjectFileIsSectionIteratorAtEnd(obj_data->binary, sec_itr)) {
  3620. if (is_relocation_section(obj_data, sec_itr)) {
  3621. name = (char *)LLVMGetSectionName(sec_itr);
  3622. relocation_group->section_name = name;
  3623. if (obj_data->comp_ctx->enable_llvm_pgo
  3624. && (!strcmp(name, ".rela__llvm_prf_data")
  3625. || !strcmp(name, ".rel__llvm_prf_data"))) {
  3626. char buf[32];
  3627. snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_data_idx);
  3628. size = (uint32)(strlen(buf) + 1);
  3629. if (!(relocation_group->section_name =
  3630. wasm_runtime_malloc(size))) {
  3631. aot_set_last_error(
  3632. "allocate memory for section name failed.");
  3633. LLVMDisposeSectionIterator(sec_itr);
  3634. return false;
  3635. }
  3636. bh_memcpy_s(relocation_group->section_name, size, buf, size);
  3637. relocation_group->is_section_name_allocated = true;
  3638. }
  3639. if (!aot_resolve_object_relocation_group(obj_data, relocation_group,
  3640. sec_itr)) {
  3641. LLVMDisposeSectionIterator(sec_itr);
  3642. return false;
  3643. }
  3644. if (obj_data->comp_ctx->enable_llvm_pgo
  3645. && (!strcmp(name, ".rela__llvm_prf_data")
  3646. || !strcmp(name, ".rel__llvm_prf_data"))) {
  3647. llvm_prf_data_idx++;
  3648. }
  3649. if (!strcmp(relocation_group->section_name, ".rela.text.unlikely.")
  3650. || !strcmp(relocation_group->section_name, ".rela.text.hot.")) {
  3651. relocation_group->section_name = ".rela.text";
  3652. }
  3653. else if (!strcmp(relocation_group->section_name,
  3654. ".rela.ltext.unlikely.")
  3655. || !strcmp(relocation_group->section_name,
  3656. ".rela.ltext.hot.")) {
  3657. relocation_group->section_name = ".rela.ltext";
  3658. }
  3659. else if (!strcmp(relocation_group->section_name,
  3660. ".rel.text.unlikely.")
  3661. || !strcmp(relocation_group->section_name,
  3662. ".rel.text.hot.")) {
  3663. relocation_group->section_name = ".rel.text";
  3664. }
  3665. else if (!strcmp(relocation_group->section_name,
  3666. ".rel.ltext.unlikely.")
  3667. || !strcmp(relocation_group->section_name,
  3668. ".rel.ltext.hot.")) {
  3669. relocation_group->section_name = ".rel.ltext";
  3670. }
  3671. /*
  3672. * Relocations in read-only sections are problematic,
  3673. * especially for XIP on platforms which don't have
  3674. * copy-on-write mappings.
  3675. */
  3676. if (obj_data->comp_ctx->is_indirect_mode
  3677. && is_readonly_section(relocation_group->section_name)) {
  3678. LOG_WARNING("%" PRIu32
  3679. " text relocations in %s section for indirect mode",
  3680. relocation_group->relocation_count,
  3681. relocation_group->section_name);
  3682. }
  3683. relocation_group++;
  3684. }
  3685. LLVMMoveToNextSection(sec_itr);
  3686. }
  3687. LLVMDisposeSectionIterator(sec_itr);
  3688. return true;
  3689. }
  3690. static void
  3691. destroy_relocation_groups(AOTRelocationGroup *relocation_groups,
  3692. uint32 relocation_group_count)
  3693. {
  3694. uint32 i, j;
  3695. AOTRelocationGroup *relocation_group = relocation_groups;
  3696. for (i = 0; i < relocation_group_count; i++, relocation_group++) {
  3697. if (relocation_group->relocations) {
  3698. for (j = 0; j < relocation_group->relocation_count; j++) {
  3699. if (relocation_group->relocations[j].is_symbol_name_allocated)
  3700. wasm_runtime_free(
  3701. relocation_group->relocations[j].symbol_name);
  3702. }
  3703. wasm_runtime_free(relocation_group->relocations);
  3704. }
  3705. if (relocation_group->is_section_name_allocated)
  3706. wasm_runtime_free(relocation_group->section_name);
  3707. }
  3708. wasm_runtime_free(relocation_groups);
  3709. }
  3710. static void
  3711. destroy_relocation_symbol_list(AOTSymbolList *symbol_list)
  3712. {
  3713. AOTSymbolNode *elem;
  3714. elem = symbol_list->head;
  3715. while (elem) {
  3716. AOTSymbolNode *next = elem->next;
  3717. wasm_runtime_free(elem);
  3718. elem = next;
  3719. }
  3720. }
  3721. void
  3722. aot_obj_data_destroy(AOTObjectData *obj_data)
  3723. {
  3724. if (obj_data->binary)
  3725. LLVMDisposeBinary(obj_data->binary);
  3726. if (obj_data->mem_buf)
  3727. LLVMDisposeMemoryBuffer(obj_data->mem_buf);
  3728. if (obj_data->funcs)
  3729. wasm_runtime_free(obj_data->funcs);
  3730. if (obj_data->data_sections) {
  3731. uint32 i;
  3732. for (i = 0; i < obj_data->data_sections_count; i++) {
  3733. if (obj_data->data_sections[i].name
  3734. && obj_data->data_sections[i].is_name_allocated) {
  3735. wasm_runtime_free(obj_data->data_sections[i].name);
  3736. }
  3737. if (obj_data->data_sections[i].data
  3738. && obj_data->data_sections[i].is_data_allocated) {
  3739. wasm_runtime_free(obj_data->data_sections[i].data);
  3740. }
  3741. }
  3742. wasm_runtime_free(obj_data->data_sections);
  3743. }
  3744. if (obj_data->relocation_groups)
  3745. destroy_relocation_groups(obj_data->relocation_groups,
  3746. obj_data->relocation_group_count);
  3747. if (obj_data->symbol_list.len)
  3748. destroy_relocation_symbol_list(&obj_data->symbol_list);
  3749. if (obj_data->stack_sizes)
  3750. wasm_runtime_free(obj_data->stack_sizes);
  3751. wasm_runtime_free(obj_data);
  3752. }
  3753. AOTObjectData *
  3754. aot_obj_data_create(AOTCompContext *comp_ctx)
  3755. {
  3756. char *err = NULL;
  3757. AOTObjectData *obj_data;
  3758. LLVMTargetRef target = LLVMGetTargetMachineTarget(comp_ctx->target_machine);
  3759. bh_print_time("Begin to emit object file to buffer");
  3760. if (!(obj_data = wasm_runtime_malloc(sizeof(AOTObjectData)))) {
  3761. aot_set_last_error("allocate memory failed.");
  3762. return false;
  3763. }
  3764. memset(obj_data, 0, sizeof(AOTObjectData));
  3765. obj_data->comp_ctx = comp_ctx;
  3766. bh_print_time("Begin to emit object file");
  3767. if (comp_ctx->external_llc_compiler || comp_ctx->external_asm_compiler) {
  3768. #if defined(_WIN32) || defined(_WIN32_)
  3769. aot_set_last_error("external toolchain not supported on Windows");
  3770. goto fail;
  3771. #else
  3772. /* Generate a temp file name */
  3773. int ret;
  3774. char obj_file_name[64];
  3775. if (!aot_generate_tempfile_name("wamrc-obj", "o", obj_file_name,
  3776. sizeof(obj_file_name))) {
  3777. goto fail;
  3778. }
  3779. if (!aot_emit_object_file(comp_ctx, obj_file_name)) {
  3780. goto fail;
  3781. }
  3782. /* create memory buffer from object file */
  3783. ret = LLVMCreateMemoryBufferWithContentsOfFile(
  3784. obj_file_name, &obj_data->mem_buf, &err);
  3785. /* remove temp object file */
  3786. unlink(obj_file_name);
  3787. if (ret != 0) {
  3788. if (err) {
  3789. LLVMDisposeMessage(err);
  3790. err = NULL;
  3791. }
  3792. aot_set_last_error("create mem buffer with file failed.");
  3793. goto fail;
  3794. }
  3795. #endif /* end of defined(_WIN32) || defined(_WIN32_) */
  3796. }
  3797. else if (!strncmp(LLVMGetTargetName(target), "arc", 3)) {
  3798. #if defined(_WIN32) || defined(_WIN32_)
  3799. aot_set_last_error("emit object file on Windows is unsupported.");
  3800. goto fail;
  3801. #else
  3802. /* Emit to assembly file instead for arc target
  3803. as it cannot emit to object file */
  3804. char file_name[] = "wasm-XXXXXX", buf[128];
  3805. int fd, ret;
  3806. if ((fd = mkstemp(file_name)) <= 0) {
  3807. aot_set_last_error("make temp file failed.");
  3808. goto fail;
  3809. }
  3810. /* close and remove temp file */
  3811. close(fd);
  3812. unlink(file_name);
  3813. snprintf(buf, sizeof(buf), "%s%s", file_name, ".s");
  3814. if (LLVMTargetMachineEmitToFile(comp_ctx->target_machine,
  3815. comp_ctx->module, buf, LLVMAssemblyFile,
  3816. &err)
  3817. != 0) {
  3818. if (err) {
  3819. LLVMDisposeMessage(err);
  3820. err = NULL;
  3821. }
  3822. aot_set_last_error("emit elf to object file failed.");
  3823. goto fail;
  3824. }
  3825. /* call arc gcc to compile assembly file to object file */
  3826. /* TODO: get arc gcc from environment variable firstly
  3827. and check whether the toolchain exists actually */
  3828. snprintf(buf, sizeof(buf), "%s%s%s%s%s%s",
  3829. "/opt/zephyr-sdk/arc-zephyr-elf/bin/arc-zephyr-elf-gcc ",
  3830. "-mcpu=arcem -o ", file_name, ".o -c ", file_name, ".s");
  3831. /* TODO: use try..catch to handle possible exceptions */
  3832. ret = system(buf);
  3833. /* remove temp assembly file */
  3834. snprintf(buf, sizeof(buf), "%s%s", file_name, ".s");
  3835. unlink(buf);
  3836. if (ret != 0) {
  3837. aot_set_last_error("failed to compile asm file to obj file "
  3838. "with arc gcc toolchain.");
  3839. goto fail;
  3840. }
  3841. /* create memory buffer from object file */
  3842. snprintf(buf, sizeof(buf), "%s%s", file_name, ".o");
  3843. ret = LLVMCreateMemoryBufferWithContentsOfFile(buf, &obj_data->mem_buf,
  3844. &err);
  3845. /* remove temp object file */
  3846. snprintf(buf, sizeof(buf), "%s%s", file_name, ".o");
  3847. unlink(buf);
  3848. if (ret != 0) {
  3849. if (err) {
  3850. LLVMDisposeMessage(err);
  3851. err = NULL;
  3852. }
  3853. aot_set_last_error("create mem buffer with file failed.");
  3854. goto fail;
  3855. }
  3856. #endif /* end of defined(_WIN32) || defined(_WIN32_) */
  3857. }
  3858. else {
  3859. if (LLVMTargetMachineEmitToMemoryBuffer(
  3860. comp_ctx->target_machine, comp_ctx->module, LLVMObjectFile,
  3861. &err, &obj_data->mem_buf)
  3862. != 0) {
  3863. if (err) {
  3864. LLVMDisposeMessage(err);
  3865. err = NULL;
  3866. }
  3867. aot_set_last_error("llvm emit to memory buffer failed.");
  3868. goto fail;
  3869. }
  3870. }
  3871. if (!(obj_data->binary = LLVMCreateBinary(obj_data->mem_buf, NULL, &err))) {
  3872. if (err) {
  3873. LLVMDisposeMessage(err);
  3874. err = NULL;
  3875. }
  3876. aot_set_last_error("llvm create binary failed.");
  3877. goto fail;
  3878. }
  3879. /* Create wasm feature flags form compile options */
  3880. obj_data->target_info.feature_flags = 0;
  3881. if (comp_ctx->enable_simd) {
  3882. obj_data->target_info.feature_flags |= WASM_FEATURE_SIMD_128BIT;
  3883. }
  3884. if (comp_ctx->enable_bulk_memory) {
  3885. obj_data->target_info.feature_flags |= WASM_FEATURE_BULK_MEMORY;
  3886. }
  3887. if (comp_ctx->enable_thread_mgr) {
  3888. obj_data->target_info.feature_flags |= WASM_FEATURE_MULTI_THREAD;
  3889. }
  3890. if (comp_ctx->enable_ref_types) {
  3891. obj_data->target_info.feature_flags |= WASM_FEATURE_REF_TYPES;
  3892. }
  3893. if (comp_ctx->enable_gc) {
  3894. obj_data->target_info.feature_flags |= WASM_FEATURE_GARBAGE_COLLECTION;
  3895. }
  3896. if (comp_ctx->aux_stack_frame_type == AOT_STACK_FRAME_TYPE_TINY) {
  3897. obj_data->target_info.feature_flags |= WASM_FEATURE_TINY_STACK_FRAME;
  3898. }
  3899. if (comp_ctx->call_stack_features.frame_per_function) {
  3900. obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_PER_FUNCTION;
  3901. }
  3902. if (!comp_ctx->call_stack_features.func_idx) {
  3903. obj_data->target_info.feature_flags |= WASM_FEATURE_FRAME_NO_FUNC_IDX;
  3904. }
  3905. bh_print_time("Begin to resolve object file info");
  3906. /* resolve target info/text/relocations/functions */
  3907. if (!aot_resolve_target_info(comp_ctx, obj_data)
  3908. || !aot_resolve_text(obj_data) || !aot_resolve_literal(obj_data)
  3909. || !aot_resolve_object_data_sections(obj_data)
  3910. || !aot_resolve_functions(comp_ctx, obj_data)
  3911. || !aot_resolve_object_relocation_groups(obj_data))
  3912. goto fail;
  3913. return obj_data;
  3914. fail:
  3915. aot_obj_data_destroy(obj_data);
  3916. return NULL;
  3917. }
  3918. uint8 *
  3919. aot_emit_aot_file_buf(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  3920. uint32 *p_aot_file_size)
  3921. {
  3922. AOTObjectData *obj_data = aot_obj_data_create(comp_ctx);
  3923. uint8 *aot_file_buf;
  3924. uint32 aot_file_size;
  3925. if (!obj_data)
  3926. return NULL;
  3927. aot_file_size = aot_get_aot_file_size(comp_ctx, comp_data, obj_data);
  3928. if (aot_file_size == 0) {
  3929. aot_set_last_error("get aot file size failed");
  3930. goto fail1;
  3931. }
  3932. if (!(aot_file_buf = wasm_runtime_malloc(aot_file_size))) {
  3933. aot_set_last_error("allocate memory failed.");
  3934. goto fail1;
  3935. }
  3936. memset(aot_file_buf, 0, aot_file_size);
  3937. if (!aot_emit_aot_file_buf_ex(comp_ctx, comp_data, obj_data, aot_file_buf,
  3938. aot_file_size))
  3939. goto fail2;
  3940. *p_aot_file_size = aot_file_size;
  3941. aot_obj_data_destroy(obj_data);
  3942. return aot_file_buf;
  3943. fail2:
  3944. wasm_runtime_free(aot_file_buf);
  3945. fail1:
  3946. aot_obj_data_destroy(obj_data);
  3947. return NULL;
  3948. }
  3949. bool
  3950. aot_emit_aot_file_buf_ex(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  3951. AOTObjectData *obj_data, uint8 *buf,
  3952. uint32 aot_file_size)
  3953. {
  3954. uint8 *buf_end = buf + aot_file_size;
  3955. uint32 offset = 0;
  3956. if (!aot_emit_file_header(buf, buf_end, &offset, comp_data, obj_data)
  3957. || !aot_emit_target_info_section(buf, buf_end, &offset, comp_data,
  3958. obj_data)
  3959. || !aot_emit_init_data_section(buf, buf_end, &offset, comp_ctx,
  3960. comp_data, obj_data)
  3961. || !aot_emit_text_section(buf, buf_end, &offset, comp_data, obj_data)
  3962. || !aot_emit_func_section(buf, buf_end, &offset, comp_ctx, comp_data,
  3963. obj_data)
  3964. || !aot_emit_export_section(buf, buf_end, &offset, comp_ctx, comp_data,
  3965. obj_data)
  3966. || !aot_emit_relocation_section(buf, buf_end, &offset, comp_ctx,
  3967. comp_data, obj_data)
  3968. || !aot_emit_native_symbol(buf, buf_end, &offset, comp_ctx)
  3969. || !aot_emit_custom_sections(buf, buf_end, &offset, comp_data, comp_ctx)
  3970. #if WASM_ENABLE_STRINGREF != 0
  3971. || !aot_emit_string_literal_section(buf, buf_end, &offset, comp_data,
  3972. comp_ctx)
  3973. #endif
  3974. )
  3975. return false;
  3976. #if 0
  3977. dump_buf(buf, offset, "sections");
  3978. #endif
  3979. if (offset != aot_file_size) {
  3980. aot_set_last_error("emit aot file failed.");
  3981. return false;
  3982. }
  3983. return true;
  3984. }
  3985. bool
  3986. aot_emit_aot_file(AOTCompContext *comp_ctx, AOTCompData *comp_data,
  3987. const char *file_name)
  3988. {
  3989. uint8 *aot_file_buf;
  3990. uint32 aot_file_size;
  3991. bool ret = false;
  3992. FILE *file;
  3993. bh_print_time("Begin to emit AOT file");
  3994. if (!(aot_file_buf =
  3995. aot_emit_aot_file_buf(comp_ctx, comp_data, &aot_file_size))) {
  3996. return false;
  3997. }
  3998. /* write buffer to file */
  3999. if (!(file = fopen(file_name, "wb"))) {
  4000. aot_set_last_error("open or create aot file failed.");
  4001. goto fail1;
  4002. }
  4003. if (!fwrite(aot_file_buf, aot_file_size, 1, file)) {
  4004. aot_set_last_error("write to aot file failed.");
  4005. goto fail2;
  4006. }
  4007. ret = true;
  4008. fail2:
  4009. fclose(file);
  4010. fail1:
  4011. wasm_runtime_free(aot_file_buf);
  4012. return ret;
  4013. }