[update] micropython/py folder

py/asmarm.c

@@ -40,7 +40,11 @@
 
 void asm_arm_end_pass(asm_arm_t *as) {
     if (as->base.pass == MP_ASM_PASS_EMIT) {
-#ifdef __arm__
+#if defined(__linux__) && defined(__GNUC__)
+        char *start = mp_asm_base_get_code(&as->base);
+        char *end = start + mp_asm_base_get_code_size(&as->base);
+        __builtin___clear_cache(start, end);
+#elif defined(__arm__)
         // flush I- and D-cache
         asm volatile(
                 "0:"
@@ -197,7 +201,16 @@ void asm_arm_mov_reg_reg(asm_arm_t *as, uint reg_dest, uint reg_src) {
     emit_al(as, asm_arm_op_mov_reg(reg_dest, reg_src));
 }
 
-void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
+size_t asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
+    // Insert immediate into code and jump over it
+    emit_al(as, 0x59f0000 | (rd << 12)); // ldr rd, [pc]
+    emit_al(as, 0xa000000); // b pc
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
+    emit(as, imm);
+    return loc;
+}
+
+void asm_arm_mov_reg_i32_optimised(asm_arm_t *as, uint rd, int imm) {
     // TODO: There are more variants of immediate values
     if ((imm & 0xFF) == imm) {
         emit_al(as, asm_arm_op_mov_imm(rd, imm));
@@ -205,10 +218,7 @@ void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
         // mvn is "move not", not "move negative"
         emit_al(as, asm_arm_op_mvn_imm(rd, ~imm));
     } else {
-        //Insert immediate into code and jump over it
-        emit_al(as, 0x59f0000 | (rd << 12)); // ldr rd, [pc]
-        emit_al(as, 0xa000000); // b pc
-        emit(as, imm);
+        asm_arm_mov_reg_i32(as, rd, imm);
     }
 }
 

py/asmarm.h

@@ -81,7 +81,8 @@ void asm_arm_bkpt(asm_arm_t *as);
 
 // mov
 void asm_arm_mov_reg_reg(asm_arm_t *as, uint reg_dest, uint reg_src);
-void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm);
+size_t asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm);
+void asm_arm_mov_reg_i32_optimised(asm_arm_t *as, uint rd, int imm);
 void asm_arm_mov_local_reg(asm_arm_t *as, int local_num, uint rd);
 void asm_arm_mov_reg_local(asm_arm_t *as, uint rd, int local_num);
 void asm_arm_setcc_reg(asm_arm_t *as, uint rd, uint cond);
@@ -177,7 +178,9 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src);
 #define ASM_CALL_IND(as, idx) asm_arm_bl_ind(as, idx, ASM_ARM_REG_R3)
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_arm_mov_local_reg((as), (local_num), (reg_src))
-#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_arm_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_arm_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_arm_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmbase.c

@@ -31,7 +31,7 @@
 #include "py/misc.h"
 #include "py/asmbase.h"
 
-#if MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
+#if MICROPY_EMIT_MACHINE_CODE
 
 void mp_asm_base_init(mp_asm_base_t *as, size_t max_num_labels) {
     as->max_num_labels = max_num_labels;
@@ -99,4 +99,4 @@ void mp_asm_base_data(mp_asm_base_t* as, unsigned int bytesize, uintptr_t val) {
     }
 }
 
-#endif // MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
+#endif // MICROPY_EMIT_MACHINE_CODE

py/asmbase.h

@@ -60,7 +60,7 @@ static inline size_t mp_asm_base_get_code_size(mp_asm_base_t *as) {
 
 static inline void *mp_asm_base_get_code(mp_asm_base_t *as) {
     #if defined(MP_PLAT_COMMIT_EXEC)
-    return MP_PLAT_COMMIT_EXEC(as->code_base, as->code_size);
+    return MP_PLAT_COMMIT_EXEC(as->code_base, as->code_size, NULL);
     #else
     return as->code_base;
     #endif

py/asmthumb.c

@@ -33,6 +33,8 @@
 // wrapper around everything in this file
 #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB
 
+#include "py/mpstate.h"
+#include "py/persistentcode.h"
 #include "py/mphal.h"
 #include "py/asmthumb.h"
 
@@ -118,6 +120,21 @@ STATIC void asm_thumb_write_word32(asm_thumb_t *as, int w32) {
 void asm_thumb_entry(asm_thumb_t *as, int num_locals) {
     assert(num_locals >= 0);
 
+    // If this Thumb machine code is run from ARM state then add a prelude
+    // to switch to Thumb state for the duration of the function.
+    #if MICROPY_DYNAMIC_COMPILER || MICROPY_EMIT_ARM || (defined(__arm__) && !defined(__thumb2__))
+    #if MICROPY_DYNAMIC_COMPILER
+    if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_ARMV6)
+    #endif
+    {
+        asm_thumb_op32(as, 0x4010, 0xe92d); // push {r4, lr}
+        asm_thumb_op32(as, 0xe009, 0xe28f); // add lr, pc, 8 + 1
+        asm_thumb_op32(as, 0xff3e, 0xe12f); // blx lr
+        asm_thumb_op32(as, 0x4010, 0xe8bd); // pop {r4, lr}
+        asm_thumb_op32(as, 0xff1e, 0xe12f); // bx lr
+    }
+    #endif
+
     // work out what to push and how many extra spaces to reserve on stack
     // so that we have enough for all locals and it's aligned an 8-byte boundary
     // we push extra regs (r1, r2, r3) to help do the stack adjustment
@@ -225,10 +242,12 @@ void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src) {
 }
 
 // if loading lo half with movw, the i16 value will be zero extended into the r32 register!
-void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) {
+size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) {
     assert(reg_dest < ASM_THUMB_REG_R15);
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     // mov[wt] reg_dest, #i16_src
     asm_thumb_op32(as, mov_op | ((i16_src >> 1) & 0x0400) | ((i16_src >> 12) & 0xf), ((i16_src << 4) & 0x7000) | (reg_dest << 8) | (i16_src & 0xff));
+    return loc;
 }
 
 #define OP_B_N(byte_offset) (0xe000 | (((byte_offset) >> 1) & 0x07ff))
@@ -271,12 +290,16 @@ bool asm_thumb_bl_label(asm_thumb_t *as, uint label) {
     return as->base.pass != MP_ASM_PASS_EMIT || SIGNED_FIT23(rel);
 }
 
-void asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) {
+size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) {
     // movw, movt does it in 8 bytes
     // ldr [pc, #], dw does it in 6 bytes, but we might not reach to end of code for dw
 
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
+
     asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, reg_dest, i32);
     asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVT, reg_dest, i32 >> 16);
+
+    return loc;
 }
 
 void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32) {

py/asmthumb.h

@@ -241,14 +241,14 @@ static inline void asm_thumb_ldrh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uin
 #define ASM_THUMB_OP_MOVT (0xf2c0)
 
 void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src);
-void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src);
+size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src);
 
 // these return true if the destination is in range, false otherwise
 bool asm_thumb_b_n_label(asm_thumb_t *as, uint label);
 bool asm_thumb_bcc_nw_label(asm_thumb_t *as, int cond, uint label, bool wide);
 bool asm_thumb_bl_label(asm_thumb_t *as, uint label);
 
-void asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32_src); // convenience
+size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32_src); // convenience
 void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32_src); // convenience
 void asm_thumb_mov_local_reg(asm_thumb_t *as, int local_num_dest, uint rlo_src); // convenience
 void asm_thumb_mov_reg_local(asm_thumb_t *as, uint rlo_dest, int local_num); // convenience
@@ -315,6 +315,8 @@ void asm_thumb_bl_ind(asm_thumb_t *as, uint fun_id, uint reg_temp); // convenien
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_thumb_mov_reg_i16((as), ASM_THUMB_OP_MOVW, (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_thumb_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_thumb_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmx64.c

@@ -334,14 +334,16 @@ void asm_x64_mov_i8_to_r8(asm_x64_t *as, int src_i8, int dest_r64) {
 }
 */
 
-STATIC void asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64) {
+size_t asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64) {
     // cpu defaults to i32 to r64, with zero extension
     if (dest_r64 < 8) {
         asm_x64_write_byte_1(as, OPCODE_MOV_I64_TO_R64 | dest_r64);
     } else {
         asm_x64_write_byte_2(as, REX_PREFIX | REX_B, OPCODE_MOV_I64_TO_R64 | (dest_r64 & 7));
     }
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     asm_x64_write_word32(as, src_i32);
+    return loc;
 }
 
 void asm_x64_mov_i64_to_r64(asm_x64_t *as, int64_t src_i64, int dest_r64) {

py/asmx64.h

@@ -83,6 +83,7 @@ void asm_x64_nop(asm_x64_t* as);
 void asm_x64_push_r64(asm_x64_t* as, int src_r64);
 void asm_x64_pop_r64(asm_x64_t* as, int dest_r64);
 void asm_x64_mov_r64_r64(asm_x64_t* as, int dest_r64, int src_r64);
+size_t asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64);
 void asm_x64_mov_i64_to_r64(asm_x64_t* as, int64_t src_i64, int dest_r64);
 void asm_x64_mov_i64_to_r64_optimised(asm_x64_t *as, int64_t src_i64, int dest_r64);
 void asm_x64_mov_r8_to_mem8(asm_x64_t *as, int src_r64, int dest_r64, int dest_disp);
@@ -181,6 +182,8 @@ void asm_x64_call_ind(asm_x64_t* as, size_t fun_id, int temp_r32);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x64_mov_r64_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_optimised((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x64_mov_local_to_r64((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x64_mov_r64_r64((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x64_mov_local_addr_to_r64((as), (local_num), (reg_dest))

py/asmx86.c

@@ -236,9 +236,11 @@ void asm_x86_mov_i8_to_r8(asm_x86_t *as, int src_i8, int dest_r32) {
 }
 #endif
 
-void asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32) {
+size_t asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32) {
     asm_x86_write_byte_1(as, OPCODE_MOV_I32_TO_R32 | dest_r32);
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     asm_x86_write_word32(as, src_i32);
+    return loc;
 }
 
 void asm_x86_and_r32_r32(asm_x86_t *as, int dest_r32, int src_r32) {

py/asmx86.h

@@ -83,7 +83,7 @@ static inline void asm_x86_end_pass(asm_x86_t *as) {
 }
 
 void asm_x86_mov_r32_r32(asm_x86_t* as, int dest_r32, int src_r32);
-void asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32);
+size_t asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32);
 void asm_x86_mov_r8_to_mem8(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
 void asm_x86_mov_r16_to_mem16(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
 void asm_x86_mov_r32_to_mem32(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
@@ -179,6 +179,8 @@ void asm_x86_call_ind(asm_x86_t* as, size_t fun_id, mp_uint_t n_args, int temp_r
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x86_mov_r32_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x86_mov_local_to_r32((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x86_mov_r32_r32((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x86_mov_local_addr_to_r32((as), (local_num), (reg_dest))

py/asmxtensa.c

@@ -30,14 +30,13 @@
 #include "py/mpconfig.h"
 
 // wrapper around everything in this file
-#if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA
+#if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN
 
 #include "py/asmxtensa.h"
 
 #define WORD_SIZE (4)
 #define SIGNED_FIT8(x) ((((x) & 0xffffff80) == 0) || (((x) & 0xffffff80) == 0xffffff80))
 #define SIGNED_FIT12(x) ((((x) & 0xfffff800) == 0) || (((x) & 0xfffff800) == 0xfffff800))
-#define NUM_REGS_SAVED (5)
 
 void asm_xtensa_end_pass(asm_xtensa_t *as) {
     as->num_const = as->cur_const;
@@ -69,7 +68,7 @@ void asm_xtensa_entry(asm_xtensa_t *as, int num_locals) {
     as->const_table = (uint32_t*)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4);
 
     // adjust the stack-pointer to store a0, a12, a13, a14, a15 and locals, 16-byte aligned
-    as->stack_adjust = (((NUM_REGS_SAVED + num_locals) * WORD_SIZE) + 15) & ~15;
+    as->stack_adjust = (((ASM_XTENSA_NUM_REGS_SAVED + num_locals) * WORD_SIZE) + 15) & ~15;
     if (SIGNED_FIT8(-as->stack_adjust)) {
         asm_xtensa_op_addi(as, ASM_XTENSA_REG_A1, ASM_XTENSA_REG_A1, -as->stack_adjust);
     } else {
@@ -79,14 +78,14 @@ void asm_xtensa_entry(asm_xtensa_t *as, int num_locals) {
 
     // save return value (a0) and callee-save registers (a12, a13, a14, a15)
     asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
-    for (int i = 1; i < NUM_REGS_SAVED; ++i) {
+    for (int i = 1; i < ASM_XTENSA_NUM_REGS_SAVED; ++i) {
         asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A11 + i, ASM_XTENSA_REG_A1, i);
     }
 }
 
 void asm_xtensa_exit(asm_xtensa_t *as) {
     // restore registers
-    for (int i = NUM_REGS_SAVED - 1; i >= 1; --i) {
+    for (int i = ASM_XTENSA_NUM_REGS_SAVED - 1; i >= 1; --i) {
         asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A11 + i, ASM_XTENSA_REG_A1, i);
     }
     asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
@@ -102,6 +101,22 @@ void asm_xtensa_exit(asm_xtensa_t *as) {
     asm_xtensa_op_ret_n(as);
 }
 
+void asm_xtensa_entry_win(asm_xtensa_t *as, int num_locals) {
+    // jump over the constants
+    asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4);
+    mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte
+    as->const_table = (uint32_t*)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4);
+
+    as->stack_adjust = 32 + ((((ASM_XTENSA_NUM_REGS_SAVED_WIN + num_locals) * WORD_SIZE) + 15) & ~15);
+    asm_xtensa_op_entry(as, ASM_XTENSA_REG_A1, as->stack_adjust);
+    asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
+}
+
+void asm_xtensa_exit_win(asm_xtensa_t *as) {
+    asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
+    asm_xtensa_op_retw_n(as);
+}
+
 STATIC uint32_t get_label_dest(asm_xtensa_t *as, uint label) {
     assert(label < as->base.max_num_labels);
     return as->base.label_offsets[label];
@@ -156,31 +171,37 @@ void asm_xtensa_setcc_reg_reg_reg(asm_xtensa_t *as, uint cond, uint reg_dest, ui
     asm_xtensa_op_movi_n(as, reg_dest, 0);
 }
 
-void asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
+size_t asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
+    // load the constant
+    uint32_t const_table_offset = (uint8_t*)as->const_table - as->base.code_base;
+    size_t loc = const_table_offset + as->cur_const * WORD_SIZE;
+    asm_xtensa_op_l32r(as, reg_dest, as->base.code_offset, loc);
+    // store the constant in the table
+    if (as->const_table != NULL) {
+        as->const_table[as->cur_const] = i32;
+    }
+    ++as->cur_const;
+    return loc;
+}
+
+void asm_xtensa_mov_reg_i32_optimised(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
     if (SIGNED_FIT12(i32)) {
         asm_xtensa_op_movi(as, reg_dest, i32);
     } else {
-        // load the constant
-        uint32_t const_table_offset = (uint8_t*)as->const_table - as->base.code_base;
-        asm_xtensa_op_l32r(as, reg_dest, as->base.code_offset, const_table_offset + as->cur_const * WORD_SIZE);
-        // store the constant in the table
-        if (as->const_table != NULL) {
-            as->const_table[as->cur_const] = i32;
-        }
-        ++as->cur_const;
+        asm_xtensa_mov_reg_i32(as, reg_dest, i32);
     }
 }
 
 void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src) {
-    asm_xtensa_op_s32i(as, reg_src, ASM_XTENSA_REG_A1, NUM_REGS_SAVED + local_num);
+    asm_xtensa_op_s32i(as, reg_src, ASM_XTENSA_REG_A1, local_num);
 }
 
 void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num) {
-    asm_xtensa_op_l32i(as, reg_dest, ASM_XTENSA_REG_A1, NUM_REGS_SAVED + local_num);
+    asm_xtensa_op_l32i(as, reg_dest, ASM_XTENSA_REG_A1, local_num);
 }
 
 void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num) {
-    uint off = (NUM_REGS_SAVED + local_num) * WORD_SIZE;
+    uint off = local_num * WORD_SIZE;
     if (SIGNED_FIT8(off)) {
         asm_xtensa_op_addi(as, reg_dest, ASM_XTENSA_REG_A1, off);
     } else {
@@ -220,4 +241,13 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx) {
     asm_xtensa_op_callx0(as, ASM_XTENSA_REG_A0);
 }
 
-#endif // MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA
+void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx) {
+    if (idx < 16) {
+        asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx);
+    } else {
+        asm_xtensa_op_l32i(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx);
+    }
+    asm_xtensa_op_callx8(as, ASM_XTENSA_REG_A8);
+}
+
+#endif // MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN

py/asmxtensa.h

@@ -26,6 +26,7 @@
 #ifndef MICROPY_INCLUDED_PY_ASMXTENSA_H
 #define MICROPY_INCLUDED_PY_ASMXTENSA_H
 
+#include "py/misc.h"
 #include "py/asmbase.h"
 
 // calling conventions:
@@ -36,6 +37,16 @@
 // callee save: a1, a12, a13, a14, a15
 // caller save: a3
 
+// With windowed registers, size 8:
+// - a0: return PC
+// - a1: stack pointer, full descending, aligned to 16 bytes
+// - a2-a7: incoming args, and essentially callee save
+// - a2: return value
+// - a8-a15: caller save temporaries
+// - a10-a15: input args to called function
+// - a10: return value of called function
+// note: a0-a7 are saved automatically via window shift of called function
+
 #define ASM_XTENSA_REG_A0  (0)
 #define ASM_XTENSA_REG_A1  (1)
 #define ASM_XTENSA_REG_A2  (2)
@@ -95,6 +106,10 @@
 #define ASM_XTENSA_ENCODE_RI7(op0, s, imm7) \
     ((((imm7) & 0xf) << 12) | ((s) << 8) | ((imm7) & 0x70) | (op0))
 
+// Number of registers saved on the stack upon entry to function
+#define ASM_XTENSA_NUM_REGS_SAVED (5)
+#define ASM_XTENSA_NUM_REGS_SAVED_WIN (1)
+
 typedef struct _asm_xtensa_t {
     mp_asm_base_t base;
     uint32_t cur_const;
@@ -108,11 +123,18 @@ void asm_xtensa_end_pass(asm_xtensa_t *as);
 void asm_xtensa_entry(asm_xtensa_t *as, int num_locals);
 void asm_xtensa_exit(asm_xtensa_t *as);
 
+void asm_xtensa_entry_win(asm_xtensa_t *as, int num_locals);
+void asm_xtensa_exit_win(asm_xtensa_t *as);
+
 void asm_xtensa_op16(asm_xtensa_t *as, uint16_t op);
 void asm_xtensa_op24(asm_xtensa_t *as, uint32_t op);
 
 // raw instructions
 
+static inline void asm_xtensa_op_entry(asm_xtensa_t *as, uint reg_src, int32_t num_bytes) {
+    asm_xtensa_op24(as, ASM_XTENSA_ENCODE_BRI12(6, reg_src, 0, 3, (num_bytes / 8) & 0xfff));
+}
+
 static inline void asm_xtensa_op_add_n(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) {
     asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(10, reg_dest, reg_src_a, reg_src_b));
 }
@@ -141,6 +163,10 @@ static inline void asm_xtensa_op_callx0(asm_xtensa_t *as, uint reg) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALLX(0, 0, 0, 0, reg, 3, 0));
 }
 
+static inline void asm_xtensa_op_callx8(asm_xtensa_t *as, uint reg) {
+    asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALLX(0, 0, 0, 0, reg, 3, 2));
+}
+
 static inline void asm_xtensa_op_j(asm_xtensa_t *as, int32_t rel18) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALL(6, 0, rel18 & 0x3ffff));
 }
@@ -193,6 +219,10 @@ static inline void asm_xtensa_op_ret_n(asm_xtensa_t *as) {
     asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(13, 15, 0, 0));
 }
 
+static inline void asm_xtensa_op_retw_n(asm_xtensa_t *as) {
+    asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(13, 15, 0, 1));
+}
+
 static inline void asm_xtensa_op_s8i(asm_xtensa_t *as, uint reg_src, uint reg_base, uint byte_offset) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 4, reg_base, reg_src, byte_offset & 0xff));
 }
@@ -238,15 +268,18 @@ void asm_xtensa_j_label(asm_xtensa_t *as, uint label);
 void asm_xtensa_bccz_reg_label(asm_xtensa_t *as, uint cond, uint reg, uint label);
 void asm_xtensa_bcc_reg_reg_label(asm_xtensa_t *as, uint cond, uint reg1, uint reg2, uint label);
 void asm_xtensa_setcc_reg_reg_reg(asm_xtensa_t *as, uint cond, uint reg_dest, uint reg_src1, uint reg_src2);
-void asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
+size_t asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
+void asm_xtensa_mov_reg_i32_optimised(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
 void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src);
 void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num);
 void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num);
 void asm_xtensa_mov_reg_pcrel(asm_xtensa_t *as, uint reg_dest, uint label);
 void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
+void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx);
 
 // Holds a pointer to mp_fun_table
 #define ASM_XTENSA_REG_FUN_TABLE ASM_XTENSA_REG_A15
+#define ASM_XTENSA_REG_FUN_TABLE_WIN ASM_XTENSA_REG_A7
 
 #if GENERIC_ASM_API
 
@@ -255,6 +288,9 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 
 #define ASM_WORD_SIZE (4)
 
+#if !GENERIC_ASM_API_WIN
+// Configuration for non-windowed calls
+
 #define REG_RET ASM_XTENSA_REG_A2
 #define REG_ARG_1 ASM_XTENSA_REG_A2
 #define REG_ARG_2 ASM_XTENSA_REG_A3
@@ -271,12 +307,47 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 #define REG_LOCAL_3 ASM_XTENSA_REG_A14
 #define REG_LOCAL_NUM (3)
 
+#define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED
 #define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE
 
+#define ASM_ENTRY(as, nlocal)   asm_xtensa_entry((as), (nlocal))
+#define ASM_EXIT(as)            asm_xtensa_exit((as))
+#define ASM_CALL_IND(as, idx)   asm_xtensa_call_ind((as), (idx))
+
+#else
+// Configuration for windowed calls with window size 8
+
+#define REG_PARENT_RET ASM_XTENSA_REG_A2
+#define REG_PARENT_ARG_1 ASM_XTENSA_REG_A2
+#define REG_PARENT_ARG_2 ASM_XTENSA_REG_A3
+#define REG_PARENT_ARG_3 ASM_XTENSA_REG_A4
+#define REG_PARENT_ARG_4 ASM_XTENSA_REG_A5
+#define REG_RET ASM_XTENSA_REG_A10
+#define REG_ARG_1 ASM_XTENSA_REG_A10
+#define REG_ARG_2 ASM_XTENSA_REG_A11
+#define REG_ARG_3 ASM_XTENSA_REG_A12
+#define REG_ARG_4 ASM_XTENSA_REG_A13
+
+#define REG_TEMP0 ASM_XTENSA_REG_A10
+#define REG_TEMP1 ASM_XTENSA_REG_A11
+#define REG_TEMP2 ASM_XTENSA_REG_A12
+
+#define REG_LOCAL_1 ASM_XTENSA_REG_A4
+#define REG_LOCAL_2 ASM_XTENSA_REG_A5
+#define REG_LOCAL_3 ASM_XTENSA_REG_A6
+#define REG_LOCAL_NUM (3)
+
+#define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED_WIN
+#define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE_WIN
+
+#define ASM_ENTRY(as, nlocal)   asm_xtensa_entry_win((as), (nlocal))
+#define ASM_EXIT(as)            asm_xtensa_exit_win((as))
+#define ASM_CALL_IND(as, idx)   asm_xtensa_call_ind_win((as), (idx))
+
+#endif
+
 #define ASM_T               asm_xtensa_t
 #define ASM_END_PASS        asm_xtensa_end_pass
-#define ASM_ENTRY           asm_xtensa_entry
-#define ASM_EXIT            asm_xtensa_exit
 
 #define ASM_JUMP            asm_xtensa_j_label
 #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \
@@ -286,13 +357,14 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 #define ASM_JUMP_IF_REG_EQ(as, reg1, reg2, label) \
     asm_xtensa_bcc_reg_reg_label(as, ASM_XTENSA_CC_EQ, reg1, reg2, label)
 #define ASM_JUMP_REG(as, reg) asm_xtensa_op_jx((as), (reg))
-#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind((as), (idx))
 
-#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), (local_num), (reg_src))
-#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
-#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), (local_num))
+#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), ASM_NUM_REGS_SAVED + (local_num), (reg_src))
+#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mov_n((as), (reg_dest), (reg_src))
-#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), (local_num))
+#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))
 #define ASM_MOV_REG_PCREL(as, reg_dest, label) asm_xtensa_mov_reg_pcrel((as), (reg_dest), (label))
 
 #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) \

py/bc.c

@@ -40,6 +40,8 @@
 #define DEBUG_printf(...) (void)0
 #endif
 
+#if !MICROPY_PERSISTENT_CODE
+
 mp_uint_t mp_decode_uint(const byte **ptr) {
     mp_uint_t unum = 0;
     byte val;
@@ -70,6 +72,8 @@ const byte *mp_decode_uint_skip(const byte *ptr) {
     return ptr;
 }
 
+#endif
+
 STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) {
 #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE
     // generic message, used also for other argument issues
@@ -119,16 +123,22 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
     code_state->prev = NULL;
     #endif
 
-    // get params
-    size_t n_state = mp_decode_uint(&code_state->ip);
-    code_state->ip = mp_decode_uint_skip(code_state->ip); // skip n_exc_stack
-    size_t scope_flags = *code_state->ip++;
-    size_t n_pos_args = *code_state->ip++;
-    size_t n_kwonly_args = *code_state->ip++;
-    size_t n_def_pos_args = *code_state->ip++;
+    #if MICROPY_PY_SYS_SETTRACE
+    code_state->prev_state = NULL;
+    code_state->frame = NULL;
+    #endif
+
+    // Get cached n_state (rather than decode it again)
+    size_t n_state = code_state->n_state;
+
+    // Decode prelude
+    size_t n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args;
+    MP_BC_PRELUDE_SIG_DECODE_INTO(code_state->ip, n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args);
+    (void)n_state_unused;
+    (void)n_exc_stack_unused;
 
     code_state->sp = &code_state->state[0] - 1;
-    code_state->exc_sp = (mp_exc_stack_t*)(code_state->state + n_state) - 1;
+    code_state->exc_sp_idx = 0;
 
     // zero out the local stack to begin with
     memset(code_state->state, 0, n_state * sizeof(*code_state->state));
@@ -263,19 +273,25 @@ continue2:;
         }
     }
 
-    // get the ip and skip argument names
+    // read the size part of the prelude
     const byte *ip = code_state->ip;
+    MP_BC_PRELUDE_SIZE_DECODE(ip);
 
     // jump over code info (source file and line-number mapping)
-    ip += mp_decode_uint_value(ip);
+    ip += n_info;
 
     // bytecode prelude: initialise closed over variables
-    size_t local_num;
-    while ((local_num = *ip++) != 255) {
+    for (; n_cell; --n_cell) {
+        size_t local_num = *ip++;
         code_state->state[n_state - 1 - local_num] =
             mp_obj_new_cell(code_state->state[n_state - 1 - local_num]);
     }
 
+    #if !MICROPY_PERSISTENT_CODE
+    // so bytecode is aligned
+    ip = MP_ALIGN(ip, sizeof(mp_uint_t));
+    #endif
+
     // now that we skipped over the prelude, set the ip for the VM
     code_state->ip = ip;
 
@@ -287,105 +303,17 @@ continue2:;
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
 // The following table encodes the number of bytes that a specific opcode
-// takes up.  There are 3 special opcodes that always have an extra byte:
-//     MP_BC_MAKE_CLOSURE
-//     MP_BC_MAKE_CLOSURE_DEFARGS
-//     MP_BC_RAISE_VARARGS
+// takes up.  Some opcodes have an extra byte, defined by MP_BC_MASK_EXTRA_BYTE.
 // There are 4 special opcodes that have an extra byte only when
 // MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE is enabled (and they take a qstr):
 //     MP_BC_LOAD_NAME
 //     MP_BC_LOAD_GLOBAL
 //     MP_BC_LOAD_ATTR
 //     MP_BC_STORE_ATTR
-#define OC4(a, b, c, d) (a | (b << 2) | (c << 4) | (d << 6))
-#define U (0) // undefined opcode
-#define B (MP_OPCODE_BYTE) // single byte
-#define Q (MP_OPCODE_QSTR) // single byte plus 2-byte qstr
-#define V (MP_OPCODE_VAR_UINT) // single byte plus variable encoded unsigned int
-#define O (MP_OPCODE_OFFSET) // single byte plus 2-byte bytecode offset
-STATIC const byte opcode_format_table[64] = {
-    OC4(U, U, U, U), // 0x00-0x03
-    OC4(U, U, U, U), // 0x04-0x07
-    OC4(U, U, U, U), // 0x08-0x0b
-    OC4(U, U, U, U), // 0x0c-0x0f
-    OC4(B, B, B, U), // 0x10-0x13
-    OC4(V, U, Q, V), // 0x14-0x17
-    OC4(B, V, V, Q), // 0x18-0x1b
-    OC4(Q, Q, Q, Q), // 0x1c-0x1f
-    OC4(B, B, V, V), // 0x20-0x23
-    OC4(Q, Q, Q, B), // 0x24-0x27
-    OC4(V, V, Q, Q), // 0x28-0x2b
-    OC4(U, U, U, U), // 0x2c-0x2f
-    OC4(B, B, B, B), // 0x30-0x33
-    OC4(B, O, O, O), // 0x34-0x37
-    OC4(O, O, U, U), // 0x38-0x3b
-    OC4(U, O, B, O), // 0x3c-0x3f
-    OC4(O, B, B, O), // 0x40-0x43
-    OC4(B, B, O, B), // 0x44-0x47
-    OC4(U, U, U, U), // 0x48-0x4b
-    OC4(U, U, U, U), // 0x4c-0x4f
-    OC4(V, V, U, V), // 0x50-0x53
-    OC4(B, U, V, V), // 0x54-0x57
-    OC4(V, V, V, B), // 0x58-0x5b
-    OC4(B, B, B, U), // 0x5c-0x5f
-    OC4(V, V, V, V), // 0x60-0x63
-    OC4(V, V, V, V), // 0x64-0x67
-    OC4(Q, Q, B, U), // 0x68-0x6b
-    OC4(U, U, U, U), // 0x6c-0x6f
-
-    OC4(B, B, B, B), // 0x70-0x73
-    OC4(B, B, B, B), // 0x74-0x77
-    OC4(B, B, B, B), // 0x78-0x7b
-    OC4(B, B, B, B), // 0x7c-0x7f
-    OC4(B, B, B, B), // 0x80-0x83
-    OC4(B, B, B, B), // 0x84-0x87
-    OC4(B, B, B, B), // 0x88-0x8b
-    OC4(B, B, B, B), // 0x8c-0x8f
-    OC4(B, B, B, B), // 0x90-0x93
-    OC4(B, B, B, B), // 0x94-0x97
-    OC4(B, B, B, B), // 0x98-0x9b
-    OC4(B, B, B, B), // 0x9c-0x9f
-    OC4(B, B, B, B), // 0xa0-0xa3
-    OC4(B, B, B, B), // 0xa4-0xa7
-    OC4(B, B, B, B), // 0xa8-0xab
-    OC4(B, B, B, B), // 0xac-0xaf
-
-    OC4(B, B, B, B), // 0xb0-0xb3
-    OC4(B, B, B, B), // 0xb4-0xb7
-    OC4(B, B, B, B), // 0xb8-0xbb
-    OC4(B, B, B, B), // 0xbc-0xbf
-
-    OC4(B, B, B, B), // 0xc0-0xc3
-    OC4(B, B, B, B), // 0xc4-0xc7
-    OC4(B, B, B, B), // 0xc8-0xcb
-    OC4(B, B, B, B), // 0xcc-0xcf
-
-    OC4(B, B, B, B), // 0xd0-0xd3
-    OC4(U, U, U, B), // 0xd4-0xd7
-    OC4(B, B, B, B), // 0xd8-0xdb
-    OC4(B, B, B, B), // 0xdc-0xdf
-
-    OC4(B, B, B, B), // 0xe0-0xe3
-    OC4(B, B, B, B), // 0xe4-0xe7
-    OC4(B, B, B, B), // 0xe8-0xeb
-    OC4(B, B, B, B), // 0xec-0xef
-
-    OC4(B, B, B, B), // 0xf0-0xf3
-    OC4(B, B, B, B), // 0xf4-0xf7
-    OC4(U, U, U, U), // 0xf8-0xfb
-    OC4(U, U, U, U), // 0xfc-0xff
-};
-#undef OC4
-#undef U
-#undef B
-#undef Q
-#undef V
-#undef O
-
-uint mp_opcode_format(const byte *ip, size_t *opcode_size) {
-    uint f = (opcode_format_table[*ip >> 2] >> (2 * (*ip & 3))) & 3;
+uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint) {
+    uint f = MP_BC_FORMAT(*ip);
     const byte *ip_start = ip;
-    if (f == MP_OPCODE_QSTR) {
+    if (f == MP_BC_FORMAT_QSTR) {
         if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
             if (*ip == MP_BC_LOAD_NAME
                 || *ip == MP_BC_LOAD_GLOBAL
@@ -396,16 +324,14 @@ uint mp_opcode_format(const byte *ip, size_t *opcode_size) {
         }
         ip += 3;
     } else {
-        int extra_byte = (
-            *ip == MP_BC_RAISE_VARARGS
-            || *ip == MP_BC_MAKE_CLOSURE
-            || *ip == MP_BC_MAKE_CLOSURE_DEFARGS
-        );
+        int extra_byte = (*ip & MP_BC_MASK_EXTRA_BYTE) == 0;
         ip += 1;
-        if (f == MP_OPCODE_VAR_UINT) {
-            while ((*ip++ & 0x80) != 0) {
+        if (f == MP_BC_FORMAT_VAR_UINT) {
+            if (count_var_uint) {
+                while ((*ip++ & 0x80) != 0) {
+                }
             }
-        } else if (f == MP_OPCODE_OFFSET) {
+        } else if (f == MP_BC_FORMAT_OFFSET) {
             ip += 2;
         }
         ip += extra_byte;

py/bc.h

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -31,24 +32,35 @@
 
 // bytecode layout:
 //
-//  n_state         : var uint
-//  n_exc_stack     : var uint
-//  scope_flags     : byte
-//  n_pos_args      : byte          number of arguments this function takes
-//  n_kwonly_args   : byte          number of keyword-only arguments this function takes
-//  n_def_pos_args  : byte          number of default positional arguments
+//  func signature  : var uint
+//      contains six values interleaved bit-wise as: xSSSSEAA [xFSSKAED repeated]
+//          x = extension           another byte follows
+//          S = n_state - 1         number of entries in Python value stack
+//          E = n_exc_stack         number of entries in exception stack
+//          F = scope_flags         four bits of flags, MP_SCOPE_FLAG_xxx
+//          A = n_pos_args          number of arguments this function takes
+//          K = n_kwonly_args       number of keyword-only arguments this function takes
+//          D = n_def_pos_args      number of default positional arguments
 //
-//  code_info_size  : var uint |    code_info_size counts bytes in this chunk
-//  simple_name     : var qstr |
-//  source_file     : var qstr |
-//  <line number info>         |
-//  <word alignment padding>   |    only needed if bytecode contains pointers
+//  prelude size    : var uint
+//      contains two values interleaved bit-wise as: xIIIIIIC repeated
+//          x = extension           another byte follows
+//          I = n_info              number of bytes in source info section
+//          C = n_cells             number of bytes/cells in closure section
 //
-//  local_num0      : byte     |
-//  ...             : byte     |
-//  local_numN      : byte     |    N = num_cells
-//  255             : byte     |    end of list sentinel
-//  <bytecode>                 |
+//  source info section:
+//      simple_name : var qstr
+//      source_file : var qstr
+//      <line number info>
+//
+//  closure section:
+//      local_num0  : byte
+//      ...         : byte
+//      local_numN  : byte          N = n_cells-1
+//
+//  <word alignment padding>        only needed if bytecode contains pointers
+//
+//  <bytecode>
 //
 //
 // constant table layout:
@@ -59,13 +71,127 @@
 //  const0          : obj
 //  constN          : obj
 
+#define MP_BC_PRELUDE_SIG_ENCODE(S, E, scope, out_byte, out_env) \
+do {                                                            \
+    /*// Get values to store in prelude */                      \
+    size_t F = scope->scope_flags & MP_SCOPE_FLAG_ALL_SIG;      \
+    size_t A = scope->num_pos_args;                             \
+    size_t K = scope->num_kwonly_args;                          \
+    size_t D = scope->num_def_pos_args;                         \
+                                                                \
+    /* Adjust S to shrink range, to compress better */          \
+    S -= 1;                                                     \
+                                                                \
+    /* Encode prelude */                                        \
+    /* xSSSSEAA */                                              \
+    uint8_t z = (S & 0xf) << 3 | (E & 1) << 2 | (A & 3);        \
+    S >>= 4;                                                    \
+    E >>= 1;                                                    \
+    A >>= 2;                                                    \
+    while (S | E | F | A | K | D) {                             \
+        out_byte(out_env, 0x80 | z);                            \
+        /* xFSSKAED */                                          \
+        z = (F & 1) << 6 | (S & 3) << 4 | (K & 1) << 3          \
+            | (A & 1) << 2 | (E & 1) << 1 | (D & 1);            \
+        S >>= 2;                                                \
+        E >>= 1;                                                \
+        F >>= 1;                                                \
+        A >>= 1;                                                \
+        K >>= 1;                                                \
+        D >>= 1;                                                \
+    }                                                           \
+    out_byte(out_env, z);                                       \
+} while (0)
+
+#define MP_BC_PRELUDE_SIG_DECODE_INTO(ip, S, E, F, A, K, D)     \
+do {                                                            \
+    uint8_t z = *(ip)++;                                        \
+    /* xSSSSEAA */                                              \
+    S = (z >> 3) & 0xf;                                         \
+    E = (z >> 2) & 0x1;                                         \
+    F = 0;                                                      \
+    A = z & 0x3;                                                \
+    K = 0;                                                      \
+    D = 0;                                                      \
+    for (unsigned n = 0; z & 0x80; ++n) {                       \
+        z = *(ip)++;                                            \
+        /* xFSSKAED */                                          \
+        S |= (z & 0x30) << (2 * n);                             \
+        E |= (z & 0x02) << n;                                   \
+        F |= ((z & 0x40) >> 6) << n;                            \
+        A |= (z & 0x4) << n;                                    \
+        K |= ((z & 0x08) >> 3) << n;                            \
+        D |= (z & 0x1) << n;                                    \
+    }                                                           \
+    S += 1;                                                     \
+} while (0)
+
+#define MP_BC_PRELUDE_SIG_DECODE(ip) \
+    size_t n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args; \
+    MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args)
+
+#define MP_BC_PRELUDE_SIZE_ENCODE(I, C, out_byte, out_env)      \
+do {                                                            \
+    /* Encode bit-wise as: xIIIIIIC */                          \
+    uint8_t z = 0;                                              \
+    do {                                                        \
+        z = (I & 0x3f) << 1 | (C & 1);                          \
+        C >>= 1;                                                \
+        I >>= 6;                                                \
+        if (C | I) {                                            \
+            z |= 0x80;                                          \
+        }                                                       \
+        out_byte(out_env, z);                                   \
+    } while (C | I);                                            \
+} while (0)
+
+#define MP_BC_PRELUDE_SIZE_DECODE_INTO(ip, I, C)                \
+do {                                                            \
+    uint8_t z;                                                  \
+    C = 0;                                                      \
+    I = 0;                                                      \
+    for (unsigned n = 0;; ++n) {                                \
+        z = *(ip)++;                                            \
+        /* xIIIIIIC */                                          \
+        C |= (z & 1) << n;                                      \
+        I |= ((z & 0x7e) >> 1) << (6 * n);                      \
+        if (!(z & 0x80)) {                                      \
+            break;                                              \
+        }                                                       \
+    }                                                           \
+} while (0)
+
+#define MP_BC_PRELUDE_SIZE_DECODE(ip) \
+    size_t n_info, n_cell; \
+    MP_BC_PRELUDE_SIZE_DECODE_INTO(ip, n_info, n_cell)
+
+// Sentinel value for mp_code_state_t.exc_sp_idx
+#define MP_CODE_STATE_EXC_SP_IDX_SENTINEL ((uint16_t)-1)
+
+// To convert mp_code_state_t.exc_sp_idx to/from a pointer to mp_exc_stack_t
+#define MP_CODE_STATE_EXC_SP_IDX_FROM_PTR(exc_stack, exc_sp) ((exc_sp) + 1 - (exc_stack))
+#define MP_CODE_STATE_EXC_SP_IDX_TO_PTR(exc_stack, exc_sp_idx) ((exc_stack) + (exc_sp_idx) - 1)
+
+typedef struct _mp_bytecode_prelude_t {
+    uint n_state;
+    uint n_exc_stack;
+    uint scope_flags;
+    uint n_pos_args;
+    uint n_kwonly_args;
+    uint n_def_pos_args;
+    qstr qstr_block_name;
+    qstr qstr_source_file;
+    const byte *line_info;
+    const byte *opcodes;
+} mp_bytecode_prelude_t;
+
 // Exception stack entry
 typedef struct _mp_exc_stack_t {
     const byte *handler;
-    // bit 0 is saved currently_in_except_block value
+    // bit 0 is currently unused
     // bit 1 is whether the opcode was SETUP_WITH or SETUP_FINALLY
     mp_obj_t *val_sp;
-    // Saved exception, valid if currently_in_except_block bit is 1
+    // Saved exception
     mp_obj_base_t *prev_exc;
 } mp_exc_stack_t;
 
@@ -77,12 +203,16 @@ typedef struct _mp_code_state_t {
     mp_obj_fun_bc_t *fun_bc;
     const byte *ip;
     mp_obj_t *sp;
-    // bit 0 is saved currently_in_except_block value
-    mp_exc_stack_t *exc_sp;
+    uint16_t n_state;
+    uint16_t exc_sp_idx;
     mp_obj_dict_t *old_globals;
     #if MICROPY_STACKLESS
     struct _mp_code_state_t *prev;
     #endif
+    #if MICROPY_PY_SYS_SETTRACE
+    struct _mp_code_state_t *prev_state;
+    struct _mp_obj_frame_t *frame;
+    #endif
     // Variable-length
     mp_obj_t state[0];
     // Variable-length, never accessed by name, only as (void*)(state + n_state)
@@ -109,13 +239,35 @@ const byte *mp_bytecode_print_str(const byte *ip);
 
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
-#define MP_OPCODE_BYTE (0)
-#define MP_OPCODE_QSTR (1)
-#define MP_OPCODE_VAR_UINT (2)
-#define MP_OPCODE_OFFSET (3)
-
-uint mp_opcode_format(const byte *ip, size_t *opcode_size);
+uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint);
 
 #endif
 
+static inline size_t mp_bytecode_get_source_line(const byte *line_info, size_t bc_offset) {
+    size_t source_line = 1;
+    size_t c;
+    while ((c = *line_info)) {
+        size_t b, l;
+        if ((c & 0x80) == 0) {
+            // 0b0LLBBBBB encoding
+            b = c & 0x1f;
+            l = c >> 5;
+            line_info += 1;
+        } else {
+            // 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte)
+            b = c & 0xf;
+            l = ((c << 4) & 0x700) | line_info[1];
+            line_info += 2;
+        }
+        if (bc_offset >= b) {
+            bc_offset -= b;
+            source_line += l;
+        } else {
+            // found source line corresponding to bytecode offset
+            break;
+        }
+    }
+    return source_line;
+}
+
 #endif // MICROPY_INCLUDED_PY_BC_H

py/bc0.h

@@ -26,94 +26,125 @@
 #ifndef MICROPY_INCLUDED_PY_BC0_H
 #define MICROPY_INCLUDED_PY_BC0_H
 
-// MicroPython byte-codes.
-// The comment at the end of the line (if it exists) tells the arguments to the byte-code.
-
-#define MP_BC_LOAD_CONST_FALSE   (0x10)
-#define MP_BC_LOAD_CONST_NONE    (0x11)
-#define MP_BC_LOAD_CONST_TRUE    (0x12)
-#define MP_BC_LOAD_CONST_SMALL_INT   (0x14) // signed var-int
-#define MP_BC_LOAD_CONST_STRING  (0x16) // qstr
-#define MP_BC_LOAD_CONST_OBJ     (0x17) // ptr
-#define MP_BC_LOAD_NULL          (0x18)
-
-#define MP_BC_LOAD_FAST_N        (0x19) // uint
-#define MP_BC_LOAD_DEREF         (0x1a) // uint
-#define MP_BC_LOAD_NAME          (0x1b) // qstr
-#define MP_BC_LOAD_GLOBAL        (0x1c) // qstr
-#define MP_BC_LOAD_ATTR          (0x1d) // qstr
-#define MP_BC_LOAD_METHOD        (0x1e) // qstr
-#define MP_BC_LOAD_SUPER_METHOD  (0x1f) // qstr
-#define MP_BC_LOAD_BUILD_CLASS   (0x20)
-#define MP_BC_LOAD_SUBSCR        (0x21)
-
-#define MP_BC_STORE_FAST_N       (0x22) // uint
-#define MP_BC_STORE_DEREF        (0x23) // uint
-#define MP_BC_STORE_NAME         (0x24) // qstr
-#define MP_BC_STORE_GLOBAL       (0x25) // qstr
-#define MP_BC_STORE_ATTR         (0x26) // qstr
-#define MP_BC_STORE_SUBSCR       (0x27)
-
-#define MP_BC_DELETE_FAST        (0x28) // uint
-#define MP_BC_DELETE_DEREF       (0x29) // uint
-#define MP_BC_DELETE_NAME        (0x2a) // qstr
-#define MP_BC_DELETE_GLOBAL      (0x2b) // qstr
-
-#define MP_BC_DUP_TOP            (0x30)
-#define MP_BC_DUP_TOP_TWO        (0x31)
-#define MP_BC_POP_TOP            (0x32)
-#define MP_BC_ROT_TWO            (0x33)
-#define MP_BC_ROT_THREE          (0x34)
-
-#define MP_BC_JUMP               (0x35) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_POP_JUMP_IF_TRUE   (0x36) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_POP_JUMP_IF_FALSE  (0x37) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_JUMP_IF_TRUE_OR_POP    (0x38) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_JUMP_IF_FALSE_OR_POP   (0x39) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_SETUP_WITH         (0x3d) // rel byte code offset, 16-bit unsigned
-#define MP_BC_WITH_CLEANUP       (0x3e)
-#define MP_BC_SETUP_EXCEPT       (0x3f) // rel byte code offset, 16-bit unsigned
-#define MP_BC_SETUP_FINALLY      (0x40) // rel byte code offset, 16-bit unsigned
-#define MP_BC_END_FINALLY        (0x41)
-#define MP_BC_GET_ITER           (0x42)
-#define MP_BC_FOR_ITER           (0x43) // rel byte code offset, 16-bit unsigned
-#define MP_BC_POP_BLOCK          (0x44)
-#define MP_BC_POP_EXCEPT         (0x45)
-#define MP_BC_UNWIND_JUMP        (0x46) // rel byte code offset, 16-bit signed, in excess; then a byte
-#define MP_BC_GET_ITER_STACK     (0x47)
-
-#define MP_BC_BUILD_TUPLE        (0x50) // uint
-#define MP_BC_BUILD_LIST         (0x51) // uint
-#define MP_BC_BUILD_MAP          (0x53) // uint
-#define MP_BC_STORE_MAP          (0x54)
-#define MP_BC_BUILD_SET          (0x56) // uint
-#define MP_BC_BUILD_SLICE        (0x58) // uint
-#define MP_BC_STORE_COMP         (0x57) // uint
-#define MP_BC_UNPACK_SEQUENCE    (0x59) // uint
-#define MP_BC_UNPACK_EX          (0x5a) // uint
-
-#define MP_BC_RETURN_VALUE       (0x5b)
-#define MP_BC_RAISE_VARARGS      (0x5c) // byte
-#define MP_BC_YIELD_VALUE        (0x5d)
-#define MP_BC_YIELD_FROM         (0x5e)
-
-#define MP_BC_MAKE_FUNCTION         (0x60) // uint
-#define MP_BC_MAKE_FUNCTION_DEFARGS (0x61) // uint
-#define MP_BC_MAKE_CLOSURE          (0x62) // uint
-#define MP_BC_MAKE_CLOSURE_DEFARGS  (0x63) // uint
-#define MP_BC_CALL_FUNCTION         (0x64) // uint
-#define MP_BC_CALL_FUNCTION_VAR_KW  (0x65) // uint
-#define MP_BC_CALL_METHOD           (0x66) // uint
-#define MP_BC_CALL_METHOD_VAR_KW    (0x67) // uint
-
-#define MP_BC_IMPORT_NAME        (0x68) // qstr
-#define MP_BC_IMPORT_FROM        (0x69) // qstr
-#define MP_BC_IMPORT_STAR        (0x6a)
-
-#define MP_BC_LOAD_CONST_SMALL_INT_MULTI (0x70) // + N(64)
-#define MP_BC_LOAD_FAST_MULTI            (0xb0) // + N(16)
-#define MP_BC_STORE_FAST_MULTI           (0xc0) // + N(16)
-#define MP_BC_UNARY_OP_MULTI             (0xd0) // + op(<MP_UNARY_OP_NUM_BYTECODE)
-#define MP_BC_BINARY_OP_MULTI            (0xd7) // + op(<MP_BINARY_OP_NUM_BYTECODE)
+// MicroPython bytecode opcodes, grouped based on the format of the opcode
+
+#define MP_BC_MASK_FORMAT                   (0xf0)
+#define MP_BC_MASK_EXTRA_BYTE               (0x9e)
+
+#define MP_BC_FORMAT_BYTE                   (0)
+#define MP_BC_FORMAT_QSTR                   (1)
+#define MP_BC_FORMAT_VAR_UINT               (2)
+#define MP_BC_FORMAT_OFFSET                 (3)
+
+// Nibbles in magic number are: BB BB BB BB BB BO VV QU
+#define MP_BC_FORMAT(op) ((0x000003a4 >> (2 * ((op) >> 4))) & 3)
+
+// Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op
+#define MP_BC_BASE_RESERVED                 (0x00) // ----------------
+#define MP_BC_BASE_QSTR_O                   (0x10) // LLLLLLSSSDDII---
+#define MP_BC_BASE_VINT_E                   (0x20) // MMLLLLSSDDBBBBBB
+#define MP_BC_BASE_VINT_O                   (0x30) // UUMMCCCC--------
+#define MP_BC_BASE_JUMP_E                   (0x40) // J-JJJJJEEEEF----
+#define MP_BC_BASE_BYTE_O                   (0x50) // LLLLSSDTTTTTEEFF
+#define MP_BC_BASE_BYTE_E                   (0x60) // --BREEEYYI------
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI    (0x70) // LLLLLLLLLLLLLLLL
+                                         // (0x80) // LLLLLLLLLLLLLLLL
+                                         // (0x90) // LLLLLLLLLLLLLLLL
+                                         // (0xa0) // LLLLLLLLLLLLLLLL
+#define MP_BC_LOAD_FAST_MULTI               (0xb0) // LLLLLLLLLLLLLLLL
+#define MP_BC_STORE_FAST_MULTI              (0xc0) // SSSSSSSSSSSSSSSS
+#define MP_BC_UNARY_OP_MULTI                (0xd0) // OOOOOOO
+#define MP_BC_BINARY_OP_MULTI               (0xd7) //        OOOOOOOOO
+                                         // (0xe0) // OOOOOOOOOOOOOOOO
+                                         // (0xf0) // OOOOOOOOOO------
+
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM (64)
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS (16)
+#define MP_BC_LOAD_FAST_MULTI_NUM           (16)
+#define MP_BC_STORE_FAST_MULTI_NUM          (16)
+#define MP_BC_UNARY_OP_MULTI_NUM            (MP_UNARY_OP_NUM_BYTECODE)
+#define MP_BC_BINARY_OP_MULTI_NUM           (MP_BINARY_OP_NUM_BYTECODE)
+
+#define MP_BC_LOAD_CONST_FALSE              (MP_BC_BASE_BYTE_O + 0x00)
+#define MP_BC_LOAD_CONST_NONE               (MP_BC_BASE_BYTE_O + 0x01)
+#define MP_BC_LOAD_CONST_TRUE               (MP_BC_BASE_BYTE_O + 0x02)
+#define MP_BC_LOAD_CONST_SMALL_INT          (MP_BC_BASE_VINT_E + 0x02) // signed var-int
+#define MP_BC_LOAD_CONST_STRING             (MP_BC_BASE_QSTR_O + 0x00) // qstr
+#define MP_BC_LOAD_CONST_OBJ                (MP_BC_BASE_VINT_E + 0x03) // ptr
+#define MP_BC_LOAD_NULL                     (MP_BC_BASE_BYTE_O + 0x03)
+
+#define MP_BC_LOAD_FAST_N                   (MP_BC_BASE_VINT_E + 0x04) // uint
+#define MP_BC_LOAD_DEREF                    (MP_BC_BASE_VINT_E + 0x05) // uint
+#define MP_BC_LOAD_NAME                     (MP_BC_BASE_QSTR_O + 0x01) // qstr
+#define MP_BC_LOAD_GLOBAL                   (MP_BC_BASE_QSTR_O + 0x02) // qstr
+#define MP_BC_LOAD_ATTR                     (MP_BC_BASE_QSTR_O + 0x03) // qstr
+#define MP_BC_LOAD_METHOD                   (MP_BC_BASE_QSTR_O + 0x04) // qstr
+#define MP_BC_LOAD_SUPER_METHOD             (MP_BC_BASE_QSTR_O + 0x05) // qstr
+#define MP_BC_LOAD_BUILD_CLASS              (MP_BC_BASE_BYTE_O + 0x04)
+#define MP_BC_LOAD_SUBSCR                   (MP_BC_BASE_BYTE_O + 0x05)
+
+#define MP_BC_STORE_FAST_N                  (MP_BC_BASE_VINT_E + 0x06) // uint
+#define MP_BC_STORE_DEREF                   (MP_BC_BASE_VINT_E + 0x07) // uint
+#define MP_BC_STORE_NAME                    (MP_BC_BASE_QSTR_O + 0x06) // qstr
+#define MP_BC_STORE_GLOBAL                  (MP_BC_BASE_QSTR_O + 0x07) // qstr
+#define MP_BC_STORE_ATTR                    (MP_BC_BASE_QSTR_O + 0x08) // qstr
+#define MP_BC_STORE_SUBSCR                  (MP_BC_BASE_BYTE_O + 0x06)
+
+#define MP_BC_DELETE_FAST                   (MP_BC_BASE_VINT_E + 0x08) // uint
+#define MP_BC_DELETE_DEREF                  (MP_BC_BASE_VINT_E + 0x09) // uint
+#define MP_BC_DELETE_NAME                   (MP_BC_BASE_QSTR_O + 0x09) // qstr
+#define MP_BC_DELETE_GLOBAL                 (MP_BC_BASE_QSTR_O + 0x0a) // qstr
+
+#define MP_BC_DUP_TOP                       (MP_BC_BASE_BYTE_O + 0x07)
+#define MP_BC_DUP_TOP_TWO                   (MP_BC_BASE_BYTE_O + 0x08)
+#define MP_BC_POP_TOP                       (MP_BC_BASE_BYTE_O + 0x09)
+#define MP_BC_ROT_TWO                       (MP_BC_BASE_BYTE_O + 0x0a)
+#define MP_BC_ROT_THREE                     (MP_BC_BASE_BYTE_O + 0x0b)
+
+#define MP_BC_JUMP                          (MP_BC_BASE_JUMP_E + 0x02) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_POP_JUMP_IF_TRUE              (MP_BC_BASE_JUMP_E + 0x03) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_POP_JUMP_IF_FALSE             (MP_BC_BASE_JUMP_E + 0x04) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_JUMP_IF_TRUE_OR_POP           (MP_BC_BASE_JUMP_E + 0x05) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_JUMP_IF_FALSE_OR_POP          (MP_BC_BASE_JUMP_E + 0x06) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_UNWIND_JUMP                   (MP_BC_BASE_JUMP_E + 0x00) // rel byte code offset, 16-bit signed, in excess; then a byte
+#define MP_BC_SETUP_WITH                    (MP_BC_BASE_JUMP_E + 0x07) // rel byte code offset, 16-bit unsigned
+#define MP_BC_SETUP_EXCEPT                  (MP_BC_BASE_JUMP_E + 0x08) // rel byte code offset, 16-bit unsigned
+#define MP_BC_SETUP_FINALLY                 (MP_BC_BASE_JUMP_E + 0x09) // rel byte code offset, 16-bit unsigned
+#define MP_BC_POP_EXCEPT_JUMP               (MP_BC_BASE_JUMP_E + 0x0a) // rel byte code offset, 16-bit unsigned
+#define MP_BC_FOR_ITER                      (MP_BC_BASE_JUMP_E + 0x0b) // rel byte code offset, 16-bit unsigned
+#define MP_BC_WITH_CLEANUP                  (MP_BC_BASE_BYTE_O + 0x0c)
+#define MP_BC_END_FINALLY                   (MP_BC_BASE_BYTE_O + 0x0d)
+#define MP_BC_GET_ITER                      (MP_BC_BASE_BYTE_O + 0x0e)
+#define MP_BC_GET_ITER_STACK                (MP_BC_BASE_BYTE_O + 0x0f)
+
+#define MP_BC_BUILD_TUPLE                   (MP_BC_BASE_VINT_E + 0x0a) // uint
+#define MP_BC_BUILD_LIST                    (MP_BC_BASE_VINT_E + 0x0b) // uint
+#define MP_BC_BUILD_MAP                     (MP_BC_BASE_VINT_E + 0x0c) // uint
+#define MP_BC_STORE_MAP                     (MP_BC_BASE_BYTE_E + 0x02)
+#define MP_BC_BUILD_SET                     (MP_BC_BASE_VINT_E + 0x0d) // uint
+#define MP_BC_BUILD_SLICE                   (MP_BC_BASE_VINT_E + 0x0e) // uint
+#define MP_BC_STORE_COMP                    (MP_BC_BASE_VINT_E + 0x0f) // uint
+#define MP_BC_UNPACK_SEQUENCE               (MP_BC_BASE_VINT_O + 0x00) // uint
+#define MP_BC_UNPACK_EX                     (MP_BC_BASE_VINT_O + 0x01) // uint
+
+#define MP_BC_RETURN_VALUE                  (MP_BC_BASE_BYTE_E + 0x03)
+#define MP_BC_RAISE_LAST                    (MP_BC_BASE_BYTE_E + 0x04)
+#define MP_BC_RAISE_OBJ                     (MP_BC_BASE_BYTE_E + 0x05)
+#define MP_BC_RAISE_FROM                    (MP_BC_BASE_BYTE_E + 0x06)
+#define MP_BC_YIELD_VALUE                   (MP_BC_BASE_BYTE_E + 0x07)
+#define MP_BC_YIELD_FROM                    (MP_BC_BASE_BYTE_E + 0x08)
+
+#define MP_BC_MAKE_FUNCTION                 (MP_BC_BASE_VINT_O + 0x02) // uint
+#define MP_BC_MAKE_FUNCTION_DEFARGS         (MP_BC_BASE_VINT_O + 0x03) // uint
+#define MP_BC_MAKE_CLOSURE                  (MP_BC_BASE_VINT_E + 0x00) // uint; extra byte
+#define MP_BC_MAKE_CLOSURE_DEFARGS          (MP_BC_BASE_VINT_E + 0x01) // uint; extra byte
+#define MP_BC_CALL_FUNCTION                 (MP_BC_BASE_VINT_O + 0x04) // uint
+#define MP_BC_CALL_FUNCTION_VAR_KW          (MP_BC_BASE_VINT_O + 0x05) // uint
+#define MP_BC_CALL_METHOD                   (MP_BC_BASE_VINT_O + 0x06) // uint
+#define MP_BC_CALL_METHOD_VAR_KW            (MP_BC_BASE_VINT_O + 0x07) // uint
+
+#define MP_BC_IMPORT_NAME                   (MP_BC_BASE_QSTR_O + 0x0b) // qstr
+#define MP_BC_IMPORT_FROM                   (MP_BC_BASE_QSTR_O + 0x0c) // qstr
+#define MP_BC_IMPORT_STAR                   (MP_BC_BASE_BYTE_E + 0x09)
 
 #endif // MICROPY_INCLUDED_PY_BC0_H

py/binary.c

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014-2017 Paul Sokolovsky
+ * Copyright (c) 2014-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -41,7 +42,7 @@
 #define alignof(type) offsetof(struct { char c; type t; }, t)
 #endif
 
-size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign) {
+size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
     size_t size = 0;
     int align = 1;
     switch (struct_type) {
@@ -112,7 +113,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign) {
     return size;
 }
 
-mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index) {
+mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index) {
     mp_int_t val = 0;
     switch (typecode) {
         case 'b':
@@ -161,7 +162,7 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index) {
 // The long long type is guaranteed to hold at least 64 bits, and size is at
 // most 8 (for q and Q), so we will always be able to parse the given data
 // and fit it into a long long.
-long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src) {
+long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const byte *src) {
     int delta;
     if (!big_endian) {
         delta = -1;
@@ -184,14 +185,14 @@ long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, con
 }
 
 #define is_signed(typecode) (typecode > 'Z')
-mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) {
+mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr) {
     byte *p = *ptr;
-    mp_uint_t align;
+    size_t align;
 
     size_t size = mp_binary_get_size(struct_type, val_type, &align);
     if (struct_type == '@') {
-        // Make pointer aligned
-        p = (byte*)MP_ALIGN(p, (size_t)align);
+        // Align p relative to p_base
+        p = p_base + (uintptr_t)MP_ALIGN(p - p_base, align);
         #if MP_ENDIANNESS_LITTLE
         struct_type = '<';
         #else
@@ -230,7 +231,7 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) {
     }
 }
 
-void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val) {
+void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val) {
     if (MP_ENDIANNESS_LITTLE && !big_endian) {
         memcpy(dest, &val, val_sz);
     } else if (MP_ENDIANNESS_BIG && big_endian) {
@@ -249,14 +250,14 @@ void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t
     }
 }
 
-void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr) {
+void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p_base, byte **ptr) {
     byte *p = *ptr;
-    mp_uint_t align;
+    size_t align;
 
     size_t size = mp_binary_get_size(struct_type, val_type, &align);
     if (struct_type == '@') {
-        // Make pointer aligned
-        p = (byte*)MP_ALIGN(p, (size_t)align);
+        // Align p relative to p_base
+        p = p_base + (uintptr_t)MP_ALIGN(p - p_base, align);
         if (MP_ENDIANNESS_LITTLE) {
             struct_type = '<';
         } else {
@@ -314,7 +315,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **
     mp_binary_set_int(MIN((size_t)size, sizeof(val)), struct_type == '>', p, val);
 }
 
-void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in) {
+void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in) {
     switch (typecode) {
 #if MICROPY_PY_BUILTINS_FLOAT
         case 'f':
@@ -341,7 +342,7 @@ void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t v
     }
 }
 
-void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, mp_int_t val) {
+void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val) {
     switch (typecode) {
         case 'b':
             ((signed char*)p)[index] = val;

py/binary.h

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
+ * Copyright (c) 2014-2017 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -33,13 +34,13 @@
 // type-specification errors due to end-of-string.
 #define BYTEARRAY_TYPECODE 1
 
-size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign);
-mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index);
-void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in);
-void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, mp_int_t val);
-mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr);
-void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr);
-long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src);
-void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val);
+size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign);
+mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index);
+void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in);
+void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val);
+mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr);
+void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p_base, byte **ptr);
+long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const byte *src);
+void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val);
 
 #endif // MICROPY_INCLUDED_PY_BINARY_H

py/builtin.h

@@ -89,7 +89,7 @@ MP_DECLARE_CONST_FUN_OBJ_2(mp_op_delitem_obj);
 
 extern const mp_obj_module_t mp_module___main__;
 extern const mp_obj_module_t mp_module_builtins;
-extern const mp_obj_module_t mp_module_array;
+extern const mp_obj_module_t mp_module_uarray;
 extern const mp_obj_module_t mp_module_collections;
 extern const mp_obj_module_t mp_module_io;
 extern const mp_obj_module_t mp_module_math;
@@ -122,6 +122,7 @@ extern const mp_obj_module_t mp_module_uwebsocket;
 extern const mp_obj_module_t mp_module_webrepl;
 extern const mp_obj_module_t mp_module_framebuf;
 extern const mp_obj_module_t mp_module_btree;
+extern const mp_obj_module_t mp_module_ubluetooth;
 
 extern const char MICROPY_PY_BUILTINS_HELP_TEXT[];
 

py/builtinhelp.c

@@ -80,10 +80,6 @@ STATIC void mp_help_print_modules(void) {
 
     mp_help_add_from_map(list, &mp_builtin_module_map);
 
-    #if MICROPY_MODULE_WEAK_LINKS
-    mp_help_add_from_map(list, &mp_builtin_module_weak_links_map);
-    #endif
-
     #if MICROPY_MODULE_FROZEN_STR
     extern const char mp_frozen_str_names[];
     mp_help_add_from_names(list, mp_frozen_str_names);

py/builtinimport.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -145,11 +145,11 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) {
 #endif
 
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY
-STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
+STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, const char* source_name) {
+    (void)source_name;
+
     #if MICROPY_PY___FILE__
-    // TODO
-    //qstr source_name = lex->source_name;
-    //mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
+    mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name)));
     #endif
 
     // execute the module in its context
@@ -206,7 +206,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     // its data) in the list of frozen files, execute it.
     #if MICROPY_MODULE_FROZEN_MPY
     if (frozen_type == MP_FROZEN_MPY) {
-        do_execute_raw_code(module_obj, modref);
+        do_execute_raw_code(module_obj, modref, file_str);
         return;
     }
     #endif
@@ -216,7 +216,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD
     if (file_str[file->len - 3] == 'm') {
         mp_raw_code_t *raw_code = mp_raw_code_load_file(file_str);
-        do_execute_raw_code(module_obj, raw_code);
+        do_execute_raw_code(module_obj, raw_code, file_str);
         return;
     }
     #endif
@@ -334,6 +334,10 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
         mod_len = new_mod_l;
     }
 
+    if (mod_len == 0) {
+        mp_raise_ValueError(NULL);
+    }
+
     // check if module already exists
     qstr module_name_qstr = mp_obj_str_get_qstr(module_name);
     mp_obj_t module_obj = mp_module_get(module_name_qstr);
@@ -381,21 +385,18 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
             DEBUG_printf("Current path: %.*s\n", vstr_len(&path), vstr_str(&path));
 
             if (stat == MP_IMPORT_STAT_NO_EXIST) {
+                module_obj = MP_OBJ_NULL;
                 #if MICROPY_MODULE_WEAK_LINKS
                 // check if there is a weak link to this module
                 if (i == mod_len) {
-                    mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(mod_name), MP_MAP_LOOKUP);
-                    if (el == NULL) {
-                        goto no_exist;
+                    module_obj = mp_module_search_umodule(mod_str);
+                    if (module_obj != MP_OBJ_NULL) {
+                        // found weak linked module
+                        mp_module_call_init(mod_name, module_obj);
                     }
-                    // found weak linked module
-                    module_obj = el->value;
-                    mp_module_call_init(mod_name, module_obj);
-                } else {
-                    no_exist:
-                #else
-                {
+                }
                 #endif
+                if (module_obj == MP_OBJ_NULL) {
                     // couldn't find the file, so fail
                     if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) {
                         mp_raise_msg(&mp_type_ImportError, "module not found");
@@ -492,11 +493,11 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
 
     #if MICROPY_MODULE_WEAK_LINKS
     // Check if there is a weak link to this module
-    mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(module_name_qstr), MP_MAP_LOOKUP);
-    if (el != NULL) {
+    module_obj = mp_module_search_umodule(qstr_str(module_name_qstr));
+    if (module_obj != MP_OBJ_NULL) {
         // Found weak-linked module
-        mp_module_call_init(module_name_qstr, el->value);
-        return el->value;
+        mp_module_call_init(module_name_qstr, module_obj);
+        return module_obj;
     }
     #endif
 

py/compile.c

@@ -35,6 +35,7 @@
 #include "py/compile.h"
 #include "py/runtime.h"
 #include "py/asmbase.h"
+#include "py/persistentcode.h"
 
 #if MICROPY_ENABLE_COMPILER
 
@@ -78,7 +79,26 @@ typedef enum {
 
 #endif
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER
+
+#define NATIVE_EMITTER(f) emit_native_table[mp_dynamic_compiler.native_arch]->emit_##f
+#define NATIVE_EMITTER_TABLE emit_native_table[mp_dynamic_compiler.native_arch]
+
+STATIC const emit_method_table_t *emit_native_table[] = {
+    NULL,
+    &emit_native_x86_method_table,
+    &emit_native_x64_method_table,
+    &emit_native_arm_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_xtensa_method_table,
+    &emit_native_xtensawin_method_table,
+};
+
+#elif MICROPY_EMIT_NATIVE
 // define a macro to access external native emitter
 #if MICROPY_EMIT_X64
 #define NATIVE_EMITTER(f) emit_native_x64_##f
@@ -90,12 +110,34 @@ typedef enum {
 #define NATIVE_EMITTER(f) emit_native_arm_##f
 #elif MICROPY_EMIT_XTENSA
 #define NATIVE_EMITTER(f) emit_native_xtensa_##f
+#elif MICROPY_EMIT_XTENSAWIN
+#define NATIVE_EMITTER(f) emit_native_xtensawin_##f
 #else
 #error "unknown native emitter"
 #endif
+#define NATIVE_EMITTER_TABLE &NATIVE_EMITTER(method_table)
 #endif
 
-#if MICROPY_EMIT_INLINE_ASM
+#if MICROPY_EMIT_INLINE_ASM && MICROPY_DYNAMIC_COMPILER
+
+#define ASM_EMITTER(f) emit_asm_table[mp_dynamic_compiler.native_arch]->asm_##f
+#define ASM_EMITTER_TABLE emit_asm_table[mp_dynamic_compiler.native_arch]
+
+STATIC const emit_inline_asm_method_table_t *emit_asm_table[] = {
+    NULL,
+    NULL,
+    NULL,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_xtensa_method_table,
+    NULL,
+};
+
+#elif MICROPY_EMIT_INLINE_ASM
 // define macros for inline assembler
 #if MICROPY_EMIT_INLINE_THUMB
 #define ASM_DECORATOR_QSTR MP_QSTR_asm_thumb
@@ -106,6 +148,7 @@ typedef enum {
 #else
 #error "unknown asm emitter"
 #endif
+#define ASM_EMITTER_TABLE &ASM_EMITTER(method_table)
 #endif
 
 #define EMIT_INLINE_ASM(fun) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm))
@@ -164,6 +207,7 @@ STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const cha
 
 STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_arglist, bool is_method_call, int n_positional_extra);
 STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, scope_kind_t kind);
+STATIC void compile_atom_brace_helper(compiler_t *comp, mp_parse_node_struct_t *pns, bool create_map);
 STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn);
 
 STATIC uint comp_next_label(compiler_t *comp) {
@@ -798,13 +842,32 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, int name_len, mp_parse_
         *emit_options = MP_EMIT_OPT_VIPER;
 #endif
     #if MICROPY_EMIT_INLINE_ASM
+    #if MICROPY_DYNAMIC_COMPILER
+    } else if (attr == MP_QSTR_asm_thumb) {
+        *emit_options = MP_EMIT_OPT_ASM;
+    } else if (attr == MP_QSTR_asm_xtensa) {
+        *emit_options = MP_EMIT_OPT_ASM;
+    #else
     } else if (attr == ASM_DECORATOR_QSTR) {
         *emit_options = MP_EMIT_OPT_ASM;
     #endif
+    #endif
     } else {
         compile_syntax_error(comp, name_nodes[1], "invalid micropython decorator");
     }
 
+    #if MICROPY_DYNAMIC_COMPILER
+    if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) {
+        if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) {
+            compile_syntax_error(comp, name_nodes[1], "invalid arch");
+        }
+    } else if (*emit_options == MP_EMIT_OPT_ASM) {
+        if (emit_asm_table[mp_dynamic_compiler.native_arch] == NULL) {
+            compile_syntax_error(comp, name_nodes[1], "invalid arch");
+        }
+    }
+    #endif
+
     return true;
 }
 
@@ -965,16 +1028,13 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
 
 STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
     uint16_t label;
-    const char *error_msg;
     if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) {
         label = comp->break_label;
-        error_msg = "'break' outside loop";
     } else {
         label = comp->continue_label;
-        error_msg = "'continue' outside loop";
     }
     if (label == INVALID_LABEL) {
-        compile_syntax_error(comp, (mp_parse_node_t)pns, error_msg);
+        compile_syntax_error(comp, (mp_parse_node_t)pns, "'break'/'continue' outside loop");
     }
     assert(comp->cur_except_level >= comp->break_continue_except_level);
     EMIT_ARG(unwind_jump, label, comp->cur_except_level - comp->break_continue_except_level);
@@ -1137,6 +1197,13 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) {
     } while (0);
 
     if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[1], MP_TOKEN_OP_STAR)) {
+        #if MICROPY_CPYTHON_COMPAT
+        if (comp->scope_cur->kind != SCOPE_MODULE) {
+            compile_syntax_error(comp, (mp_parse_node_t)pns, "import * not at module level");
+            return;
+        }
+        #endif
+
         EMIT_ARG(load_const_small_int, import_level);
 
         // build the "fromlist" tuple
@@ -1146,7 +1213,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) {
         // do the import
         qstr dummy_q;
         do_import_name(comp, pn_import_source, &dummy_q);
-        EMIT_ARG(import, MP_QSTR_NULL, MP_EMIT_IMPORT_STAR);
+        EMIT_ARG(import, MP_QSTRnull, MP_EMIT_IMPORT_STAR);
 
     } else {
         EMIT_ARG(load_const_small_int, import_level);
@@ -1534,8 +1601,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
     compile_increase_except_level(comp, l1, MP_EMIT_SETUP_BLOCK_EXCEPT);
 
     compile_node(comp, pn_body); // body
-    EMIT(pop_block);
-    EMIT_ARG(jump, success_label); // jump over exception handler
+    EMIT_ARG(pop_except_jump, success_label, false); // jump over exception handler
 
     EMIT_ARG(label_assign, l1); // start of exception handler
     EMIT(start_except_handler);
@@ -1550,6 +1616,9 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
 
         qstr qstr_exception_local = 0;
         uint end_finally_label = comp_next_label(comp);
+        #if MICROPY_PY_SYS_SETTRACE
+        EMIT_ARG(set_source_line, pns_except->source_line);
+        #endif
 
         if (MP_PARSE_NODE_IS_NULL(pns_except->nodes[0])) {
             // this is a catch all exception handler
@@ -1598,7 +1667,6 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
         }
         compile_node(comp, pns_except->nodes[1]); // the <body>
         if (qstr_exception_local != 0) {
-            EMIT(pop_block);
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
             EMIT_ARG(label_assign, l3);
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
@@ -1607,8 +1675,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
             compile_decrease_except_level(comp);
         }
 
-        EMIT(pop_except);
-        EMIT_ARG(jump, l2);
+        EMIT_ARG(pop_except_jump, l2, true);
         EMIT_ARG(label_assign, end_finally_label);
         EMIT_ARG(adjust_stack_size, 1); // stack adjust for the exception instance
     }
@@ -1634,7 +1701,6 @@ STATIC void compile_try_finally(compiler_t *comp, mp_parse_node_t pn_body, int n
     } else {
         compile_try_except(comp, pn_body, n_except, pn_except, pn_else);
     }
-    EMIT(pop_block);
     EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
     EMIT_ARG(label_assign, l_finally_block);
     compile_node(comp, pn_finally);
@@ -1742,8 +1808,7 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns
     compile_load_id(comp, context);
     compile_await_object_method(comp, MP_QSTR___anext__);
     c_assign(comp, pns->nodes[0], ASSIGN_STORE); // variable
-    EMIT(pop_block);
-    EMIT_ARG(jump, try_else_label);
+    EMIT_ARG(pop_except_jump, try_else_label, false);
 
     EMIT_ARG(label_assign, try_exception_label);
     EMIT(start_except_handler);
@@ -1752,8 +1817,7 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns
     EMIT_ARG(binary_op, MP_BINARY_OP_EXCEPTION_MATCH);
     EMIT_ARG(pop_jump_if, false, try_finally_label);
     EMIT(pop_top); // pop exception instance
-    EMIT(pop_except);
-    EMIT_ARG(jump, while_else_label);
+    EMIT_ARG(pop_except_jump, while_else_label, true);
 
     EMIT_ARG(label_assign, try_finally_label);
     EMIT_ARG(adjust_stack_size, 1); // if we jump here, the exc is on the stack
@@ -1810,8 +1874,7 @@ STATIC void compile_async_with_stmt_helper(compiler_t *comp, int n, mp_parse_nod
         compile_async_with_stmt_helper(comp, n - 1, nodes + 1, body);
         EMIT_ARG(adjust_stack_size, -3);
 
-        // Finish the "try" block
-        EMIT(pop_block);
+        // We have now finished the "try" block and fall through to the "finally"
 
         // At this point, after the with body has executed, we have 3 cases:
         // 1. no exception, we just fall through to this point; stack: (..., ctx_mgr)
@@ -1941,21 +2004,8 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
             c_assign(comp, pns->nodes[0], ASSIGN_AUG_LOAD); // lhs load for aug assign
             compile_node(comp, pns1->nodes[1]); // rhs
             assert(MP_PARSE_NODE_IS_TOKEN(pns1->nodes[0]));
-            mp_binary_op_t op;
-            switch (MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])) {
-                case MP_TOKEN_DEL_PIPE_EQUAL: op = MP_BINARY_OP_INPLACE_OR; break;
-                case MP_TOKEN_DEL_CARET_EQUAL: op = MP_BINARY_OP_INPLACE_XOR; break;
-                case MP_TOKEN_DEL_AMPERSAND_EQUAL: op = MP_BINARY_OP_INPLACE_AND; break;
-                case MP_TOKEN_DEL_DBL_LESS_EQUAL: op = MP_BINARY_OP_INPLACE_LSHIFT; break;
-                case MP_TOKEN_DEL_DBL_MORE_EQUAL: op = MP_BINARY_OP_INPLACE_RSHIFT; break;
-                case MP_TOKEN_DEL_PLUS_EQUAL: op = MP_BINARY_OP_INPLACE_ADD; break;
-                case MP_TOKEN_DEL_MINUS_EQUAL: op = MP_BINARY_OP_INPLACE_SUBTRACT; break;
-                case MP_TOKEN_DEL_STAR_EQUAL: op = MP_BINARY_OP_INPLACE_MULTIPLY; break;
-                case MP_TOKEN_DEL_DBL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_FLOOR_DIVIDE; break;
-                case MP_TOKEN_DEL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_TRUE_DIVIDE; break;
-                case MP_TOKEN_DEL_PERCENT_EQUAL: op = MP_BINARY_OP_INPLACE_MODULO; break;
-                case MP_TOKEN_DEL_DBL_STAR_EQUAL: default: op = MP_BINARY_OP_INPLACE_POWER; break;
-            }
+            mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]);
+            mp_binary_op_t op = MP_BINARY_OP_INPLACE_OR + (tok - MP_TOKEN_DEL_PIPE_EQUAL);
             EMIT_ARG(binary_op, op);
             c_assign(comp, pns->nodes[0], ASSIGN_AUG_STORE); // lhs store for aug assign
         } else if (kind == PN_expr_stmt_assign_list) {
@@ -2089,15 +2139,12 @@ STATIC void compile_comparison(compiler_t *comp, mp_parse_node_struct_t *pns) {
             EMIT(rot_three);
         }
         if (MP_PARSE_NODE_IS_TOKEN(pns->nodes[i])) {
+            mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]);
             mp_binary_op_t op;
-            switch (MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])) {
-                case MP_TOKEN_OP_LESS: op = MP_BINARY_OP_LESS; break;
-                case MP_TOKEN_OP_MORE: op = MP_BINARY_OP_MORE; break;
-                case MP_TOKEN_OP_DBL_EQUAL: op = MP_BINARY_OP_EQUAL; break;
-                case MP_TOKEN_OP_LESS_EQUAL: op = MP_BINARY_OP_LESS_EQUAL; break;
-                case MP_TOKEN_OP_MORE_EQUAL: op = MP_BINARY_OP_MORE_EQUAL; break;
-                case MP_TOKEN_OP_NOT_EQUAL: op = MP_BINARY_OP_NOT_EQUAL; break;
-                case MP_TOKEN_KW_IN: default: op = MP_BINARY_OP_IN; break;
+            if (tok == MP_TOKEN_KW_IN) {
+                op = MP_BINARY_OP_IN;
+            } else {
+                op = MP_BINARY_OP_LESS + (tok - MP_TOKEN_OP_LESS);
             }
             EMIT_ARG(binary_op, op);
         } else {
@@ -2151,36 +2198,21 @@ STATIC void compile_term(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_node(comp, pns->nodes[0]);
     for (int i = 1; i + 1 < num_nodes; i += 2) {
         compile_node(comp, pns->nodes[i + 1]);
-        mp_binary_op_t op;
         mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]);
-        switch (tok) {
-            case MP_TOKEN_OP_PLUS:      op = MP_BINARY_OP_ADD; break;
-            case MP_TOKEN_OP_MINUS:     op = MP_BINARY_OP_SUBTRACT; break;
-            case MP_TOKEN_OP_STAR:      op = MP_BINARY_OP_MULTIPLY; break;
-            case MP_TOKEN_OP_DBL_SLASH: op = MP_BINARY_OP_FLOOR_DIVIDE; break;
-            case MP_TOKEN_OP_SLASH:     op = MP_BINARY_OP_TRUE_DIVIDE; break;
-            case MP_TOKEN_OP_PERCENT:   op = MP_BINARY_OP_MODULO; break;
-            case MP_TOKEN_OP_DBL_LESS:  op = MP_BINARY_OP_LSHIFT; break;
-            default:
-                assert(tok == MP_TOKEN_OP_DBL_MORE);
-                op = MP_BINARY_OP_RSHIFT;
-                break;
-        }
+        mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS);
         EMIT_ARG(binary_op, op);
     }
 }
 
 STATIC void compile_factor_2(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_node(comp, pns->nodes[1]);
-    mp_unary_op_t op;
     mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
-    switch (tok) {
-        case MP_TOKEN_OP_PLUS:  op = MP_UNARY_OP_POSITIVE; break;
-        case MP_TOKEN_OP_MINUS: op = MP_UNARY_OP_NEGATIVE; break;
-        default:
-            assert(tok == MP_TOKEN_OP_TILDE);
-            op = MP_UNARY_OP_INVERT;
-            break;
+    mp_unary_op_t op;
+    if (tok == MP_TOKEN_OP_TILDE) {
+        op = MP_UNARY_OP_INVERT;
+    } else {
+        assert(tok == MP_TOKEN_OP_PLUS || tok == MP_TOKEN_OP_MINUS);
+        op = MP_UNARY_OP_POSITIVE + (tok - MP_TOKEN_OP_PLUS);
     }
     EMIT_ARG(unary_op, op);
 }
@@ -2247,6 +2279,20 @@ STATIC void compile_atom_expr_normal(compiler_t *comp, mp_parse_node_struct_t *p
             EMIT_ARG(call_function, 2, 0, 0);
             i = 1;
         }
+
+    #if MICROPY_COMP_CONST_LITERAL && MICROPY_PY_COLLECTIONS_ORDEREDDICT
+    // handle special OrderedDict constructor
+    } else if (MP_PARSE_NODE_IS_ID(pns->nodes[0])
+        && MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]) == MP_QSTR_OrderedDict
+        && MP_PARSE_NODE_STRUCT_KIND(pns_trail[0]) == PN_trailer_paren
+        && MP_PARSE_NODE_IS_STRUCT_KIND(pns_trail[0]->nodes[0], PN_atom_brace)) {
+        // at this point we have matched "OrderedDict({...})"
+
+        EMIT_ARG(call_function, 0, 0, 0);
+        mp_parse_node_struct_t *pns_dict = (mp_parse_node_struct_t*)pns_trail[0]->nodes[0];
+        compile_atom_brace_helper(comp, pns_dict, false);
+        i = 1;
+    #endif
     }
 
     // compile the remaining trailers
@@ -2455,16 +2501,20 @@ STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns)
     }
 }
 
-STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
+STATIC void compile_atom_brace_helper(compiler_t *comp, mp_parse_node_struct_t *pns, bool create_map) {
     mp_parse_node_t pn = pns->nodes[0];
     if (MP_PARSE_NODE_IS_NULL(pn)) {
         // empty dict
-        EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP);
+        if (create_map) {
+            EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP);
+        }
     } else if (MP_PARSE_NODE_IS_STRUCT(pn)) {
         pns = (mp_parse_node_struct_t*)pn;
         if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker_item) {
             // dict with one element
-            EMIT_ARG(build, 1, MP_EMIT_BUILD_MAP);
+            if (create_map) {
+                EMIT_ARG(build, 1, MP_EMIT_BUILD_MAP);
+            }
             compile_node(comp, pn);
             EMIT(store_map);
         } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker) {
@@ -2481,7 +2531,9 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
                 bool is_dict;
                 if (!MICROPY_PY_BUILTINS_SET || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_dictorsetmaker_item)) {
                     // a dictionary
-                    EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_MAP);
+                    if (create_map) {
+                        EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_MAP);
+                    }
                     compile_node(comp, pns->nodes[0]);
                     EMIT(store_map);
                     is_dict = true;
@@ -2551,6 +2603,10 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
     }
 }
 
+STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
+    compile_atom_brace_helper(comp, pns, true);
+}
+
 STATIC void compile_trailer_paren(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_trailer_paren_helper(comp, pns->nodes[0], false, 0);
 }
@@ -2675,9 +2731,6 @@ STATIC mp_obj_t get_const_object(mp_parse_node_struct_t *pns) {
 }
 
 STATIC void compile_const_object(compiler_t *comp, mp_parse_node_struct_t *pns) {
-    #if MICROPY_EMIT_NATIVE
-    comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-    #endif
     EMIT_ARG(load_const_obj, get_const_object(pns));
 }
 
@@ -2712,9 +2765,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
             } else {
                 EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg));
             }
-            #if MICROPY_EMIT_NATIVE
-            comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-            #endif
         }
         #else
         EMIT_ARG(load_const_small_int, arg);
@@ -2733,9 +2783,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
                     const byte *data = qstr_data(arg, &len);
                     EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len));
                 }
-                #if MICROPY_EMIT_NATIVE
-                comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-                #endif
                 break;
             case MP_PARSE_NODE_TOKEN: default:
                 if (arg == MP_TOKEN_NEWLINE) {
@@ -2784,7 +2831,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn
         return;
     }
 
-    qstr param_name = MP_QSTR_NULL;
+    qstr param_name = MP_QSTRnull;
     uint param_flag = ID_FLAG_IS_PARAM;
     mp_parse_node_struct_t *pns = NULL;
     if (MP_PARSE_NODE_IS_ID(pn)) {
@@ -2843,7 +2890,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn
         }
     }
 
-    if (param_name != MP_QSTR_NULL) {
+    if (param_name != MP_QSTRnull) {
         id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, param_name, ID_INFO_KIND_UNDECIDED);
         if (id_info->kind != ID_INFO_KIND_UNDECIDED) {
             compile_syntax_error(comp, pn, "argument name reused");
@@ -3011,6 +3058,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
         mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)scope->pn;
         assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 3);
 
+        // Set the source line number for the start of the lambda
+        EMIT_ARG(set_source_line, pns->source_line);
+
         // work out number of parameters, keywords and default parameters, and add them to the id_info array
         // must be done before compiling the body so that arguments are numbered first (for LOAD_FAST etc)
         if (comp->pass == MP_PASS_SCOPE) {
@@ -3045,6 +3095,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
             scope->num_pos_args = 1;
         }
 
+        // Set the source line number for the start of the comprehension
+        EMIT_ARG(set_source_line, pns->source_line);
+
         if (scope->kind == SCOPE_LIST_COMP) {
             EMIT_ARG(build, 0, MP_EMIT_BUILD_LIST);
         } else if (scope->kind == SCOPE_DICT_COMP) {
@@ -3084,6 +3137,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
             scope_find_or_add_id(scope, MP_QSTR___class__, ID_INFO_KIND_LOCAL);
         }
 
+        #if MICROPY_PY_SYS_SETTRACE
+        EMIT_ARG(set_source_line, pns->source_line);
+        #endif
         compile_load_id(comp, MP_QSTR___name__);
         compile_store_id(comp, MP_QSTR___module__);
         EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pns->nodes[0])); // 0 is class name
@@ -3257,7 +3313,11 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind
             void *f = mp_asm_base_get_code((mp_asm_base_t*)comp->emit_inline_asm);
             mp_emit_glue_assign_native(comp->scope_cur->raw_code, MP_CODE_NATIVE_ASM,
                 f, mp_asm_base_get_code_size((mp_asm_base_t*)comp->emit_inline_asm),
-                NULL, comp->scope_cur->num_pos_args, 0, type_sig);
+                NULL,
+                #if MICROPY_PERSISTENT_CODE_SAVE
+                0, 0, 0, 0, NULL,
+                #endif
+                comp->scope_cur->num_pos_args, 0, type_sig);
         }
     }
 
@@ -3363,7 +3423,7 @@ STATIC void scope_compute_things(scope_t *scope) {
 #if !MICROPY_PERSISTENT_CODE_SAVE
 STATIC
 #endif
-mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
+mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) {
     // put compiler state on the stack, it's relatively small
     compiler_t comp_state = {0};
     compiler_t *comp = &comp_state;
@@ -3374,6 +3434,11 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     comp->continue_label = INVALID_LABEL;
 
     // create the module scope
+    #if MICROPY_EMIT_NATIVE
+    const uint emit_opt = MP_STATE_VM(default_emit_opt);
+    #else
+    const uint emit_opt = MP_EMIT_OPT_NONE;
+    #endif
     scope_t *module_scope = scope_new_and_link(comp, SCOPE_MODULE, parse_tree->root, emit_opt);
 
     // create standard emitter; it's used at least for MP_PASS_SCOPE
@@ -3386,12 +3451,12 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     #endif
     uint max_num_labels = 0;
     for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
-        if (false) {
         #if MICROPY_EMIT_INLINE_ASM
-        } else if (s->emit_options == MP_EMIT_OPT_ASM) {
+        if (s->emit_options == MP_EMIT_OPT_ASM) {
             compile_scope_inline_asm(comp, s, MP_PASS_SCOPE);
+        } else
         #endif
-        } else {
+        {
             compile_scope(comp, s, MP_PASS_SCOPE);
 
             // Check if any implicitly declared variables should be closed over
@@ -3422,30 +3487,32 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     emit_t *emit_native = NULL;
 #endif
     for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
-        if (false) {
-            // dummy
-
         #if MICROPY_EMIT_INLINE_ASM
-        } else if (s->emit_options == MP_EMIT_OPT_ASM) {
+        if (s->emit_options == MP_EMIT_OPT_ASM) {
             // inline assembly
             if (comp->emit_inline_asm == NULL) {
                 comp->emit_inline_asm = ASM_EMITTER(new)(max_num_labels);
             }
             comp->emit = NULL;
-            comp->emit_inline_asm_method_table = &ASM_EMITTER(method_table);
+            comp->emit_inline_asm_method_table = ASM_EMITTER_TABLE;
             compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
             #if MICROPY_EMIT_INLINE_XTENSA
             // Xtensa requires an extra pass to compute size of l32r const table
             // TODO this can be improved by calculating it during SCOPE pass
             // but that requires some other structural changes to the asm emitters
-            compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
+            #if MICROPY_DYNAMIC_COMPILER
+            if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSA)
+            #endif
+            {
+                compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
+            }
             #endif
             if (comp->compile_error == MP_OBJ_NULL) {
                 compile_scope_inline_asm(comp, s, MP_PASS_EMIT);
             }
+        } else
         #endif
-
-        } else {
+        {
 
             // choose the emit type
 
@@ -3457,7 +3524,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
                     if (emit_native == NULL) {
                         emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels);
                     }
-                    comp->emit_method_table = &NATIVE_EMITTER(method_table);
+                    comp->emit_method_table = NATIVE_EMITTER_TABLE;
                     comp->emit = emit_native;
                     break;
 #endif // MICROPY_EMIT_NATIVE
@@ -3526,8 +3593,8 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     }
 }
 
-mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
-    mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl);
+mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) {
+    mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl);
     // return function that executes the outer module
     return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
 }

py/compile.h

@@ -32,11 +32,11 @@
 
 // the compiler will raise an exception if an error occurred
 // the compiler will clear the parse tree before it returns
-mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl);
+mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl);
 
 #if MICROPY_PERSISTENT_CODE_SAVE
 // this has the same semantics as mp_compile
-mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl);
+mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl);
 #endif
 
 // this is implemented in runtime.c

py/dynruntime.h

@@ -0,0 +1,218 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H
+#define MICROPY_INCLUDED_PY_DYNRUNTIME_H
+
+// This header file contains definitions to dynamically implement the static
+// MicroPython runtime API defined in py/obj.h and py/runtime.h.
+
+#include "py/nativeglue.h"
+#include "py/objstr.h"
+
+#undef MP_ROM_QSTR
+#undef MP_OBJ_QSTR_VALUE
+#undef MP_OBJ_NEW_QSTR
+#undef mp_const_none
+#undef mp_const_false
+#undef mp_const_true
+#undef mp_const_empty_tuple
+#undef nlr_raise
+
+/******************************************************************************/
+// Memory allocation
+
+#define m_malloc(n)                     (m_malloc_dyn((n)))
+#define m_free(ptr)                     (m_free_dyn((ptr)))
+#define m_realloc(ptr, new_num_bytes)   (m_realloc_dyn((ptr), (new_num_bytes)))
+
+static inline void *m_malloc_dyn(size_t n) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(NULL, n, false);
+}
+
+static inline void m_free_dyn(void *ptr) {
+    mp_fun_table.realloc_(ptr, 0, false);
+}
+
+static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(ptr, new_num_bytes, true);
+}
+
+/******************************************************************************/
+// Printing
+
+#define mp_plat_print               (*mp_fun_table.plat_print)
+#define mp_printf(p, ...)           (mp_fun_table.printf_((p), __VA_ARGS__))
+#define mp_vprintf(p, fmt, args)    (mp_fun_table.vprintf_((p), (fmt), (args)))
+
+/******************************************************************************/
+// Types and objects
+
+#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x
+
+#define mp_type_type                        (*mp_fun_table.type_type)
+#define mp_type_str                         (*mp_fun_table.type_str)
+#define mp_type_list                        (*mp_fun_table.type_list)
+#define mp_type_EOFError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError)))
+#define mp_type_IndexError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError)))
+#define mp_type_KeyError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError)))
+#define mp_type_NotImplementedError         (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError)))
+#define mp_type_RuntimeError                (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError)))
+#define mp_type_TypeError                   (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError)))
+#define mp_type_ValueError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError)))
+
+#define mp_stream_read_obj                  (*mp_fun_table.stream_read_obj)
+#define mp_stream_readinto_obj              (*mp_fun_table.stream_readinto_obj)
+#define mp_stream_unbuffered_readline_obj   (*mp_fun_table.stream_unbuffered_readline_obj)
+#define mp_stream_write_obj                 (*mp_fun_table.stream_write_obj)
+
+#define mp_const_none                       ((mp_obj_t)mp_fun_table.const_none)
+#define mp_const_false                      ((mp_obj_t)mp_fun_table.const_false)
+#define mp_const_true                       ((mp_obj_t)mp_fun_table.const_true)
+#define mp_const_empty_tuple                (mp_fun_table.new_tuple(0, NULL))
+
+#define mp_obj_new_bool(b)                  ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false)
+#define mp_obj_new_int(i)                   (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT))
+#define mp_obj_new_int_from_uint(i)         (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_UINT))
+#define mp_obj_new_str(data, len)           (mp_fun_table.obj_new_str((data), (len)))
+#define mp_obj_new_str_of_type(t, d, l)     (mp_obj_new_str_of_type_dyn((t), (d), (l)))
+#define mp_obj_new_bytes(data, len)         (mp_fun_table.obj_new_bytes((data), (len)))
+#define mp_obj_new_bytearray_by_ref(n, i)   (mp_fun_table.obj_new_bytearray_by_ref((n), (i)))
+#define mp_obj_new_tuple(n, items)          (mp_fun_table.new_tuple((n), (items)))
+#define mp_obj_new_list(n, items)           (mp_fun_table.new_list((n), (items)))
+
+#define mp_obj_get_type(o)                  (mp_fun_table.obj_get_type((o)))
+#define mp_obj_get_int(o)                   (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT))
+#define mp_obj_get_int_truncated(o)         (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_UINT))
+#define mp_obj_str_get_str(s)               ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR))
+#define mp_obj_str_get_data(o, len)         (mp_obj_str_get_data_dyn((o), (len)))
+#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
+#define mp_get_stream_raise(s, flags)       (mp_fun_table.get_stream_raise((s), (flags)))
+
+#define mp_obj_len(o)                       (mp_obj_len_dyn(o))
+#define mp_obj_subscr(base, index, val)     (mp_fun_table.obj_subscr((base), (index), (val)))
+#define mp_obj_list_append(list, item)      (mp_fun_table.list_append((list), (item)))
+
+static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) {
+    if (type == &mp_type_str) {
+        return mp_obj_new_str((const char*)data, len);
+    } else {
+        return mp_obj_new_bytes(data, len);
+    }
+}
+
+static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ);
+    *l = bufinfo.len;
+    return bufinfo.buf;
+}
+
+static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) {
+    // If bytes implemented MP_UNARY_OP_LEN could use: mp_unary_op(MP_UNARY_OP_LEN, o)
+    return mp_fun_table.call_function_n_kw(mp_fun_table.load_name(MP_QSTR_len), 1, &o);
+}
+
+/******************************************************************************/
+// General runtime functions
+
+#define mp_load_name(qst)           (mp_fun_table.load_name(qst))
+#define mp_load_global(qst)         (mp_fun_table.load_global(qst))
+#define mp_store_global(qst, obj)   (mp_fun_table.store_global((qst), (obj)))
+#define mp_unary_op(op, obj)        (mp_fun_table.unary_op((op), (obj)))
+#define mp_binary_op(op, lhs, rhs)  (mp_fun_table.binary_op((op), (lhs), (rhs)))
+
+#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \
+    (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args)))
+
+#define mp_call_function_n_kw(fun, n_args, n_kw, args) \
+    (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args))
+
+#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \
+    (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw))))
+
+#define MP_DYNRUNTIME_INIT_ENTRY \
+    mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \
+    mp_raw_code_t rc; \
+    rc.kind = MP_CODE_NATIVE_VIPER; \
+    rc.scope_flags = 0; \
+    rc.const_table = (void*)self->const_table; \
+    (void)rc;
+
+#define MP_DYNRUNTIME_INIT_EXIT \
+    mp_fun_table.swap_globals(old_globals); \
+    return mp_const_none;
+
+#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \
+    (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL))
+
+/******************************************************************************/
+// Exceptions
+
+#define mp_obj_new_exception(o)                 ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then
+#define mp_obj_new_exception_arg1(e_type, arg)  (mp_obj_new_exception_arg1_dyn((e_type), (arg)))
+
+#define nlr_raise(o)                            (mp_raise_dyn(o))
+#define mp_raise_msg(type, msg)                 (mp_fun_table.raise_msg((type), (msg)))
+#define mp_raise_OSError(er)                    (mp_raise_OSError_dyn(er))
+#define mp_raise_NotImplementedError(msg)       (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
+#define mp_raise_TypeError(msg)                 (mp_raise_msg(&mp_type_TypeError, (msg)))
+#define mp_raise_ValueError(msg)                (mp_raise_msg(&mp_type_ValueError, (msg)))
+
+static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) {
+    mp_obj_t args[1] = { arg };
+    return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]);
+}
+
+static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
+    mp_fun_table.raise(o);
+    for (;;) {
+    }
+}
+
+static inline void mp_raise_OSError_dyn(int er) {
+    mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) };
+    nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0]));
+}
+
+/******************************************************************************/
+// Floating point
+
+#define mp_obj_new_float_from_f(f)  (mp_fun_table.obj_new_float_from_f((f)))
+#define mp_obj_new_float_from_d(d)  (mp_fun_table.obj_new_float_from_d((d)))
+#define mp_obj_get_float_to_f(o)    (mp_fun_table.obj_get_float_to_f((o)))
+#define mp_obj_get_float_to_d(o)    (mp_fun_table.obj_get_float_to_d((o)))
+
+#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT
+#define mp_obj_new_float(f)         (mp_obj_new_float_from_f((f)))
+#define mp_obj_get_float(o)         (mp_obj_get_float_to_f((o)))
+#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE
+#define mp_obj_new_float(f)         (mp_obj_new_float_from_d((f)))
+#define mp_obj_get_float(o)         (mp_obj_get_float_to_d((o)))
+#endif
+
+#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H

py/dynruntime.mk

@@ -0,0 +1,144 @@
+# Makefile fragment for generating native .mpy files from C source
+# MPY_DIR must be set to the top of the MicroPython source tree
+
+BUILD ?= build
+
+ECHO = @echo
+RM = /bin/rm
+MKDIR = /bin/mkdir
+PYTHON = python3
+MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross
+MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py
+MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py
+
+Q = @
+ifeq ("$(origin V)", "command line")
+ifeq ($(V),1)
+Q =
+MPY_LD += '-vvv'
+endif
+endif
+
+ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]')
+CONFIG_H = $(BUILD)/$(MOD).config.h
+
+CFLAGS += -I. -I$(MPY_DIR)
+CFLAGS += -std=c99
+CFLAGS += -Os
+CFLAGS += -Wall -Werror -DNDEBUG
+CFLAGS += -DNO_QSTR
+CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>'
+CFLAGS += -fpic -fno-common
+CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions
+#CFLAGS += -fdata-sections -ffunction-sections
+
+MPY_CROSS_FLAGS += -march=$(ARCH)
+
+SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))))
+SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC))))
+
+################################################################################
+# Architecture configuration
+
+ifeq ($(ARCH),x86)
+
+# x86
+CROSS =
+CFLAGS += -m32 -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),x64)
+
+# x64
+CROSS =
+CFLAGS += -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),armv7m)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m3
+MICROPY_FLOAT_IMPL ?= none
+
+else ifeq ($(ARCH),armv7emsp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m4
+CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard
+MICROPY_FLOAT_IMPL ?= float
+
+else ifeq ($(ARCH),armv7emdp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m7
+CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),xtensa)
+
+# xtensa
+CROSS = xtensa-lx106-elf-
+CFLAGS += -mforce-l32
+MICROPY_FLOAT_IMPL ?= none
+
+else ifeq ($(ARCH),xtensawin)
+
+# xtensawin
+CROSS = xtensa-esp32-elf-
+CFLAGS +=
+MICROPY_FLOAT_IMPL ?= float
+
+else
+$(error architecture '$(ARCH)' not supported)
+endif
+
+MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]')
+CFLAGS += -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER)
+
+CFLAGS += $(CFLAGS_EXTRA)
+
+################################################################################
+# Build rules
+
+.PHONY: all clean
+
+all: $(MOD).mpy
+
+clean:
+	$(RM) -rf $(BUILD) $(CLEAN_EXTRA)
+
+# Create build destination directories first
+BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY)))
+$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS)
+$(BUILD_DIRS):
+	$(Q)$(MKDIR) -p $@
+
+# Preprocess all source files to generate $(CONFIG_H)
+$(CONFIG_H): $(SRC)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^
+
+# Build .o from .c source files
+$(BUILD)/%.o: %.c $(CONFIG_H) Makefile
+	$(ECHO) "CC $<"
+	$(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $<
+
+# Build .mpy from .py source files
+$(BUILD)/%.mpy: %.py
+	$(ECHO) "MPY $<"
+	$(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $<
+
+# Build native .mpy from object files
+$(BUILD)/$(MOD).native.mpy: $(SRC_O)
+	$(ECHO) "LINK $<"
+	$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^
+
+# Build final .mpy from all intermediate .mpy files
+$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_TOOL) --merge -o $@ $^

py/emit.h

@@ -76,15 +76,15 @@ typedef enum {
 
 // Kind for emit->setup_block()
 #define MP_EMIT_SETUP_BLOCK_WITH (0)
-#define MP_EMIT_SETUP_BLOCK_EXCEPT (2)
-#define MP_EMIT_SETUP_BLOCK_FINALLY (3)
+#define MP_EMIT_SETUP_BLOCK_EXCEPT (1)
+#define MP_EMIT_SETUP_BLOCK_FINALLY (2)
 
 // Kind for emit->build()
 #define MP_EMIT_BUILD_TUPLE (0)
 #define MP_EMIT_BUILD_LIST (1)
-#define MP_EMIT_BUILD_MAP (3)
-#define MP_EMIT_BUILD_SET (6)
-#define MP_EMIT_BUILD_SLICE (8)
+#define MP_EMIT_BUILD_MAP (2)
+#define MP_EMIT_BUILD_SET (3)
+#define MP_EMIT_BUILD_SLICE (4)
 
 // Kind for emit->yield()
 #define MP_EMIT_YIELD_VALUE (0)
@@ -98,6 +98,11 @@ typedef struct _mp_emit_method_table_id_ops_t {
 } mp_emit_method_table_id_ops_t;
 
 typedef struct _emit_method_table_t {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_t *(*emit_new)(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
+    void (*emit_free)(emit_t *emit);
+    #endif
+
     void (*start_pass)(emit_t *emit, pass_kind_t pass, scope_t *scope);
     void (*end_pass)(emit_t *emit);
     bool (*last_emit_was_return_value)(emit_t *emit);
@@ -134,8 +139,7 @@ typedef struct _emit_method_table_t {
     void (*get_iter)(emit_t *emit, bool use_stack);
     void (*for_iter)(emit_t *emit, mp_uint_t label);
     void (*for_iter_end)(emit_t *emit);
-    void (*pop_block)(emit_t *emit);
-    void (*pop_except)(emit_t *emit);
+    void (*pop_except_jump)(emit_t *emit, mp_uint_t label, bool within_exc_handler);
     void (*unary_op)(emit_t *emit, mp_unary_op_t op);
     void (*binary_op)(emit_t *emit, mp_binary_op_t op);
     void (*build)(emit_t *emit, mp_uint_t n_args, int kind);
@@ -157,8 +161,6 @@ typedef struct _emit_method_table_t {
     void (*end_except_handler)(emit_t *emit);
 } emit_method_table_t;
 
-int mp_native_type_from_qstr(qstr qst);
-
 static inline void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) {
     scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT);
 }
@@ -172,6 +174,7 @@ extern const emit_method_table_t emit_native_x86_method_table;
 extern const emit_method_table_t emit_native_thumb_method_table;
 extern const emit_method_table_t emit_native_arm_method_table;
 extern const emit_method_table_t emit_native_xtensa_method_table;
+extern const emit_method_table_t emit_native_xtensawin_method_table;
 
 extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops;
 extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops;
@@ -183,6 +186,7 @@ emit_t *emit_native_x86_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t ma
 emit_t *emit_native_thumb_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 emit_t *emit_native_arm_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 emit_t *emit_native_xtensa_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
+emit_t *emit_native_xtensawin_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 
 void emit_bc_set_max_num_labels(emit_t* emit, mp_uint_t max_num_labels);
 
@@ -192,6 +196,7 @@ void emit_native_x86_free(emit_t *emit);
 void emit_native_thumb_free(emit_t *emit);
 void emit_native_arm_free(emit_t *emit);
 void emit_native_xtensa_free(emit_t *emit);
+void emit_native_xtensawin_free(emit_t *emit);
 
 void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope);
 void mp_emit_bc_end_pass(emit_t *emit);
@@ -232,8 +237,7 @@ void mp_emit_bc_end_finally(emit_t *emit);
 void mp_emit_bc_get_iter(emit_t *emit, bool use_stack);
 void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label);
 void mp_emit_bc_for_iter_end(emit_t *emit);
-void mp_emit_bc_pop_block(emit_t *emit);
-void mp_emit_bc_pop_except(emit_t *emit);
+void mp_emit_bc_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler);
 void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op);
 void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op);
 void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind);
@@ -254,6 +258,11 @@ void mp_emit_bc_end_except_handler(emit_t *emit);
 typedef struct _emit_inline_asm_t emit_inline_asm_t;
 
 typedef struct _emit_inline_asm_method_table_t {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_asm_t *(*asm_new)(mp_uint_t max_num_labels);
+    void (*asm_free)(emit_inline_asm_t *emit);
+    #endif
+
     void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, mp_obj_t *error_slot);
     void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
     mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params);

py/emitbc.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -64,6 +64,9 @@ struct _emit_t {
     size_t bytecode_size;
     byte *code_base; // stores both byte code and code info
 
+    size_t n_info;
+    size_t n_cell;
+
     #if MICROPY_PERSISTENT_CODE
     uint16_t ct_cur_obj;
     uint16_t ct_num_obj;
@@ -123,10 +126,6 @@ STATIC void emit_write_code_info_byte(emit_t* emit, byte val) {
     *emit_get_cur_to_write_code_info(emit, 1) = val;
 }
 
-STATIC void emit_write_code_info_uint(emit_t* emit, mp_uint_t val) {
-    emit_write_uint(emit, emit_get_cur_to_write_code_info, val);
-}
-
 STATIC void emit_write_code_info_qstr(emit_t *emit, qstr qst) {
     #if MICROPY_PERSISTENT_CODE
     assert((qst >> 16) == 0);
@@ -182,20 +181,20 @@ STATIC byte *emit_get_cur_to_write_bytecode(emit_t *emit, int num_bytes_to_write
     }
 }
 
-STATIC void emit_write_bytecode_byte(emit_t *emit, byte b1) {
+STATIC void emit_write_bytecode_raw_byte(emit_t *emit, byte b1) {
     byte *c = emit_get_cur_to_write_bytecode(emit, 1);
     c[0] = b1;
 }
 
-STATIC void emit_write_bytecode_byte_byte(emit_t* emit, byte b1, byte b2) {
-    byte *c = emit_get_cur_to_write_bytecode(emit, 2);
+STATIC void emit_write_bytecode_byte(emit_t *emit, int stack_adj, byte b1) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
+    byte *c = emit_get_cur_to_write_bytecode(emit, 1);
     c[0] = b1;
-    c[1] = b2;
 }
 
 // Similar to emit_write_bytecode_uint(), just some extra handling to encode sign
-STATIC void emit_write_bytecode_byte_int(emit_t *emit, byte b1, mp_int_t num) {
-    emit_write_bytecode_byte(emit, b1);
+STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, mp_int_t num) {
+    emit_write_bytecode_byte(emit, stack_adj, b1);
 
     // We store each 7 bits in a separate byte, and that's how many bytes needed
     byte buf[BYTES_FOR_INT];
@@ -220,40 +219,41 @@ STATIC void emit_write_bytecode_byte_int(emit_t *emit, byte b1, mp_int_t num) {
     *c = *p;
 }
 
-STATIC void emit_write_bytecode_byte_uint(emit_t *emit, byte b, mp_uint_t val) {
-    emit_write_bytecode_byte(emit, b);
+STATIC void emit_write_bytecode_byte_uint(emit_t *emit, int stack_adj, byte b, mp_uint_t val) {
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit_write_uint(emit, emit_get_cur_to_write_bytecode, val);
 }
 
 #if MICROPY_PERSISTENT_CODE
-STATIC void emit_write_bytecode_byte_const(emit_t *emit, byte b, mp_uint_t n, mp_uint_t c) {
+STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n, mp_uint_t c) {
     if (emit->pass == MP_PASS_EMIT) {
         emit->const_table[n] = c;
     }
-    emit_write_bytecode_byte_uint(emit, b, n);
+    emit_write_bytecode_byte_uint(emit, stack_adj, b, n);
 }
 #endif
 
-STATIC void emit_write_bytecode_byte_qstr(emit_t* emit, byte b, qstr qst) {
+STATIC void emit_write_bytecode_byte_qstr(emit_t* emit, int stack_adj, byte b, qstr qst) {
     #if MICROPY_PERSISTENT_CODE
     assert((qst >> 16) == 0);
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     byte *c = emit_get_cur_to_write_bytecode(emit, 3);
     c[0] = b;
     c[1] = qst;
     c[2] = qst >> 8;
     #else
-    emit_write_bytecode_byte_uint(emit, b, qst);
+    emit_write_bytecode_byte_uint(emit, stack_adj, b, qst);
     #endif
 }
 
-STATIC void emit_write_bytecode_byte_obj(emit_t *emit, byte b, mp_obj_t obj) {
+STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) {
     #if MICROPY_PERSISTENT_CODE
-    emit_write_bytecode_byte_const(emit, b,
+    emit_write_bytecode_byte_const(emit, stack_adj, b,
         emit->scope->num_pos_args + emit->scope->num_kwonly_args
         + emit->ct_cur_obj++, (mp_uint_t)obj);
     #else
     // aligns the pointer so it is friendly to GC
-    emit_write_bytecode_byte(emit, b);
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(mp_obj_t));
     mp_obj_t *c = (mp_obj_t*)emit_get_cur_to_write_bytecode(emit, sizeof(mp_obj_t));
     // Verify thar c is already uint-aligned
@@ -262,24 +262,28 @@ STATIC void emit_write_bytecode_byte_obj(emit_t *emit, byte b, mp_obj_t obj) {
     #endif
 }
 
-STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, byte b, mp_raw_code_t *rc) {
+STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) {
     #if MICROPY_PERSISTENT_CODE
-    emit_write_bytecode_byte_const(emit, b,
+    emit_write_bytecode_byte_const(emit, stack_adj, b,
         emit->scope->num_pos_args + emit->scope->num_kwonly_args
         + emit->ct_num_obj + emit->ct_cur_raw_code++, (mp_uint_t)(uintptr_t)rc);
     #else
     // aligns the pointer so it is friendly to GC
-    emit_write_bytecode_byte(emit, b);
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(void*));
     void **c = (void**)emit_get_cur_to_write_bytecode(emit, sizeof(void*));
     // Verify thar c is already uint-aligned
     assert(c == MP_ALIGN(c, sizeof(void*)));
     *c = rc;
     #endif
+    #if MICROPY_PY_SYS_SETTRACE
+    rc->line_of_definition = emit->last_source_line;
+    #endif
 }
 
 // unsigned labels are relative to ip following this instruction, stored as 16 bits
-STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, byte b1, mp_uint_t label) {
+STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     mp_uint_t bytecode_offset;
     if (emit->pass < MP_PASS_EMIT) {
         bytecode_offset = 0;
@@ -293,7 +297,8 @@ STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, byte b1, mp_ui
 }
 
 // signed labels are relative to ip following this instruction, stored as 16 bits, in excess
-STATIC void emit_write_bytecode_byte_signed_label(emit_t *emit, byte b1, mp_uint_t label) {
+STATIC void emit_write_bytecode_byte_signed_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     int bytecode_offset;
     if (emit->pass < MP_PASS_EMIT) {
         bytecode_offset = 0;
@@ -322,7 +327,7 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
     emit->bytecode_offset = 0;
     emit->code_info_offset = 0;
 
-    // Write local state size and exception stack size.
+    // Write local state size, exception stack size, scope flags and number of arguments
     {
         mp_uint_t n_state = scope->num_locals + scope->stack_size;
         if (n_state == 0) {
@@ -335,40 +340,22 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
         // An extra slot in the stack is needed to detect VM stack overflow
         n_state += 1;
         #endif
-        emit_write_code_info_uint(emit, n_state);
-        emit_write_code_info_uint(emit, scope->exc_stack_size);
-    }
 
-    // Write scope flags and number of arguments.
-    // TODO check that num args all fit in a byte
-    emit_write_code_info_byte(emit, emit->scope->scope_flags);
-    emit_write_code_info_byte(emit, emit->scope->num_pos_args);
-    emit_write_code_info_byte(emit, emit->scope->num_kwonly_args);
-    emit_write_code_info_byte(emit, emit->scope->num_def_pos_args);
+        size_t n_exc_stack = scope->exc_stack_size;
+        MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, scope, emit_write_code_info_byte, emit);
+    }
 
-    // Write size of the rest of the code info.  We don't know how big this
-    // variable uint will be on the MP_PASS_CODE_SIZE pass so we reserve 2 bytes
-    // for it and hope that is enough!  TODO assert this or something.
-    if (pass == MP_PASS_EMIT) {
-        emit_write_code_info_uint(emit, emit->code_info_size - emit->code_info_offset);
-    } else  {
-        emit_get_cur_to_write_code_info(emit, 2);
+    // Write number of cells and size of the source code info
+    if (pass >= MP_PASS_CODE_SIZE) {
+        MP_BC_PRELUDE_SIZE_ENCODE(emit->n_info, emit->n_cell, emit_write_code_info_byte, emit);
     }
 
+    emit->n_info = emit->code_info_offset;
+
     // Write the name and source file of this function.
     emit_write_code_info_qstr(emit, scope->simple_name);
     emit_write_code_info_qstr(emit, scope->source_file);
 
-    // bytecode prelude: initialise closed over variables
-    for (int i = 0; i < scope->id_info_len; i++) {
-        id_info_t *id = &scope->id_info[i];
-        if (id->kind == ID_INFO_KIND_CELL) {
-            assert(id->local_num < 255);
-            emit_write_bytecode_byte(emit, id->local_num); // write the local which should be converted to a cell
-        }
-    }
-    emit_write_bytecode_byte(emit, 255); // end of list sentinel
-
     #if MICROPY_PERSISTENT_CODE
     emit->ct_cur_obj = 0;
     emit->ct_cur_raw_code = 0;
@@ -414,6 +401,20 @@ void mp_emit_bc_end_pass(emit_t *emit) {
 
     emit_write_code_info_byte(emit, 0); // end of line number info
 
+    // Calculate size of source code info section
+    emit->n_info = emit->code_info_offset - emit->n_info;
+
+    // Emit closure section of prelude
+    emit->n_cell = 0;
+    for (size_t i = 0; i < emit->scope->id_info_len; ++i) {
+        id_info_t *id = &emit->scope->id_info[i];
+        if (id->kind == ID_INFO_KIND_CELL) {
+            assert(id->local_num <= 255);
+            emit_write_code_info_byte(emit, id->local_num); // write the local which should be converted to a cell
+            ++emit->n_cell;
+        }
+    }
+
     #if MICROPY_PERSISTENT_CODE
     assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->ct_num_obj == emit->ct_cur_obj));
     emit->ct_num_obj = emit->ct_cur_obj;
@@ -468,10 +469,6 @@ void mp_emit_bc_adjust_stack_size(emit_t *emit, mp_int_t delta) {
     emit->last_emit_was_return_value = false;
 }
 
-static inline void emit_bc_pre(emit_t *emit, mp_int_t stack_size_delta) {
-    mp_emit_bc_adjust_stack_size(emit, stack_size_delta);
-}
-
 void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t source_line) {
     //printf("source: line %d -> %d  offset %d -> %d\n", emit->last_source_line, source_line, emit->last_source_line_offset, emit->bytecode_offset);
 #if MICROPY_ENABLE_SOURCE_LINE
@@ -493,7 +490,7 @@ void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t source_line) {
 }
 
 void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) {
-    emit_bc_pre(emit, 0);
+    mp_emit_bc_adjust_stack_size(emit, 0);
     if (emit->pass == MP_PASS_SCOPE) {
         return;
     }
@@ -511,64 +508,54 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) {
 void mp_emit_bc_import(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_NAME == MP_BC_IMPORT_NAME);
     MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_FROM == MP_BC_IMPORT_FROM);
-    if (kind == MP_EMIT_IMPORT_FROM) {
-        emit_bc_pre(emit, 1);
-    } else {
-        emit_bc_pre(emit, -1);
-    }
+    int stack_adj = kind == MP_EMIT_IMPORT_FROM ? 1 : -1;
     if (kind == MP_EMIT_IMPORT_STAR) {
-        emit_write_bytecode_byte(emit, MP_BC_IMPORT_STAR);
+        emit_write_bytecode_byte(emit, stack_adj, MP_BC_IMPORT_STAR);
     } else {
-        emit_write_bytecode_byte_qstr(emit, MP_BC_IMPORT_NAME + kind, qst);
+        emit_write_bytecode_byte_qstr(emit, stack_adj, MP_BC_IMPORT_NAME + kind, qst);
     }
 }
 
 void mp_emit_bc_load_const_tok(emit_t *emit, mp_token_kind_t tok) {
-    emit_bc_pre(emit, 1);
-    switch (tok) {
-        case MP_TOKEN_KW_FALSE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_FALSE); break;
-        case MP_TOKEN_KW_NONE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_NONE); break;
-        case MP_TOKEN_KW_TRUE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_TRUE); break;
-        default:
-            assert(tok == MP_TOKEN_ELLIPSIS);
-            emit_write_bytecode_byte_obj(emit, MP_BC_LOAD_CONST_OBJ, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
-            break;
+    MP_STATIC_ASSERT(MP_BC_LOAD_CONST_FALSE + (MP_TOKEN_KW_NONE - MP_TOKEN_KW_FALSE) == MP_BC_LOAD_CONST_NONE);
+    MP_STATIC_ASSERT(MP_BC_LOAD_CONST_FALSE + (MP_TOKEN_KW_TRUE - MP_TOKEN_KW_FALSE) == MP_BC_LOAD_CONST_TRUE);
+    if (tok == MP_TOKEN_ELLIPSIS) {
+        emit_write_bytecode_byte_obj(emit, 1, MP_BC_LOAD_CONST_OBJ, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+    } else {
+        emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_CONST_FALSE + (tok - MP_TOKEN_KW_FALSE));
     }
 }
 
 void mp_emit_bc_load_const_small_int(emit_t *emit, mp_int_t arg) {
-    emit_bc_pre(emit, 1);
-    if (-16 <= arg && arg <= 47) {
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_SMALL_INT_MULTI + 16 + arg);
+    if (-MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS <= arg
+        && arg < MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) {
+        emit_write_bytecode_byte(emit, 1,
+            MP_BC_LOAD_CONST_SMALL_INT_MULTI + MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS + arg);
     } else {
-        emit_write_bytecode_byte_int(emit, MP_BC_LOAD_CONST_SMALL_INT, arg);
+        emit_write_bytecode_byte_int(emit, 1, MP_BC_LOAD_CONST_SMALL_INT, arg);
     }
 }
 
 void mp_emit_bc_load_const_str(emit_t *emit, qstr qst) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_CONST_STRING, qst);
+    emit_write_bytecode_byte_qstr(emit, 1, MP_BC_LOAD_CONST_STRING, qst);
 }
 
 void mp_emit_bc_load_const_obj(emit_t *emit, mp_obj_t obj) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_obj(emit, MP_BC_LOAD_CONST_OBJ, obj);
+    emit_write_bytecode_byte_obj(emit, 1, MP_BC_LOAD_CONST_OBJ, obj);
 }
 
 void mp_emit_bc_load_null(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_LOAD_NULL);
+    emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_NULL);
 }
 
 void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) {
     MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_LOAD_FAST_N);
     MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_LOAD_DEREF);
     (void)qst;
-    emit_bc_pre(emit, 1);
     if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) {
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_FAST_MULTI + local_num);
+        emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_FAST_MULTI + local_num);
     } else {
-        emit_write_bytecode_byte_uint(emit, MP_BC_LOAD_FAST_N + kind, local_num);
+        emit_write_bytecode_byte_uint(emit, 1, MP_BC_LOAD_FAST_N + kind, local_num);
     }
 }
 
@@ -576,51 +563,45 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_LOAD_NAME);
     MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_LOAD_GLOBAL);
     (void)qst;
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, 1, MP_BC_LOAD_NAME + kind, qst);
     if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
-        emit_write_bytecode_byte(emit, 0);
+        emit_write_bytecode_raw_byte(emit, 0);
     }
 }
 
 void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super) {
-    emit_bc_pre(emit, 1 - 2 * is_super);
-    emit_write_bytecode_byte_qstr(emit, is_super ? MP_BC_LOAD_SUPER_METHOD : MP_BC_LOAD_METHOD, qst);
+    int stack_adj = 1 - 2 * is_super;
+    emit_write_bytecode_byte_qstr(emit, stack_adj, is_super ? MP_BC_LOAD_SUPER_METHOD : MP_BC_LOAD_METHOD, qst);
 }
 
 void mp_emit_bc_load_build_class(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_LOAD_BUILD_CLASS);
+    emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_BUILD_CLASS);
 }
 
 void mp_emit_bc_subscr(emit_t *emit, int kind) {
     if (kind == MP_EMIT_SUBSCR_LOAD) {
-        emit_bc_pre(emit, -1);
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_SUBSCR);
+        emit_write_bytecode_byte(emit, -1, MP_BC_LOAD_SUBSCR);
     } else {
         if (kind == MP_EMIT_SUBSCR_DELETE) {
             mp_emit_bc_load_null(emit);
             mp_emit_bc_rot_three(emit);
         }
-        emit_bc_pre(emit, -3);
-        emit_write_bytecode_byte(emit, MP_BC_STORE_SUBSCR);
+        emit_write_bytecode_byte(emit, -3, MP_BC_STORE_SUBSCR);
     }
 }
 
 void mp_emit_bc_attr(emit_t *emit, qstr qst, int kind) {
     if (kind == MP_EMIT_ATTR_LOAD) {
-        emit_bc_pre(emit, 0);
-        emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst);
+        emit_write_bytecode_byte_qstr(emit, 0, MP_BC_LOAD_ATTR, qst);
     } else {
         if (kind == MP_EMIT_ATTR_DELETE) {
             mp_emit_bc_load_null(emit);
             mp_emit_bc_rot_two(emit);
         }
-        emit_bc_pre(emit, -2);
-        emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst);
+        emit_write_bytecode_byte_qstr(emit, -2, MP_BC_STORE_ATTR, qst);
     }
     if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
-        emit_write_bytecode_byte(emit, 0);
+        emit_write_bytecode_raw_byte(emit, 0);
     }
 }
 
@@ -628,98 +609,86 @@ void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kin
     MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_STORE_FAST_N);
     MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_STORE_DEREF);
     (void)qst;
-    emit_bc_pre(emit, -1);
     if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) {
-        emit_write_bytecode_byte(emit, MP_BC_STORE_FAST_MULTI + local_num);
+        emit_write_bytecode_byte(emit, -1, MP_BC_STORE_FAST_MULTI + local_num);
     } else {
-        emit_write_bytecode_byte_uint(emit, MP_BC_STORE_FAST_N + kind, local_num);
+        emit_write_bytecode_byte_uint(emit, -1, MP_BC_STORE_FAST_N + kind, local_num);
     }
 }
 
 void mp_emit_bc_store_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_STORE_NAME);
     MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_STORE_GLOBAL);
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, -1, MP_BC_STORE_NAME + kind, qst);
 }
 
 void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) {
     MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_DELETE_FAST);
     MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_DELETE_DEREF);
     (void)qst;
-    emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_FAST + kind, local_num);
+    emit_write_bytecode_byte_uint(emit, 0, MP_BC_DELETE_FAST + kind, local_num);
 }
 
 void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_DELETE_NAME);
     MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_DELETE_GLOBAL);
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, 0, MP_BC_DELETE_NAME + kind, qst);
 }
 
 void mp_emit_bc_dup_top(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_DUP_TOP);
+    emit_write_bytecode_byte(emit, 1, MP_BC_DUP_TOP);
 }
 
 void mp_emit_bc_dup_top_two(emit_t *emit) {
-    emit_bc_pre(emit, 2);
-    emit_write_bytecode_byte(emit, MP_BC_DUP_TOP_TWO);
+    emit_write_bytecode_byte(emit, 2, MP_BC_DUP_TOP_TWO);
 }
 
 void mp_emit_bc_pop_top(emit_t *emit) {
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+    emit_write_bytecode_byte(emit, -1, MP_BC_POP_TOP);
 }
 
 void mp_emit_bc_rot_two(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_ROT_TWO);
+    emit_write_bytecode_byte(emit, 0, MP_BC_ROT_TWO);
 }
 
 void mp_emit_bc_rot_three(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_ROT_THREE);
+    emit_write_bytecode_byte(emit, 0, MP_BC_ROT_THREE);
 }
 
 void mp_emit_bc_jump(emit_t *emit, mp_uint_t label) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP, label);
+    emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label);
 }
 
 void mp_emit_bc_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label) {
-    emit_bc_pre(emit, -1);
     if (cond) {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_POP_JUMP_IF_TRUE, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_TRUE, label);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_POP_JUMP_IF_FALSE, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_FALSE, label);
     }
 }
 
 void mp_emit_bc_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label) {
-    emit_bc_pre(emit, -1);
     if (cond) {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP_IF_TRUE_OR_POP, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_TRUE_OR_POP, label);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP_IF_FALSE_OR_POP, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_FALSE_OR_POP, label);
     }
 }
 
 void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) {
     if (except_depth == 0) {
-        emit_bc_pre(emit, 0);
         if (label & MP_EMIT_BREAK_FROM_FOR) {
             // need to pop the iterator if we are breaking out of a for loop
-            emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+            emit_write_bytecode_raw_byte(emit, MP_BC_POP_TOP);
             // also pop the iter_buf
             for (size_t i = 0; i < MP_OBJ_ITER_BUF_NSLOTS - 1; ++i) {
-                emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+                emit_write_bytecode_raw_byte(emit, MP_BC_POP_TOP);
             }
         }
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
+        emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
-        emit_write_bytecode_byte(emit, ((label & MP_EMIT_BREAK_FROM_FOR) ? 0x80 : 0) | except_depth);
+        emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
+        emit_write_bytecode_raw_byte(emit, ((label & MP_EMIT_BREAK_FROM_FOR) ? 0x80 : 0) | except_depth);
     }
 }
 
@@ -727,57 +696,45 @@ void mp_emit_bc_setup_block(emit_t *emit, mp_uint_t label, int kind) {
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_WITH == MP_BC_SETUP_WITH);
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_EXCEPT == MP_BC_SETUP_EXCEPT);
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_FINALLY == MP_BC_SETUP_FINALLY);
-    if (kind == MP_EMIT_SETUP_BLOCK_WITH) {
     // The SETUP_WITH opcode pops ctx_mgr from the top of the stack
     // and then pushes 3 entries: __exit__, ctx_mgr, as_value.
-        emit_bc_pre(emit, 2);
-    } else {
-        emit_bc_pre(emit, 0);
-    }
-    emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_WITH + kind, label);
+    int stack_adj = kind == MP_EMIT_SETUP_BLOCK_WITH ? 2 : 0;
+    emit_write_bytecode_byte_unsigned_label(emit, stack_adj, MP_BC_SETUP_WITH + kind, label);
 }
 
 void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label) {
-    mp_emit_bc_pop_block(emit);
     mp_emit_bc_load_const_tok(emit, MP_TOKEN_KW_NONE);
     mp_emit_bc_label_assign(emit, label);
-    emit_bc_pre(emit, 2); // ensure we have enough stack space to call the __exit__ method
-    emit_write_bytecode_byte(emit, MP_BC_WITH_CLEANUP);
-    emit_bc_pre(emit, -4); // cancel the 2 above, plus the 2 from mp_emit_bc_setup_block(MP_EMIT_SETUP_BLOCK_WITH)
+    // The +2 is to ensure we have enough stack space to call the __exit__ method
+    emit_write_bytecode_byte(emit, 2, MP_BC_WITH_CLEANUP);
+    // Cancel the +2 above, plus the +2 from mp_emit_bc_setup_block(MP_EMIT_SETUP_BLOCK_WITH)
+    mp_emit_bc_adjust_stack_size(emit, -4);
 }
 
 void mp_emit_bc_end_finally(emit_t *emit) {
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_END_FINALLY);
+    emit_write_bytecode_byte(emit, -1, MP_BC_END_FINALLY);
 }
 
 void mp_emit_bc_get_iter(emit_t *emit, bool use_stack) {
-    emit_bc_pre(emit, use_stack ? MP_OBJ_ITER_BUF_NSLOTS - 1 : 0);
-    emit_write_bytecode_byte(emit, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER);
+    int stack_adj = use_stack ? MP_OBJ_ITER_BUF_NSLOTS - 1 : 0;
+    emit_write_bytecode_byte(emit, stack_adj, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER);
 }
 
 void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_unsigned_label(emit, MP_BC_FOR_ITER, label);
+    emit_write_bytecode_byte_unsigned_label(emit, 1, MP_BC_FOR_ITER, label);
 }
 
 void mp_emit_bc_for_iter_end(emit_t *emit) {
-    emit_bc_pre(emit, -MP_OBJ_ITER_BUF_NSLOTS);
-}
-
-void mp_emit_bc_pop_block(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_POP_BLOCK);
+    mp_emit_bc_adjust_stack_size(emit, -MP_OBJ_ITER_BUF_NSLOTS);
 }
 
-void mp_emit_bc_pop_except(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_POP_EXCEPT);
+void mp_emit_bc_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) {
+    (void)within_exc_handler;
+    emit_write_bytecode_byte_unsigned_label(emit, 0, MP_BC_POP_EXCEPT_JUMP, label);
 }
 
 void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_UNARY_OP_MULTI + op);
+    emit_write_bytecode_byte(emit, 0, MP_BC_UNARY_OP_MULTI + op);
 }
 
 void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op) {
@@ -789,11 +746,9 @@ void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op) {
         invert = true;
         op = MP_BINARY_OP_IS;
     }
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_BINARY_OP_MULTI + op);
+    emit_write_bytecode_byte(emit, -1, MP_BC_BINARY_OP_MULTI + op);
     if (invert) {
-        emit_bc_pre(emit, 0);
-        emit_write_bytecode_byte(emit, MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NOT);
+        emit_write_bytecode_byte(emit, 0, MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NOT);
     }
 }
 
@@ -803,17 +758,12 @@ void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind) {
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_BC_BUILD_MAP);
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SET == MP_BC_BUILD_SET);
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SLICE == MP_BC_BUILD_SLICE);
-    if (kind == MP_EMIT_BUILD_MAP) {
-        emit_bc_pre(emit, 1);
-    } else {
-        emit_bc_pre(emit, 1 - n_args);
-    }
-    emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_TUPLE + kind, n_args);
+    int stack_adj = kind == MP_EMIT_BUILD_MAP ? 1 : 1 - n_args;
+    emit_write_bytecode_byte_uint(emit, stack_adj, MP_BC_BUILD_TUPLE + kind, n_args);
 }
 
 void mp_emit_bc_store_map(emit_t *emit) {
-    emit_bc_pre(emit, -2);
-    emit_write_bytecode_byte(emit, MP_BC_STORE_MAP);
+    emit_write_bytecode_byte(emit, -2, MP_BC_STORE_MAP);
 }
 
 void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t collection_stack_index) {
@@ -829,51 +779,46 @@ void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t collection
         n = 0;
         t = 2;
     }
-    emit_bc_pre(emit, -1 - n);
     // the lower 2 bits of the opcode argument indicate the collection type
-    emit_write_bytecode_byte_uint(emit, MP_BC_STORE_COMP, ((collection_stack_index + n) << 2) | t);
+    emit_write_bytecode_byte_uint(emit, -1 - n, MP_BC_STORE_COMP, ((collection_stack_index + n) << 2) | t);
 }
 
 void mp_emit_bc_unpack_sequence(emit_t *emit, mp_uint_t n_args) {
-    emit_bc_pre(emit, -1 + n_args);
-    emit_write_bytecode_byte_uint(emit, MP_BC_UNPACK_SEQUENCE, n_args);
+    emit_write_bytecode_byte_uint(emit, -1 + n_args, MP_BC_UNPACK_SEQUENCE, n_args);
 }
 
 void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) {
-    emit_bc_pre(emit, -1 + n_left + n_right + 1);
-    emit_write_bytecode_byte_uint(emit, MP_BC_UNPACK_EX, n_left | (n_right << 8));
+    emit_write_bytecode_byte_uint(emit, -1 + n_left + n_right + 1, MP_BC_UNPACK_EX, n_left | (n_right << 8));
 }
 
 void mp_emit_bc_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) {
     if (n_pos_defaults == 0 && n_kw_defaults == 0) {
-        emit_bc_pre(emit, 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_FUNCTION, scope->raw_code);
+        emit_write_bytecode_byte_raw_code(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code);
     } else {
-        emit_bc_pre(emit, -1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code);
+        emit_write_bytecode_byte_raw_code(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code);
     }
 }
 
 void mp_emit_bc_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) {
     if (n_pos_defaults == 0 && n_kw_defaults == 0) {
-        emit_bc_pre(emit, -n_closed_over + 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_CLOSURE, scope->raw_code);
-        emit_write_bytecode_byte(emit, n_closed_over);
+        int stack_adj = -n_closed_over + 1;
+        emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code);
+        emit_write_bytecode_raw_byte(emit, n_closed_over);
     } else {
         assert(n_closed_over <= 255);
-        emit_bc_pre(emit, -2 - (mp_int_t)n_closed_over + 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code);
-        emit_write_bytecode_byte(emit, n_closed_over);
+        int stack_adj = -2 - (mp_int_t)n_closed_over + 1;
+        emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code);
+        emit_write_bytecode_raw_byte(emit, n_closed_over);
     }
 }
 
-STATIC void emit_bc_call_function_method_helper(emit_t *emit, mp_int_t stack_adj, mp_uint_t bytecode_base, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) {
+STATIC void emit_bc_call_function_method_helper(emit_t *emit, int stack_adj, mp_uint_t bytecode_base, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) {
     if (star_flags) {
-        emit_bc_pre(emit, stack_adj - (mp_int_t)n_positional - 2 * (mp_int_t)n_keyword - 2);
-        emit_write_bytecode_byte_uint(emit, bytecode_base + 1, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
+        stack_adj -= (int)n_positional + 2 * (int)n_keyword + 2;
+        emit_write_bytecode_byte_uint(emit, stack_adj, bytecode_base + 1, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
     } else {
-        emit_bc_pre(emit, stack_adj - (mp_int_t)n_positional - 2 * (mp_int_t)n_keyword);
-        emit_write_bytecode_byte_uint(emit, bytecode_base, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
+        stack_adj -= (int)n_positional + 2 * (int)n_keyword;
+        emit_write_bytecode_byte_uint(emit, stack_adj, bytecode_base, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
     }
 }
 
@@ -886,22 +831,21 @@ void mp_emit_bc_call_method(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_ke
 }
 
 void mp_emit_bc_return_value(emit_t *emit) {
-    emit_bc_pre(emit, -1);
+    emit_write_bytecode_byte(emit, -1, MP_BC_RETURN_VALUE);
     emit->last_emit_was_return_value = true;
-    emit_write_bytecode_byte(emit, MP_BC_RETURN_VALUE);
 }
 
 void mp_emit_bc_raise_varargs(emit_t *emit, mp_uint_t n_args) {
+    MP_STATIC_ASSERT(MP_BC_RAISE_LAST + 1 == MP_BC_RAISE_OBJ);
+    MP_STATIC_ASSERT(MP_BC_RAISE_LAST + 2 == MP_BC_RAISE_FROM);
     assert(n_args <= 2);
-    emit_bc_pre(emit, -n_args);
-    emit_write_bytecode_byte_byte(emit, MP_BC_RAISE_VARARGS, n_args);
+    emit_write_bytecode_byte(emit, -n_args, MP_BC_RAISE_LAST + n_args);
 }
 
 void mp_emit_bc_yield(emit_t *emit, int kind) {
     MP_STATIC_ASSERT(MP_BC_YIELD_VALUE + 1 == MP_BC_YIELD_FROM);
-    emit_bc_pre(emit, -kind);
+    emit_write_bytecode_byte(emit, -kind, MP_BC_YIELD_VALUE + kind);
     emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR;
-    emit_write_bytecode_byte(emit, MP_BC_YIELD_VALUE + kind);
 }
 
 void mp_emit_bc_start_except_handler(emit_t *emit) {
@@ -914,6 +858,11 @@ void mp_emit_bc_end_except_handler(emit_t *emit) {
 
 #if MICROPY_EMIT_NATIVE
 const emit_method_table_t emit_bc_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    NULL,
+    NULL,
+    #endif
+
     mp_emit_bc_start_pass,
     mp_emit_bc_end_pass,
     mp_emit_bc_last_emit_was_return_value,
@@ -959,8 +908,7 @@ const emit_method_table_t emit_bc_method_table = {
     mp_emit_bc_get_iter,
     mp_emit_bc_for_iter,
     mp_emit_bc_for_iter_end,
-    mp_emit_bc_pop_block,
-    mp_emit_bc_pop_except,
+    mp_emit_bc_pop_except_jump,
     mp_emit_bc_unary_op,
     mp_emit_bc_binary_op,
     mp_emit_bc_build,

py/emitglue.c

@@ -34,6 +34,7 @@
 #include "py/emitglue.h"
 #include "py/runtime0.h"
 #include "py/bc.h"
+#include "py/profile.h"
 
 #if MICROPY_DEBUG_VERBOSE // print debugging info
 #define DEBUG_PRINT (1)
@@ -52,6 +53,9 @@ mp_uint_t mp_verbose_flag = 0;
 mp_raw_code_t *mp_emit_glue_new_raw_code(void) {
     mp_raw_code_t *rc = m_new0(mp_raw_code_t, 1);
     rc->kind = MP_CODE_RESERVED;
+    #if MICROPY_PY_SYS_SETTRACE
+    rc->line_of_definition = 0;
+    #endif
     return rc;
 }
 
@@ -67,12 +71,17 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
 
     rc->kind = MP_CODE_BYTECODE;
     rc->scope_flags = scope_flags;
-    rc->data.u_byte.bytecode = code;
-    rc->data.u_byte.const_table = const_table;
+    rc->fun_data = code;
+    rc->const_table = const_table;
     #if MICROPY_PERSISTENT_CODE_SAVE
-    rc->data.u_byte.bc_len = len;
-    rc->data.u_byte.n_obj = n_obj;
-    rc->data.u_byte.n_raw_code = n_raw_code;
+    rc->fun_data_len = len;
+    rc->n_obj = n_obj;
+    rc->n_raw_code = n_raw_code;
+    #endif
+
+    #if MICROPY_PY_SYS_SETTRACE
+    mp_bytecode_prelude_t *prelude = &rc->prelude;
+    mp_prof_extract_prelude(code, prelude);
     #endif
 
 #ifdef DEBUG_PRINT
@@ -88,15 +97,32 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
 #endif
 }
 
-#if MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
-void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) {
+#if MICROPY_EMIT_MACHINE_CODE
+void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table,
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t prelude_offset,
+    uint16_t n_obj, uint16_t n_raw_code,
+    uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link,
+    #endif
+    mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) {
+
     assert(kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER || kind == MP_CODE_NATIVE_ASM);
+
     rc->kind = kind;
     rc->scope_flags = scope_flags;
     rc->n_pos_args = n_pos_args;
-    rc->data.u_native.fun_data = fun_data;
-    rc->data.u_native.const_table = const_table;
-    rc->data.u_native.type_sig = type_sig;
+    rc->fun_data = fun_data;
+    rc->const_table = const_table;
+    rc->type_sig = type_sig;
+
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    rc->fun_data_len = fun_len;
+    rc->prelude_offset = prelude_offset;
+    rc->n_obj = n_obj;
+    rc->n_raw_code = n_raw_code;
+    rc->n_qstr= n_qstr;
+    rc->qstr_link = qstr_link;
+    #endif
 
 #ifdef DEBUG_PRINT
     DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags);
@@ -135,7 +161,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
         #if MICROPY_EMIT_NATIVE
         case MP_CODE_NATIVE_PY:
         case MP_CODE_NATIVE_VIPER:
-            fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->data.u_native.fun_data, rc->data.u_native.const_table);
+            fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->fun_data, rc->const_table);
             // Check for a generator function, and if so change the type of the object
             if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
                 ((mp_obj_base_t*)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap;
@@ -144,17 +170,23 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
         #endif
         #if MICROPY_EMIT_INLINE_ASM
         case MP_CODE_NATIVE_ASM:
-            fun = mp_obj_new_fun_asm(rc->n_pos_args, rc->data.u_native.fun_data, rc->data.u_native.type_sig);
+            fun = mp_obj_new_fun_asm(rc->n_pos_args, rc->fun_data, rc->type_sig);
             break;
         #endif
         default:
             // rc->kind should always be set and BYTECODE is the only remaining case
             assert(rc->kind == MP_CODE_BYTECODE);
-            fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->data.u_byte.bytecode, rc->data.u_byte.const_table);
+            fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->fun_data, rc->const_table);
             // check for generator functions and if so change the type of the object
             if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
                 ((mp_obj_base_t*)MP_OBJ_TO_PTR(fun))->type = &mp_type_gen_wrap;
             }
+
+            #if MICROPY_PY_SYS_SETTRACE
+            mp_obj_fun_bc_t *self_fun = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(fun);
+            self_fun->rc = rc;
+            #endif
+
             break;
     }
 

py/emitglue.h

@@ -27,6 +27,7 @@
 #define MICROPY_INCLUDED_PY_EMITGLUE_H
 
 #include "py/obj.h"
+#include "py/bc.h"
 
 // These variables and functions glue the code emitters to the runtime.
 
@@ -48,26 +49,38 @@ typedef enum {
     MP_CODE_NATIVE_ASM,
 } mp_raw_code_kind_t;
 
+typedef struct _mp_qstr_link_entry_t {
+    uint16_t off;
+    uint16_t qst;
+} mp_qstr_link_entry_t;
+
 typedef struct _mp_raw_code_t {
     mp_uint_t kind : 3; // of type mp_raw_code_kind_t
     mp_uint_t scope_flags : 7;
     mp_uint_t n_pos_args : 11;
-    union {
-        struct {
-            const byte *bytecode;
-            const mp_uint_t *const_table;
-            #if MICROPY_PERSISTENT_CODE_SAVE
-            mp_uint_t bc_len;
-            uint16_t n_obj;
-            uint16_t n_raw_code;
-            #endif
-        } u_byte;
-        struct {
-            void *fun_data;
-            const mp_uint_t *const_table;
-            mp_uint_t type_sig; // for viper, compressed as 2-bit types; ret is MSB, then arg0, arg1, etc
-        } u_native;
-    } data;
+    const void *fun_data;
+    const mp_uint_t *const_table;
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t fun_data_len;
+    uint16_t n_obj;
+    uint16_t n_raw_code;
+    #if MICROPY_PY_SYS_SETTRACE
+    mp_bytecode_prelude_t prelude;
+    // line_of_definition is a Python source line where the raw_code was
+    // created e.g. MP_BC_MAKE_FUNCTION. This is different from lineno info
+    // stored in prelude, which provides line number for first statement of
+    // a function. Required to properly implement "call" trace event.
+    mp_uint_t line_of_definition;
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    uint16_t prelude_offset;
+    uint16_t n_qstr;
+    mp_qstr_link_entry_t *qstr_link;
+    #endif
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    mp_uint_t type_sig; // for viper, compressed as 2-bit types; ret is MSB, then arg0, arg1, etc
+    #endif
 } mp_raw_code_t;
 
 mp_raw_code_t *mp_emit_glue_new_raw_code(void);
@@ -81,7 +94,15 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
     uint16_t n_obj, uint16_t n_raw_code,
     #endif
     mp_uint_t scope_flags);
-void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig);
+
+void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len,
+    const mp_uint_t *const_table,
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t prelude_offset,
+    uint16_t n_obj, uint16_t n_raw_code,
+    uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link,
+    #endif
+    mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig);
 
 mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args);
 mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args);

py/emitinlinethumb.c

@@ -812,6 +812,11 @@ branch_not_in_range:
 }
 
 const emit_inline_asm_method_table_t emit_inline_thumb_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_thumb_new,
+    emit_inline_thumb_free,
+    #endif
+
     emit_inline_thumb_start_pass,
     emit_inline_thumb_end_pass,
     emit_inline_thumb_count_params,

py/emitinlinextensa.c

@@ -335,6 +335,11 @@ branch_not_in_range:
 }
 
 const emit_inline_asm_method_table_t emit_inline_xtensa_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_xtensa_new,
+    emit_inline_xtensa_free,
+    #endif
+
     emit_inline_xtensa_start_pass,
     emit_inline_xtensa_end_pass,
     emit_inline_xtensa_count_params,

py/emitnative.c

@@ -47,7 +47,8 @@
 #include <assert.h>
 
 #include "py/emit.h"
-#include "py/bc.h"
+#include "py/nativeglue.h"
+#include "py/objstr.h"
 
 #if MICROPY_DEBUG_VERBOSE // print debugging info
 #define DEBUG_PRINT (1)
@@ -57,7 +58,7 @@
 #endif
 
 // wrapper around everything in this file
-#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA
+#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA || N_XTENSAWIN
 
 // C stack layout for native functions:
 //  0:                          nlr_buf_t [optional]
@@ -80,6 +81,30 @@
 //                              locals (reversed, L0 at end)    |
 //                              (L0-L2 may be in regs instead)
 
+// Native emitter needs to know the following sizes and offsets of C structs (on the target):
+#if MICROPY_DYNAMIC_COMPILER
+#define SIZEOF_NLR_BUF (2 + mp_dynamic_compiler.nlr_buf_num_regs + 1) // the +1 is conservative in case MICROPY_ENABLE_PYSTACK enabled
+#else
+#define SIZEOF_NLR_BUF (sizeof(nlr_buf_t) / sizeof(uintptr_t))
+#endif
+#define SIZEOF_CODE_STATE (sizeof(mp_code_state_t) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_t, state) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_CONST_TABLE (offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t))
+
+// If not already defined, set parent args to same as child call registers
+#ifndef REG_PARENT_RET
+#define REG_PARENT_RET REG_RET
+#define REG_PARENT_ARG_1 REG_ARG_1
+#define REG_PARENT_ARG_2 REG_ARG_2
+#define REG_PARENT_ARG_3 REG_ARG_3
+#define REG_PARENT_ARG_4 REG_ARG_4
+#endif
+
 // Word index of nlr_buf_t.ret_val
 #define NLR_BUF_IDX_RET_VAL (1)
 
@@ -101,9 +126,9 @@
 #define LOCAL_IDX_EXC_HANDLER_PC(emit) (NLR_BUF_IDX_LOCAL_1)
 #define LOCAL_IDX_EXC_HANDLER_UNWIND(emit) (NLR_BUF_IDX_LOCAL_2)
 #define LOCAL_IDX_RET_VAL(emit) (NLR_BUF_IDX_LOCAL_3)
-#define LOCAL_IDX_FUN_OBJ(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t))
-#define LOCAL_IDX_OLD_GLOBALS(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
-#define LOCAL_IDX_GEN_PC(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
+#define LOCAL_IDX_FUN_OBJ(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_FUN_BC)
+#define LOCAL_IDX_OLD_GLOBALS(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP)
+#define LOCAL_IDX_GEN_PC(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP)
 #define LOCAL_IDX_LOCAL_VAR(emit, local_num) ((emit)->stack_start + (emit)->n_state - 1 - (local_num))
 
 #define REG_GENERATOR_STATE (REG_LOCAL_3)
@@ -136,20 +161,6 @@ typedef enum {
     VTYPE_BUILTIN_CAST = 0x70 | MP_NATIVE_TYPE_OBJ,
 } vtype_kind_t;
 
-int mp_native_type_from_qstr(qstr qst) {
-    switch (qst) {
-        case MP_QSTR_object: return MP_NATIVE_TYPE_OBJ;
-        case MP_QSTR_bool: return MP_NATIVE_TYPE_BOOL;
-        case MP_QSTR_int: return MP_NATIVE_TYPE_INT;
-        case MP_QSTR_uint: return MP_NATIVE_TYPE_UINT;
-        case MP_QSTR_ptr: return MP_NATIVE_TYPE_PTR;
-        case MP_QSTR_ptr8: return MP_NATIVE_TYPE_PTR8;
-        case MP_QSTR_ptr16: return MP_NATIVE_TYPE_PTR16;
-        case MP_QSTR_ptr32: return MP_NATIVE_TYPE_PTR32;
-        default: return -1;
-    }
-}
-
 STATIC qstr vtype_to_qstr(vtype_kind_t vtype) {
     switch (vtype) {
         case VTYPE_PYOBJ: return MP_QSTR_object;
@@ -208,12 +219,18 @@ struct _emit_t {
     uint16_t code_state_start;
     uint16_t stack_start;
     int stack_size;
+    uint16_t n_cell;
 
     uint16_t const_table_cur_obj;
     uint16_t const_table_num_obj;
     uint16_t const_table_cur_raw_code;
     mp_uint_t *const_table;
 
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t qstr_link_cur;
+    mp_qstr_link_entry_t *qstr_link;
+    #endif
+
     bool last_emit_was_return_value;
 
     scope_t *scope;
@@ -225,6 +242,7 @@ STATIC const uint8_t reg_local_table[REG_LOCAL_NUM] = {REG_LOCAL_1, REG_LOCAL_2,
 
 STATIC void emit_native_global_exc_entry(emit_t *emit);
 STATIC void emit_native_global_exc_exit(emit_t *emit);
+STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj);
 
 emit_t *EXPORT_FUN(new)(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) {
     emit_t *emit = m_new0(emit_t, 1);
@@ -279,6 +297,32 @@ STATIC void emit_native_mov_reg_state_addr(emit_t *emit, int reg_dest, int local
     }
 }
 
+STATIC void emit_native_mov_reg_qstr(emit_t *emit, int arg_reg, qstr qst) {
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t loc = ASM_MOV_REG_IMM_FIX_U16(emit->as, arg_reg, qst);
+    size_t link_idx = emit->qstr_link_cur++;
+    if (emit->pass == MP_PASS_EMIT) {
+        emit->qstr_link[link_idx].off = loc << 2 | 1;
+        emit->qstr_link[link_idx].qst = qst;
+    }
+    #else
+    ASM_MOV_REG_IMM(emit->as, arg_reg, qst);
+    #endif
+}
+
+STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) {
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t loc = ASM_MOV_REG_IMM_FIX_WORD(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+    size_t link_idx = emit->qstr_link_cur++;
+    if (emit->pass == MP_PASS_EMIT) {
+        emit->qstr_link[link_idx].off = loc << 2 | 2;
+        emit->qstr_link[link_idx].qst = qst;
+    }
+    #else
+    ASM_MOV_REG_IMM(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+    #endif
+}
+
 #define emit_native_mov_state_imm_via(emit, local_num, imm, reg_temp) \
     do { \
         ASM_MOV_REG_IMM((emit)->as, (reg_temp), (imm)); \
@@ -291,8 +335,15 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
     emit->pass = pass;
     emit->do_viper_types = scope->emit_options == MP_EMIT_OPT_VIPER;
     emit->stack_size = 0;
-    emit->const_table_cur_obj = 1; // first entry is for mp_fun_table
+    #if N_PRELUDE_AS_BYTES_OBJ
+    emit->const_table_cur_obj = emit->do_viper_types ? 0 : 1; // reserve first obj for prelude bytes obj
+    #else
+    emit->const_table_cur_obj = 0;
+    #endif
     emit->const_table_cur_raw_code = 0;
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    emit->qstr_link_cur = 0;
+    #endif
     emit->last_emit_was_return_value = false;
     emit->scope = scope;
 
@@ -343,7 +394,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
     // Work out start of code state (mp_code_state_t or reduced version for viper)
     emit->code_state_start = 0;
     if (NEED_GLOBAL_EXC_HANDLER(emit)) {
-        emit->code_state_start = sizeof(nlr_buf_t) / sizeof(uintptr_t);
+        emit->code_state_start = SIZEOF_NLR_BUF;
     }
 
     if (emit->do_viper_types) {
@@ -377,16 +428,16 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
         ASM_ENTRY(emit->as, emit->stack_start + emit->n_state - num_locals_in_regs);
 
         #if N_X86
-        asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_1);
+        asm_x86_mov_arg_to_r32(emit->as, 0, REG_PARENT_ARG_1);
         #endif
 
         // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
-        ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_ARG_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+        ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
         ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, 0);
 
         // Store function object (passed as first arg) to stack if needed
         if (NEED_FUN_OBJ(emit)) {
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_ARG_1);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1);
         }
 
         // Put n_args in REG_ARG_1, n_kw in REG_ARG_2, args array in REG_LOCAL_3
@@ -395,9 +446,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
         asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_2);
         asm_x86_mov_arg_to_r32(emit->as, 3, REG_LOCAL_3);
         #else
-        ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_ARG_2);
-        ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_ARG_3);
-        ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_ARG_4);
+        ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_PARENT_ARG_2);
+        ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_PARENT_ARG_3);
+        ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_4);
         #endif
 
         // Check number of args matches this function, and call mp_arg_check_num_sig if not
@@ -437,31 +488,36 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
 
         if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
             emit->code_state_start = 0;
-            emit->stack_start = sizeof(mp_code_state_t) / sizeof(mp_uint_t);
+            emit->stack_start = SIZEOF_CODE_STATE;
+            #if N_PRELUDE_AS_BYTES_OBJ
+            // Load index of prelude bytes object in const_table
+            mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)(emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1));
+            #else
             mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_offset);
+            #endif
             mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->start_offset);
-            ASM_ENTRY(emit->as, sizeof(nlr_buf_t) / sizeof(uintptr_t));
+            ASM_ENTRY(emit->as, SIZEOF_NLR_BUF);
 
             // Put address of code_state into REG_GENERATOR_STATE
             #if N_X86
             asm_x86_mov_arg_to_r32(emit->as, 0, REG_GENERATOR_STATE);
             #else
-            ASM_MOV_REG_REG(emit->as, REG_GENERATOR_STATE, REG_ARG_1);
+            ASM_MOV_REG_REG(emit->as, REG_GENERATOR_STATE, REG_PARENT_ARG_1);
             #endif
 
             // Put throw value into LOCAL_IDX_EXC_VAL slot, for yield/yield-from
             #if N_X86
-            asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_2);
+            asm_x86_mov_arg_to_r32(emit->as, 1, REG_PARENT_ARG_2);
             #endif
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_ARG_2);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_PARENT_ARG_2);
 
             // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit));
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
         } else {
             // The locals and stack start after the code_state structure
-            emit->stack_start = emit->code_state_start + sizeof(mp_code_state_t) / sizeof(mp_uint_t);
+            emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE;
 
             // Allocate space on C-stack for code_state structure, which includes state
             ASM_ENTRY(emit->as, emit->stack_start + emit->n_state);
@@ -469,26 +525,49 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
             // Prepare incoming arguments for call to mp_setup_code_state
 
             #if N_X86
-            asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_1);
-            asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_2);
-            asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_3);
-            asm_x86_mov_arg_to_r32(emit->as, 3, REG_ARG_4);
+            asm_x86_mov_arg_to_r32(emit->as, 0, REG_PARENT_ARG_1);
+            asm_x86_mov_arg_to_r32(emit->as, 1, REG_PARENT_ARG_2);
+            asm_x86_mov_arg_to_r32(emit->as, 2, REG_PARENT_ARG_3);
+            asm_x86_mov_arg_to_r32(emit->as, 3, REG_PARENT_ARG_4);
             #endif
 
             // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_ARG_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
 
             // Set code_state.fun_bc
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_ARG_1);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1);
 
             // Set code_state.ip (offset from start of this function to prelude info)
+            #if N_PRELUDE_AS_BYTES_OBJ
+            // Prelude is a bytes object in const_table; store ip = prelude->data - fun_bc->bytecode
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1);
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, offsetof(mp_obj_str_t, data) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE);
+            ASM_SUB_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1);
+            emit_native_mov_state_reg(emit, emit->code_state_start + OFFSETOF_CODE_STATE_IP, REG_LOCAL_3);
+            #else
             // TODO this encoding may change size in the final pass, need to make it fixed
-            emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), emit->prelude_offset, REG_ARG_1);
+            emit_native_mov_state_imm_via(emit, emit->code_state_start + OFFSETOF_CODE_STATE_IP, emit->prelude_offset, REG_PARENT_ARG_1);
+            #endif
+
+            // Set code_state.n_state (only works on little endian targets due to n_state being uint16_t)
+            emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t), emit->n_state, REG_ARG_1);
 
             // Put address of code_state into first arg
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, emit->code_state_start);
 
+            // Copy next 3 args if needed
+            #if REG_ARG_2 != REG_PARENT_ARG_2
+            ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_PARENT_ARG_2);
+            #endif
+            #if REG_ARG_3 != REG_PARENT_ARG_3
+            ASM_MOV_REG_REG(emit->as, REG_ARG_3, REG_PARENT_ARG_3);
+            #endif
+            #if REG_ARG_4 != REG_PARENT_ARG_4
+            ASM_MOV_REG_REG(emit->as, REG_ARG_4, REG_PARENT_ARG_4);
+            #endif
+
             // Call mp_setup_code_state to prepare code_state structure
             #if N_THUMB
             asm_thumb_bl_ind(emit->as, MP_F_SETUP_CODE_STATE, ASM_THUMB_REG_R4);
@@ -535,22 +614,28 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
 
 }
 
+static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) {
+    mp_asm_base_data(&emit->as->base, 1, val);
+}
+
 STATIC void emit_native_end_pass(emit_t *emit) {
     emit_native_global_exc_exit(emit);
 
     if (!emit->do_viper_types) {
         emit->prelude_offset = mp_asm_base_get_code_pos(&emit->as->base);
-        mp_asm_base_data(&emit->as->base, 1, 0x80 | ((emit->n_state >> 7) & 0x7f));
-        mp_asm_base_data(&emit->as->base, 1, emit->n_state & 0x7f);
-        mp_asm_base_data(&emit->as->base, 1, 0); // n_exc_stack
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->scope_flags);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_pos_args);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_kwonly_args);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_def_pos_args);
-
-        // write code info
+
+        size_t n_state = emit->n_state;
+        size_t n_exc_stack = 0; // exc-stack not needed for native code
+        MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, emit->scope, emit_native_write_code_info_byte, emit);
+
+        #if MICROPY_PERSISTENT_CODE
+        size_t n_info = 4;
+        #else
+        size_t n_info = 1;
+        #endif
+        MP_BC_PRELUDE_SIZE_ENCODE(n_info, emit->n_cell, emit_native_write_code_info_byte, emit);
+
         #if MICROPY_PERSISTENT_CODE
-        mp_asm_base_data(&emit->as->base, 1, 5);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name >> 8);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file);
@@ -560,14 +645,25 @@ STATIC void emit_native_end_pass(emit_t *emit) {
         #endif
 
         // bytecode prelude: initialise closed over variables
+        size_t cell_start = mp_asm_base_get_code_pos(&emit->as->base);
         for (int i = 0; i < emit->scope->id_info_len; i++) {
             id_info_t *id = &emit->scope->id_info[i];
             if (id->kind == ID_INFO_KIND_CELL) {
-                assert(id->local_num < 255);
+                assert(id->local_num <= 255);
                 mp_asm_base_data(&emit->as->base, 1, id->local_num); // write the local which should be converted to a cell
             }
         }
-        mp_asm_base_data(&emit->as->base, 1, 255); // end of list sentinel
+        emit->n_cell = mp_asm_base_get_code_pos(&emit->as->base) - cell_start;
+
+        #if N_PRELUDE_AS_BYTES_OBJ
+        // Prelude bytes object is after qstr arg names and mp_fun_table
+        size_t table_off = emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1;
+        if (emit->pass == MP_PASS_EMIT) {
+            void *buf = emit->as->base.code_base + emit->prelude_offset;
+            size_t n = emit->as->base.code_offset - emit->prelude_offset;
+            emit->const_table[table_off] = (uintptr_t)mp_obj_new_bytes(buf, n);
+        }
+        #endif
     }
 
     ASM_END_PASS(emit->as);
@@ -580,7 +676,7 @@ STATIC void emit_native_end_pass(emit_t *emit) {
     assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->const_table_num_obj == emit->const_table_cur_obj));
     emit->const_table_num_obj = emit->const_table_cur_obj;
     if (emit->pass == MP_PASS_CODE_SIZE) {
-        size_t const_table_alloc = emit->const_table_num_obj + emit->const_table_cur_raw_code;
+        size_t const_table_alloc = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code;
         size_t nqstr = 0;
         if (!emit->do_viper_types) {
             // Add room for qstr names of arguments
@@ -588,8 +684,18 @@ STATIC void emit_native_end_pass(emit_t *emit) {
             const_table_alloc += nqstr;
         }
         emit->const_table = m_new(mp_uint_t, const_table_alloc);
+        #if !MICROPY_DYNAMIC_COMPILER
         // Store mp_fun_table pointer just after qstrs
-        emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)mp_fun_table;
+        // (but in dynamic-compiler mode eliminate dependency on mp_fun_table)
+        emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)&mp_fun_table;
+        #endif
+
+        #if MICROPY_PERSISTENT_CODE_SAVE
+        size_t qstr_link_alloc = emit->qstr_link_cur;
+        if (qstr_link_alloc > 0) {
+            emit->qstr_link = m_new(mp_qstr_link_entry_t, qstr_link_alloc);
+        }
+        #endif
     }
 
     if (emit->pass == MP_PASS_EMIT) {
@@ -599,6 +705,11 @@ STATIC void emit_native_end_pass(emit_t *emit) {
         mp_emit_glue_assign_native(emit->scope->raw_code,
             emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY,
             f, f_len, emit->const_table,
+            #if MICROPY_PERSISTENT_CODE_SAVE
+            emit->prelude_offset,
+            emit->const_table_cur_obj, emit->const_table_cur_raw_code,
+            emit->qstr_link_cur, emit->qstr_link,
+            #endif
             emit->scope->num_pos_args, emit->scope->scope_flags, 0);
     }
 }
@@ -883,6 +994,12 @@ STATIC void emit_call_with_2_imm_args(emit_t *emit, mp_fun_kind_t fun_kind, mp_i
     ASM_CALL_IND(emit->as, fun_kind);
 }
 
+STATIC void emit_call_with_qstr_arg(emit_t *emit, mp_fun_kind_t fun_kind, qstr qst, int arg_reg) {
+    need_reg_all(emit);
+    emit_native_mov_reg_qstr(emit, arg_reg, qst);
+    ASM_CALL_IND(emit->as, fun_kind);
+}
+
 // vtype of all n_pop objects is VTYPE_PYOBJ
 // Will convert any items that are not VTYPE_PYOBJ to this type and put them back on the stack.
 // If any conversions of non-immediate values are needed, then it uses REG_ARG_1, REG_ARG_2 and REG_RET.
@@ -994,17 +1111,19 @@ STATIC void emit_load_reg_with_ptr(emit_t *emit, int reg, mp_uint_t ptr, size_t
         emit->const_table[table_off] = ptr;
     }
     emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit));
-    ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+    ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
     ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off);
 }
 
 STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) {
-    size_t table_off = emit->const_table_cur_obj++;
+    // First entry is for mp_fun_table
+    size_t table_off = 1 + emit->const_table_cur_obj++;
     emit_load_reg_with_ptr(emit, reg, (mp_uint_t)obj, table_off);
 }
 
 STATIC void emit_load_reg_with_raw_code(emit_t *emit, int reg, mp_raw_code_t *rc) {
-    size_t table_off = emit->const_table_num_obj + emit->const_table_cur_raw_code++;
+    // First entry is for mp_fun_table, then constant objects
+    size_t table_off = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code++;
     emit_load_reg_with_ptr(emit, reg, (mp_uint_t)rc, table_off);
 }
 
@@ -1049,7 +1168,7 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
         if (!(emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR)) {
             // Set new globals
             emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit));
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_GLOBALS);
             emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS);
 
             // Save old globals (or NULL if globals didn't change)
@@ -1065,6 +1184,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
             // Wrap everything in an nlr context
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0);
             emit_call(emit, MP_F_NLR_PUSH);
+            #if N_NLR_SETJMP
+            ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 2);
+            emit_call(emit, MP_F_SETJMP);
+            #endif
             ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, start_label, true);
         } else {
             // Clear the unwind state
@@ -1079,6 +1202,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
             ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_2, LOCAL_IDX_EXC_HANDLER_UNWIND(emit));
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0);
             emit_call(emit, MP_F_NLR_PUSH);
+            #if N_NLR_SETJMP
+            ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 2);
+            emit_call(emit, MP_F_SETJMP);
+            #endif
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_LOCAL_2);
             ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true);
 
@@ -1089,6 +1216,12 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
 
             // Global exception handler: check for valid exception handler
             emit_native_label_assign(emit, global_except_label);
+            #if N_NLR_SETJMP
+            // Reload REG_FUN_TABLE, since it may be clobbered by longjmp
+            emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
+            #endif
             ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit));
             ASM_JUMP_IF_REG_NONZERO(emit->as, REG_LOCAL_1, nlr_label, false);
         }
@@ -1102,10 +1235,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
         if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
             // Store return value in state[0]
             ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, LOCAL_IDX_EXC_VAL(emit));
-            ASM_STORE_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, offsetof(mp_code_state_t, state) / sizeof(uintptr_t));
+            ASM_STORE_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, OFFSETOF_CODE_STATE_STATE);
 
             // Load return kind
-            ASM_MOV_REG_IMM(emit->as, REG_RET, MP_VM_RETURN_EXCEPTION);
+            ASM_MOV_REG_IMM(emit->as, REG_PARENT_RET, MP_VM_RETURN_EXCEPTION);
 
             ASM_EXIT(emit->as);
         } else {
@@ -1160,7 +1293,7 @@ STATIC void emit_native_global_exc_exit(emit_t *emit) {
         }
 
         // Load return value
-        ASM_MOV_REG_LOCAL(emit->as, REG_RET, LOCAL_IDX_RET_VAL(emit));
+        ASM_MOV_REG_LOCAL(emit->as, REG_PARENT_RET, LOCAL_IDX_RET_VAL(emit));
     }
 
     ASM_EXIT(emit->as);
@@ -1181,7 +1314,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) {
     assert(vtype_level == VTYPE_PYOBJ);
     emit->do_viper_types = orig_do_viper_types;
 
-    emit_call_with_imm_arg(emit, MP_F_IMPORT_NAME, qst, REG_ARG_1); // arg1 = import name
+    emit_call_with_qstr_arg(emit, MP_F_IMPORT_NAME, qst, REG_ARG_1); // arg1 = import name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1191,7 +1324,7 @@ STATIC void emit_native_import_from(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_module;
     emit_access_stack(emit, 1, &vtype_module, REG_ARG_1); // arg1 = module
     assert(vtype_module == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_IMPORT_FROM, qst, REG_ARG_2); // arg2 = import name
+    emit_call_with_qstr_arg(emit, MP_F_IMPORT_FROM, qst, REG_ARG_2); // arg2 = import name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1217,7 +1350,11 @@ STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) {
 STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) {
     DEBUG_printf("load_const_tok(tok=%u)\n", tok);
     if (tok == MP_TOKEN_ELLIPSIS) {
+        #if MICROPY_PERSISTENT_CODE_SAVE
+        emit_native_load_const_obj(emit, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+        #else
         emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+        #endif
     } else {
         emit_native_pre(emit);
         if (tok == MP_TOKEN_KW_NONE) {
@@ -1245,11 +1382,14 @@ STATIC void emit_native_load_const_str(emit_t *emit, qstr qst) {
     } else
     */
     {
-        emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+        need_reg_single(emit, REG_TEMP0, 0);
+        emit_native_mov_reg_qstr_obj(emit, REG_TEMP0, qst);
+        emit_post_push_reg(emit, VTYPE_PYOBJ, REG_TEMP0);
     }
 }
 
 STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj) {
+    emit->scope->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
     emit_native_pre(emit);
     need_reg_single(emit, REG_RET, 0);
     emit_load_reg_with_object(emit, REG_RET, obj);
@@ -1308,13 +1448,13 @@ STATIC void emit_native_load_global(emit_t *emit, qstr qst, int kind) {
         if (emit->do_viper_types) {
             // check for builtin casting operators
             int native_type = mp_native_type_from_qstr(qst);
-            if (native_type >= MP_NATIVE_TYPE_INT) {
+            if (native_type >= MP_NATIVE_TYPE_BOOL) {
                 emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, native_type);
                 return;
             }
         }
     }
-    emit_call_with_imm_arg(emit, MP_F_LOAD_NAME + kind, qst, REG_ARG_1);
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_NAME + kind, qst, REG_ARG_1);
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1326,7 +1466,7 @@ STATIC void emit_native_load_attr(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_base;
     emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
     assert(vtype_base == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_LOAD_ATTR, qst, REG_ARG_2); // arg2 = attribute name
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1334,13 +1474,13 @@ STATIC void emit_native_load_method(emit_t *emit, qstr qst, bool is_super) {
     if (is_super) {
         emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, 3); // arg2 = dest ptr
         emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_2, 2); // arg2 = dest ptr
-        emit_call_with_imm_arg(emit, MP_F_LOAD_SUPER_METHOD, qst, REG_ARG_1); // arg1 = method name
+        emit_call_with_qstr_arg(emit, MP_F_LOAD_SUPER_METHOD, qst, REG_ARG_1); // arg1 = method name
     } else {
         vtype_kind_t vtype_base;
         emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
         assert(vtype_base == VTYPE_PYOBJ);
         emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-        emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, qst, REG_ARG_2); // arg2 = method name
+        emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, qst, REG_ARG_2); // arg2 = method name
     }
 }
 
@@ -1547,7 +1687,7 @@ STATIC void emit_native_store_global(emit_t *emit, qstr qst, int kind) {
             ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_RET);
         }
     }
-    emit_call_with_imm_arg(emit, MP_F_STORE_NAME + kind, qst, REG_ARG_1); // arg1 = name
+    emit_call_with_qstr_arg(emit, MP_F_STORE_NAME + kind, qst, REG_ARG_1); // arg1 = name
     emit_post(emit);
 }
 
@@ -1556,7 +1696,7 @@ STATIC void emit_native_store_attr(emit_t *emit, qstr qst) {
     emit_pre_pop_reg_reg(emit, &vtype_base, REG_ARG_1, &vtype_val, REG_ARG_3); // arg1 = base, arg3 = value
     assert(vtype_base == VTYPE_PYOBJ);
     assert(vtype_val == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
+    emit_call_with_qstr_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post(emit);
 }
 
@@ -1753,7 +1893,7 @@ STATIC void emit_native_delete_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_F_DELETE_NAME);
     MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_F_DELETE_GLOBAL);
     emit_native_pre(emit);
-    emit_call_with_imm_arg(emit, MP_F_DELETE_NAME + kind, qst, REG_ARG_1);
+    emit_call_with_qstr_arg(emit, MP_F_DELETE_NAME + kind, qst, REG_ARG_1);
     emit_post(emit);
 }
 
@@ -1761,7 +1901,8 @@ STATIC void emit_native_delete_attr(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_base;
     emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
     assert(vtype_base == VTYPE_PYOBJ);
-    emit_call_with_2_imm_args(emit, MP_F_STORE_ATTR, qst, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL, REG_ARG_3); // arg2 = attribute name, arg3 = value (null for delete)
+    ASM_XOR_REG_REG(emit->as, REG_ARG_3, REG_ARG_3); // arg3 = value (null for delete)
+    emit_call_with_qstr_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post(emit);
 }
 
@@ -1916,7 +2057,7 @@ STATIC void emit_native_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t exc
             prev_finally->unwind_label = UNWIND_LABEL_DO_FINAL_UNWIND;
             ASM_MOV_REG_PCREL(emit->as, REG_RET, label & ~MP_EMIT_BREAK_FROM_FOR);
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_RET);
-            // Cancel any active exception (see also emit_native_pop_except)
+            // Cancel any active exception (see also emit_native_pop_except_jump)
             emit_native_mov_reg_const(emit, REG_RET, MP_F_CONST_NONE_OBJ);
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_RET);
             // Jump to the innermost active finally
@@ -1935,7 +2076,7 @@ STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) {
     emit_access_stack(emit, 1, &vtype, REG_ARG_1); // arg1 = ctx_mgr
     assert(vtype == VTYPE_PYOBJ);
     emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-    emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___exit__, REG_ARG_2);
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___exit__, REG_ARG_2);
     // stack: (..., ctx_mgr, __exit__, self)
 
     emit_pre_pop_reg(emit, &vtype, REG_ARG_3); // self
@@ -1948,7 +2089,7 @@ STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) {
 
     // get __enter__ method
     emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-    emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___enter__, REG_ARG_2); // arg2 = method name
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___enter__, REG_ARG_2); // arg2 = method name
     // stack: (..., __exit__, self, __enter__, self)
 
     // call __enter__ method
@@ -2094,12 +2235,12 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) {
     emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_1, MP_OBJ_ITER_BUF_NSLOTS);
     adjust_stack(emit, MP_OBJ_ITER_BUF_NSLOTS);
     emit_call(emit, MP_F_NATIVE_ITERNEXT);
-    #ifdef NDEBUG
-    MP_STATIC_ASSERT(MP_OBJ_STOP_ITERATION == 0);
-    ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, label, false);
-    #else
+    #if MICROPY_DEBUG_MP_OBJ_SENTINELS
     ASM_MOV_REG_IMM(emit->as, REG_TEMP1, (mp_uint_t)MP_OBJ_STOP_ITERATION);
     ASM_JUMP_IF_REG_EQ(emit->as, REG_RET, REG_TEMP1, label);
+    #else
+    MP_STATIC_ASSERT(MP_OBJ_STOP_ITERATION == 0);
+    ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, label, false);
     #endif
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
@@ -2111,18 +2252,15 @@ STATIC void emit_native_for_iter_end(emit_t *emit) {
     emit_post(emit);
 }
 
-STATIC void emit_native_pop_block(emit_t *emit) {
-    emit_native_pre(emit);
-    if (!emit->exc_stack[emit->exc_stack_size - 1].is_finally) {
+STATIC void emit_native_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) {
+    if (within_exc_handler) {
+        // Cancel any active exception so subsequent handlers don't see it
+        emit_native_mov_reg_const(emit, REG_TEMP0, MP_F_CONST_NONE_OBJ);
+        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0);
+    } else {
         emit_native_leave_exc_stack(emit, false);
     }
-    emit_post(emit);
-}
-
-STATIC void emit_native_pop_except(emit_t *emit) {
-    // Cancel any active exception so subsequent handlers don't see it
-    emit_native_mov_reg_const(emit, REG_TEMP0, MP_F_CONST_NONE_OBJ);
-    ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0);
+    emit_native_jump(emit, label);
 }
 
 STATIC void emit_native_unary_op(emit_t *emit, mp_unary_op_t op) {
@@ -2181,17 +2319,16 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) {
         int reg_rhs = REG_ARG_3;
         emit_pre_pop_reg_flexible(emit, &vtype_rhs, &reg_rhs, REG_RET, REG_ARG_2);
         emit_pre_pop_reg(emit, &vtype_lhs, REG_ARG_2);
-        if (0) {
-            // dummy
         #if !(N_X64 || N_X86)
-        } else if (op == MP_BINARY_OP_LSHIFT) {
+        if (op == MP_BINARY_OP_LSHIFT) {
             ASM_LSL_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
         } else if (op == MP_BINARY_OP_RSHIFT) {
             ASM_ASR_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
+        } else
         #endif
-        } else if (op == MP_BINARY_OP_OR) {
+        if (op == MP_BINARY_OP_OR) {
             ASM_OR_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
         } else if (op == MP_BINARY_OP_XOR) {
@@ -2267,7 +2404,7 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) {
                 ASM_ARM_CC_NE,
             };
             asm_arm_setcc_reg(emit->as, REG_RET, ccs[op - MP_BINARY_OP_LESS]);
-            #elif N_XTENSA
+            #elif N_XTENSA || N_XTENSAWIN
             static uint8_t ccs[6] = {
                 ASM_XTENSA_CC_LT,
                 0x80 | ASM_XTENSA_CC_LT, // for GT we'll swap args
@@ -2527,7 +2664,7 @@ STATIC void emit_native_return_value(emit_t *emit) {
     if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
         // Save pointer to current stack position for caller to access return value
         emit_get_stack_pointer_to_reg_for_pop(emit, REG_TEMP0, 1);
-        emit_native_mov_state_reg(emit, offsetof(mp_code_state_t, sp) / sizeof(uintptr_t), REG_TEMP0);
+        emit_native_mov_state_reg(emit, OFFSETOF_CODE_STATE_SP, REG_TEMP0);
 
         // Put return type in return value slot
         ASM_MOV_REG_IMM(emit->as, REG_TEMP0, MP_VM_RETURN_NORMAL);
@@ -2544,13 +2681,13 @@ STATIC void emit_native_return_value(emit_t *emit) {
         if (peek_vtype(emit, 0) == VTYPE_PTR_NONE) {
             emit_pre_pop_discard(emit);
             if (return_vtype == VTYPE_PYOBJ) {
-                emit_native_mov_reg_const(emit, REG_RET, MP_F_CONST_NONE_OBJ);
+                emit_native_mov_reg_const(emit, REG_PARENT_RET, MP_F_CONST_NONE_OBJ);
             } else {
                 ASM_MOV_REG_IMM(emit->as, REG_ARG_1, 0);
             }
         } else {
             vtype_kind_t vtype;
-            emit_pre_pop_reg(emit, &vtype, return_vtype == VTYPE_PYOBJ ? REG_RET : REG_ARG_1);
+            emit_pre_pop_reg(emit, &vtype, return_vtype == VTYPE_PYOBJ ? REG_PARENT_RET : REG_ARG_1);
             if (vtype != return_vtype) {
                 EMIT_NATIVE_VIPER_TYPE_ERROR(emit,
                     "return expected '%q' but got '%q'",
@@ -2559,15 +2696,18 @@ STATIC void emit_native_return_value(emit_t *emit) {
         }
         if (return_vtype != VTYPE_PYOBJ) {
             emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, return_vtype, REG_ARG_2);
+            #if REG_RET != REG_PARENT_RET
+            ASM_MOV_REG_REG(emit->as, REG_PARENT_RET, REG_RET);
+            #endif
         }
     } else {
         vtype_kind_t vtype;
-        emit_pre_pop_reg(emit, &vtype, REG_RET);
+        emit_pre_pop_reg(emit, &vtype, REG_PARENT_RET);
         assert(vtype == VTYPE_PYOBJ);
     }
     if (NEED_GLOBAL_EXC_HANDLER(emit)) {
         // Save return value for the global exception handler to use
-        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_RET_VAL(emit), REG_RET);
+        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_RET_VAL(emit), REG_PARENT_RET);
     }
     emit_native_unwind_jump(emit, emit->exit_label, emit->exc_stack_size);
     emit->last_emit_was_return_value = true;
@@ -2610,7 +2750,7 @@ STATIC void emit_native_yield(emit_t *emit, int kind) {
 
     // Save pointer to current stack position for caller to access yielded value
     emit_get_stack_pointer_to_reg_for_pop(emit, REG_TEMP0, 1);
-    emit_native_mov_state_reg(emit, offsetof(mp_code_state_t, sp) / sizeof(uintptr_t), REG_TEMP0);
+    emit_native_mov_state_reg(emit, OFFSETOF_CODE_STATE_SP, REG_TEMP0);
 
     // Put return type in return value slot
     ASM_MOV_REG_IMM(emit->as, REG_TEMP0, MP_VM_RETURN_YIELD);
@@ -2681,6 +2821,11 @@ STATIC void emit_native_end_except_handler(emit_t *emit) {
 }
 
 const emit_method_table_t EXPORT_FUN(method_table) = {
+    #if MICROPY_DYNAMIC_COMPILER
+    EXPORT_FUN(new),
+    EXPORT_FUN(free),
+    #endif
+
     emit_native_start_pass,
     emit_native_end_pass,
     emit_native_last_emit_was_return_value,
@@ -2726,8 +2871,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = {
     emit_native_get_iter,
     emit_native_for_iter,
     emit_native_for_iter_end,
-    emit_native_pop_block,
-    emit_native_pop_except,
+    emit_native_pop_except_jump,
     emit_native_unary_op,
     emit_native_binary_op,
     emit_native_build,

py/emitnx86.c

@@ -1,7 +1,7 @@
 // x86 specific stuff
 
 #include "py/mpconfig.h"
-#include "py/runtime0.h"
+#include "py/nativeglue.h"
 
 #if MICROPY_EMIT_X86
 
@@ -34,13 +34,11 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = {
     [MP_F_BINARY_OP] = 3,
     [MP_F_BUILD_TUPLE] = 2,
     [MP_F_BUILD_LIST] = 2,
-    [MP_F_LIST_APPEND] = 2,
     [MP_F_BUILD_MAP] = 1,
-    [MP_F_STORE_MAP] = 3,
-    #if MICROPY_PY_BUILTINS_SET
     [MP_F_BUILD_SET] = 2,
     [MP_F_STORE_SET] = 2,
-    #endif
+    [MP_F_LIST_APPEND] = 2,
+    [MP_F_STORE_MAP] = 3,
     [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3,
     [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3,
     [MP_F_CALL_METHOD_N_KW] = 3,
@@ -53,20 +51,18 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = {
     [MP_F_IMPORT_NAME] = 3,
     [MP_F_IMPORT_FROM] = 2,
     [MP_F_IMPORT_ALL] = 1,
-    #if MICROPY_PY_BUILTINS_SLICE
     [MP_F_NEW_SLICE] = 3,
-    #endif
     [MP_F_UNPACK_SEQUENCE] = 3,
     [MP_F_UNPACK_EX] = 3,
     [MP_F_DELETE_NAME] = 1,
     [MP_F_DELETE_GLOBAL] = 1,
-    [MP_F_NEW_CELL] = 1,
     [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3,
     [MP_F_ARG_CHECK_NUM_SIG] = 3,
     [MP_F_SETUP_CODE_STATE] = 4,
     [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2,
     [MP_F_SMALL_INT_MODULO] = 2,
     [MP_F_NATIVE_YIELD_FROM] = 3,
+    [MP_F_SETJMP] = 1,
 };
 
 #define N_X86 (1)

py/emitnxtensawin.c

@@ -0,0 +1,23 @@
+// Xtensa-Windowed specific stuff
+
+#include "py/mpconfig.h"
+
+#if MICROPY_EMIT_XTENSAWIN
+
+// this is defined so that the assembler exports generic assembler API macros
+#define GENERIC_ASM_API (1)
+#define GENERIC_ASM_API_WIN (1)
+#include "py/asmxtensa.h"
+
+// Word indices of REG_LOCAL_x in nlr_buf_t
+#define NLR_BUF_IDX_LOCAL_1 (2 + 4) // a4
+#define NLR_BUF_IDX_LOCAL_2 (2 + 5) // a5
+#define NLR_BUF_IDX_LOCAL_3 (2 + 6) // a6
+
+#define N_NLR_SETJMP (1)
+#define N_PRELUDE_AS_BYTES_OBJ (1)
+#define N_XTENSAWIN (1)
+#define EXPORT_FUN(name) emit_native_xtensawin_##name
+#include "py/emitnative.c"
+
+#endif

py/frozenmod.h

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2015 Paul Sokolovsky
+ * Copyright (c) 2016 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

py/gc.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

py/grammar.h

@@ -55,7 +55,7 @@ DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE))
 // varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
 // vfpdef: NAME
 
-DEF_RULE_NC(decorator, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
+DEF_RULE_NC(decorator, and(4), tok(OP_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
 DEF_RULE_NC(decorators, one_or_more, rule(decorator))
 DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body))
 #if MICROPY_PY_ASYNC_AWAIT
@@ -96,7 +96,7 @@ DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), t
 // small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
 // expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*)
 // testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
-// augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
+// augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
 // # For normal assignments, additional restrictions enforced by the interpreter
 
 DEF_RULE_NC(small_stmt, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt))
@@ -108,7 +108,7 @@ DEF_RULE_NC(expr_stmt_assign, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6))
 DEF_RULE_NC(expr_stmt_6, or(2), rule(yield_expr), rule(testlist_star_expr))
 DEF_RULE(testlist_star_expr, c(generic_tuple), list_with_end, rule(testlist_star_expr_2), tok(DEL_COMMA))
 DEF_RULE_NC(testlist_star_expr_2, or(2), rule(star_expr), rule(test))
-DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
+DEF_RULE_NC(augassign, or(13), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_AT_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
 
 // del_stmt: 'del' exprlist
 // pass_stmt: 'pass'
@@ -226,7 +226,7 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara
 // and_expr: shift_expr ('&' shift_expr)*
 // shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 // arith_expr: term (('+'|'-') term)*
-// term: factor (('*'|'/'|'%'|'//') factor)*
+// term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 // factor: ('+'|'-'|'~') factor | power
 // power: atom_expr ['**' factor]
 // atom_expr: 'await' atom trailer* | atom trailer*
@@ -249,7 +249,7 @@ DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE))
 DEF_RULE(arith_expr, c(term), list, rule(term), rule(arith_op))
 DEF_RULE_NC(arith_op, or(2), tok(OP_PLUS), tok(OP_MINUS))
 DEF_RULE(term, c(term), list, rule(factor), rule(term_op))
-DEF_RULE_NC(term_op, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
+DEF_RULE_NC(term_op, or(5), tok(OP_STAR), tok(OP_AT), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
 DEF_RULE_NC(factor, or(2), rule(factor_2), rule(power))
 DEF_RULE(factor_2, c(factor_2), and_ident(2), rule(factor_op), rule(factor))
 DEF_RULE_NC(factor_op, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE))

py/lexer.c

@@ -174,7 +174,7 @@ STATIC void indent_pop(mp_lexer_t *lex) {
 // this means if the start of two ops are the same then they are equal til the last char
 
 STATIC const char *const tok_enc =
-    "()[]{},:;@~" // singles
+    "()[]{},:;~" // singles
     "<e=c<e="     // < <= << <<=
     ">e=c>e="     // > >= >> >>=
     "*e=c*e="     // * *= ** **=
@@ -185,6 +185,7 @@ STATIC const char *const tok_enc =
     "/e=c/e="     // / /= // //=
     "%e="         // % %=
     "^e="         // ^ ^=
+    "@e="         // @ @=
     "=e="         // = ==
     "!.";         // start of special cases: != . ...
 
@@ -193,7 +194,7 @@ STATIC const uint8_t tok_enc_kind[] = {
     MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
     MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
     MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE,
-    MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE,
+    MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_OP_TILDE,
 
     MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL,
     MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL,
@@ -205,6 +206,7 @@ STATIC const uint8_t tok_enc_kind[] = {
     MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL,
     MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL,
     MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
+    MP_TOKEN_OP_AT, MP_TOKEN_DEL_AT_EQUAL,
     MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
 };
 

py/lexer.h

@@ -96,26 +96,46 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_KW_WITH,
     MP_TOKEN_KW_YIELD,
 
-    MP_TOKEN_OP_PLUS,
-    MP_TOKEN_OP_MINUS,
-    MP_TOKEN_OP_STAR,
-    MP_TOKEN_OP_DBL_STAR,
-    MP_TOKEN_OP_SLASH,
-    MP_TOKEN_OP_DBL_SLASH,
-    MP_TOKEN_OP_PERCENT,
+    MP_TOKEN_OP_TILDE,
+
+    // Order of these 6 matches corresponding mp_binary_op_t operator
     MP_TOKEN_OP_LESS,
-    MP_TOKEN_OP_DBL_LESS,
     MP_TOKEN_OP_MORE,
-    MP_TOKEN_OP_DBL_MORE,
-    MP_TOKEN_OP_AMPERSAND,
-    MP_TOKEN_OP_PIPE,
-    MP_TOKEN_OP_CARET,
-    MP_TOKEN_OP_TILDE,
+    MP_TOKEN_OP_DBL_EQUAL,
     MP_TOKEN_OP_LESS_EQUAL,
     MP_TOKEN_OP_MORE_EQUAL,
-    MP_TOKEN_OP_DBL_EQUAL,
     MP_TOKEN_OP_NOT_EQUAL,
 
+    // Order of these 13 matches corresponding mp_binary_op_t operator
+    MP_TOKEN_OP_PIPE,
+    MP_TOKEN_OP_CARET,
+    MP_TOKEN_OP_AMPERSAND,
+    MP_TOKEN_OP_DBL_LESS,
+    MP_TOKEN_OP_DBL_MORE,
+    MP_TOKEN_OP_PLUS,
+    MP_TOKEN_OP_MINUS,
+    MP_TOKEN_OP_STAR,
+    MP_TOKEN_OP_AT,
+    MP_TOKEN_OP_DBL_SLASH,
+    MP_TOKEN_OP_SLASH,
+    MP_TOKEN_OP_PERCENT,
+    MP_TOKEN_OP_DBL_STAR,
+
+    // Order of these 13 matches corresponding mp_binary_op_t operator
+    MP_TOKEN_DEL_PIPE_EQUAL,
+    MP_TOKEN_DEL_CARET_EQUAL,
+    MP_TOKEN_DEL_AMPERSAND_EQUAL,
+    MP_TOKEN_DEL_DBL_LESS_EQUAL,
+    MP_TOKEN_DEL_DBL_MORE_EQUAL,
+    MP_TOKEN_DEL_PLUS_EQUAL,
+    MP_TOKEN_DEL_MINUS_EQUAL,
+    MP_TOKEN_DEL_STAR_EQUAL,
+    MP_TOKEN_DEL_AT_EQUAL,
+    MP_TOKEN_DEL_DBL_SLASH_EQUAL,
+    MP_TOKEN_DEL_SLASH_EQUAL,
+    MP_TOKEN_DEL_PERCENT_EQUAL,
+    MP_TOKEN_DEL_DBL_STAR_EQUAL,
+
     MP_TOKEN_DEL_PAREN_OPEN,
     MP_TOKEN_DEL_PAREN_CLOSE,
     MP_TOKEN_DEL_BRACKET_OPEN,
@@ -126,20 +146,7 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_DEL_COLON,
     MP_TOKEN_DEL_PERIOD,
     MP_TOKEN_DEL_SEMICOLON,
-    MP_TOKEN_DEL_AT,
     MP_TOKEN_DEL_EQUAL,
-    MP_TOKEN_DEL_PLUS_EQUAL,
-    MP_TOKEN_DEL_MINUS_EQUAL,
-    MP_TOKEN_DEL_STAR_EQUAL,
-    MP_TOKEN_DEL_SLASH_EQUAL,
-    MP_TOKEN_DEL_DBL_SLASH_EQUAL,
-    MP_TOKEN_DEL_PERCENT_EQUAL,
-    MP_TOKEN_DEL_AMPERSAND_EQUAL,
-    MP_TOKEN_DEL_PIPE_EQUAL,
-    MP_TOKEN_DEL_CARET_EQUAL,
-    MP_TOKEN_DEL_DBL_MORE_EQUAL,
-    MP_TOKEN_DEL_DBL_LESS_EQUAL,
-    MP_TOKEN_DEL_DBL_STAR_EQUAL,
     MP_TOKEN_DEL_MINUS_MORE,
 } mp_token_kind_t;
 
@@ -177,8 +184,6 @@ void mp_lexer_to_next(mp_lexer_t *lex);
 // platform specific import function; must be implemented for a specific port
 // TODO tidy up, rename, or put elsewhere
 
-//mp_lexer_t *mp_import_open_file(qstr mod_name);
-
 typedef enum {
     MP_IMPORT_STAT_NO_EXIST,
     MP_IMPORT_STAT_DIR,

py/makemoduledefs.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+# This pre-processor parses provided objects' c files for
+# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+# These are used to generate a header with the required entries for
+# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
+
+from __future__ import print_function
+
+import re
+import io
+import os
+import argparse
+
+
+pattern = re.compile(
+    r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
+    flags=re.DOTALL
+)
+
+
+def find_c_file(obj_file, vpath):
+    """ Search vpaths for the c file that matches the provided object_file.
+
+    :param str obj_file: object file to find the matching c file for
+    :param List[str] vpath: List of base paths, similar to gcc vpath
+    :return: str path to c file or None
+    """
+    c_file = None
+    relative_c_file = os.path.splitext(obj_file)[0] + ".c"
+    relative_c_file = relative_c_file.lstrip('/\\')
+    for p in vpath:
+        possible_c_file = os.path.join(p, relative_c_file)
+        if os.path.exists(possible_c_file):
+            c_file = possible_c_file
+            break
+
+    return c_file
+
+
+def find_module_registrations(c_file):
+    """ Find any MP_REGISTER_MODULE definitions in the provided c file.
+
+    :param str c_file: path to c file to check
+    :return: List[(module_name, obj_module, enabled_define)]
+    """
+    global pattern
+
+    if c_file is None:
+        # No c file to match the object file, skip
+        return set()
+
+    with io.open(c_file, encoding='utf-8') as c_file_obj:
+        return set(re.findall(pattern, c_file_obj.read()))
+
+
+def generate_module_table_header(modules):
+    """ Generate header with module table entries for builtin modules.
+
+    :param List[(module_name, obj_module, enabled_define)] modules: module defs
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    mod_defs = []
+    print("// Automatically generated by makemoduledefs.py.\n")
+    for module_name, obj_module, enabled_define in modules:
+        mod_def = "MODULE_DEF_{}".format(module_name.upper())
+        mod_defs.append(mod_def)
+        print((
+            "#if ({enabled_define})\n"
+            "    extern const struct _mp_obj_module_t {obj_module};\n"
+            "    #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
+            "#else\n"
+            "    #define {mod_def}\n"
+            "#endif\n"
+            ).format(module_name=module_name, obj_module=obj_module,
+                     enabled_define=enabled_define, mod_def=mod_def)
+        )
+
+    print("\n#define MICROPY_REGISTERED_MODULES \\")
+
+    for mod_def in mod_defs:
+        print("    {mod_def} \\".format(mod_def=mod_def))
+
+    print("// MICROPY_REGISTERED_MODULES")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vpath", default=".",
+                        help="comma separated list of folders to search for c files in")
+    parser.add_argument("files", nargs="*",
+                        help="list of c files to search")
+    args = parser.parse_args()
+
+    vpath = [p.strip() for p in args.vpath.split(',')]
+
+    modules = set()
+    for obj_file in args.files:
+        c_file = find_c_file(obj_file, vpath)
+        modules |= find_module_registrations(c_file)
+
+    generate_module_table_header(sorted(modules))
+
+
+if __name__ == '__main__':
+    main()

py/makeqstrdata.py

@@ -51,6 +51,176 @@ codepoint2name[ord('^')] = 'caret'
 codepoint2name[ord('|')] = 'pipe'
 codepoint2name[ord('~')] = 'tilde'
 
+# static qstrs, should be sorted
+
+static_qstr_list = [
+    "",
+    "__dir__", # Put __dir__ after empty qstr for builtin dir() to work
+    "\n",
+    " ",
+    "*",
+    "/",
+    "<module>",
+    "_",
+    "__call__",
+    "__class__",
+    "__delitem__",
+    "__enter__",
+    "__exit__",
+    "__getattr__",
+    "__getitem__",
+    "__hash__",
+    "__init__",
+    "__int__",
+    "__iter__",
+    "__len__",
+    "__main__",
+    "__module__",
+    "__name__",
+    "__new__",
+    "__next__",
+    "__qualname__",
+    "__repr__",
+    "__setitem__",
+    "__str__",
+    "ArithmeticError",
+    "AssertionError",
+    "AttributeError",
+    "BaseException",
+    "EOFError",
+    "Ellipsis",
+    "Exception",
+    "GeneratorExit",
+    "ImportError",
+    "IndentationError",
+    "IndexError",
+    "KeyError",
+    "KeyboardInterrupt",
+    "LookupError",
+    "MemoryError",
+    "NameError",
+    "NoneType",
+    "NotImplementedError",
+    "OSError",
+    "OverflowError",
+    "RuntimeError",
+    "StopIteration",
+    "SyntaxError",
+    "SystemExit",
+    "TypeError",
+    "ValueError",
+    "ZeroDivisionError",
+    "abs",
+    "all",
+    "any",
+    "append",
+    "args",
+    "bool",
+    "builtins",
+    "bytearray",
+    "bytecode",
+    "bytes",
+    "callable",
+    "chr",
+    "classmethod",
+    "clear",
+    "close",
+    "const",
+    "copy",
+    "count",
+    "dict",
+    "dir",
+    "divmod",
+    "end",
+    "endswith",
+    "eval",
+    "exec",
+    "extend",
+    "find",
+    "format",
+    "from_bytes",
+    "get",
+    "getattr",
+    "globals",
+    "hasattr",
+    "hash",
+    "id",
+    "index",
+    "insert",
+    "int",
+    "isalpha",
+    "isdigit",
+    "isinstance",
+    "islower",
+    "isspace",
+    "issubclass",
+    "isupper",
+    "items",
+    "iter",
+    "join",
+    "key",
+    "keys",
+    "len",
+    "list",
+    "little",
+    "locals",
+    "lower",
+    "lstrip",
+    "main",
+    "map",
+    "micropython",
+    "next",
+    "object",
+    "open",
+    "ord",
+    "pop",
+    "popitem",
+    "pow",
+    "print",
+    "range",
+    "read",
+    "readinto",
+    "readline",
+    "remove",
+    "replace",
+    "repr",
+    "reverse",
+    "rfind",
+    "rindex",
+    "round",
+    "rsplit",
+    "rstrip",
+    "self",
+    "send",
+    "sep",
+    "set",
+    "setattr",
+    "setdefault",
+    "sort",
+    "sorted",
+    "split",
+    "start",
+    "startswith",
+    "staticmethod",
+    "step",
+    "stop",
+    "str",
+    "strip",
+    "sum",
+    "super",
+    "throw",
+    "to_bytes",
+    "tuple",
+    "type",
+    "update",
+    "upper",
+    "utf-8",
+    "value",
+    "values",
+    "write",
+    "zip",
+]
+
 # this must match the equivalent function in qstr.c
 def compute_hash(qstr, bytes_hash):
     hash = 5381
@@ -70,9 +240,22 @@ def qstr_escape(qst):
     return re.sub(r'[^A-Za-z0-9_]', esc_char, qst)
 
 def parse_input_headers(infiles):
-    # read the qstrs in from the input files
     qcfgs = {}
     qstrs = {}
+
+    # add static qstrs
+    for qstr in static_qstr_list:
+        # work out the corresponding qstr name
+        ident = qstr_escape(qstr)
+
+        # don't add duplicates
+        assert ident not in qstrs
+
+        # add the qstr to the list, with order number to retain original order in file
+        order = len(qstrs) - 300000
+        qstrs[ident] = (order, ident, qstr)
+
+    # read the qstrs in from the input files
     for infile in infiles:
         with open(infile, 'rt') as f:
             for line in f:
@@ -96,9 +279,11 @@ def parse_input_headers(infiles):
                 # get the qstr value
                 qstr = match.group(1)
 
-                # special case to specify control characters
+                # special cases to specify control characters
                 if qstr == '\\n':
                     qstr = '\n'
+                elif qstr == '\\r\\n':
+                    qstr = '\r\n'
 
                 # work out the corresponding qstr name
                 ident = qstr_escape(qstr)
@@ -154,7 +339,7 @@ def print_qstr_data(qcfgs, qstrs):
     print('')
 
     # add NULL qstr with no hash or data
-    print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
+    print('QDEF(MP_QSTRnull, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
 
     # go through each qstr and print it out
     for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):

py/makeqstrdefs.py

@@ -9,12 +9,9 @@ from __future__ import print_function
 
 import re
 import sys
+import io
 import os
 
-# Blacklist of qstrings that are specially handled in further
-# processing and should be ignored
-QSTRING_BLACK_LIST = set(['NULL', 'number_of'])
-
 
 def write_out(fname, output):
     if output:
@@ -45,8 +42,7 @@ def process_file(f):
             continue
         for match in re_qstr.findall(line):
             name = match.replace('MP_QSTR_', '')
-            if name not in QSTRING_BLACK_LIST:
-                output.append('Q(' + name + ')')
+            output.append('Q(' + name + ')')
 
     write_out(last_fname, output)
     return ""
@@ -108,7 +104,7 @@ if __name__ == "__main__":
         pass
 
     if args.command == "split":
-        with open(args.input_filename) as infile:
+        with io.open(args.input_filename, encoding='utf-8') as infile:
             process_file(infile)
 
     if args.command == "cat":

py/misc.h

@@ -47,8 +47,8 @@ typedef unsigned int uint;
 #endif
 
 // Classical double-indirection stringification of preprocessor macro's value
-#define _MP_STRINGIFY(x) #x
-#define MP_STRINGIFY(x) _MP_STRINGIFY(x)
+#define MP_STRINGIFY_HELPER(x) #x
+#define MP_STRINGIFY(x) MP_STRINGIFY_HELPER(x)
 
 // Static assertion macro
 #define MP_STATIC_ASSERT(cond) ((void)sizeof(char[1 - 2 * !(cond)]))

py/mkenv.mk

@@ -42,11 +42,14 @@ ECHO = @echo
 CP = cp
 MKDIR = mkdir
 SED = sed
+CAT = cat
+TOUCH = touch
 PYTHON = python3
 
 AS = $(CROSS_COMPILE)as
 CC = $(CROSS_COMPILE)gcc
 CXX = $(CROSS_COMPILE)g++
+GDB = $(CROSS_COMPILE)gdb
 LD = $(CROSS_COMPILE)ld
 OBJCOPY = $(CROSS_COMPILE)objcopy
 SIZE = $(CROSS_COMPILE)size
@@ -58,10 +61,13 @@ CXX += -m32
 LD += -m32
 endif
 
+MAKE_MANIFEST = $(PYTHON) $(TOP)/tools/makemanifest.py
 MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py
 MPY_CROSS = $(TOP)/mpy-cross/mpy-cross
 MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py
 
+MPY_LIB_DIR = $(TOP)/../micropython-lib
+
 all:
 .PHONY: all
 

py/mkrules.mk

@@ -20,12 +20,12 @@ endif
 # can be located. By following this scheme, it allows a single build rule
 # to be used to compile all .c files.
 
-vpath %.S . $(TOP)
+vpath %.S . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.S
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
-vpath %.s . $(TOP)
+vpath %.s . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.s
 	$(ECHO) "AS $<"
 	$(Q)$(AS) -o $@ $<
@@ -42,18 +42,18 @@ $(Q)$(CC) $(CFLAGS) -c -MD -o $@ $<
   $(RM) -f $(@:.o=.d)
 endef
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.c
 	$(call compile_c)
 
 QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR
 QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 
 $(BUILD)/%.pp: %.c
 	$(ECHO) "PreProcess $<"
-	$(Q)$(CC) $(CFLAGS) -E -Wp,-C,-dD,-dI -o $@ $<
+	$(Q)$(CPP) $(CFLAGS) -Wp,-C,-dD,-dI -o $@ $<
 
 # The following rule uses | to create an order only prerequisite. Order only
 # prerequisites only get built if they don't exist. They don't cause timestamp
@@ -70,14 +70,15 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h
 # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^)
 # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?)
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
-$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
+# See more information about this process in docs/develop/qstr.rst.
+$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS)
 	$(ECHO) "GEN $@"
-	$(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last;
+	$(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last
 
 $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last
 	$(ECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
-	$(Q)touch $@
+	$(Q)$(TOUCH) $@
 
 $(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split
 	$(ECHO) "GEN $@"
@@ -96,23 +97,35 @@ $(OBJ_DIRS):
 $(HEADER_BUILD):
 	$(MKDIR) -p $@
 
+ifneq ($(FROZEN_MANIFEST),)
+# to build frozen_content.c from a manifest
+$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h
+	$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) $(FROZEN_MANIFEST)
+
 ifneq ($(FROZEN_DIR),)
+$(error FROZEN_DIR cannot be used in conjunction with FROZEN_MANIFEST)
+endif
+
+ifneq ($(FROZEN_MPY_DIR),)
+$(error FROZEN_MPY_DIR cannot be used in conjunction with FROZEN_MANIFEST)
+endif
+endif
+
+ifneq ($(FROZEN_DIR),)
+$(info Warning: FROZEN_DIR is deprecated in favour of FROZEN_MANIFEST)
 $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DEPS)
 	$(ECHO) "GEN $@"
 	$(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@
 endif
 
 ifneq ($(FROZEN_MPY_DIR),)
-# to build the MicroPython cross compiler
-$(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/ports/windows/fmode.c
-	$(Q)$(MAKE) -C $(TOP)/mpy-cross
-
+$(info Warning: FROZEN_MPY_DIR is deprecated in favour of FROZEN_MANIFEST)
 # make a list of all the .py files that need compiling and freezing
 FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==')
 FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy))
 
 # to build .mpy files from .py files
-$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py $(TOP)/mpy-cross/mpy-cross
+$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py
 	@$(ECHO) "MPY $<"
 	$(Q)$(MKDIR) -p $(dir $@)
 	$(Q)$(MPY_CROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $<
@@ -146,6 +159,13 @@ clean-prog:
 .PHONY: clean-prog
 endif
 
+submodules:
+	$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
+ifneq ($(GIT_SUBMODULES),)
+	$(Q)git submodule update --init $(addprefix $(TOP)/,$(GIT_SUBMODULES))
+endif
+.PHONY: submodules
+
 LIBMICROPYTHON = libmicropython.a
 
 # We can execute extra commands after library creation using
@@ -190,7 +210,7 @@ print-cfg:
 
 print-def:
 	@$(ECHO) "The following defines are built into the $(CC) compiler"
-	touch __empty__.c
+	$(TOUCH) __empty__.c
 	@$(CC) -E -Wp,-dM __empty__.c
 	@$(RM) -f __empty__.c
 

py/modarray.c

@@ -29,15 +29,17 @@
 #if MICROPY_PY_ARRAY
 
 STATIC const mp_rom_map_elem_t mp_module_array_globals_table[] = {
-    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_array) },
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uarray) },
     { MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_type_array) },
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_array_globals, mp_module_array_globals_table);
 
-const mp_obj_module_t mp_module_array = {
+const mp_obj_module_t mp_module_uarray = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_array_globals,
 };
 
+MP_REGISTER_MODULE(MP_QSTR_uarray, mp_module_uarray, MICROPY_PY_ARRAY);
+
 #endif

py/modio.c

@@ -208,15 +208,8 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) {
     // package parameter being None, the path_in is interpreted as a
     // raw path.
     if (package_in != mp_const_none) {
-        mp_obj_t args[5];
-        args[0] = package_in;
-        args[1] = mp_const_none; // TODO should be globals
-        args[2] = mp_const_none; // TODO should be locals
-        args[3] = mp_const_true; // Pass sentinel "non empty" value to force returning of leaf module
-        args[4] = MP_OBJ_NEW_SMALL_INT(0);
-
-        // TODO lookup __import__ and call that instead of going straight to builtin implementation
-        mp_obj_t pkg = mp_builtin___import__(5, args);
+        // Pass "True" as sentinel value in fromlist to force returning of leaf module
+        mp_obj_t pkg = mp_import_name(mp_obj_str_get_qstr(package_in), mp_const_true, MP_OBJ_NEW_SMALL_INT(0));
 
         mp_obj_t dest[2];
         mp_load_method_maybe(pkg, MP_QSTR___path__, dest);

py/modmath.c

@@ -171,6 +171,42 @@ MATH_FUN_1(lgamma, lgamma)
 #endif
 //TODO: fsum
 
+#if MICROPY_PY_MATH_ISCLOSE
+STATIC mp_obj_t mp_math_isclose(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_a, ARG_b, ARG_rel_tol, ARG_abs_tol };
+    static const mp_arg_t allowed_args[] = {
+        {MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ},
+        {MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ},
+        {MP_QSTR_rel_tol, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL}},
+        {MP_QSTR_abs_tol, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NEW_SMALL_INT(0)}},
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+    const mp_float_t a = mp_obj_get_float(args[ARG_a].u_obj);
+    const mp_float_t b = mp_obj_get_float(args[ARG_b].u_obj);
+    const mp_float_t rel_tol = args[ARG_rel_tol].u_obj == MP_OBJ_NULL
+        ? (mp_float_t)1e-9 : mp_obj_get_float(args[ARG_rel_tol].u_obj);
+    const mp_float_t abs_tol = mp_obj_get_float(args[ARG_abs_tol].u_obj);
+    if (rel_tol < (mp_float_t)0.0 || abs_tol < (mp_float_t)0.0) {
+        math_error();
+    }
+    if (a == b) {
+        return mp_const_true;
+    }
+    const mp_float_t difference = MICROPY_FLOAT_C_FUN(fabs)(a - b);
+    if (isinf(difference)) { // Either a or b is inf
+        return mp_const_false;
+    }
+    if ((difference <= abs_tol) ||
+        (difference <= MICROPY_FLOAT_C_FUN(fabs)(rel_tol * a)) ||
+        (difference <= MICROPY_FLOAT_C_FUN(fabs)(rel_tol * b))) {
+        return mp_const_true;
+    }
+    return mp_const_false;
+}
+MP_DEFINE_CONST_FUN_OBJ_KW(mp_math_isclose_obj, 2, mp_math_isclose);
+#endif
+
 // Function that takes a variable number of arguments
 
 // log(x[, base])
@@ -335,6 +371,9 @@ STATIC const mp_rom_map_elem_t mp_module_math_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_isfinite), MP_ROM_PTR(&mp_math_isfinite_obj) },
     { MP_ROM_QSTR(MP_QSTR_isinf), MP_ROM_PTR(&mp_math_isinf_obj) },
     { MP_ROM_QSTR(MP_QSTR_isnan), MP_ROM_PTR(&mp_math_isnan_obj) },
+    #if MICROPY_PY_MATH_ISCLOSE
+    { MP_ROM_QSTR(MP_QSTR_isclose), MP_ROM_PTR(&mp_math_isclose_obj) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_trunc), MP_ROM_PTR(&mp_math_trunc_obj) },
     { MP_ROM_QSTR(MP_QSTR_radians), MP_ROM_PTR(&mp_math_radians_obj) },
     { MP_ROM_QSTR(MP_QSTR_degrees), MP_ROM_PTR(&mp_math_degrees_obj) },

py/modmicropython.c

@@ -150,7 +150,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_micropython_kbd_intr_obj, mp_micropython_kbd
 #if MICROPY_ENABLE_SCHEDULER
 STATIC mp_obj_t mp_micropython_schedule(mp_obj_t function, mp_obj_t arg) {
     if (!mp_sched_schedule(function, arg)) {
-        mp_raise_msg(&mp_type_RuntimeError, "schedule stack full");
+        mp_raise_msg(&mp_type_RuntimeError, "schedule queue full");
     }
     return mp_const_none;
 }

py/modstruct.c

@@ -97,7 +97,7 @@ STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) {
             size += cnt;
         } else {
             total_cnt += cnt;
-            mp_uint_t align;
+            size_t align;
             size_t sz = mp_binary_get_size(fmt_type, *fmt, &align);
             while (cnt--) {
                 // Apply alignment
@@ -146,6 +146,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
         }
         p += offset;
     }
+    byte *p_base = p;
 
     // Check that the input buffer is big enough to unpack all the values
     if (p + total_sz > end_p) {
@@ -164,7 +165,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
             res->items[i++] = item;
         } else {
             while (cnt--) {
-                item = mp_binary_get_val(fmt_type, *fmt, &p);
+                item = mp_binary_get_val(fmt_type, *fmt, p_base, &p);
                 res->items[i++] = item;
             }
         }
@@ -179,6 +180,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c
     const char *fmt = mp_obj_str_get_str(fmt_in);
     char fmt_type = get_fmt_type(&fmt);
 
+    byte *p_base = p;
     size_t i;
     for (i = 0; i < n_args;) {
         mp_uint_t cnt = 1;
@@ -203,7 +205,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c
         } else {
             // If we run out of args then we just finish; CPython would raise struct.error
             while (cnt-- && i < n_args) {
-                mp_binary_set_val(fmt_type, *fmt, args[i++], &p);
+                mp_binary_set_val(fmt_type, *fmt, args[i++], p_base, &p);
             }
         }
         fmt++;

py/modsys.c

@@ -34,6 +34,12 @@
 #include "py/stream.h"
 #include "py/smallint.h"
 #include "py/runtime.h"
+#include "py/persistentcode.h"
+
+#if MICROPY_PY_SYS_SETTRACE
+#include "py/objmodule.h"
+#include "py/profile.h"
+#endif
 
 #if MICROPY_PY_SYS
 
@@ -61,22 +67,36 @@ STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = {
     3,
     { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) }
 };
+#if MICROPY_PERSISTENT_CODE_LOAD
+#define SYS_IMPLEMENTATION_ELEMS \
+    MP_ROM_QSTR(MP_QSTR_micropython), \
+    MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \
+    MP_ROM_INT(MPY_FILE_HEADER_INT)
+#else
+#define SYS_IMPLEMENTATION_ELEMS \
+    MP_ROM_QSTR(MP_QSTR_micropython), \
+    MP_ROM_PTR(&mp_sys_implementation_version_info_obj)
+#endif
 #if MICROPY_PY_ATTRTUPLE
-STATIC const qstr impl_fields[] = { MP_QSTR_name, MP_QSTR_version };
+STATIC const qstr impl_fields[] = {
+    MP_QSTR_name,
+    MP_QSTR_version,
+    #if MICROPY_PERSISTENT_CODE_LOAD
+    MP_QSTR_mpy,
+    #endif
+};
 STATIC MP_DEFINE_ATTRTUPLE(
     mp_sys_implementation_obj,
     impl_fields,
-    2,
-        MP_ROM_QSTR(MP_QSTR_micropython),
-        MP_ROM_PTR(&mp_sys_implementation_version_info_obj)
+    2 + MICROPY_PERSISTENT_CODE_LOAD,
+        SYS_IMPLEMENTATION_ELEMS
 );
 #else
 STATIC const mp_rom_obj_tuple_t mp_sys_implementation_obj = {
     {&mp_type_tuple},
-    2,
+    2 + MICROPY_PERSISTENT_CODE_LOAD,
     {
-        MP_ROM_QSTR(MP_QSTR_micropython),
-        MP_ROM_PTR(&mp_sys_implementation_version_info_obj),
+        SYS_IMPLEMENTATION_ELEMS
     }
 };
 #endif
@@ -146,6 +166,24 @@ STATIC mp_obj_t mp_sys_getsizeof(mp_obj_t obj) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof);
 #endif
 
+#if MICROPY_PY_SYS_ATEXIT
+// atexit(callback): Callback is called when sys.exit is called.
+STATIC mp_obj_t mp_sys_atexit(mp_obj_t obj) {
+    mp_obj_t old = MP_STATE_VM(sys_exitfunc);
+    MP_STATE_VM(sys_exitfunc) = obj;
+    return old;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_atexit_obj, mp_sys_atexit);
+#endif
+
+#if MICROPY_PY_SYS_SETTRACE
+// settrace(tracefunc): Set the system’s trace function.
+STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) {
+    return mp_prof_settrace(obj);
+}
+MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace);
+#endif // MICROPY_PY_SYS_SETTRACE
+
 STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) },
 
@@ -180,6 +218,10 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_exit), MP_ROM_PTR(&mp_sys_exit_obj) },
     #endif
 
+    #if MICROPY_PY_SYS_SETTRACE
+    { MP_ROM_QSTR(MP_QSTR_settrace), MP_ROM_PTR(&mp_sys_settrace_obj) },
+    #endif
+
     #if MICROPY_PY_SYS_STDFILES
     { MP_ROM_QSTR(MP_QSTR_stdin), MP_ROM_PTR(&mp_sys_stdin_obj) },
     { MP_ROM_QSTR(MP_QSTR_stdout), MP_ROM_PTR(&mp_sys_stdout_obj) },
@@ -201,6 +243,9 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
      */
 
     { MP_ROM_QSTR(MP_QSTR_print_exception), MP_ROM_PTR(&mp_sys_print_exception_obj) },
+    #if MICROPY_PY_SYS_ATEXIT
+    { MP_ROM_QSTR(MP_QSTR_atexit), MP_ROM_PTR(&mp_sys_atexit_obj) },
+    #endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_sys_globals, mp_module_sys_globals_table);

py/moduerrno.c

@@ -104,7 +104,7 @@ qstr mp_errno_to_str(mp_obj_t errno_val) {
     // We have the errorcode dict so can do a lookup using the hash map
     mp_map_elem_t *elem = mp_map_lookup((mp_map_t*)&errorcode_dict.map, errno_val, MP_MAP_LOOKUP);
     if (elem == NULL) {
-        return MP_QSTR_NULL;
+        return MP_QSTRnull;
     } else {
         return MP_OBJ_QSTR_VALUE(elem->value);
     }
@@ -115,7 +115,7 @@ qstr mp_errno_to_str(mp_obj_t errno_val) {
             return MP_OBJ_QSTR_VALUE(mp_module_uerrno_globals_table[i].key);
         }
     }
-    return MP_QSTR_NULL;
+    return MP_QSTRnull;
     #endif
 }
 

py/mpconfig.h

@@ -28,7 +28,7 @@
 
 // Current version of MicroPython
 #define MICROPY_VERSION_MAJOR 1
-#define MICROPY_VERSION_MINOR 10
+#define MICROPY_VERSION_MINOR 12
 #define MICROPY_VERSION_MICRO 0
 
 // Combined version as a 32-bit number for convenience
@@ -323,12 +323,23 @@
 #define MICROPY_EMIT_INLINE_XTENSA (0)
 #endif
 
+// Whether to emit Xtensa-Windowed native code
+#ifndef MICROPY_EMIT_XTENSAWIN
+#define MICROPY_EMIT_XTENSAWIN (0)
+#endif
+
 // Convenience definition for whether any native emitter is enabled
-#define MICROPY_EMIT_NATIVE (MICROPY_EMIT_X64 || MICROPY_EMIT_X86 || MICROPY_EMIT_THUMB || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA)
+#define MICROPY_EMIT_NATIVE (MICROPY_EMIT_X64 || MICROPY_EMIT_X86 || MICROPY_EMIT_THUMB || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN)
+
+// Select prelude-as-bytes-object for certain emitters
+#define MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ (MICROPY_EMIT_XTENSAWIN)
 
 // Convenience definition for whether any inline assembler emitter is enabled
 #define MICROPY_EMIT_INLINE_ASM (MICROPY_EMIT_INLINE_THUMB || MICROPY_EMIT_INLINE_XTENSA)
 
+// Convenience definition for whether any native or inline assembler emitter is enabled
+#define MICROPY_EMIT_MACHINE_CODE (MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM)
+
 /*****************************************************************************/
 /* Compiler configuration                                                    */
 
@@ -338,6 +349,7 @@
 #endif
 
 // Whether the compiler is dynamically configurable (ie at runtime)
+// This will disable the ability to execute native/viper code
 #ifndef MICROPY_DYNAMIC_COMPILER
 #define MICROPY_DYNAMIC_COMPILER (0)
 #endif
@@ -356,6 +368,11 @@
 #define MICROPY_COMP_CONST_FOLDING (1)
 #endif
 
+// Whether to enable optimisations for constant literals, eg OrderedDict
+#ifndef MICROPY_COMP_CONST_LITERAL
+#define MICROPY_COMP_CONST_LITERAL (1)
+#endif
+
 // Whether to enable lookup of constants in modules; eg module.CONST
 #ifndef MICROPY_COMP_MODULE_CONST
 #define MICROPY_COMP_MODULE_CONST (0)
@@ -409,6 +426,11 @@
 #define MICROPY_DEBUG_VERBOSE (0)
 #endif
 
+// Whether to enable debugging versions of MP_OBJ_NULL/STOP_ITERATION/SENTINEL
+#ifndef MICROPY_DEBUG_MP_OBJ_SENTINELS
+#define MICROPY_DEBUG_MP_OBJ_SENTINELS (0)
+#endif
+
 // Whether to enable a simple VM stack overflow check
 #ifndef MICROPY_DEBUG_VM_STACK_OVERFLOW
 #define MICROPY_DEBUG_VM_STACK_OVERFLOW (0)
@@ -846,6 +868,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_BUILTINS_MEMORYVIEW (0)
 #endif
 
+// Whether to support memoryview.itemsize attribute
+#ifndef MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE
+#define MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE (0)
+#endif
+
 // Whether to support set object
 #ifndef MICROPY_PY_BUILTINS_SET
 #define MICROPY_PY_BUILTINS_SET (1)
@@ -1060,6 +1087,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_MATH_FACTORIAL (0)
 #endif
 
+// Whether to provide math.isclose function
+#ifndef MICROPY_PY_MATH_ISCLOSE
+#define MICROPY_PY_MATH_ISCLOSE (0)
+#endif
+
 // Whether to provide "cmath" module
 #ifndef MICROPY_PY_CMATH
 #define MICROPY_PY_CMATH (0)
@@ -1142,6 +1174,16 @@ typedef double mp_float_t;
 #define MICROPY_PY_SYS_EXIT (1)
 #endif
 
+// Whether to provide "sys.atexit" function (MicroPython extension)
+#ifndef MICROPY_PY_SYS_ATEXIT
+#define MICROPY_PY_SYS_ATEXIT (0)
+#endif
+
+// Whether to provide "sys.settrace" function
+#ifndef MICROPY_PY_SYS_SETTRACE
+#define MICROPY_PY_SYS_SETTRACE (0)
+#endif
+
 // Whether to provide "sys.getsizeof" function
 #ifndef MICROPY_PY_SYS_GETSIZEOF
 #define MICROPY_PY_SYS_GETSIZEOF (0)
@@ -1230,6 +1272,10 @@ typedef double mp_float_t;
 #define MICROPY_PY_URE (0)
 #endif
 
+#ifndef MICROPY_PY_URE_DEBUG
+#define MICROPY_PY_URE_DEBUG (0)
+#endif
+
 #ifndef MICROPY_PY_URE_MATCH_GROUPS
 #define MICROPY_PY_URE_MATCH_GROUPS (0)
 #endif
@@ -1271,6 +1317,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_UCRYPTOLIB (0)
 #endif
 
+// Depends on MICROPY_PY_UCRYPTOLIB
+#ifndef MICROPY_PY_UCRYPTOLIB_CTR
+#define MICROPY_PY_UCRYPTOLIB_CTR (0)
+#endif
+
 #ifndef MICROPY_PY_UCRYPTOLIB_CONSTS
 #define MICROPY_PY_UCRYPTOLIB_CONSTS (0)
 #endif
@@ -1341,11 +1392,6 @@ typedef double mp_float_t;
 #define MICROPY_PORT_BUILTIN_MODULES
 #endif
 
-// Any module weak links - see objmodule.c:mp_builtin_module_weak_links_table.
-#ifndef MICROPY_PORT_BUILTIN_MODULE_WEAK_LINKS
-#define MICROPY_PORT_BUILTIN_MODULE_WEAK_LINKS
-#endif
-
 // Additional constant definitions for the compiler - see compile.c:mp_constants_table.
 #ifndef MICROPY_PORT_CONSTANTS
 #define MICROPY_PORT_CONSTANTS
@@ -1493,6 +1539,15 @@ typedef double mp_float_t;
 #define MP_UNLIKELY(x) __builtin_expect((x), 0)
 #endif
 
+// To annotate that code is unreachable
+#ifndef MP_UNREACHABLE
+#if defined(__GNUC__)
+#define MP_UNREACHABLE __builtin_unreachable();
+#else
+#define MP_UNREACHABLE for (;;);
+#endif
+#endif
+
 #ifndef MP_HTOBE16
 #if MP_ENDIANNESS_LITTLE
 # define MP_HTOBE16(x) ((uint16_t)( (((x) & 0xff) << 8) | (((x) >> 8) & 0xff) ))
@@ -1524,4 +1579,14 @@ typedef double mp_float_t;
 # define MP_WARN_CAT(x) (NULL)
 #endif
 
+// Feature dependency check.
+#if MICROPY_PY_SYS_SETTRACE
+#if !MICROPY_PERSISTENT_CODE_SAVE
+#error "MICROPY_PY_SYS_SETTRACE requires MICROPY_PERSISTENT_CODE_SAVE to be enabled"
+#endif
+#if MICROPY_COMP_CONST
+#error "MICROPY_PY_SYS_SETTRACE requires MICROPY_COMP_CONST to be disabled"
+#endif
+#endif
+
 #endif // MICROPY_INCLUDED_PY_MPCONFIG_H

py/mphal.h

@@ -34,6 +34,10 @@
 #include <mphalport.h>
 #endif
 
+#ifndef mp_hal_stdio_poll
+uintptr_t mp_hal_stdio_poll(uintptr_t poll_flags);
+#endif
+
 #ifndef mp_hal_stdin_rx_chr
 int mp_hal_stdin_rx_chr(void);
 #endif

py/mpprint.c

@@ -503,22 +503,28 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) {
                 chrs += mp_print_strn(print, str, prec, flags, fill, width);
                 break;
             }
-            case 'u':
-                chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 10, 'a', flags, fill, width);
-                break;
-            case 'd':
-                chrs += mp_print_int(print, va_arg(args, int), 1, 10, 'a', flags, fill, width);
+            case 'd': {
+                mp_int_t val;
+                if (long_arg) {
+                    val = va_arg(args, long int);
+                } else {
+                    val = va_arg(args, int);
+                }
+                chrs += mp_print_int(print, val, 1, 10, 'a', flags, fill, width);
                 break;
+            }
+            case 'u':
             case 'x':
             case 'X': {
-                char fmt_c = *fmt - 'X' + 'A';
+                int base = 16 - ((*fmt + 1) & 6); // maps char u/x/X to base 10/16/16
+                char fmt_c = (*fmt & 0xf0) - 'P' + 'A'; // maps char u/x/X to char a/a/A
                 mp_uint_t val;
                 if (long_arg) {
                     val = va_arg(args, unsigned long int);
                 } else {
                     val = va_arg(args, unsigned int);
                 }
-                chrs += mp_print_int(print, val, 0, 16, fmt_c, flags, fill, width);
+                chrs += mp_print_int(print, val, 0, base, fmt_c, flags, fill, width);
                 break;
             }
             case 'p':

py/mpstate.h

@@ -46,6 +46,8 @@ typedef struct mp_dynamic_compiler_t {
     uint8_t small_int_bits; // must be <= host small_int_bits
     bool opt_cache_map_lookup_in_bytecode;
     bool py_builtins_str_unicode;
+    uint8_t native_arch;
+    uint8_t nlr_buf_num_regs;
 } mp_dynamic_compiler_t;
 extern mp_dynamic_compiler_t mp_dynamic_compiler;
 #endif
@@ -140,7 +142,7 @@ typedef struct _mp_state_vm_t {
     volatile mp_obj_t mp_pending_exception;
 
     #if MICROPY_ENABLE_SCHEDULER
-    mp_sched_item_t sched_stack[MICROPY_SCHEDULER_DEPTH];
+    mp_sched_item_t sched_queue[MICROPY_SCHEDULER_DEPTH];
     #endif
 
     // current exception being handled, for sys.exc_info()
@@ -148,6 +150,11 @@ typedef struct _mp_state_vm_t {
     mp_obj_base_t *cur_exception;
     #endif
 
+    #if MICROPY_PY_SYS_ATEXIT
+    // exposed through sys.atexit function
+    mp_obj_t sys_exitfunc;
+    #endif
+
     // dictionary for the __main__ module
     mp_obj_dict_t dict_main;
 
@@ -182,6 +189,10 @@ typedef struct _mp_state_vm_t {
     struct _mp_vfs_mount_t *vfs_mount_table;
     #endif
 
+    #if MICROPY_PY_BLUETOOTH
+    mp_obj_t bluetooth;
+    #endif
+
     //
     // END ROOT POINTER SECTION
     ////////////////////////////////////////////////////////////
@@ -199,6 +210,9 @@ typedef struct _mp_state_vm_t {
 
     #if MICROPY_ENABLE_COMPILER
     mp_uint_t mp_optimise_value;
+    #if MICROPY_EMIT_NATIVE
+    uint8_t default_emit_opt; // one of MP_EMIT_OPT_xxx
+    #endif
     #endif
 
     // size of the emergency exception buf, if it's dynamically allocated
@@ -208,7 +222,8 @@ typedef struct _mp_state_vm_t {
 
     #if MICROPY_ENABLE_SCHEDULER
     volatile int16_t sched_state;
-    uint16_t sched_sp;
+    uint8_t sched_len;
+    uint8_t sched_idx;
     #endif
 
     #if MICROPY_PY_THREAD_GIL
@@ -243,6 +258,12 @@ typedef struct _mp_state_thread_t {
     mp_obj_dict_t *dict_globals;
 
     nlr_buf_t *nlr_top;
+
+    #if MICROPY_PY_SYS_SETTRACE
+    mp_obj_t prof_trace_callback;
+    bool prof_callback_is_executing;
+    struct _mp_code_state_t *current_code_state;
+    #endif
 } mp_state_thread_t;
 
 // This structure combines the above 3 structures.

py/mpy_scheduler.c

@@ -30,6 +30,19 @@
 
 #if MICROPY_ENABLE_SCHEDULER
 
+#define IDX_MASK(i) ((i) & (MICROPY_SCHEDULER_DEPTH - 1))
+
+static inline bool mp_sched_full(void) {
+    MP_STATIC_ASSERT(MICROPY_SCHEDULER_DEPTH <= 255); // MICROPY_SCHEDULER_DEPTH must fit in 8 bits
+    MP_STATIC_ASSERT((IDX_MASK(MICROPY_SCHEDULER_DEPTH) == 0)); // MICROPY_SCHEDULER_DEPTH must be a power of 2
+
+    return mp_sched_num_pending() == MICROPY_SCHEDULER_DEPTH;
+}
+
+static inline bool mp_sched_empty(void) {
+    return mp_sched_num_pending() == 0;
+}
+
 // A variant of this is inlined in the VM at the pending exception check
 void mp_handle_pending(void) {
     if (MP_STATE_VM(sched_state) == MP_SCHED_PENDING) {
@@ -51,8 +64,10 @@ void mp_handle_pending(void) {
 // or by the VM's inlined version of that function.
 void mp_handle_pending_tail(mp_uint_t atomic_state) {
     MP_STATE_VM(sched_state) = MP_SCHED_LOCKED;
-    if (MP_STATE_VM(sched_sp) > 0) {
-        mp_sched_item_t item = MP_STATE_VM(sched_stack)[--MP_STATE_VM(sched_sp)];
+    if (!mp_sched_empty()) {
+        mp_sched_item_t item = MP_STATE_VM(sched_queue)[MP_STATE_VM(sched_idx)];
+        MP_STATE_VM(sched_idx) = IDX_MASK(MP_STATE_VM(sched_idx) + 1);
+        --MP_STATE_VM(sched_len);
         MICROPY_END_ATOMIC_SECTION(atomic_state);
         mp_call_function_1_protected(item.func, item.arg);
     } else {
@@ -87,16 +102,16 @@ void mp_sched_unlock(void) {
 bool mp_sched_schedule(mp_obj_t function, mp_obj_t arg) {
     mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION();
     bool ret;
-    if (MP_STATE_VM(sched_sp) < MICROPY_SCHEDULER_DEPTH) {
+    if (!mp_sched_full()) {
         if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) {
             MP_STATE_VM(sched_state) = MP_SCHED_PENDING;
         }
-        MP_STATE_VM(sched_stack)[MP_STATE_VM(sched_sp)].func = function;
-        MP_STATE_VM(sched_stack)[MP_STATE_VM(sched_sp)].arg = arg;
-        ++MP_STATE_VM(sched_sp);
+        uint8_t iput = IDX_MASK(MP_STATE_VM(sched_idx) + MP_STATE_VM(sched_len)++);
+        MP_STATE_VM(sched_queue)[iput].func = function;
+        MP_STATE_VM(sched_queue)[iput].arg = arg;
         ret = true;
     } else {
-        // schedule stack is full
+        // schedule queue is full
         ret = false;
     }
     MICROPY_END_ATOMIC_SECTION(atomic_state);

py/nativeglue.c

@@ -24,14 +24,15 @@
  * THE SOFTWARE.
  */
 
+#include <stdarg.h>
 #include <stdio.h>
 #include <string.h>
 #include <assert.h>
 
 #include "py/runtime.h"
 #include "py/smallint.h"
-#include "py/emitglue.h"
-#include "py/bc.h"
+#include "py/nativeglue.h"
+#include "py/gc.h"
 
 #if MICROPY_DEBUG_VERBOSE // print debugging info
 #define DEBUG_printf DEBUG_printf
@@ -41,17 +42,31 @@
 
 #if MICROPY_EMIT_NATIVE
 
+int mp_native_type_from_qstr(qstr qst) {
+    switch (qst) {
+        case MP_QSTR_object: return MP_NATIVE_TYPE_OBJ;
+        case MP_QSTR_bool: return MP_NATIVE_TYPE_BOOL;
+        case MP_QSTR_int: return MP_NATIVE_TYPE_INT;
+        case MP_QSTR_uint: return MP_NATIVE_TYPE_UINT;
+        case MP_QSTR_ptr: return MP_NATIVE_TYPE_PTR;
+        case MP_QSTR_ptr8: return MP_NATIVE_TYPE_PTR8;
+        case MP_QSTR_ptr16: return MP_NATIVE_TYPE_PTR16;
+        case MP_QSTR_ptr32: return MP_NATIVE_TYPE_PTR32;
+        default: return -1;
+    }
+}
+
 // convert a MicroPython object to a valid native value based on type
-mp_uint_t mp_convert_obj_to_native(mp_obj_t obj, mp_uint_t type) {
-    DEBUG_printf("mp_convert_obj_to_native(%p, " UINT_FMT ")\n", obj, type);
+mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type) {
+    DEBUG_printf("mp_native_from_obj(%p, " UINT_FMT ")\n", obj, type);
     switch (type & 0xf) {
         case MP_NATIVE_TYPE_OBJ: return (mp_uint_t)obj;
-        case MP_NATIVE_TYPE_BOOL:
+        case MP_NATIVE_TYPE_BOOL: return mp_obj_is_true(obj);
         case MP_NATIVE_TYPE_INT:
         case MP_NATIVE_TYPE_UINT: return mp_obj_get_int_truncated(obj);
         default: { // cast obj to a pointer
             mp_buffer_info_t bufinfo;
-            if (mp_get_buffer(obj, &bufinfo, MP_BUFFER_RW)) {
+            if (mp_get_buffer(obj, &bufinfo, MP_BUFFER_READ)) {
                 return (mp_uint_t)bufinfo.buf;
             } else {
                 // assume obj is an integer that represents an address
@@ -63,11 +78,11 @@ mp_uint_t mp_convert_obj_to_native(mp_obj_t obj, mp_uint_t type) {
 
 #endif
 
-#if MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
+#if MICROPY_EMIT_MACHINE_CODE
 
 // convert a native value to a MicroPython object based on type
-mp_obj_t mp_convert_native_to_obj(mp_uint_t val, mp_uint_t type) {
-    DEBUG_printf("mp_convert_native_to_obj(" UINT_FMT ", " UINT_FMT ")\n", val, type);
+mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) {
+    DEBUG_printf("mp_native_to_obj(" UINT_FMT ", " UINT_FMT ")\n", val, type);
     switch (type & 0xf) {
         case MP_NATIVE_TYPE_OBJ: return (mp_obj_t)val;
         case MP_NATIVE_TYPE_BOOL: return mp_obj_new_bool(val);
@@ -81,9 +96,32 @@ mp_obj_t mp_convert_native_to_obj(mp_uint_t val, mp_uint_t type) {
 
 #endif
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER
+
+#if !MICROPY_PY_BUILTINS_SET
+mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items) {
+    (void)n_args;
+    (void)items;
+    mp_raise_msg(&mp_type_RuntimeError, "set unsupported");
+}
+
+void mp_obj_set_store(mp_obj_t self_in, mp_obj_t item) {
+    (void)self_in;
+    (void)item;
+    mp_raise_msg(&mp_type_RuntimeError, "set unsupported");
+}
+#endif
+
+#if !MICROPY_PY_BUILTINS_SLICE
+mp_obj_t mp_obj_new_slice(mp_obj_t ostart, mp_obj_t ostop, mp_obj_t ostep) {
+    (void)ostart;
+    (void)ostop;
+    (void)ostep;
+    mp_raise_msg(&mp_type_RuntimeError, "slice unsupported");
+}
+#endif
 
-mp_obj_dict_t *mp_native_swap_globals(mp_obj_dict_t *new_globals) {
+STATIC mp_obj_dict_t *mp_native_swap_globals(mp_obj_dict_t *new_globals) {
     if (new_globals == NULL) {
         // Globals were the originally the same so don't restore them
         return NULL;
@@ -99,13 +137,13 @@ mp_obj_dict_t *mp_native_swap_globals(mp_obj_dict_t *new_globals) {
 
 // wrapper that accepts n_args and n_kw in one argument
 // (native emitter can only pass at most 3 arguments to a function)
-mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args) {
+STATIC mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args) {
     return mp_call_function_n_kw(fun_in, n_args_kw & 0xff, (n_args_kw >> 8) & 0xff, args);
 }
 
 // wrapper that makes raise obj and raises it
 // END_FINALLY opcode requires that we don't raise if o==None
-void mp_native_raise(mp_obj_t o) {
+STATIC void mp_native_raise(mp_obj_t o) {
     if (o != MP_OBJ_NULL && o != mp_const_none) {
         nlr_raise(mp_make_raise_obj(o));
     }
@@ -173,13 +211,55 @@ STATIC bool mp_native_yield_from(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *re
     return false;
 }
 
+#if MICROPY_PY_BUILTINS_FLOAT
+
+STATIC mp_obj_t mp_obj_new_float_from_f(float f) {
+    return mp_obj_new_float((mp_float_t)f);
+}
+
+STATIC mp_obj_t mp_obj_new_float_from_d(double d) {
+    return mp_obj_new_float((mp_float_t)d);
+}
+
+STATIC float mp_obj_get_float_to_f(mp_obj_t o) {
+    return (float)mp_obj_get_float(o);
+}
+
+STATIC double mp_obj_get_float_to_d(mp_obj_t o) {
+    return (double)mp_obj_get_float(o);
+}
+
+#else
+
+STATIC mp_obj_t mp_obj_new_float_from_f(float f) {
+    (void)f;
+    mp_raise_msg(&mp_type_RuntimeError, "float unsupported");
+}
+
+STATIC mp_obj_t mp_obj_new_float_from_d(double d) {
+    (void)d;
+    mp_raise_msg(&mp_type_RuntimeError, "float unsupported");
+}
+
+STATIC float mp_obj_get_float_to_f(mp_obj_t o) {
+    (void)o;
+    mp_raise_msg(&mp_type_RuntimeError, "float unsupported");
+}
+
+STATIC double mp_obj_get_float_to_d(mp_obj_t o) {
+    (void)o;
+    mp_raise_msg(&mp_type_RuntimeError, "float unsupported");
+}
+
+#endif
+
 // these must correspond to the respective enum in runtime0.h
-const void *const mp_fun_table[MP_F_NUMBER_OF] = {
+const mp_fun_table_t mp_fun_table = {
     &mp_const_none_obj,
     &mp_const_false_obj,
     &mp_const_true_obj,
-    mp_convert_obj_to_native,
-    mp_convert_native_to_obj,
+    mp_native_from_obj,
+    mp_native_to_obj,
     mp_native_swap_globals,
     mp_load_name,
     mp_load_global,
@@ -196,45 +276,74 @@ const void *const mp_fun_table[MP_F_NUMBER_OF] = {
     mp_binary_op,
     mp_obj_new_tuple,
     mp_obj_new_list,
-    mp_obj_list_append,
     mp_obj_new_dict,
-    mp_obj_dict_store,
-#if MICROPY_PY_BUILTINS_SET
-    mp_obj_set_store,
     mp_obj_new_set,
-#endif
+    mp_obj_set_store,
+    mp_obj_list_append,
+    mp_obj_dict_store,
     mp_make_function_from_raw_code,
     mp_native_call_function_n_kw,
     mp_call_method_n_kw,
     mp_call_method_n_kw_var,
     mp_native_getiter,
     mp_native_iternext,
+    #if MICROPY_NLR_SETJMP
+    nlr_push_tail,
+    #else
     nlr_push,
+    #endif
     nlr_pop,
     mp_native_raise,
     mp_import_name,
     mp_import_from,
     mp_import_all,
-#if MICROPY_PY_BUILTINS_SLICE
     mp_obj_new_slice,
-#endif
     mp_unpack_sequence,
     mp_unpack_ex,
     mp_delete_name,
     mp_delete_global,
-    mp_obj_new_cell,
     mp_make_closure_from_raw_code,
     mp_arg_check_num_sig,
     mp_setup_code_state,
     mp_small_int_floor_divide,
     mp_small_int_modulo,
     mp_native_yield_from,
+    #if MICROPY_NLR_SETJMP
+    setjmp,
+    #else
+    NULL,
+    #endif
+    // Additional entries for dynamic runtime, starts at index 50
+    memset,
+    memmove,
+    gc_realloc,
+    mp_printf,
+    mp_vprintf,
+    mp_raise_msg,
+    mp_obj_get_type,
+    mp_obj_new_str,
+    mp_obj_new_bytes,
+    mp_obj_new_bytearray_by_ref,
+    mp_obj_new_float_from_f,
+    mp_obj_new_float_from_d,
+    mp_obj_get_float_to_f,
+    mp_obj_get_float_to_d,
+    mp_get_buffer_raise,
+    mp_get_stream_raise,
+    &mp_plat_print,
+    &mp_type_type,
+    &mp_type_str,
+    &mp_type_list,
+    &mp_type_dict,
+    &mp_type_fun_builtin_0,
+    &mp_type_fun_builtin_1,
+    &mp_type_fun_builtin_2,
+    &mp_type_fun_builtin_3,
+    &mp_type_fun_builtin_var,
+    &mp_stream_read_obj,
+    &mp_stream_readinto_obj,
+    &mp_stream_unbuffered_readline_obj,
+    &mp_stream_write_obj,
 };
 
-/*
-void mp_f_vector(mp_fun_kind_t fun_kind) {
-    (mp_f_table[fun_kind])();
-}
-*/
-
 #endif // MICROPY_EMIT_NATIVE

py/nativeglue.h

@@ -0,0 +1,177 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2013-2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_PY_NATIVEGLUE_H
+#define MICROPY_INCLUDED_PY_NATIVEGLUE_H
+
+#include <stdarg.h>
+#include "py/obj.h"
+#include "py/persistentcode.h"
+#include "py/stream.h"
+
+typedef enum {
+    MP_F_CONST_NONE_OBJ = 0,
+    MP_F_CONST_FALSE_OBJ,
+    MP_F_CONST_TRUE_OBJ,
+    MP_F_CONVERT_OBJ_TO_NATIVE,
+    MP_F_CONVERT_NATIVE_TO_OBJ,
+    MP_F_NATIVE_SWAP_GLOBALS,
+    MP_F_LOAD_NAME,
+    MP_F_LOAD_GLOBAL,
+    MP_F_LOAD_BUILD_CLASS,
+    MP_F_LOAD_ATTR,
+    MP_F_LOAD_METHOD,
+    MP_F_LOAD_SUPER_METHOD,
+    MP_F_STORE_NAME,
+    MP_F_STORE_GLOBAL,
+    MP_F_STORE_ATTR,
+    MP_F_OBJ_SUBSCR,
+    MP_F_OBJ_IS_TRUE,
+    MP_F_UNARY_OP,
+    MP_F_BINARY_OP,
+    MP_F_BUILD_TUPLE,
+    MP_F_BUILD_LIST,
+    MP_F_BUILD_MAP,
+    MP_F_BUILD_SET,
+    MP_F_STORE_SET,
+    MP_F_LIST_APPEND,
+    MP_F_STORE_MAP,
+    MP_F_MAKE_FUNCTION_FROM_RAW_CODE,
+    MP_F_NATIVE_CALL_FUNCTION_N_KW,
+    MP_F_CALL_METHOD_N_KW,
+    MP_F_CALL_METHOD_N_KW_VAR,
+    MP_F_NATIVE_GETITER,
+    MP_F_NATIVE_ITERNEXT,
+    MP_F_NLR_PUSH,
+    MP_F_NLR_POP,
+    MP_F_NATIVE_RAISE,
+    MP_F_IMPORT_NAME,
+    MP_F_IMPORT_FROM,
+    MP_F_IMPORT_ALL,
+    MP_F_NEW_SLICE,
+    MP_F_UNPACK_SEQUENCE,
+    MP_F_UNPACK_EX,
+    MP_F_DELETE_NAME,
+    MP_F_DELETE_GLOBAL,
+    MP_F_MAKE_CLOSURE_FROM_RAW_CODE,
+    MP_F_ARG_CHECK_NUM_SIG,
+    MP_F_SETUP_CODE_STATE,
+    MP_F_SMALL_INT_FLOOR_DIVIDE,
+    MP_F_SMALL_INT_MODULO,
+    MP_F_NATIVE_YIELD_FROM,
+    MP_F_SETJMP,
+    MP_F_NUMBER_OF,
+} mp_fun_kind_t;
+
+typedef struct _mp_fun_table_t {
+    mp_const_obj_t const_none;
+    mp_const_obj_t const_false;
+    mp_const_obj_t const_true;
+    mp_uint_t (*native_from_obj)(mp_obj_t obj, mp_uint_t type);
+    mp_obj_t (*native_to_obj)(mp_uint_t val, mp_uint_t type);
+    mp_obj_dict_t *(*swap_globals)(mp_obj_dict_t *new_globals);
+    mp_obj_t (*load_name)(qstr qst);
+    mp_obj_t (*load_global)(qstr qst);
+    mp_obj_t (*load_build_class)(void);
+    mp_obj_t (*load_attr)(mp_obj_t base, qstr attr);
+    void (*load_method)(mp_obj_t base, qstr attr, mp_obj_t *dest);
+    void (*load_super_method)(qstr attr, mp_obj_t *dest);
+    void (*store_name)(qstr qst, mp_obj_t obj);
+    void (*store_global)(qstr qst, mp_obj_t obj);
+    void (*store_attr)(mp_obj_t base, qstr attr, mp_obj_t val);
+    mp_obj_t (*obj_subscr)(mp_obj_t base, mp_obj_t index, mp_obj_t val);
+    bool (*obj_is_true)(mp_obj_t arg);
+    mp_obj_t (*unary_op)(mp_unary_op_t op, mp_obj_t arg);
+    mp_obj_t (*binary_op)(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs);
+    mp_obj_t (*new_tuple)(size_t n, const mp_obj_t *items);
+    mp_obj_t (*new_list)(size_t n, mp_obj_t *items);
+    mp_obj_t (*new_dict)(size_t n_args);
+    mp_obj_t (*new_set)(size_t n_args, mp_obj_t *items);
+    void (*set_store)(mp_obj_t self_in, mp_obj_t item);
+    mp_obj_t (*list_append)(mp_obj_t self_in, mp_obj_t arg);
+    mp_obj_t (*dict_store)(mp_obj_t self_in, mp_obj_t key, mp_obj_t value);
+    mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args);
+    mp_obj_t (*call_function_n_kw)(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args);
+    mp_obj_t (*call_method_n_kw)(size_t n_args, size_t n_kw, const mp_obj_t *args);
+    mp_obj_t (*call_method_n_kw_var)(bool have_self, size_t n_args_n_kw, const mp_obj_t *args);
+    mp_obj_t (*getiter)(mp_obj_t obj, mp_obj_iter_buf_t *iter);
+    mp_obj_t (*iternext)(mp_obj_iter_buf_t *iter);
+    unsigned int (*nlr_push)(nlr_buf_t *);
+    void (*nlr_pop)(void);
+    void (*raise)(mp_obj_t o);
+    mp_obj_t (*import_name)(qstr name, mp_obj_t fromlist, mp_obj_t level);
+    mp_obj_t (*import_from)(mp_obj_t module, qstr name);
+    void (*import_all)(mp_obj_t module);
+    mp_obj_t (*new_slice)(mp_obj_t start, mp_obj_t stop, mp_obj_t step);
+    void (*unpack_sequence)(mp_obj_t seq, size_t num, mp_obj_t *items);
+    void (*unpack_ex)(mp_obj_t seq, size_t num, mp_obj_t *items);
+    void (*delete_name)(qstr qst);
+    void (*delete_global)(qstr qst);
+    mp_obj_t (*make_closure_from_raw_code)(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args);
+    void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig);
+    void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args);
+    mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom);
+    mp_int_t (*small_int_modulo)(mp_int_t dividend, mp_int_t divisor);
+    bool (*yield_from)(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *ret_value);
+    void *setjmp;
+    // Additional entries for dynamic runtime, starts at index 50
+    void *(*memset_)(void *s, int c, size_t n);
+    void *(*memmove_)(void *dest, const void *src, size_t n);
+    void *(*realloc_)(void *ptr, size_t n_bytes, bool allow_move);
+    int (*printf_)(const mp_print_t *print, const char *fmt, ...);
+    int (*vprintf_)(const mp_print_t *print, const char *fmt, va_list args);
+    #if defined(__GNUC__)
+    NORETURN // Only certain compilers support no-return attributes in function pointer declarations
+    #endif
+    void (*raise_msg)(const mp_obj_type_t *exc_type, const char *msg);
+    mp_obj_type_t *(*obj_get_type)(mp_const_obj_t o_in);
+    mp_obj_t (*obj_new_str)(const char* data, size_t len);
+    mp_obj_t (*obj_new_bytes)(const byte* data, size_t len);
+    mp_obj_t (*obj_new_bytearray_by_ref)(size_t n, void *items);
+    mp_obj_t (*obj_new_float_from_f)(float f);
+    mp_obj_t (*obj_new_float_from_d)(double d);
+    float (*obj_get_float_to_f)(mp_obj_t o);
+    double (*obj_get_float_to_d)(mp_obj_t o);
+    void (*get_buffer_raise)(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flags);
+    const mp_stream_p_t *(*get_stream_raise)(mp_obj_t self_in, int flags);
+    const mp_print_t *plat_print;
+    const mp_obj_type_t *type_type;
+    const mp_obj_type_t *type_str;
+    const mp_obj_type_t *type_list;
+    const mp_obj_type_t *type_dict;
+    const mp_obj_type_t *type_fun_builtin_0;
+    const mp_obj_type_t *type_fun_builtin_1;
+    const mp_obj_type_t *type_fun_builtin_2;
+    const mp_obj_type_t *type_fun_builtin_3;
+    const mp_obj_type_t *type_fun_builtin_var;
+    const mp_obj_fun_builtin_var_t *stream_read_obj;
+    const mp_obj_fun_builtin_var_t *stream_readinto_obj;
+    const mp_obj_fun_builtin_var_t *stream_unbuffered_readline_obj;
+    const mp_obj_fun_builtin_var_t *stream_write_obj;
+} mp_fun_table_t;
+
+extern const mp_fun_table_t mp_fun_table;
+
+#endif // MICROPY_INCLUDED_PY_NATIVEGLUE_H

py/nlr.h

@@ -34,6 +34,14 @@
 
 #include "py/mpconfig.h"
 
+#define MICROPY_NLR_NUM_REGS_X86            (6)
+#define MICROPY_NLR_NUM_REGS_X64            (8)
+#define MICROPY_NLR_NUM_REGS_X64_WIN        (10)
+#define MICROPY_NLR_NUM_REGS_ARM_THUMB      (10)
+#define MICROPY_NLR_NUM_REGS_ARM_THUMB_FP   (10 + 6)
+#define MICROPY_NLR_NUM_REGS_XTENSA         (10)
+#define MICROPY_NLR_NUM_REGS_XTENSAWIN      (17)
+
 // If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch
 #if !MICROPY_NLR_SETJMP
 // A lot of nlr-related things need different treatment on Windows
@@ -44,20 +52,31 @@
 #endif
 #if defined(__i386__)
     #define MICROPY_NLR_X86 (1)
-    #define MICROPY_NLR_NUM_REGS (6)
+    #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_X86)
 #elif defined(__x86_64__)
     #define MICROPY_NLR_X64 (1)
     #if MICROPY_NLR_OS_WINDOWS
-        #define MICROPY_NLR_NUM_REGS (10)
+        #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_X64_WIN)
     #else
-        #define MICROPY_NLR_NUM_REGS (8)
+        #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_X64)
     #endif
 #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__)
     #define MICROPY_NLR_THUMB (1)
-    #define MICROPY_NLR_NUM_REGS (10)
+    #if defined(__SOFTFP__)
+        #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_ARM_THUMB)
+    #else
+        // With hardware FP registers s16-s31 are callee save so in principle
+        // should be saved and restored by the NLR code.  gcc only uses s16-s21
+        // so only save/restore those as an optimisation.
+        #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_ARM_THUMB_FP)
+    #endif
 #elif defined(__xtensa__)
     #define MICROPY_NLR_XTENSA (1)
-    #define MICROPY_NLR_NUM_REGS (10)
+    #define MICROPY_NLR_NUM_REGS (MICROPY_NLR_NUM_REGS_XTENSA)
+#elif defined(__powerpc__)
+    #define MICROPY_NLR_POWERPC (1)
+    // this could be less but using 128 for safety
+    #define MICROPY_NLR_NUM_REGS (128)
 #else
     #define MICROPY_NLR_SETJMP (1)
     //#warning "No native NLR support for this arch, using setjmp implementation"

py/nlrpowerpc.c

@@ -0,0 +1,121 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019, Michael Neuling, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/mpstate.h"
+
+#if MICROPY_NLR_POWERPC
+
+#undef nlr_push
+
+// Saving all ABI non-vol registers here
+
+unsigned int nlr_push(nlr_buf_t *nlr) {
+
+    __asm__ volatile(
+    "li     4, 0x4eed ; "      // Store canary
+    "std    4,  0x00(%0) ;"
+    "std    0,  0x08(%0) ;"
+    "std    1,  0x10(%0) ;"
+    "std    2,  0x18(%0) ;"
+    "std    14, 0x20(%0) ;"
+    "std    15, 0x28(%0) ;"
+    "std    16, 0x30(%0) ;"
+    "std    17, 0x38(%0) ;"
+    "std    18, 0x40(%0) ;"
+    "std    19, 0x48(%0) ;"
+    "std    20, 0x50(%0) ;"
+    "std    21, 0x58(%0) ;"
+    "std    22, 0x60(%0) ;"
+    "std    23, 0x68(%0) ;"
+    "std    24, 0x70(%0) ;"
+    "std    25, 0x78(%0) ;"
+    "std    26, 0x80(%0) ;"
+    "std    27, 0x88(%0) ;"
+    "std    28, 0x90(%0) ;"
+    "std    29, 0x98(%0) ;"
+    "std    30, 0xA0(%0) ;"
+    "std    31, 0xA8(%0) ;"
+
+    "mfcr   4 ; "
+    "std    4, 0xB0(%0) ;"
+    "mflr   4 ;"
+    "std    4, 0xB8(%0) ;"
+    "li     4, nlr_push_tail@l ;"
+    "oris   4, 4, nlr_push_tail@h ;"
+    "mtctr  4 ;"
+    "mr    3, %1 ; "
+    "bctr  ;"
+    :
+    : "r"(&nlr->regs), "r"(nlr)
+    :
+    );
+
+    return 0;
+}
+
+NORETURN void nlr_jump(void *val) {
+    MP_NLR_JUMP_HEAD(val, top)
+
+    __asm__ volatile(
+    "ld    3, 0x0(%0) ;"
+    "cmpdi 3, 0x4eed ; " // Check canary
+    "bne   . ; "
+    "ld    0,  0x08(%0) ;"
+    "ld    1,  0x10(%0) ;"
+    "ld    2,  0x18(%0) ;"
+    "ld    14, 0x20(%0) ;"
+    "ld    15, 0x28(%0) ;"
+    "ld    16, 0x30(%0) ;"
+    "ld    17, 0x38(%0) ;"
+    "ld    18, 0x40(%0) ;"
+    "ld    19, 0x48(%0) ;"
+    "ld    20, 0x50(%0) ;"
+    "ld    21, 0x58(%0) ;"
+    "ld    22, 0x60(%0) ;"
+    "ld    23, 0x68(%0) ;"
+    "ld    24, 0x70(%0) ;"
+    "ld    25, 0x78(%0) ;"
+    "ld    26, 0x80(%0) ;"
+    "ld    27, 0x88(%0) ;"
+    "ld    28, 0x90(%0) ;"
+    "ld    29, 0x98(%0) ;"
+    "ld    30, 0xA0(%0) ;"
+    "ld    31, 0xA8(%0) ;"
+    "ld    3,  0xB0(%0) ;"
+    "mtcr  3 ;"
+    "ld    3, 0xB8(%0) ;"
+    "mtlr  3 ; "
+    "li    3, 1;"
+    "blr ;"
+    :
+    : "r"(&top->regs)
+    :
+    );
+
+    MP_UNREACHABLE;
+}
+
+#endif // MICROPY_NLR_POWERPC

py/nlrthumb.c

@@ -44,7 +44,7 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) {
     "str    r6, [r0, #20]       \n" // store r6 into nlr_buf
     "str    r7, [r0, #24]       \n" // store r7 into nlr_buf
 
-#if defined(__ARM_ARCH_6M__)
+#if !defined(__thumb2__)
     "mov    r1, r8              \n"
     "str    r1, [r0, #28]       \n" // store r8 into nlr_buf
     "mov    r1, r9              \n"
@@ -63,16 +63,25 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) {
     "str    r10, [r0, #36]      \n" // store r10 into nlr_buf
     "str    r11, [r0, #40]      \n" // store r11 into nlr_buf
     "str    r13, [r0, #44]      \n" // store r13=sp into nlr_buf
+    #if MICROPY_NLR_NUM_REGS == 16
+    "vstr   d8, [r0, #48]       \n" // store s16-s17 into nlr_buf
+    "vstr   d9, [r0, #56]       \n" // store s18-s19 into nlr_buf
+    "vstr   d10, [r0, #64]      \n" // store s20-s21 into nlr_buf
+    #endif
     "str    lr, [r0, #8]        \n" // store lr into nlr_buf
 #endif
 
-#if defined(__ARM_ARCH_6M__)
+#if !defined(__thumb2__)
     "ldr    r1, nlr_push_tail_var \n"
     "bx     r1                  \n" // do the rest in C
     ".align 2                   \n"
     "nlr_push_tail_var: .word nlr_push_tail \n"
 #else
+    #if defined(__APPLE__) || defined(__MACH__)
+    "b      _nlr_push_tail      \n" // do the rest in C
+    #else
     "b      nlr_push_tail       \n" // do the rest in C
+    #endif
 #endif
     );
 
@@ -93,7 +102,7 @@ NORETURN void nlr_jump(void *val) {
     "ldr    r6, [r0, #20]       \n" // load r6 from nlr_buf
     "ldr    r7, [r0, #24]       \n" // load r7 from nlr_buf
 
-#if defined(__ARM_ARCH_6M__)
+#if !defined(__thumb2__)
     "ldr    r1, [r0, #28]       \n" // load r8 from nlr_buf
     "mov    r8, r1              \n"
     "ldr    r1, [r0, #32]       \n" // load r9 from nlr_buf
@@ -112,6 +121,11 @@ NORETURN void nlr_jump(void *val) {
     "ldr    r10, [r0, #36]      \n" // load r10 from nlr_buf
     "ldr    r11, [r0, #40]      \n" // load r11 from nlr_buf
     "ldr    r13, [r0, #44]      \n" // load r13=sp from nlr_buf
+    #if MICROPY_NLR_NUM_REGS == 16
+    "vldr   d8, [r0, #48]       \n" // load s16-s17 from nlr_buf
+    "vldr   d9, [r0, #56]       \n" // load s18-s19 from nlr_buf
+    "vldr   d10, [r0, #64]      \n" // load s20-s21 from nlr_buf
+    #endif
     "ldr    lr, [r0, #8]        \n" // load lr from nlr_buf
 #endif
     "movs   r0, #1              \n" // return 1, non-local return
@@ -121,11 +135,7 @@ NORETURN void nlr_jump(void *val) {
     :                               // clobbered registers
     );
 
-    #if defined(__GNUC__)
-    __builtin_unreachable();
-    #else
-    for (;;); // needed to silence compiler warning
-    #endif
+    MP_UNREACHABLE
 }
 
 #endif // MICROPY_NLR_THUMB

py/nlrx64.c

@@ -108,7 +108,7 @@ NORETURN void nlr_jump(void *val) {
     :                               // clobbered registers
     );
 
-    for (;;); // needed to silence compiler warning
+    MP_UNREACHABLE
 }
 
 #endif // MICROPY_NLR_X64

py/nlrx86.c

@@ -100,7 +100,7 @@ NORETURN void nlr_jump(void *val) {
     :                               // clobbered registers
     );
 
-    for (;;); // needed to silence compiler warning
+    MP_UNREACHABLE
 }
 
 #endif // MICROPY_NLR_X86

py/nlrxtensa.c

@@ -77,7 +77,7 @@ NORETURN void nlr_jump(void *val) {
     :                               // clobbered registers
     );
 
-    for (;;); // needed to silence compiler warning
+    MP_UNREACHABLE
 }
 
 #endif // MICROPY_NLR_XTENSA

py/obj.c

@@ -93,7 +93,7 @@ void mp_obj_print_exception(const mp_print_t *print, mp_obj_t exc) {
 #endif
                 // the block name can be NULL if it's unknown
                 qstr block = values[i + 2];
-                if (block == MP_QSTR_NULL) {
+                if (block == MP_QSTRnull) {
                     mp_print_str(print, "\n");
                 } else {
                     mp_printf(print, ", in %q\n", block);
@@ -113,7 +113,7 @@ bool mp_obj_is_true(mp_obj_t arg) {
     } else if (arg == mp_const_none) {
         return 0;
     } else if (mp_obj_is_small_int(arg)) {
-        if (MP_OBJ_SMALL_INT_VALUE(arg) == 0) {
+        if (arg == MP_OBJ_NEW_SMALL_INT(0)) {
             return 0;
         } else {
             return 1;

py/obj.h

@@ -65,14 +65,14 @@ typedef struct _mp_obj_base_t mp_obj_base_t;
 // For debugging purposes they are all different.  For non-debug mode, we alias
 // as many as we can to MP_OBJ_NULL because it's cheaper to load/compare 0.
 
-#ifdef NDEBUG
-#define MP_OBJ_NULL             (MP_OBJ_FROM_PTR((void*)0))
-#define MP_OBJ_STOP_ITERATION   (MP_OBJ_FROM_PTR((void*)0))
-#define MP_OBJ_SENTINEL         (MP_OBJ_FROM_PTR((void*)4))
-#else
+#if MICROPY_DEBUG_MP_OBJ_SENTINELS
 #define MP_OBJ_NULL             (MP_OBJ_FROM_PTR((void*)0))
 #define MP_OBJ_STOP_ITERATION   (MP_OBJ_FROM_PTR((void*)4))
 #define MP_OBJ_SENTINEL         (MP_OBJ_FROM_PTR((void*)8))
+#else
+#define MP_OBJ_NULL             (MP_OBJ_FROM_PTR((void*)0))
+#define MP_OBJ_STOP_ITERATION   (MP_OBJ_FROM_PTR((void*)0))
+#define MP_OBJ_SENTINEL         (MP_OBJ_FROM_PTR((void*)4))
 #endif
 
 // These macros/inline functions operate on objects and depend on the
@@ -326,6 +326,13 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
 #define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
 #define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
 
+// Declare a module as a builtin, processed by makemoduledefs.py
+// param module_name: MP_QSTR_<module name>
+// param obj_module: mp_obj_module_t instance
+// prarm enabled_define: used as `#if (enabled_define) around entry`
+
+#define MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+
 // Underlying map/hash table implementation (not dict object or map function)
 
 typedef struct _mp_map_elem_t {
@@ -684,7 +691,6 @@ mp_float_t mp_obj_get_float(mp_obj_t self_in);
 bool mp_obj_get_float_maybe(mp_obj_t arg, mp_float_t *value);
 void mp_obj_get_complex(mp_obj_t self_in, mp_float_t *real, mp_float_t *imag);
 #endif
-//qstr mp_obj_get_qstr(mp_obj_t arg);
 void mp_obj_get_array(mp_obj_t o, size_t *len, mp_obj_t **items); // *items may point inside a GC block
 void mp_obj_get_array_fixed_n(mp_obj_t o, size_t len, mp_obj_t **items); // *items may point inside a GC block
 size_t mp_get_index(const mp_obj_type_t *type, size_t len, mp_obj_t index, bool is_slice);

py/objarray.c

@@ -231,6 +231,19 @@ STATIC mp_obj_t memoryview_make_new(const mp_obj_type_t *type_in, size_t n_args,
 
     return MP_OBJ_FROM_PTR(self);
 }
+
+#if MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE
+STATIC void memoryview_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
+    if (dest[0] != MP_OBJ_NULL) {
+        return;
+    }
+    if (attr == MP_QSTR_itemsize) {
+        mp_obj_array_t *self = MP_OBJ_TO_PTR(self_in);
+        dest[0] = MP_OBJ_NEW_SMALL_INT(mp_binary_get_size('@', self->typecode & TYPECODE_MASK, NULL));
+    }
+}
+#endif
+
 #endif
 
 STATIC mp_obj_t array_unary_op(mp_unary_op_t op, mp_obj_t o_in) {
@@ -375,9 +388,8 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
         return MP_OBJ_NULL; // op not supported
     } else {
         mp_obj_array_t *o = MP_OBJ_TO_PTR(self_in);
-        if (0) {
 #if MICROPY_PY_BUILTINS_SLICE
-        } else if (mp_obj_is_type(index_in, &mp_type_slice)) {
+        if (mp_obj_is_type(index_in, &mp_type_slice)) {
             mp_bound_slice_t slice;
             if (!mp_seq_get_fast_slice_indexes(o->len, index_in, &slice)) {
                 mp_raise_NotImplementedError("only slices with step=1 (aka None) are supported");
@@ -433,7 +445,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     if (len_adj > o->free) {
                         // TODO: alloc policy; at the moment we go conservative
                         o->items = m_renew(byte, o->items, (o->len + o->free) * item_sz, (o->len + len_adj) * item_sz);
-                        o->free = 0;
+                        o->free = len_adj;
                         dest_items = o->items;
                     }
                     mp_seq_replace_slice_grow_inplace(dest_items, o->len,
@@ -446,6 +458,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
                     mp_seq_clear(dest_items, o->len + len_adj, o->len, item_sz);
                     // TODO: alloc policy after shrinking
                 }
+                o->free -= len_adj;
                 o->len += len_adj;
                 return mp_const_none;
                 #else
@@ -456,22 +469,22 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value
             mp_obj_array_t *res;
             size_t sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL);
             assert(sz > 0);
-            if (0) {
-                // dummy
             #if MICROPY_PY_BUILTINS_MEMORYVIEW
-            } else if (o->base.type == &mp_type_memoryview) {
+            if (o->base.type == &mp_type_memoryview) {
                 res = m_new_obj(mp_obj_array_t);
                 *res = *o;
                 res->memview_offset += slice.start;
                 res->len = slice.stop - slice.start;
+            } else
             #endif
-            } else {
+            {
                 res = array_new(o->typecode, slice.stop - slice.start);
                 memcpy(res->items, (uint8_t*)o->items + slice.start * sz, (slice.stop - slice.start) * sz);
             }
             return MP_OBJ_FROM_PTR(res);
+        } else
 #endif
-        } else {
+        {
             size_t index = mp_get_index(o->base.type, o->len, index_in, false);
             #if MICROPY_PY_BUILTINS_MEMORYVIEW
             if (o->base.type == &mp_type_memoryview) {
@@ -518,6 +531,9 @@ STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_ui
 STATIC const mp_rom_map_elem_t array_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_append), MP_ROM_PTR(&array_append_obj) },
     { MP_ROM_QSTR(MP_QSTR_extend), MP_ROM_PTR(&array_extend_obj) },
+    #if MICROPY_CPYTHON_COMPAT
+    { MP_ROM_QSTR(MP_QSTR_decode), MP_ROM_PTR(&bytes_decode_obj) },
+    #endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(array_locals_dict, array_locals_dict_table);
@@ -561,6 +577,9 @@ const mp_obj_type_t mp_type_memoryview = {
     .getiter = array_iterator_new,
     .unary_op = array_unary_op,
     .binary_op = array_binary_op,
+    #if MICROPY_PY_BUILTINS_MEMORYVIEW_ITEMSIZE
+    .attr = memoryview_attr,
+    #endif
     .subscr = array_subscr,
     .buffer_p = { .get_buffer = array_get_buffer },
 };

py/objdict.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014-2017 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -30,6 +31,7 @@
 #include "py/runtime.h"
 #include "py/builtin.h"
 #include "py/objtype.h"
+#include "py/objstr.h"
 
 #define mp_obj_is_dict_type(o) (mp_obj_is_obj(o) && ((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->make_new == dict_make_new)
 
@@ -58,7 +60,7 @@ STATIC void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_
     if (!(MICROPY_PY_UJSON && kind == PRINT_JSON)) {
         kind = PRINT_REPR;
     }
-    if (MICROPY_PY_COLLECTIONS_ORDEREDDICT && self->base.type != &mp_type_dict) {
+    if (MICROPY_PY_COLLECTIONS_ORDEREDDICT && self->base.type != &mp_type_dict && kind != PRINT_JSON) {
         mp_printf(print, "%q(", self->base.type->name);
     }
     mp_print_str(print, "{");
@@ -69,12 +71,19 @@ STATIC void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_
             mp_print_str(print, ", ");
         }
         first = false;
+        bool add_quote = MICROPY_PY_UJSON && kind == PRINT_JSON && !mp_obj_is_str_or_bytes(next->key);
+        if (add_quote) {
+            mp_print_str(print, "\"");
+        }
         mp_obj_print_helper(print, next->key, kind);
+        if (add_quote) {
+            mp_print_str(print, "\"");
+        }
         mp_print_str(print, ": ");
         mp_obj_print_helper(print, next->value, kind);
     }
     mp_print_str(print, "}");
-    if (MICROPY_PY_COLLECTIONS_ORDEREDDICT && self->base.type != &mp_type_dict) {
+    if (MICROPY_PY_COLLECTIONS_ORDEREDDICT && self->base.type != &mp_type_dict && kind != PRINT_JSON) {
         mp_print_str(print, ")");
     }
 }

py/objenumerate.c

@@ -59,7 +59,7 @@ STATIC mp_obj_t enumerate_make_new(const mp_obj_type_t *type, size_t n_args, siz
     o->iter = mp_getiter(arg_vals.iterable.u_obj, NULL);
     o->cur = arg_vals.start.u_int;
 #else
-    (void)n_kw;
+    mp_arg_check_num(n_args, n_kw, 1, 2, false);
     mp_obj_enumerate_t *o = m_new_obj(mp_obj_enumerate_t);
     o->base.type = type;
     o->iter = mp_getiter(args[0], NULL);

py/objexcept.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014-2016 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -99,11 +100,6 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) {
 #endif
 #endif  // MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
 
-// Instance of GeneratorExit exception - needed by generator.close()
-// This would belong to objgenerator.c, but to keep mp_obj_exception_t
-// definition module-private so far, have it here.
-const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj};
-
 void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) {
     mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in);
     mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS;
@@ -125,7 +121,7 @@ void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kin
             // try to provide a nice OSError error message
             if (o->base.type == &mp_type_OSError && mp_obj_is_small_int(o->args->items[0])) {
                 qstr qst = mp_errno_to_str(o->args->items[0]);
-                if (qst != MP_QSTR_NULL) {
+                if (qst != MP_QSTRnull) {
                     mp_printf(print, "[Errno " INT_FMT "] %q", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), qst);
                     return;
                 }
@@ -348,9 +344,9 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg
 
     // Create the string object and call mp_obj_exception_make_new to create the exception
     o_str->base.type = &mp_type_str;
-    o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
     o_str->len = strlen(msg);
     o_str->data = (const byte*)msg;
+    o_str->hash = qstr_compute_hash(o_str->data, o_str->len);
     mp_obj_t arg = MP_OBJ_FROM_PTR(o_str);
     return mp_obj_exception_make_new(exc_type, 1, 0, &arg);
 }

py/objfun.c

@@ -139,7 +139,7 @@ const mp_obj_type_t mp_type_fun_builtin_var = {
 /* byte code functions                                                        */
 
 qstr mp_obj_code_get_name(const byte *code_info) {
-    code_info = mp_decode_uint_skip(code_info); // skip code_info_size entry
+    MP_BC_PRELUDE_SIZE_DECODE(code_info);
     #if MICROPY_PERSISTENT_CODE
     return code_info[0] | (code_info[1] << 8);
     #else
@@ -161,12 +161,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) {
     #endif
 
     const byte *bc = fun->bytecode;
-    bc = mp_decode_uint_skip(bc); // skip n_state
-    bc = mp_decode_uint_skip(bc); // skip n_exc_stack
-    bc++; // skip scope_params
-    bc++; // skip n_pos_args
-    bc++; // skip n_kwonly_args
-    bc++; // skip n_def_pos_args
+    MP_BC_PRELUDE_SIG_DECODE(bc);
     return mp_obj_code_get_name(bc);
 }
 
@@ -197,18 +192,19 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) {
 
 #define DECODE_CODESTATE_SIZE(bytecode, n_state_out_var, state_size_out_var) \
     { \
-        /* bytecode prelude: state size and exception stack size */               \
-        n_state_out_var = mp_decode_uint_value(bytecode);                         \
-        size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(bytecode)); \
-                                                                                  \
+        const uint8_t *ip = bytecode; \
+        size_t n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_args; \
+        MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state_out_var, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_args); \
+                                    \
         /* state size in bytes */                                                 \
         state_size_out_var = n_state_out_var * sizeof(mp_obj_t)                   \
                            + n_exc_stack * sizeof(mp_exc_stack_t);                \
     }
 
-#define INIT_CODESTATE(code_state, _fun_bc, n_args, n_kw, args) \
+#define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \
     code_state->fun_bc = _fun_bc; \
     code_state->ip = 0; \
+    code_state->n_state = _n_state; \
     mp_setup_code_state(code_state, n_args, n_kw, args); \
     code_state->old_globals = mp_globals_get();
 
@@ -235,7 +231,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args
     }
     #endif
 
-    INIT_CODESTATE(code_state, self, n_args, n_kw, args);
+    INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args);
 
     // execute the byte code with the correct globals context
     mp_globals_set(self->globals);
@@ -280,7 +276,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
     }
     #endif
 
-    INIT_CODESTATE(code_state, self, n_args, n_kw, args);
+    INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args);
 
     // execute the byte code with the correct globals context
     mp_globals_set(self->globals);
@@ -294,9 +290,11 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
             assert(0);
         }
     }
-    const byte *bytecode_ptr = mp_decode_uint_skip(mp_decode_uint_skip(self->bytecode));
-    size_t n_pos_args = bytecode_ptr[1];
-    size_t n_kwonly_args = bytecode_ptr[2];
+    const byte *bytecode_ptr = self->bytecode;
+    size_t n_state_unused, n_exc_stack_unused, scope_flags_unused;
+    size_t n_pos_args, n_kwonly_args, n_def_args_unused;
+    MP_BC_PRELUDE_SIG_DECODE_INTO(bytecode_ptr, n_state_unused, n_exc_stack_unused,
+        scope_flags_unused, n_pos_args, n_kwonly_args, n_def_args_unused);
     // We can't check the case when an exception is returned in state[0]
     // and there are no arguments, because in this case our detection slot may have
     // been overwritten by the returned exception (which is allowed).
@@ -458,13 +456,13 @@ STATIC mp_uint_t convert_obj_for_inline_asm(mp_obj_t obj) {
         return (mp_uint_t)mp_obj_str_get_data(obj, &l);
     } else {
         mp_obj_type_t *type = mp_obj_get_type(obj);
-        if (0) {
 #if MICROPY_PY_BUILTINS_FLOAT
-        } else if (type == &mp_type_float) {
+        if (type == &mp_type_float) {
             // convert float to int (could also pass in float registers)
             return (mp_int_t)mp_obj_float_get(obj);
+        } else
 #endif
-        } else if (type == &mp_type_tuple || type == &mp_type_list) {
+        if (type == &mp_type_tuple || type == &mp_type_list) {
             // pointer to start of tuple (could pass length, but then could use len(x) for that)
             size_t len;
             mp_obj_t *items;
@@ -472,7 +470,7 @@ STATIC mp_uint_t convert_obj_for_inline_asm(mp_obj_t obj) {
             return (mp_uint_t)items;
         } else {
             mp_buffer_info_t bufinfo;
-            if (mp_get_buffer(obj, &bufinfo, MP_BUFFER_WRITE)) {
+            if (mp_get_buffer(obj, &bufinfo, MP_BUFFER_READ)) {
                 // supports the buffer protocol, return a pointer to the data
                 return (mp_uint_t)bufinfo.buf;
             } else {
@@ -510,7 +508,7 @@ STATIC mp_obj_t fun_asm_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const
         );
     }
 
-    return mp_convert_native_to_obj(ret, self->type_sig);
+    return mp_native_to_obj(ret, self->type_sig);
 }
 
 STATIC const mp_obj_type_t mp_type_fun_asm = {

py/objfun.h

@@ -33,6 +33,9 @@ typedef struct _mp_obj_fun_bc_t {
     mp_obj_dict_t *globals;         // the context within which this function was defined
     const byte *bytecode;           // bytecode for the function
     const mp_uint_t *const_table;   // constant table
+    #if MICROPY_PY_SYS_SETTRACE
+    const struct _mp_raw_code_t *rc;
+    #endif
     // the following extra_args array is allocated space to take (in order):
     //  - values of positional default args (if any)
     //  - a single slot for default kw args dict (if it has them)

py/objgenerator.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  * Copyright (c) 2014-2017 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -30,16 +30,23 @@
 
 #include "py/runtime.h"
 #include "py/bc.h"
+#include "py/objstr.h"
 #include "py/objgenerator.h"
 #include "py/objfun.h"
 #include "py/stackctrl.h"
 
+// Instance of GeneratorExit exception - needed by generator.close()
+const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj};
+
 /******************************************************************************/
 /* generator wrapper                                                          */
 
 typedef struct _mp_obj_gen_instance_t {
     mp_obj_base_t base;
-    mp_obj_dict_t *globals;
+    // mp_const_none: Not-running, no exception.
+    // MP_OBJ_NULL: Running, no exception.
+    // other: Not running, pending exception.
+    mp_obj_t pend_exc;
     mp_code_state_t code_state;
 } mp_obj_gen_instance_t;
 
@@ -48,17 +55,18 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons
     mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in);
 
     // bytecode prelude: get state size and exception stack size
-    size_t n_state = mp_decode_uint_value(self_fun->bytecode);
-    size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(self_fun->bytecode));
+    const uint8_t *ip = self_fun->bytecode;
+    MP_BC_PRELUDE_SIG_DECODE(ip);
 
     // allocate the generator object, with room for local stack and exception stack
     mp_obj_gen_instance_t *o = m_new_obj_var(mp_obj_gen_instance_t, byte,
         n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t));
     o->base.type = &mp_type_gen_instance;
 
-    o->globals = self_fun->globals;
+    o->pend_exc = mp_const_none;
     o->code_state.fun_bc = self_fun;
     o->code_state.ip = 0;
+    o->code_state.n_state = n_state;
     mp_setup_code_state(&o->code_state, n_args, n_kw, args);
     return MP_OBJ_FROM_PTR(o);
 }
@@ -84,7 +92,14 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k
 
     // Determine start of prelude, and extract n_state from it
     uintptr_t prelude_offset = ((uintptr_t*)self_fun->bytecode)[0];
-    size_t n_state = mp_decode_uint_value(self_fun->bytecode + prelude_offset);
+    #if MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ
+    // Prelude is in bytes object in const_table, at index prelude_offset
+    mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->const_table[prelude_offset]);
+    prelude_offset = (const byte*)prelude_bytes->data - self_fun->bytecode;
+    #endif
+    const uint8_t *ip = self_fun->bytecode + prelude_offset;
+    size_t n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args;
+    MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args);
     size_t n_exc_stack = 0;
 
     // Allocate the generator object, with room for local stack and exception stack
@@ -93,13 +108,14 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k
     o->base.type = &mp_type_gen_instance;
 
     // Parse the input arguments and set up the code state
-    o->globals = self_fun->globals;
+    o->pend_exc = mp_const_none;
     o->code_state.fun_bc = self_fun;
     o->code_state.ip = (const byte*)prelude_offset;
+    o->code_state.n_state = n_state;
     mp_setup_code_state(&o->code_state, n_args, n_kw, args);
 
     // Indicate we are a native function, which doesn't use this variable
-    o->code_state.exc_sp = NULL;
+    o->code_state.exc_sp_idx = MP_CODE_STATE_EXC_SP_IDX_SENTINEL;
 
     // Prepare the generator instance for execution
     uintptr_t start_offset = ((uintptr_t*)self_fun->bytecode)[1];
@@ -138,38 +154,39 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
         *ret_val = MP_OBJ_STOP_ITERATION;
         return MP_VM_RETURN_NORMAL;
     }
+
+    // Ensure the generator cannot be reentered during execution
+    if (self->pend_exc == MP_OBJ_NULL) {
+        mp_raise_ValueError("generator already executing");
+    }
+
+    #if MICROPY_PY_GENERATOR_PEND_THROW
+    // If exception is pending (set using .pend_throw()), process it now.
+    if (self->pend_exc != mp_const_none) {
+        throw_value = self->pend_exc;
+    }
+    #endif
+
+    // If the generator is started, allow sending a value.
     if (self->code_state.sp == self->code_state.state - 1) {
         if (send_value != mp_const_none) {
             mp_raise_TypeError("can't send non-None value to a just-started generator");
         }
     } else {
-        #if MICROPY_PY_GENERATOR_PEND_THROW
-        // If exception is pending (set using .pend_throw()), process it now.
-        if (*self->code_state.sp != mp_const_none) {
-            throw_value = *self->code_state.sp;
-            *self->code_state.sp = MP_OBJ_NULL;
-        } else
-        #endif
-        {
-            *self->code_state.sp = send_value;
-        }
+        *self->code_state.sp = send_value;
     }
 
-    // We set self->globals=NULL while executing, for a sentinel to ensure the generator
-    // cannot be reentered during execution
-    if (self->globals == NULL) {
-        mp_raise_ValueError("generator already executing");
-    }
+    // Mark as running
+    self->pend_exc = MP_OBJ_NULL;
 
     // Set up the correct globals context for the generator and execute it
     self->code_state.old_globals = mp_globals_get();
-    mp_globals_set(self->globals);
-    self->globals = NULL;
+    mp_globals_set(self->code_state.fun_bc->globals);
 
     mp_vm_return_kind_t ret_kind;
 
     #if MICROPY_EMIT_NATIVE
-    if (self->code_state.exc_sp == NULL) {
+    if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
         // A native generator, with entry point 2 words into the "bytecode" pointer
         typedef uintptr_t (*mp_fun_native_gen_t)(void*, mp_obj_t);
         mp_fun_native_gen_t fun = MICROPY_MAKE_POINTER_CALLABLE((const void*)(self->code_state.fun_bc->bytecode + 2 * sizeof(uintptr_t)));
@@ -181,9 +198,11 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
         ret_kind = mp_execute_bytecode(&self->code_state, throw_value);
     }
 
-    self->globals = mp_globals_get();
     mp_globals_set(self->code_state.old_globals);
 
+    // Mark as not running
+    self->pend_exc = mp_const_none;
+
     switch (ret_kind) {
         case MP_VM_RETURN_NORMAL:
         default:
@@ -247,12 +266,25 @@ STATIC mp_obj_t gen_instance_send(mp_obj_t self_in, mp_obj_t send_value) {
         return ret;
     }
 }
-
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_send_obj, gen_instance_send);
 
-STATIC mp_obj_t gen_instance_close(mp_obj_t self_in);
 STATIC mp_obj_t gen_instance_throw(size_t n_args, const mp_obj_t *args) {
-    mp_obj_t exc = (n_args == 2) ? args[1] : args[2];
+    // The signature of this function is: throw(type[, value[, traceback]])
+    // CPython will pass all given arguments through the call chain and process them
+    // at the point they are used (native generators will handle them differently to
+    // user-defined generators with a throw() method).  To save passing multiple
+    // values, MicroPython instead does partial processing here to reduce it down to
+    // one argument and passes that through:
+    // - if only args[1] is given, or args[2] is given but is None, args[1] is
+    //   passed through (in the standard case it is an exception class or instance)
+    // - if args[2] is given and not None it is passed through (in the standard
+    //   case it would be an exception instance and args[1] its corresponding class)
+    // - args[3] is always ignored
+
+    mp_obj_t exc = args[1];
+    if (n_args > 2 && args[2] != mp_const_none) {
+        exc = args[2];
+    }
 
     mp_obj_t ret = gen_resume_and_raise(args[0], mp_const_none, exc);
     if (ret == MP_OBJ_STOP_ITERATION) {
@@ -261,7 +293,6 @@ STATIC mp_obj_t gen_instance_throw(size_t n_args, const mp_obj_t *args) {
         return ret;
     }
 }
-
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(gen_instance_throw_obj, 2, 4, gen_instance_throw);
 
 STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) {
@@ -283,19 +314,20 @@ STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) {
             return mp_const_none;
     }
 }
-
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_close_obj, gen_instance_close);
 
+#if MICROPY_PY_GENERATOR_PEND_THROW
 STATIC mp_obj_t gen_instance_pend_throw(mp_obj_t self_in, mp_obj_t exc_in) {
     mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in);
-    if (self->code_state.sp == self->code_state.state - 1) {
-        mp_raise_TypeError("can't pend throw to just-started generator");
+    if (self->pend_exc == MP_OBJ_NULL) {
+        mp_raise_ValueError("generator already executing");
     }
-    mp_obj_t prev = *self->code_state.sp;
-    *self->code_state.sp = exc_in;
+    mp_obj_t prev = self->pend_exc;
+    self->pend_exc = exc_in;
     return prev;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_pend_throw_obj, gen_instance_pend_throw);
+#endif
 
 STATIC const mp_rom_map_elem_t gen_instance_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&gen_instance_close_obj) },

py/objint.c

@@ -137,7 +137,7 @@ STATIC mp_fp_as_int_class_t mp_classify_fp_as_int(mp_float_t val) {
 mp_obj_t mp_obj_new_int_from_float(mp_float_t val) {
     int cl = fpclassify(val);
     if (cl == FP_INFINITE) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_OverflowError, "can't convert inf to int"));
+        mp_raise_msg(&mp_type_OverflowError, "can't convert inf to int");
     } else if (cl == FP_NAN) {
         mp_raise_ValueError("can't convert NaN to int");
     } else {

py/objint_mpz.c

@@ -194,18 +194,18 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i
         return mp_obj_int_binary_op_extra_cases(op, lhs_in, rhs_in);
     }
 
-    if (0) {
 #if MICROPY_PY_BUILTINS_FLOAT
-    } else if (op == MP_BINARY_OP_TRUE_DIVIDE || op == MP_BINARY_OP_INPLACE_TRUE_DIVIDE) {
+    if (op == MP_BINARY_OP_TRUE_DIVIDE || op == MP_BINARY_OP_INPLACE_TRUE_DIVIDE) {
         if (mpz_is_zero(zrhs)) {
             goto zero_division_error;
         }
         mp_float_t flhs = mpz_as_float(zlhs);
         mp_float_t frhs = mpz_as_float(zrhs);
         return mp_obj_new_float(flhs / frhs);
+    } else
 #endif
 
-    } else if (op >= MP_BINARY_OP_INPLACE_OR && op < MP_BINARY_OP_CONTAINS) {
+    if (op >= MP_BINARY_OP_INPLACE_OR && op < MP_BINARY_OP_CONTAINS) {
         mp_obj_int_t *res = mp_obj_int_new_mpz();
 
         switch (op) {

py/objmodule.c

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
+ * Copyright (c) 2014-2015 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -25,12 +26,15 @@
  */
 
 #include <stdlib.h>
+#include <string.h>
 #include <assert.h>
 
 #include "py/objmodule.h"
 #include "py/runtime.h"
 #include "py/builtin.h"
 
+#include "genhdr/moduledefs.h"
+
 STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
     mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in);
@@ -136,9 +140,6 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
     { MP_ROM_QSTR(MP_QSTR_builtins), MP_ROM_PTR(&mp_module_builtins) },
     { MP_ROM_QSTR(MP_QSTR_micropython), MP_ROM_PTR(&mp_module_micropython) },
 
-#if MICROPY_PY_ARRAY
-    { MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_module_array) },
-#endif
 #if MICROPY_PY_IO
     { MP_ROM_QSTR(MP_QSTR_uio), MP_ROM_PTR(&mp_module_io) },
 #endif
@@ -223,20 +224,20 @@ STATIC const mp_rom_map_elem_t mp_builtin_module_table[] = {
 #if MICROPY_PY_BTREE
     { MP_ROM_QSTR(MP_QSTR_btree), MP_ROM_PTR(&mp_module_btree) },
 #endif
+#if MICROPY_PY_BLUETOOTH
+    { MP_ROM_QSTR(MP_QSTR_ubluetooth), MP_ROM_PTR(&mp_module_ubluetooth) },
+#endif
 
     // extra builtin modules as defined by a port
     MICROPY_PORT_BUILTIN_MODULES
-};
 
-MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);
-
-#if MICROPY_MODULE_WEAK_LINKS
-STATIC const mp_rom_map_elem_t mp_builtin_module_weak_links_table[] = {
-    MICROPY_PORT_BUILTIN_MODULE_WEAK_LINKS
+    #ifdef MICROPY_REGISTERED_MODULES
+    // builtin modules declared with MP_REGISTER_MODULE()
+    MICROPY_REGISTERED_MODULES
+    #endif
 };
 
-MP_DEFINE_CONST_MAP(mp_builtin_module_weak_links_map, mp_builtin_module_weak_links_table);
-#endif
+MP_DEFINE_CONST_MAP(mp_builtin_module_map, mp_builtin_module_table);
 
 // returns MP_OBJ_NULL if not found
 mp_obj_t mp_module_get(qstr module_name) {
@@ -262,6 +263,21 @@ void mp_module_register(qstr qst, mp_obj_t module) {
     mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module;
 }
 
+#if MICROPY_MODULE_WEAK_LINKS
+// Search for u"foo" in built-in modules, return MP_OBJ_NULL if not found
+mp_obj_t mp_module_search_umodule(const char *module_str) {
+    for (size_t i = 0; i < MP_ARRAY_SIZE(mp_builtin_module_table); ++i) {
+        const mp_map_elem_t *entry = (const mp_map_elem_t*)&mp_builtin_module_table[i];
+        const char *key = qstr_str(MP_OBJ_QSTR_VALUE(entry->key));
+        if (key[0] == 'u' && strcmp(&key[1], module_str) == 0) {
+            return (mp_obj_t)entry->value;
+        }
+
+    }
+    return MP_OBJ_NULL;
+}
+#endif
+
 #if MICROPY_MODULE_BUILTIN_INIT
 void mp_module_call_init(qstr module_name, mp_obj_t module_obj) {
     // Look for __init__ and call it if it exists

py/objmodule.h

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -34,6 +34,8 @@ extern const mp_map_t mp_builtin_module_weak_links_map;
 mp_obj_t mp_module_get(qstr module_name);
 void mp_module_register(qstr qstr, mp_obj_t module);
 
+mp_obj_t mp_module_search_umodule(const char *module_str);
+
 #if MICROPY_MODULE_BUILTIN_INIT
 void mp_module_call_init(qstr module_name, mp_obj_t module_obj);
 #else

py/objset.c

@@ -45,8 +45,6 @@ typedef struct _mp_obj_set_it_t {
     size_t cur;
 } mp_obj_set_it_t;
 
-STATIC mp_obj_t set_it_iternext(mp_obj_t self_in);
-
 STATIC bool is_set_or_frozenset(mp_obj_t o) {
     return mp_obj_is_type(o, &mp_type_set)
 #if MICROPY_PY_BUILTINS_FROZENSET
@@ -154,7 +152,6 @@ STATIC mp_obj_t set_getiter(mp_obj_t set_in, mp_obj_iter_buf_t *iter_buf) {
     return MP_OBJ_FROM_PTR(o);
 }
 
-
 /******************************************************************************/
 /* set methods                                                                */
 
@@ -169,9 +166,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(set_add_obj, set_add);
 STATIC mp_obj_t set_clear(mp_obj_t self_in) {
     check_set(self_in);
     mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in);
-
     mp_set_clear(&self->set);
-
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(set_clear_obj, set_clear);
@@ -332,6 +327,7 @@ STATIC mp_obj_t set_issubset_internal(mp_obj_t self_in, mp_obj_t other_in, bool
     }
     return out;
 }
+
 STATIC mp_obj_t set_issubset(mp_obj_t self_in, mp_obj_t other_in) {
     return set_issubset_internal(self_in, other_in, false);
 }
@@ -518,7 +514,6 @@ STATIC mp_obj_t set_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) {
 /******************************************************************************/
 /* set constructors & public C API                                            */
 
-
 STATIC const mp_rom_map_elem_t set_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_add), MP_ROM_PTR(&set_add_obj) },
     { MP_ROM_QSTR(MP_QSTR_clear), MP_ROM_PTR(&set_clear_obj) },
@@ -539,7 +534,6 @@ STATIC const mp_rom_map_elem_t set_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_update), MP_ROM_PTR(&set_update_obj) },
     { MP_ROM_QSTR(MP_QSTR___contains__), MP_ROM_PTR(&mp_op_contains_obj) },
 };
-
 STATIC MP_DEFINE_CONST_DICT(set_locals_dict, set_locals_dict_table);
 
 const mp_obj_type_t mp_type_set = {

py/objstr.c

@@ -169,7 +169,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_
 
                 // Check if a qstr with this data already exists
                 qstr q = qstr_find_strn((const char*)str_data, str_len);
-                if (q != MP_QSTR_NULL) {
+                if (q != MP_QSTRnull) {
                     return MP_OBJ_NEW_QSTR(q);
                 }
 
@@ -1913,9 +1913,6 @@ mp_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_u
         return 0;
     } else {
         // can't write to a string
-        bufinfo->buf = NULL;
-        bufinfo->len = 0;
-        bufinfo->typecode = -1;
         return 1;
     }
 }
@@ -2042,7 +2039,7 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) {
     // if not a bytes object, look if a qstr with this data already exists
     if (type == &mp_type_str) {
         qstr q = qstr_find_strn(vstr->buf, vstr->len);
-        if (q != MP_QSTR_NULL) {
+        if (q != MP_QSTRnull) {
             vstr_clear(vstr);
             vstr->alloc = 0;
             return MP_OBJ_NEW_QSTR(q);
@@ -2067,7 +2064,7 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) {
 
 mp_obj_t mp_obj_new_str(const char* data, size_t len) {
     qstr q = qstr_find_strn(data, len);
-    if (q != MP_QSTR_NULL) {
+    if (q != MP_QSTRnull) {
         // qstr with this data already exists
         return MP_OBJ_NEW_QSTR(q);
     } else {

py/objstr.h

@@ -102,5 +102,6 @@ MP_DECLARE_CONST_FUN_OBJ_1(str_isalpha_obj);
 MP_DECLARE_CONST_FUN_OBJ_1(str_isdigit_obj);
 MP_DECLARE_CONST_FUN_OBJ_1(str_isupper_obj);
 MP_DECLARE_CONST_FUN_OBJ_1(str_islower_obj);
+MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(bytes_decode_obj);
 
 #endif // MICROPY_INCLUDED_PY_OBJSTR_H

py/objstringio.c

@@ -70,9 +70,9 @@ STATIC mp_uint_t stringio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *er
 STATIC void stringio_copy_on_write(mp_obj_stringio_t *o) {
     const void *buf = o->vstr->buf;
     o->vstr->buf = m_new(char, o->vstr->len);
-    memcpy(o->vstr->buf, buf, o->vstr->len);
     o->vstr->fixed_buf = false;
     o->ref_obj = MP_OBJ_NULL;
+    memcpy(o->vstr->buf, buf, o->vstr->len);
 }
 
 STATIC mp_uint_t stringio_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) {
@@ -244,12 +244,6 @@ STATIC const mp_stream_p_t stringio_stream_p = {
     .is_text = true,
 };
 
-STATIC const mp_stream_p_t bytesio_stream_p = {
-    .read = stringio_read,
-    .write = stringio_write,
-    .ioctl = stringio_ioctl,
-};
-
 const mp_obj_type_t mp_type_stringio = {
     { &mp_type_type },
     .name = MP_QSTR_StringIO,
@@ -262,6 +256,12 @@ const mp_obj_type_t mp_type_stringio = {
 };
 
 #if MICROPY_PY_IO_BYTESIO
+STATIC const mp_stream_p_t bytesio_stream_p = {
+    .read = stringio_read,
+    .write = stringio_write,
+    .ioctl = stringio_ioctl,
+};
+
 const mp_obj_type_t mp_type_bytesio = {
     { &mp_type_type },
     .name = MP_QSTR_BytesIO,

py/objtuple.c

@@ -31,6 +31,9 @@
 #include "py/objtuple.h"
 #include "py/runtime.h"
 
+// type check is done on getiter method to allow tuple, namedtuple, attrtuple
+#define mp_obj_is_tuple_compatible(o) (mp_obj_get_type(o)->getiter == mp_obj_tuple_getiter)
+
 /******************************************************************************/
 /* tuple                                                                      */
 
@@ -101,8 +104,7 @@ STATIC mp_obj_t mp_obj_tuple_make_new(const mp_obj_type_t *type_in, size_t n_arg
 
 // Don't pass MP_BINARY_OP_NOT_EQUAL here
 STATIC mp_obj_t tuple_cmp_helper(mp_uint_t op, mp_obj_t self_in, mp_obj_t another_in) {
-    // type check is done on getiter method to allow tuple, namedtuple, attrtuple
-    mp_check_self(mp_obj_get_type(self_in)->getiter == mp_obj_tuple_getiter);
+    mp_check_self(mp_obj_is_tuple_compatible(self_in));
     mp_obj_type_t *another_type = mp_obj_get_type(another_in);
     mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in);
     if (another_type->getiter != mp_obj_tuple_getiter) {
@@ -249,7 +251,7 @@ mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items) {
 }
 
 void mp_obj_tuple_get(mp_obj_t self_in, size_t *len, mp_obj_t **items) {
-    assert(mp_obj_is_type(self_in, &mp_type_tuple));
+    assert(mp_obj_is_tuple_compatible(self_in));
     mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in);
     *len = self->len;
     *items = &self->items[0];

py/objtype.c

@@ -460,7 +460,7 @@ STATIC mp_obj_t instance_unary_op(mp_unary_op_t op, mp_obj_t self_in) {
 }
 
 // Binary-op enum values not listed here will have the default value of 0 in the
-// table, corresponding to MP_QSTR_NULL, and are therefore unsupported (a lookup will
+// table, corresponding to MP_QSTRnull, and are therefore unsupported (a lookup will
 // fail).  They can be added at the expense of code size for the qstr.
 // Qstrs for special methods are guaranteed to have a small value, so we use byte
 // type to represent them.
@@ -478,6 +478,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
     [MP_BINARY_OP_INPLACE_SUBTRACT] = MP_QSTR___isub__,
     #if MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS
     [MP_BINARY_OP_INPLACE_MULTIPLY] = MP_QSTR___imul__,
+    [MP_BINARY_OP_INPLACE_MAT_MULTIPLY] = MP_QSTR___imatmul__,
     [MP_BINARY_OP_INPLACE_FLOOR_DIVIDE] = MP_QSTR___ifloordiv__,
     [MP_BINARY_OP_INPLACE_TRUE_DIVIDE] = MP_QSTR___itruediv__,
     [MP_BINARY_OP_INPLACE_MODULO] = MP_QSTR___imod__,
@@ -493,6 +494,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
     [MP_BINARY_OP_SUBTRACT] = MP_QSTR___sub__,
     #if MICROPY_PY_ALL_SPECIAL_METHODS
     [MP_BINARY_OP_MULTIPLY] = MP_QSTR___mul__,
+    [MP_BINARY_OP_MAT_MULTIPLY] = MP_QSTR___matmul__,
     [MP_BINARY_OP_FLOOR_DIVIDE] = MP_QSTR___floordiv__,
     [MP_BINARY_OP_TRUE_DIVIDE] = MP_QSTR___truediv__,
     [MP_BINARY_OP_MODULO] = MP_QSTR___mod__,
@@ -510,6 +512,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
     [MP_BINARY_OP_REVERSE_SUBTRACT] = MP_QSTR___rsub__,
     #if MICROPY_PY_ALL_SPECIAL_METHODS
     [MP_BINARY_OP_REVERSE_MULTIPLY] = MP_QSTR___rmul__,
+    [MP_BINARY_OP_REVERSE_MAT_MULTIPLY] = MP_QSTR___rmatmul__,
     [MP_BINARY_OP_REVERSE_FLOOR_DIVIDE] = MP_QSTR___rfloordiv__,
     [MP_BINARY_OP_REVERSE_TRUE_DIVIDE] = MP_QSTR___rtruediv__,
     [MP_BINARY_OP_REVERSE_MODULO] = MP_QSTR___rmod__,
@@ -1011,6 +1014,21 @@ STATIC void type_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
             dest[0] = MP_OBJ_NEW_QSTR(self->name);
             return;
         }
+        if (attr == MP_QSTR___bases__) {
+            if (self == &mp_type_object) {
+                dest[0] = mp_const_empty_tuple;
+                return;
+            }
+            mp_obj_t parent_obj = self->parent ? MP_OBJ_FROM_PTR(self->parent) : MP_OBJ_FROM_PTR(&mp_type_object);
+            #if MICROPY_MULTIPLE_INHERITANCE
+            if (mp_obj_is_type(parent_obj, &mp_type_tuple)) {
+                dest[0] = parent_obj;
+                return;
+            }
+            #endif
+            dest[0] = mp_obj_new_tuple(1, &parent_obj);
+            return;
+        }
         #endif
         struct class_lookup_data lookup = {
             .obj = (mp_obj_instance_t*)self,

py/parse.c

@@ -135,8 +135,8 @@ STATIC const uint16_t rule_arg_combined_table[] = {
 #define RULE_EXPAND(x) x
 #define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule))
 #define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__))
-#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__
-#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r,
+#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) __VA_ARGS__
+#define RULE_PADDING_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r,
 
 // Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table
 enum {
@@ -155,8 +155,8 @@ enum {
 // Macro to compute the start of a rule in rule_arg_combined_table
 #define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule))
 #define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__))
-#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13
-#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r,
+#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, ...) _14
+#define RULE_ARG_OFFSET_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r,
 
 // Use the above enum values to create a table of offsets for each rule's arg
 // data, which indexes rule_arg_combined_table.  The offsets require 9 bits of
@@ -502,7 +502,7 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) {
     } else if (lex->tok_kind == MP_TOKEN_STRING || lex->tok_kind == MP_TOKEN_BYTES) {
         // Don't automatically intern all strings/bytes.  doc strings (which are usually large)
         // will be discarded by the compiler, and so we shouldn't intern them.
-        qstr qst = MP_QSTR_NULL;
+        qstr qst = MP_QSTRnull;
         if (lex->vstr.len <= MICROPY_ALLOC_PARSE_INTERN_STRING_LEN) {
             // intern short strings
             qst = qstr_from_strn(lex->vstr.buf, lex->vstr.len);
@@ -510,7 +510,7 @@ STATIC void push_result_token(parser_t *parser, uint8_t rule_id) {
             // check if this string is already interned
             qst = qstr_find_strn(lex->vstr.buf, lex->vstr.len);
         }
-        if (qst != MP_QSTR_NULL) {
+        if (qst != MP_QSTRnull) {
             // qstr exists, make a leaf node
             pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst);
         } else {
@@ -632,7 +632,7 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
     } else if (rule_id == RULE_shift_expr
         || rule_id == RULE_arith_expr
         || rule_id == RULE_term) {
-        // folding for binary ops: << >> + - * / % //
+        // folding for binary ops: << >> + - * @ / % //
         mp_parse_node_t pn = peek_result(parser, num_args - 1);
         if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
             return false;
@@ -644,23 +644,11 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
                 return false;
             }
             mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, i));
-            static const uint8_t token_to_op[] = {
-                MP_BINARY_OP_ADD,
-                MP_BINARY_OP_SUBTRACT,
-                MP_BINARY_OP_MULTIPLY,
-                255,//MP_BINARY_OP_POWER,
-                255,//MP_BINARY_OP_TRUE_DIVIDE,
-                MP_BINARY_OP_FLOOR_DIVIDE,
-                MP_BINARY_OP_MODULO,
-                255,//MP_BINARY_OP_LESS
-                MP_BINARY_OP_LSHIFT,
-                255,//MP_BINARY_OP_MORE
-                MP_BINARY_OP_RSHIFT,
-            };
-            mp_binary_op_t op = token_to_op[tok - MP_TOKEN_OP_PLUS];
-            if (op == (mp_binary_op_t)255) {
+            if (tok == MP_TOKEN_OP_AT || tok == MP_TOKEN_OP_SLASH || tok == MP_TOKEN_OP_DBL_STAR) {
+                // Can't fold @ or / or **
                 return false;
             }
+            mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS);
             int rhs_sign = mp_obj_int_sign(arg1);
             if (op <= MP_BINARY_OP_RSHIFT) {
                 // << and >> can't have negative rhs
@@ -683,13 +671,11 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
         }
         mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, 1));
         mp_unary_op_t op;
-        if (tok == MP_TOKEN_OP_PLUS) {
-            op = MP_UNARY_OP_POSITIVE;
-        } else if (tok == MP_TOKEN_OP_MINUS) {
-            op = MP_UNARY_OP_NEGATIVE;
-        } else {
-            assert(tok == MP_TOKEN_OP_TILDE); // should be
+        if (tok == MP_TOKEN_OP_TILDE) {
             op = MP_UNARY_OP_INVERT;
+        } else {
+            assert(tok == MP_TOKEN_OP_PLUS || tok == MP_TOKEN_OP_MINUS); // should be
+            op = MP_UNARY_OP_POSITIVE + (tok - MP_TOKEN_OP_PLUS);
         }
         arg0 = mp_unary_op(op, arg0);
 
@@ -719,7 +705,7 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
                     mp_obj_t exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
                         "constant must be an integer");
                     mp_obj_exception_add_traceback(exc, parser->lexer->source_name,
-                        ((mp_parse_node_struct_t*)pn1)->source_line, MP_QSTR_NULL);
+                        ((mp_parse_node_struct_t*)pn1)->source_line, MP_QSTRnull);
                     nlr_raise(exc);
                 }
 
@@ -1152,7 +1138,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
         }
         // add traceback to give info about file name and location
         // we don't have a 'block' name, so just pass the NULL qstr to indicate this
-        mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
+        mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTRnull);
         nlr_raise(exc);
     }
 

py/parsenum.c

@@ -41,7 +41,7 @@ STATIC NORETURN void raise_exc(mp_obj_t exc, mp_lexer_t *lex) {
     // exception's type from ValueError to SyntaxError and add traceback info
     if (lex != NULL) {
         ((mp_obj_base_t*)MP_OBJ_TO_PTR(exc))->type = &mp_type_SyntaxError;
-        mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTR_NULL);
+        mp_obj_exception_add_traceback(exc, lex->source_name, lex->tok_line, MP_QSTRnull);
     }
     nlr_raise(exc);
 }

py/persistentcode.c

@@ -30,28 +30,22 @@
 #include <assert.h>
 
 #include "py/reader.h"
-#include "py/emitglue.h"
+#include "py/nativeglue.h"
 #include "py/persistentcode.h"
-#include "py/bc.h"
+#include "py/bc0.h"
+#include "py/objstr.h"
 
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
 #include "py/smallint.h"
 
-// The current version of .mpy files
-#define MPY_VERSION (3)
-
-// The feature flags byte encodes the compile-time config options that
-// affect the generate bytecode.
-#define MPY_FEATURE_FLAGS ( \
-    ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) << 0) \
-    | ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \
-    )
-// This is a version of the flags that can be configured at runtime.
-#define MPY_FEATURE_FLAGS_DYNAMIC ( \
-    ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) << 0) \
-    | ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \
-    )
+#define QSTR_LAST_STATIC MP_QSTR_zip
+
+#if MICROPY_DYNAMIC_COMPILER
+#define MPY_FEATURE_ARCH_DYNAMIC mp_dynamic_compiler.native_arch
+#else
+#define MPY_FEATURE_ARCH_DYNAMIC MPY_FEATURE_ARCH
+#endif
 
 #if MICROPY_PERSISTENT_CODE_LOAD || (MICROPY_PERSISTENT_CODE_SAVE && !MICROPY_DYNAMIC_COMPILER)
 // The bytecode will depend on the number of bits in a small-int, and
@@ -68,6 +62,57 @@ STATIC int mp_small_int_bits(void) {
 }
 #endif
 
+#define QSTR_WINDOW_SIZE (32)
+
+typedef struct _qstr_window_t {
+    uint16_t idx; // indexes the head of the window
+    uint16_t window[QSTR_WINDOW_SIZE];
+} qstr_window_t;
+
+// Push a qstr to the head of the window, and the tail qstr is overwritten
+STATIC void qstr_window_push(qstr_window_t *qw, qstr qst) {
+    qw->idx = (qw->idx + 1) % QSTR_WINDOW_SIZE;
+    qw->window[qw->idx] = qst;
+}
+
+// Pull an existing qstr from within the window to the head of the window
+STATIC qstr qstr_window_pull(qstr_window_t *qw, size_t idx) {
+    qstr qst = qw->window[idx];
+    if (idx > qw->idx) {
+        memmove(&qw->window[idx], &qw->window[idx + 1], (QSTR_WINDOW_SIZE - idx - 1) * sizeof(uint16_t));
+        qw->window[QSTR_WINDOW_SIZE - 1] = qw->window[0];
+        idx = 0;
+    }
+    memmove(&qw->window[idx], &qw->window[idx + 1], (qw->idx - idx) * sizeof(uint16_t));
+    qw->window[qw->idx] = qst;
+    return qst;
+}
+
+#if MICROPY_PERSISTENT_CODE_LOAD
+
+// Access a qstr at the given index, relative to the head of the window (0=head)
+STATIC qstr qstr_window_access(qstr_window_t *qw, size_t idx) {
+    return qstr_window_pull(qw, (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE);
+}
+
+#endif
+
+#if MICROPY_PERSISTENT_CODE_SAVE
+
+// Insert a qstr at the head of the window, either by pulling an existing one or pushing a new one
+STATIC size_t qstr_window_insert(qstr_window_t *qw, qstr qst) {
+    for (size_t idx = 0; idx < QSTR_WINDOW_SIZE; ++idx) {
+        if (qw->window[idx] == qst) {
+            qstr_window_pull(qw, idx);
+            return (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE;
+        }
+    }
+    qstr_window_push(qw, qst);
+    return QSTR_WINDOW_SIZE;
+}
+
+#endif
+
 typedef struct _bytecode_prelude_t {
     uint n_state;
     uint n_exc_stack;
@@ -79,19 +124,20 @@ typedef struct _bytecode_prelude_t {
 } bytecode_prelude_t;
 
 // ip will point to start of opcodes
-// ip2 will point to simple_name, source_file qstrs
-STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_t *prelude) {
-    prelude->n_state = mp_decode_uint(ip);
-    prelude->n_exc_stack = mp_decode_uint(ip);
-    prelude->scope_flags = *(*ip)++;
-    prelude->n_pos_args = *(*ip)++;
-    prelude->n_kwonly_args = *(*ip)++;
-    prelude->n_def_pos_args = *(*ip)++;
-    *ip2 = *ip;
-    prelude->code_info_size = mp_decode_uint(ip2);
-    *ip += prelude->code_info_size;
-    while (*(*ip)++ != 255) {
-    }
+// return value will point to simple_name, source_file qstrs
+STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) {
+    MP_BC_PRELUDE_SIG_DECODE(*ip);
+    prelude->n_state = n_state;
+    prelude->n_exc_stack = n_exc_stack;
+    prelude->scope_flags = scope_flags;
+    prelude->n_pos_args = n_pos_args;
+    prelude->n_kwonly_args = n_kwonly_args;
+    prelude->n_def_pos_args = n_def_pos_args;
+    MP_BC_PRELUDE_SIZE_DECODE(*ip);
+    byte *ip_info = (byte*)*ip;
+    *ip += n_info;
+    *ip += n_cell;
+    return ip_info;
 }
 
 #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
@@ -100,6 +146,96 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_
 
 #include "py/parsenum.h"
 
+STATIC int read_byte(mp_reader_t *reader);
+STATIC size_t read_uint(mp_reader_t *reader, byte **out);
+
+#if MICROPY_EMIT_MACHINE_CODE
+
+typedef struct _reloc_info_t {
+    mp_reader_t *reader;
+    mp_uint_t *const_table;
+} reloc_info_t;
+
+#if MICROPY_EMIT_THUMB
+STATIC void asm_thumb_rewrite_mov(uint8_t *pc, uint16_t val) {
+    // high part
+    *(uint16_t*)pc = (*(uint16_t*)pc & 0xfbf0) | (val >> 1 & 0x0400) | (val >> 12);
+    // low part
+    *(uint16_t*)(pc + 2) = (*(uint16_t*)(pc + 2) & 0x0f00) | (val << 4 & 0x7000) | (val & 0x00ff);
+
+}
+#endif
+
+STATIC void arch_link_qstr(uint8_t *pc, bool is_obj, qstr qst) {
+    mp_uint_t val = qst;
+    if (is_obj) {
+        val = (mp_uint_t)MP_OBJ_NEW_QSTR(qst);
+    }
+    #if MICROPY_EMIT_X86 || MICROPY_EMIT_X64 || MICROPY_EMIT_ARM || MICROPY_EMIT_XTENSA || MICROPY_EMIT_XTENSAWIN
+    pc[0] = val & 0xff;
+    pc[1] = (val >> 8) & 0xff;
+    pc[2] = (val >> 16) & 0xff;
+    pc[3] = (val >> 24) & 0xff;
+    #elif MICROPY_EMIT_THUMB
+    if (is_obj) {
+        // qstr object, movw and movt
+        asm_thumb_rewrite_mov(pc, val); // movw
+        asm_thumb_rewrite_mov(pc + 4, val >> 16); // movt
+    } else {
+        // qstr number, movw instruction
+        asm_thumb_rewrite_mov(pc, val); // movw
+    }
+    #endif
+}
+
+void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) {
+    // Relocate native code
+    reloc_info_t *ri = ri_in;
+    uint8_t op;
+    uintptr_t *addr_to_adjust = NULL;
+    while ((op = read_byte(ri->reader)) != 0xff) {
+        if (op & 1) {
+            // Point to new location to make adjustments
+            size_t addr = read_uint(ri->reader, NULL);
+            if ((addr & 1) == 0) {
+                // Point to somewhere in text
+                addr_to_adjust = &((uintptr_t*)text)[addr >> 1];
+            } else {
+                // Point to somewhere in rodata
+                addr_to_adjust = &((uintptr_t*)ri->const_table[1])[addr >> 1];
+            }
+        }
+        op >>= 1;
+        uintptr_t dest;
+        size_t n = 1;
+        if (op <= 5) {
+            if (op & 1) {
+                // Read in number of adjustments to make
+                n = read_uint(ri->reader, NULL);
+            }
+            op >>= 1;
+            if (op == 0) {
+                // Destination is text
+                dest = reloc_text;
+            } else {
+                // Destination is rodata (op=1) or bss (op=1 if no rodata, else op=2)
+                dest = ri->const_table[op];
+            }
+        } else if (op == 6) {
+            // Destination is mp_fun_table itself
+            dest = (uintptr_t)&mp_fun_table;
+        } else {
+            // Destination is an entry in mp_fun_table
+            dest = ((uintptr_t*)&mp_fun_table)[op - 7];
+        }
+        while (n--) {
+            *addr_to_adjust++ += dest;
+        }
+    }
+}
+
+#endif
+
 STATIC int read_byte(mp_reader_t *reader) {
     return reader->readbyte(reader->data);
 }
@@ -110,10 +246,14 @@ STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) {
     }
 }
 
-STATIC size_t read_uint(mp_reader_t *reader) {
+STATIC size_t read_uint(mp_reader_t *reader, byte **out) {
     size_t unum = 0;
     for (;;) {
         byte b = reader->readbyte(reader->data);
+        if (out != NULL) {
+            **out = b;
+            ++*out;
+        }
         unum = (unum << 7) | (b & 0x7f);
         if ((b & 0x80) == 0) {
             break;
@@ -122,12 +262,22 @@ STATIC size_t read_uint(mp_reader_t *reader) {
     return unum;
 }
 
-STATIC qstr load_qstr(mp_reader_t *reader) {
-    size_t len = read_uint(reader);
+STATIC qstr load_qstr(mp_reader_t *reader, qstr_window_t *qw) {
+    size_t len = read_uint(reader, NULL);
+    if (len == 0) {
+        // static qstr
+        return read_byte(reader);
+    }
+    if (len & 1) {
+        // qstr in window
+        return qstr_window_access(qw, len >> 1);
+    }
+    len >>= 1;
     char *str = m_new(char, len);
     read_bytes(reader, (byte*)str, len);
     qstr qst = qstr_from_strn(str, len);
     m_del(char, str, len);
+    qstr_window_push(qw, qst);
     return qst;
 }
 
@@ -136,7 +286,7 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
     if (obj_type == 'e') {
         return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj);
     } else {
-        size_t len = read_uint(reader);
+        size_t len = read_uint(reader, NULL);
         vstr_t vstr;
         vstr_init_len(&vstr, len);
         read_bytes(reader, (byte*)vstr.buf, len);
@@ -151,64 +301,233 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) {
     }
 }
 
-STATIC void load_bytecode_qstrs(mp_reader_t *reader, byte *ip, byte *ip_top) {
+STATIC void load_prelude_qstrs(mp_reader_t *reader, qstr_window_t *qw, byte *ip) {
+    qstr simple_name = load_qstr(reader, qw);
+    ip[0] = simple_name; ip[1] = simple_name >> 8;
+    qstr source_file = load_qstr(reader, qw);
+    ip[2] = source_file; ip[3] = source_file >> 8;
+}
+
+STATIC void load_prelude(mp_reader_t *reader, qstr_window_t *qw, byte **ip, bytecode_prelude_t *prelude) {
+    // Read in the prelude header
+    byte *ip_read = *ip;
+    read_uint(reader, &ip_read);                    // read in n_state/etc (is effectively a var-uint)
+    read_uint(reader, &ip_read);                    // read in n_info/n_cell (is effectively a var-uint)
+
+    // Prelude header has been read into *ip, now decode and extract values from it
+    extract_prelude((const byte**)ip, prelude);
+
+    // Load qstrs in prelude
+    load_prelude_qstrs(reader, qw, ip_read);
+    ip_read += 4;
+
+    // Read remaining code info
+    read_bytes(reader, ip_read, *ip - ip_read);
+}
+
+STATIC void load_bytecode(mp_reader_t *reader, qstr_window_t *qw, byte *ip, byte *ip_top) {
     while (ip < ip_top) {
+        *ip = read_byte(reader);
         size_t sz;
-        uint f = mp_opcode_format(ip, &sz);
-        if (f == MP_OPCODE_QSTR) {
-            qstr qst = load_qstr(reader);
-            ip[1] = qst;
-            ip[2] = qst >> 8;
+        uint f = mp_opcode_format(ip, &sz, false);
+        ++ip;
+        --sz;
+        if (f == MP_BC_FORMAT_QSTR) {
+            qstr qst = load_qstr(reader, qw);
+            *ip++ = qst;
+            *ip++ = qst >> 8;
+            sz -= 2;
+        } else if (f == MP_BC_FORMAT_VAR_UINT) {
+            while ((*ip++ = read_byte(reader)) & 0x80) {
+            }
         }
+        read_bytes(reader, ip, sz);
         ip += sz;
     }
 }
 
-STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) {
-    // load bytecode
-    size_t bc_len = read_uint(reader);
-    byte *bytecode = m_new(byte, bc_len);
-    read_bytes(reader, bytecode, bc_len);
+STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) {
+    // Load function kind and data length
+    size_t kind_len = read_uint(reader, NULL);
+    int kind = (kind_len & 3) + MP_CODE_BYTECODE;
+    size_t fun_data_len = kind_len >> 2;
 
-    // extract prelude
-    const byte *ip = bytecode;
-    const byte *ip2;
-    bytecode_prelude_t prelude;
-    extract_prelude(&ip, &ip2, &prelude);
-
-    // load qstrs and link global qstr ids into bytecode
-    qstr simple_name = load_qstr(reader);
-    qstr source_file = load_qstr(reader);
-    ((byte*)ip2)[0] = simple_name; ((byte*)ip2)[1] = simple_name >> 8;
-    ((byte*)ip2)[2] = source_file; ((byte*)ip2)[3] = source_file >> 8;
-    load_bytecode_qstrs(reader, (byte*)ip, bytecode + bc_len);
-
-    // load constant table
-    size_t n_obj = read_uint(reader);
-    size_t n_raw_code = read_uint(reader);
-    mp_uint_t *const_table = m_new(mp_uint_t, prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code);
-    mp_uint_t *ct = const_table;
-    for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) {
-        *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader));
+    #if !MICROPY_EMIT_MACHINE_CODE
+    if (kind != MP_CODE_BYTECODE) {
+        mp_raise_ValueError("incompatible .mpy file");
     }
-    for (size_t i = 0; i < n_obj; ++i) {
-        *ct++ = (mp_uint_t)load_obj(reader);
+    #endif
+
+    uint8_t *fun_data = NULL;
+    bytecode_prelude_t prelude = {0};
+    #if MICROPY_EMIT_MACHINE_CODE
+    size_t prelude_offset = 0;
+    mp_uint_t type_sig = 0;
+    size_t n_qstr_link = 0;
+    #endif
+
+    if (kind == MP_CODE_BYTECODE) {
+        // Allocate memory for the bytecode
+        fun_data = m_new(uint8_t, fun_data_len);
+
+        // Load prelude
+        byte *ip = fun_data;
+        load_prelude(reader, qw, &ip, &prelude);
+
+        // Load bytecode
+        load_bytecode(reader, qw, ip, fun_data + fun_data_len);
+
+    #if MICROPY_EMIT_MACHINE_CODE
+    } else {
+        // Allocate memory for native data and load it
+        size_t fun_alloc;
+        MP_PLAT_ALLOC_EXEC(fun_data_len, (void**)&fun_data, &fun_alloc);
+        read_bytes(reader, fun_data, fun_data_len);
+
+        if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) {
+            // Parse qstr link table and link native code
+            n_qstr_link = read_uint(reader, NULL);
+            for (size_t i = 0; i < n_qstr_link; ++i) {
+                size_t off = read_uint(reader, NULL);
+                qstr qst = load_qstr(reader, qw);
+                uint8_t *dest = fun_data + (off >> 2);
+                if ((off & 3) == 0) {
+                    // Generic 16-bit link
+                    dest[0] = qst & 0xff;
+                    dest[1] = (qst >> 8) & 0xff;
+                } else if ((off & 3) == 3) {
+                    // Generic, aligned qstr-object link
+                    *(mp_obj_t*)dest = MP_OBJ_NEW_QSTR(qst);
+                } else {
+                    // Architecture-specific link
+                    arch_link_qstr(dest, (off & 3) == 2, qst);
+                }
+            }
+        }
+
+        if (kind == MP_CODE_NATIVE_PY) {
+            // Extract prelude for later use
+            prelude_offset = read_uint(reader, NULL);
+            const byte *ip = fun_data + prelude_offset;
+            byte *ip_info = extract_prelude(&ip, &prelude);
+            // Load qstrs in prelude
+            load_prelude_qstrs(reader, qw, ip_info);
+        } else {
+            // Load basic scope info for viper and asm
+            prelude.scope_flags = read_uint(reader, NULL);
+            prelude.n_pos_args = 0;
+            prelude.n_kwonly_args = 0;
+            if (kind == MP_CODE_NATIVE_ASM) {
+                prelude.n_pos_args = read_uint(reader, NULL);
+                type_sig = read_uint(reader, NULL);
+            }
+        }
+    #endif
     }
-    for (size_t i = 0; i < n_raw_code; ++i) {
-        *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader);
+
+    size_t n_obj = 0;
+    size_t n_raw_code = 0;
+    mp_uint_t *const_table = NULL;
+
+    if (kind != MP_CODE_NATIVE_ASM) {
+        // Load constant table for bytecode, native and viper
+
+        // Number of entries in constant table
+        n_obj = read_uint(reader, NULL);
+        n_raw_code = read_uint(reader, NULL);
+
+        // Allocate constant table
+        size_t n_alloc = prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code;
+        #if MICROPY_EMIT_MACHINE_CODE
+        if (kind != MP_CODE_BYTECODE) {
+            ++n_alloc; // additional entry for mp_fun_table
+            if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) {
+                ++n_alloc; // additional entry for rodata
+            }
+            if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) {
+                ++n_alloc; // additional entry for BSS
+            }
+        }
+        #endif
+
+        const_table = m_new(mp_uint_t, n_alloc);
+        mp_uint_t *ct = const_table;
+
+        // Load function argument names (initial entries in const_table)
+        // (viper has n_pos_args=n_kwonly_args=0 so doesn't load any qstrs here)
+        for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) {
+            *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader, qw));
+        }
+
+        #if MICROPY_EMIT_MACHINE_CODE
+        if (kind != MP_CODE_BYTECODE) {
+            // Populate mp_fun_table entry
+            *ct++ = (mp_uint_t)(uintptr_t)&mp_fun_table;
+
+            // Allocate and load rodata if needed
+            if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) {
+                size_t size = read_uint(reader, NULL);
+                uint8_t *rodata = m_new(uint8_t, size);
+                read_bytes(reader, rodata, size);
+                *ct++ = (uintptr_t)rodata;
+            }
+
+            // Allocate BSS if needed
+            if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) {
+                size_t size = read_uint(reader, NULL);
+                uint8_t *bss = m_new0(uint8_t, size);
+                *ct++ = (uintptr_t)bss;
+            }
+        }
+        #endif
+
+        // Load constant objects and raw code children
+        for (size_t i = 0; i < n_obj; ++i) {
+            *ct++ = (mp_uint_t)load_obj(reader);
+        }
+        for (size_t i = 0; i < n_raw_code; ++i) {
+            *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader, qw);
+        }
     }
 
-    // create raw_code and return it
+    // Create raw_code and return it
     mp_raw_code_t *rc = mp_emit_glue_new_raw_code();
-    mp_emit_glue_assign_bytecode(rc, bytecode,
-        #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS
-        bc_len,
-        #endif
-        const_table,
-        #if MICROPY_PERSISTENT_CODE_SAVE
-        n_obj, n_raw_code,
+    if (kind == MP_CODE_BYTECODE) {
+        // Assign bytecode to raw code object
+        mp_emit_glue_assign_bytecode(rc, fun_data,
+            #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS
+            fun_data_len,
+            #endif
+            const_table,
+            #if MICROPY_PERSISTENT_CODE_SAVE
+            n_obj, n_raw_code,
+            #endif
+            prelude.scope_flags);
+
+    #if MICROPY_EMIT_MACHINE_CODE
+    } else {
+        // Relocate and commit code to executable address space
+        reloc_info_t ri = {reader, const_table};
+        #if defined(MP_PLAT_COMMIT_EXEC)
+        void *opt_ri = (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL;
+        fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri);
+        #else
+        if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) {
+            mp_native_relocate(&ri, fun_data, (uintptr_t)fun_data);
+        }
         #endif
-        prelude.scope_flags);
+
+        // Assign native code to raw code object
+        mp_emit_glue_assign_native(rc, kind,
+            fun_data, fun_data_len, const_table,
+            #if MICROPY_PERSISTENT_CODE_SAVE
+            prelude_offset,
+            n_obj, n_raw_code,
+            n_qstr_link, NULL,
+            #endif
+            prelude.n_pos_args, prelude.scope_flags, type_sig);
+    #endif
+    }
     return rc;
 }
 
@@ -217,11 +536,20 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) {
     read_bytes(reader, header, sizeof(header));
     if (header[0] != 'M'
         || header[1] != MPY_VERSION
-        || header[2] != MPY_FEATURE_FLAGS
-        || header[3] > mp_small_int_bits()) {
+        || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS
+        || header[3] > mp_small_int_bits()
+        || read_uint(reader, NULL) > QSTR_WINDOW_SIZE) {
         mp_raise_ValueError("incompatible .mpy file");
     }
-    mp_raw_code_t *rc = load_raw_code(reader);
+    if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) {
+        byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
+        if (!MPY_FEATURE_ARCH_TEST(arch)) {
+            mp_raise_ValueError("incompatible .mpy arch");
+        }
+    }
+    qstr_window_t qw;
+    qw.idx = 0;
+    mp_raw_code_t *rc = load_raw_code(reader, &qw);
     reader->close(reader->data);
     return rc;
 }
@@ -264,10 +592,22 @@ STATIC void mp_print_uint(mp_print_t *print, size_t n) {
     print->print_strn(print->data, (char*)p, buf + sizeof(buf) - p);
 }
 
-STATIC void save_qstr(mp_print_t *print, qstr qst) {
+STATIC void save_qstr(mp_print_t *print, qstr_window_t *qw, qstr qst) {
+    if (qst <= QSTR_LAST_STATIC) {
+        // encode static qstr
+        byte buf[2] = {0, qst & 0xff};
+        mp_print_bytes(print, buf, 2);
+        return;
+    }
+    size_t idx = qstr_window_insert(qw, qst);
+    if (idx < QSTR_WINDOW_SIZE) {
+        // qstr found in window, encode index to it
+        mp_print_uint(print, idx << 1 | 1);
+        return;
+    }
     size_t len;
     const byte *str = qstr_data(qst, &len);
-    mp_print_uint(print, len);
+    mp_print_uint(print, len << 1);
     mp_print_bytes(print, str, len);
 }
 
@@ -279,7 +619,7 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) {
         } else {
             obj_type = 'b';
         }
-        mp_uint_t len;
+        size_t len;
         const char *str = mp_obj_str_get_data(o, &len);
         mp_print_bytes(print, &obj_type, 1);
         mp_print_uint(print, len);
@@ -312,52 +652,133 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) {
     }
 }
 
-STATIC void save_bytecode_qstrs(mp_print_t *print, const byte *ip, const byte *ip_top) {
+STATIC void save_prelude_qstrs(mp_print_t *print, qstr_window_t *qw, const byte *ip) {
+    save_qstr(print, qw, ip[0] | (ip[1] << 8)); // simple_name
+    save_qstr(print, qw, ip[2] | (ip[3] << 8)); // source_file
+}
+
+STATIC void save_bytecode(mp_print_t *print, qstr_window_t *qw, const byte *ip, const byte *ip_top) {
     while (ip < ip_top) {
         size_t sz;
-        uint f = mp_opcode_format(ip, &sz);
-        if (f == MP_OPCODE_QSTR) {
+        uint f = mp_opcode_format(ip, &sz, true);
+        if (f == MP_BC_FORMAT_QSTR) {
+            mp_print_bytes(print, ip, 1);
             qstr qst = ip[1] | (ip[2] << 8);
-            save_qstr(print, qst);
+            save_qstr(print, qw, qst);
+            ip += 3;
+            sz -= 3;
         }
+        mp_print_bytes(print, ip, sz);
         ip += sz;
     }
 }
 
-STATIC void save_raw_code(mp_print_t *print, mp_raw_code_t *rc) {
-    if (rc->kind != MP_CODE_BYTECODE) {
-        mp_raise_ValueError("can only save bytecode");
+STATIC void save_raw_code(mp_print_t *print, mp_raw_code_t *rc, qstr_window_t *qstr_window) {
+    // Save function kind and data length
+    mp_print_uint(print, (rc->fun_data_len << 2) | (rc->kind - MP_CODE_BYTECODE));
+
+    bytecode_prelude_t prelude;
+
+    if (rc->kind == MP_CODE_BYTECODE) {
+        // Extract prelude
+        const byte *ip = rc->fun_data;
+        const byte *ip_info = extract_prelude(&ip, &prelude);
+
+        // Save prelude
+        mp_print_bytes(print, rc->fun_data, ip_info - (const byte*)rc->fun_data);
+        save_prelude_qstrs(print, qstr_window, ip_info);
+        ip_info += 4;
+        mp_print_bytes(print, ip_info, ip - ip_info);
+
+        // Save bytecode
+        const byte *ip_top = (const byte*)rc->fun_data + rc->fun_data_len;
+        save_bytecode(print, qstr_window, ip, ip_top);
+    #if MICROPY_EMIT_MACHINE_CODE
+    } else {
+        // Save native code
+        mp_print_bytes(print, rc->fun_data, rc->fun_data_len);
+
+        if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) {
+            // Save qstr link table for native code
+            mp_print_uint(print, rc->n_qstr);
+            for (size_t i = 0; i < rc->n_qstr; ++i) {
+                mp_print_uint(print, rc->qstr_link[i].off);
+                save_qstr(print, qstr_window, rc->qstr_link[i].qst);
+            }
+        }
+
+        if (rc->kind == MP_CODE_NATIVE_PY) {
+            // Save prelude size
+            mp_print_uint(print, rc->prelude_offset);
+
+            // Extract prelude and save qstrs in prelude
+            const byte *ip = (const byte*)rc->fun_data + rc->prelude_offset;
+            const byte *ip_info = extract_prelude(&ip, &prelude);
+            save_prelude_qstrs(print, qstr_window, ip_info);
+        } else {
+            // Save basic scope info for viper and asm
+            mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG);
+            prelude.n_pos_args = 0;
+            prelude.n_kwonly_args = 0;
+            if (rc->kind == MP_CODE_NATIVE_ASM) {
+                mp_print_uint(print, rc->n_pos_args);
+                mp_print_uint(print, rc->type_sig);
+            }
+        }
+    #endif
     }
 
-    // save bytecode
-    mp_print_uint(print, rc->data.u_byte.bc_len);
-    mp_print_bytes(print, rc->data.u_byte.bytecode, rc->data.u_byte.bc_len);
+    if (rc->kind != MP_CODE_NATIVE_ASM) {
+        // Save constant table for bytecode, native and viper
 
-    // extract prelude
-    const byte *ip = rc->data.u_byte.bytecode;
-    const byte *ip2;
-    bytecode_prelude_t prelude;
-    extract_prelude(&ip, &ip2, &prelude);
-
-    // save qstrs
-    save_qstr(print, ip2[0] | (ip2[1] << 8)); // simple_name
-    save_qstr(print, ip2[2] | (ip2[3] << 8)); // source_file
-    save_bytecode_qstrs(print, ip, rc->data.u_byte.bytecode + rc->data.u_byte.bc_len);
-
-    // save constant table
-    mp_print_uint(print, rc->data.u_byte.n_obj);
-    mp_print_uint(print, rc->data.u_byte.n_raw_code);
-    const mp_uint_t *const_table = rc->data.u_byte.const_table;
-    for (uint i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) {
-        mp_obj_t o = (mp_obj_t)*const_table++;
-        save_qstr(print, MP_OBJ_QSTR_VALUE(o));
+        // Number of entries in constant table
+        mp_print_uint(print, rc->n_obj);
+        mp_print_uint(print, rc->n_raw_code);
+
+        const mp_uint_t *const_table = rc->const_table;
+
+        // Save function argument names (initial entries in const_table)
+        // (viper has n_pos_args=n_kwonly_args=0 so doesn't save any qstrs here)
+        for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) {
+            mp_obj_t o = (mp_obj_t)*const_table++;
+            save_qstr(print, qstr_window, MP_OBJ_QSTR_VALUE(o));
+        }
+
+        if (rc->kind != MP_CODE_BYTECODE) {
+            // Skip saving mp_fun_table entry
+            ++const_table;
+        }
+
+        // Save constant objects and raw code children
+        for (size_t i = 0; i < rc->n_obj; ++i) {
+            save_obj(print, (mp_obj_t)*const_table++);
+        }
+        for (size_t i = 0; i < rc->n_raw_code; ++i) {
+            save_raw_code(print, (mp_raw_code_t*)(uintptr_t)*const_table++, qstr_window);
+        }
     }
-    for (uint i = 0; i < rc->data.u_byte.n_obj; ++i) {
-        save_obj(print, (mp_obj_t)*const_table++);
+}
+
+STATIC bool mp_raw_code_has_native(mp_raw_code_t *rc) {
+    if (rc->kind != MP_CODE_BYTECODE) {
+        return true;
     }
-    for (uint i = 0; i < rc->data.u_byte.n_raw_code; ++i) {
-        save_raw_code(print, (mp_raw_code_t*)(uintptr_t)*const_table++);
+
+    const byte *ip = rc->fun_data;
+    bytecode_prelude_t prelude;
+    extract_prelude(&ip, &prelude);
+
+    const mp_uint_t *const_table = rc->const_table
+        + prelude.n_pos_args + prelude.n_kwonly_args
+        + rc->n_obj;
+
+    for (size_t i = 0; i < rc->n_raw_code; ++i) {
+        if (mp_raw_code_has_native((mp_raw_code_t*)(uintptr_t)*const_table++)) {
+            return true;
+        }
     }
+
+    return false;
 }
 
 void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) {
@@ -366,22 +787,33 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) {
     //  byte  version
     //  byte  feature flags
     //  byte  number of bits in a small int
-    byte header[4] = {'M', MPY_VERSION, MPY_FEATURE_FLAGS_DYNAMIC,
+    //  uint  size of qstr window
+    byte header[4] = {
+        'M',
+        MPY_VERSION,
+        MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC),
         #if MICROPY_DYNAMIC_COMPILER
         mp_dynamic_compiler.small_int_bits,
         #else
         mp_small_int_bits(),
         #endif
     };
+    if (mp_raw_code_has_native(rc)) {
+        header[2] |= MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC);
+    }
     mp_print_bytes(print, header, sizeof(header));
+    mp_print_uint(print, QSTR_WINDOW_SIZE);
 
-    save_raw_code(print, rc);
+    qstr_window_t qw;
+    qw.idx = 0;
+    memset(qw.window, 0, sizeof(qw.window));
+    save_raw_code(print, rc, &qw);
 }
 
 // here we define mp_raw_code_save_file depending on the port
 // TODO abstract this away properly
 
-#if defined(__i386__) || defined(__x86_64__) || defined(__unix__)
+#if defined(__i386__) || defined(__x86_64__) || defined(_WIN32) || defined(__unix__)
 
 #include <unistd.h>
 #include <sys/stat.h>

py/persistentcode.h

@@ -30,6 +30,79 @@
 #include "py/reader.h"
 #include "py/emitglue.h"
 
+// The current version of .mpy files
+#define MPY_VERSION 5
+
+// Macros to encode/decode flags to/from the feature byte
+#define MPY_FEATURE_ENCODE_FLAGS(flags) (flags)
+#define MPY_FEATURE_DECODE_FLAGS(feat) ((feat) & 3)
+
+// Macros to encode/decode native architecture to/from the feature byte
+#define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2)
+#define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2)
+
+// The feature flag bits encode the compile-time config options that
+// affect the generate bytecode.
+#define MPY_FEATURE_FLAGS ( \
+    ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) << 0) \
+    | ((MICROPY_PY_BUILTINS_STR_UNICODE) << 1) \
+    )
+// This is a version of the flags that can be configured at runtime.
+#define MPY_FEATURE_FLAGS_DYNAMIC ( \
+    ((MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) << 0) \
+    | ((MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) << 1) \
+    )
+
+// Define the host architecture
+#if MICROPY_EMIT_X86
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86)
+#elif MICROPY_EMIT_X64
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X64)
+#elif MICROPY_EMIT_THUMB
+    #if defined(__thumb2__)
+        #if defined(__ARM_FP) && (__ARM_FP & 8) == 8
+            #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMDP)
+        #elif defined(__ARM_FP) && (__ARM_FP & 4) == 4
+            #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EMSP)
+        #else
+            #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7EM)
+        #endif
+    #else
+        #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV7M)
+    #endif
+    #define MPY_FEATURE_ARCH_TEST(x) (MP_NATIVE_ARCH_ARMV6M <= (x) && (x) <= MPY_FEATURE_ARCH)
+#elif MICROPY_EMIT_ARM
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_ARMV6)
+#elif MICROPY_EMIT_XTENSA
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSA)
+#elif MICROPY_EMIT_XTENSAWIN
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_XTENSAWIN)
+#else
+    #define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_NONE)
+#endif
+
+#ifndef MPY_FEATURE_ARCH_TEST
+#define MPY_FEATURE_ARCH_TEST(x) ((x) == MPY_FEATURE_ARCH)
+#endif
+
+// 16-bit little-endian integer with the second and third bytes of supported .mpy files
+#define MPY_FILE_HEADER_INT (MPY_VERSION \
+    | (MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
+
+enum {
+    MP_NATIVE_ARCH_NONE = 0,
+    MP_NATIVE_ARCH_X86,
+    MP_NATIVE_ARCH_X64,
+    MP_NATIVE_ARCH_ARMV6,
+    MP_NATIVE_ARCH_ARMV6M,
+    MP_NATIVE_ARCH_ARMV7M,
+    MP_NATIVE_ARCH_ARMV7EM,
+    MP_NATIVE_ARCH_ARMV7EMSP,
+    MP_NATIVE_ARCH_ARMV7EMDP,
+    MP_NATIVE_ARCH_XTENSA,
+    MP_NATIVE_ARCH_XTENSAWIN,
+};
+
 mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader);
 mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len);
 mp_raw_code_t *mp_raw_code_load_file(const char *filename);
@@ -37,4 +110,6 @@ mp_raw_code_t *mp_raw_code_load_file(const char *filename);
 void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print);
 void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename);
 
+void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text);
+
 #endif // MICROPY_INCLUDED_PY_PERSISTENTCODE_H

py/profile.c

@@ -0,0 +1,984 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) SatoshiLabs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/profile.h"
+#include "py/bc0.h"
+#include "py/gc.h"
+
+#if MICROPY_PY_SYS_SETTRACE
+
+#define prof_trace_cb MP_STATE_THREAD(prof_trace_callback)
+
+STATIC uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) {
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    return mp_bytecode_get_source_line(prelude->line_info, bc);
+}
+
+void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude) {
+    const byte *ip = bytecode;
+
+    MP_BC_PRELUDE_SIG_DECODE(ip);
+    prelude->n_state = n_state;
+    prelude->n_exc_stack = n_exc_stack;
+    prelude->scope_flags = scope_flags;
+    prelude->n_pos_args = n_pos_args;
+    prelude->n_kwonly_args = n_kwonly_args;
+    prelude->n_def_pos_args = n_def_pos_args;
+
+    MP_BC_PRELUDE_SIZE_DECODE(ip);
+
+    prelude->line_info = ip + 4;
+    prelude->opcodes = ip + n_info + n_cell;
+
+    qstr block_name = ip[0] | (ip[1] << 8);
+    qstr source_file = ip[2] | (ip[3] << 8);
+    prelude->qstr_block_name = block_name;
+    prelude->qstr_source_file = source_file;
+}
+
+/******************************************************************************/
+// code object
+
+STATIC void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) {
+    (void)kind;
+    mp_obj_code_t *o = MP_OBJ_TO_PTR(o_in);
+    const mp_raw_code_t *rc = o->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    mp_printf(print,
+        "<code object %q at 0x%p, file \"%q\", line %d>",
+        prelude->qstr_block_name,
+        o,
+        prelude->qstr_source_file,
+        rc->line_of_definition
+    );
+}
+
+STATIC mp_obj_tuple_t* code_consts(const mp_raw_code_t *rc) {
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    int start = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj;
+    int stop  = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj + rc->n_raw_code;
+    mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(stop - start + 1, NULL));
+
+    size_t const_no = 0;
+    for (int i = start; i < stop; ++i) {
+        mp_obj_t code = mp_obj_new_code((const mp_raw_code_t*)MP_OBJ_TO_PTR(rc->const_table[i]));
+        if (code == MP_OBJ_NULL) {
+            m_malloc_fail(sizeof(mp_obj_code_t));
+        }
+        consts->items[const_no++] = code;
+    }
+    consts->items[const_no++] = mp_const_none;
+
+    return consts;
+}
+
+STATIC mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) {
+    // const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    uint start = 0;
+    uint stop = rc->fun_data_len - start;
+
+    uint last_lineno = mp_prof_bytecode_lineno(rc, start);
+    uint lasti = 0;
+
+    const uint buffer_chunk_size = (stop-start) >> 2; // heuristic magic
+    uint buffer_size = buffer_chunk_size;
+    byte *buffer = m_new(byte, buffer_size);
+    uint buffer_index = 0;
+
+    for (uint i = start; i < stop; ++i) {
+        uint lineno = mp_prof_bytecode_lineno(rc, i);
+        size_t line_diff = lineno - last_lineno;
+        if (line_diff > 0) {
+            uint instr_diff = (i - start) - lasti;
+
+            assert(instr_diff < 256);
+            assert(line_diff < 256);
+
+            if (buffer_index + 2 > buffer_size) {
+                buffer = m_renew(byte, buffer, buffer_size, buffer_size + buffer_chunk_size);
+                buffer_size = buffer_size + buffer_chunk_size;
+            }
+            last_lineno = lineno;
+            lasti = i - start;
+            buffer[buffer_index++] = instr_diff;
+            buffer[buffer_index++] = line_diff;
+        }
+    }
+
+    mp_obj_t o = mp_obj_new_bytes(buffer, buffer_index);
+    m_del(byte, buffer, buffer_size);
+    return o;
+}
+
+STATIC void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
+    if (dest[0] != MP_OBJ_NULL) {
+        // not load attribute
+        return;
+    }
+    mp_obj_code_t *o = MP_OBJ_TO_PTR(self_in);
+    const mp_raw_code_t *rc = o->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    switch(attr) {
+        case MP_QSTR_co_code:
+            dest[0] = mp_obj_new_bytes(
+                (void*)prelude->opcodes,
+                rc->fun_data_len - (prelude->opcodes - (const byte*)rc->fun_data)
+            );
+            break;
+        case MP_QSTR_co_consts:
+            dest[0] = MP_OBJ_FROM_PTR(code_consts(rc));
+            break;
+        case MP_QSTR_co_filename:
+            dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_source_file);
+            break;
+        case MP_QSTR_co_firstlineno:
+            dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0));
+            break;
+        case MP_QSTR_co_name:
+            dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_block_name);
+            break;
+        case MP_QSTR_co_names:
+            dest[0] = MP_OBJ_FROM_PTR(o->dict_locals);
+            break;
+        case MP_QSTR_co_lnotab:
+            if (!o->lnotab) {
+                 o->lnotab = raw_code_lnotab(rc);
+            }
+            dest[0] = o->lnotab;
+            break;
+    }
+}
+
+const mp_obj_type_t mp_type_code = {
+    { &mp_type_type },
+    .name = MP_QSTR_code,
+    .print = code_print,
+    .unary_op = mp_generic_unary_op,
+    .attr = code_attr,
+};
+
+mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc) {
+    mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t);
+    if (o == NULL) {
+        return MP_OBJ_NULL;
+    }
+    o->base.type = &mp_type_code;
+    o->rc = rc;
+    o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly?
+    o->lnotab = MP_OBJ_NULL;
+    return MP_OBJ_FROM_PTR(o);
+}
+
+/******************************************************************************/
+// frame object
+
+STATIC void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) {
+    (void)kind;
+    mp_obj_frame_t *frame = MP_OBJ_TO_PTR(o_in);
+    mp_obj_code_t *code = frame->code;
+    const mp_raw_code_t *rc = code->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    mp_printf(print,
+        "<frame at 0x%p, file '%q', line %d, code %q>",
+        frame,
+        prelude->qstr_source_file,
+        frame->lineno,
+        prelude->qstr_block_name
+    );
+}
+
+STATIC void frame_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) {
+    if (dest[0] != MP_OBJ_NULL) {
+        // not load attribute
+        return;
+    }
+
+    mp_obj_frame_t *o = MP_OBJ_TO_PTR(self_in);
+
+    switch(attr) {
+        case MP_QSTR_f_back:
+            dest[0] = mp_const_none;
+            if (o->code_state->prev_state) {
+                dest[0] = MP_OBJ_FROM_PTR(o->code_state->prev_state->frame);
+            }
+            break;
+        case MP_QSTR_f_code:
+            dest[0] = MP_OBJ_FROM_PTR(o->code);
+            break;
+        case MP_QSTR_f_globals:
+            dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->globals);
+            break;
+        case MP_QSTR_f_lasti:
+            dest[0] = MP_OBJ_NEW_SMALL_INT(o->lasti);
+            break;
+        case MP_QSTR_f_lineno:
+            dest[0] = MP_OBJ_NEW_SMALL_INT(o->lineno);
+            break;
+    }
+}
+
+const mp_obj_type_t mp_type_frame = {
+    { &mp_type_type },
+    .name = MP_QSTR_frame,
+    .print = frame_print,
+    .unary_op = mp_generic_unary_op,
+    .attr = frame_attr,
+};
+
+mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) {
+    if (gc_is_locked()) {
+        return MP_OBJ_NULL;
+    }
+
+    mp_obj_frame_t *o = m_new_obj_maybe(mp_obj_frame_t);
+    if (o == NULL) {
+        return MP_OBJ_NULL;
+    }
+
+    mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->rc));
+    if (code == NULL) {
+        return MP_OBJ_NULL;
+    }
+
+    const mp_raw_code_t *rc = code->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    o->code_state = code_state;
+    o->base.type = &mp_type_frame;
+    o->back = NULL;
+    o->code = code;
+    o->lasti = code_state->ip - prelude->opcodes;
+    o->lineno = mp_prof_bytecode_lineno(rc, o->lasti);
+    o->trace_opcodes = false;
+    o->callback = MP_OBJ_NULL;
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+
+/******************************************************************************/
+// Trace logic
+
+typedef struct {
+    struct _mp_obj_frame_t * frame;
+    mp_obj_t event;
+    mp_obj_t arg;
+} prof_callback_args_t;
+
+STATIC mp_obj_t mp_prof_callback_invoke(mp_obj_t callback, prof_callback_args_t *args) {
+    assert(mp_obj_is_callable(callback));
+
+    mp_prof_is_executing = true;
+
+    mp_obj_t a[3] = {MP_OBJ_FROM_PTR(args->frame), args->event, args->arg};
+    mp_obj_t top = mp_call_function_n_kw(callback, 3, 0, a);
+
+    mp_prof_is_executing = false;
+
+    if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL) {
+        mp_obj_t obj = MP_STATE_VM(mp_pending_exception);
+        MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL;
+        nlr_raise(obj);
+    }
+    return top;
+}
+
+mp_obj_t mp_prof_settrace(mp_obj_t callback) {
+    if (mp_obj_is_callable(callback)) {
+        prof_trace_cb = callback;
+    } else {
+        prof_trace_cb = MP_OBJ_NULL;
+    }
+    return mp_const_none;
+}
+
+mp_obj_t mp_prof_frame_enter(mp_code_state_t *code_state) {
+    assert(!mp_prof_is_executing);
+
+    mp_obj_frame_t *frame = MP_OBJ_TO_PTR(mp_obj_new_frame(code_state));
+    if (frame == NULL) {
+        // Couldn't allocate a frame object
+        return MP_OBJ_NULL;
+    }
+
+    if (code_state->prev_state && code_state->frame == NULL) {
+        // We are entering not-yet-traced frame
+        // which means it's a CALL event (not a GENERATOR)
+        // so set the function definition line.
+        const mp_raw_code_t *rc = code_state->fun_bc->rc;
+        frame->lineno = rc->line_of_definition;
+        if (!rc->line_of_definition) {
+            frame->lineno = mp_prof_bytecode_lineno(rc, 0);
+        }
+    }
+    code_state->frame = frame;
+
+    if (!prof_trace_cb) {
+        return MP_OBJ_NULL;
+    }
+
+    mp_obj_t top;
+    prof_callback_args_t _args, *args=&_args;
+    args->frame = code_state->frame;
+
+    // SETTRACE event CALL
+    args->event = MP_OBJ_NEW_QSTR(MP_QSTR_call);
+    args->arg = mp_const_none;
+    top = mp_prof_callback_invoke(prof_trace_cb, args);
+
+    code_state->frame->callback = mp_obj_is_callable(top) ? top : MP_OBJ_NULL;
+
+    // Invalidate the last executed line number so the LINE trace can trigger after this CALL.
+    frame->lineno = 0;
+
+    return top;
+}
+
+mp_obj_t mp_prof_frame_update(const mp_code_state_t *code_state) {
+    mp_obj_frame_t *frame = code_state->frame;
+    if (frame == NULL) {
+        // Frame was not allocated (eg because there was no memory available)
+        return MP_OBJ_NULL;
+    }
+
+    mp_obj_frame_t *o = frame;
+    mp_obj_code_t *code = o->code;
+    const mp_raw_code_t *rc = code->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+
+    assert(o->code_state == code_state);
+
+    o->lasti = code_state->ip - prelude->opcodes;
+    o->lineno = mp_prof_bytecode_lineno(rc, o->lasti);
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+mp_obj_t mp_prof_instr_tick(mp_code_state_t *code_state, bool is_exception) {
+    // Detect execution recursion
+    assert(!mp_prof_is_executing);
+    assert(code_state->frame);
+    assert(mp_obj_get_type(code_state->frame) == &mp_type_frame);
+
+    // Detect data recursion
+    assert(code_state != code_state->prev_state);
+
+    mp_obj_t top = mp_const_none;
+    mp_obj_t callback = code_state->frame->callback;
+
+    prof_callback_args_t _args, *args=&_args;
+    args->frame = code_state->frame;
+    args->event = mp_const_none;
+    args->arg = mp_const_none;
+
+    // Call event's are handled inside mp_prof_frame_enter
+
+    // SETTRACE event EXCEPTION
+    if (is_exception) {
+        args->event = MP_OBJ_NEW_QSTR(MP_QSTR_exception);
+        top = mp_prof_callback_invoke(callback, args);
+        return top;
+    }
+
+    // SETTRACE event LINE
+    const mp_raw_code_t *rc = code_state->fun_bc->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+    size_t prev_line_no = args->frame->lineno;
+    size_t current_line_no = mp_prof_bytecode_lineno(rc, code_state->ip - prelude->opcodes);
+    if (prev_line_no != current_line_no) {
+        args->frame->lineno = current_line_no;
+        args->event = MP_OBJ_NEW_QSTR(MP_QSTR_line);
+        top = mp_prof_callback_invoke(callback, args);
+    }
+
+    // SETTRACE event RETURN
+    const byte *ip = code_state->ip;
+    if (*ip == MP_BC_RETURN_VALUE || *ip == MP_BC_YIELD_VALUE) {
+        args->event = MP_OBJ_NEW_QSTR(MP_QSTR_return);
+        top = mp_prof_callback_invoke(callback, args);
+        if (code_state->prev_state && *ip == MP_BC_RETURN_VALUE) {
+            code_state->frame->callback = MP_OBJ_NULL;
+        }
+    }
+
+    // SETTRACE event OPCODE
+    // TODO: frame.f_trace_opcodes=True
+    if (false) {
+        args->event = MP_OBJ_NEW_QSTR(MP_QSTR_opcode);
+    }
+
+    return top;
+}
+
+/******************************************************************************/
+// DEBUG
+
+// This section is for debugging the settrace feature itself, and is not intended
+// to be included in production/release builds.  The code structure for this block
+// was taken from py/showbc.c and should not be used as a reference.  To enable
+// this debug feature enable MICROPY_PROF_INSTR_DEBUG_PRINT_ENABLE in py/profile.h.
+#if MICROPY_PROF_INSTR_DEBUG_PRINT_ENABLE
+
+#include "runtime0.h"
+
+#define DECODE_UINT { \
+    unum = 0; \
+    do { \
+        unum = (unum << 7) + (*ip & 0x7f); \
+    } while ((*ip++ & 0x80) != 0); \
+}
+#define DECODE_ULABEL do { unum = (ip[0] | (ip[1] << 8)); ip += 2; } while (0)
+#define DECODE_SLABEL do { unum = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2; } while (0)
+
+#define DECODE_QSTR \
+    qst = ip[0] | ip[1] << 8; \
+    ip += 2;
+#define DECODE_PTR \
+    DECODE_UINT; \
+    ptr = (const byte*)const_table[unum]
+#define DECODE_OBJ \
+    DECODE_UINT; \
+    obj = (mp_obj_t)const_table[unum]
+
+typedef struct _mp_dis_instruction_t {
+    mp_uint_t qstr_opname;
+    mp_uint_t arg;
+    mp_obj_t argobj;
+    mp_obj_t argobjex_cache;
+} mp_dis_instruction_t;
+
+STATIC const byte *mp_prof_opcode_decode(const byte *ip, const mp_uint_t *const_table, mp_dis_instruction_t *instruction) {
+    mp_uint_t unum;
+    const byte* ptr;
+    mp_obj_t obj;
+    qstr qst;
+
+    instruction->qstr_opname = MP_QSTR_;
+    instruction->arg = 0;
+    instruction->argobj= mp_const_none;
+    instruction->argobjex_cache = mp_const_none;
+
+    switch (*ip++) {
+        case MP_BC_LOAD_CONST_FALSE:
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_FALSE;
+            break;
+
+        case MP_BC_LOAD_CONST_NONE:
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_NONE;
+            break;
+
+        case MP_BC_LOAD_CONST_TRUE:
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_TRUE;
+            break;
+
+        case MP_BC_LOAD_CONST_SMALL_INT: {
+            mp_int_t num = 0;
+            if ((ip[0] & 0x40) != 0) {
+                // Number is negative
+                num--;
+            }
+            do {
+                num = (num << 7) | (*ip & 0x7f);
+            } while ((*ip++ & 0x80) != 0);
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_SMALL_INT;
+            instruction->arg = num;
+            break;
+        }
+
+        case MP_BC_LOAD_CONST_STRING:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_STRING;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_LOAD_CONST_OBJ:
+            DECODE_OBJ;
+            instruction->qstr_opname = MP_QSTR_LOAD_CONST_OBJ;
+            instruction->arg = unum;
+            instruction->argobj= obj;
+            break;
+
+        case MP_BC_LOAD_NULL:
+            instruction->qstr_opname = MP_QSTR_LOAD_NULL;
+            break;
+
+        case MP_BC_LOAD_FAST_N:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_LOAD_FAST_N;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_LOAD_DEREF:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_LOAD_DEREF;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_LOAD_NAME:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_NAME;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+                instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++);
+            }
+            break;
+
+        case MP_BC_LOAD_GLOBAL:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_GLOBAL;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+                instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++);
+            }
+            break;
+
+        case MP_BC_LOAD_ATTR:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_ATTR;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+                instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++);
+            }
+            break;
+
+        case MP_BC_LOAD_METHOD:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_METHOD;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_LOAD_SUPER_METHOD:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_LOAD_SUPER_METHOD;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_LOAD_BUILD_CLASS:
+            instruction->qstr_opname = MP_QSTR_LOAD_BUILD_CLASS;
+            break;
+
+        case MP_BC_LOAD_SUBSCR:
+            instruction->qstr_opname = MP_QSTR_LOAD_SUBSCR;
+            break;
+
+        case MP_BC_STORE_FAST_N:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_STORE_FAST_N;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_STORE_DEREF:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_STORE_DEREF;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_STORE_NAME:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_STORE_NAME;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_STORE_GLOBAL:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_STORE_GLOBAL;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_STORE_ATTR:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_STORE_ATTR;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE) {
+                instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(*ip++);
+            }
+            break;
+
+        case MP_BC_STORE_SUBSCR:
+            instruction->qstr_opname = MP_QSTR_STORE_SUBSCR;
+            break;
+
+        case MP_BC_DELETE_FAST:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_DELETE_FAST;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_DELETE_DEREF:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_DELETE_DEREF;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_DELETE_NAME:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_DELETE_NAME;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_DELETE_GLOBAL:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_DELETE_GLOBAL;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_DUP_TOP:
+            instruction->qstr_opname = MP_QSTR_DUP_TOP;
+            break;
+
+        case MP_BC_DUP_TOP_TWO:
+            instruction->qstr_opname = MP_QSTR_DUP_TOP_TWO;
+            break;
+
+        case MP_BC_POP_TOP:
+            instruction->qstr_opname = MP_QSTR_POP_TOP;
+            break;
+
+        case MP_BC_ROT_TWO:
+            instruction->qstr_opname = MP_QSTR_ROT_TWO;
+            break;
+
+        case MP_BC_ROT_THREE:
+            instruction->qstr_opname = MP_QSTR_ROT_THREE;
+            break;
+
+        case MP_BC_JUMP:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_JUMP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_POP_JUMP_IF_TRUE:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_POP_JUMP_IF_TRUE;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_POP_JUMP_IF_FALSE:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_POP_JUMP_IF_FALSE;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_JUMP_IF_TRUE_OR_POP:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_JUMP_IF_TRUE_OR_POP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_JUMP_IF_FALSE_OR_POP:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_JUMP_IF_FALSE_OR_POP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_SETUP_WITH:
+            DECODE_ULABEL; // loop-like labels are always forward
+            instruction->qstr_opname = MP_QSTR_SETUP_WITH;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_WITH_CLEANUP:
+            instruction->qstr_opname = MP_QSTR_WITH_CLEANUP;
+            break;
+
+        case MP_BC_UNWIND_JUMP:
+            DECODE_SLABEL;
+            instruction->qstr_opname = MP_QSTR_UNWIND_JUMP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_SETUP_EXCEPT:
+            DECODE_ULABEL; // except labels are always forward
+            instruction->qstr_opname = MP_QSTR_SETUP_EXCEPT;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_SETUP_FINALLY:
+            DECODE_ULABEL; // except labels are always forward
+            instruction->qstr_opname = MP_QSTR_SETUP_FINALLY;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_END_FINALLY:
+            // if TOS is an exception, reraises the exception (3 values on TOS)
+            // if TOS is an integer, does something else
+            // if TOS is None, just pops it and continues
+            // else error
+            instruction->qstr_opname = MP_QSTR_END_FINALLY;
+            break;
+
+        case MP_BC_GET_ITER:
+            instruction->qstr_opname = MP_QSTR_GET_ITER;
+            break;
+
+        case MP_BC_GET_ITER_STACK:
+            instruction->qstr_opname = MP_QSTR_GET_ITER_STACK;
+            break;
+
+        case MP_BC_FOR_ITER:
+            DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward
+            instruction->qstr_opname = MP_QSTR_FOR_ITER;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_BUILD_TUPLE:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_BUILD_TUPLE;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_BUILD_LIST:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_BUILD_LIST;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_BUILD_MAP:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_BUILD_MAP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_STORE_MAP:
+            instruction->qstr_opname = MP_QSTR_STORE_MAP;
+            break;
+
+        case MP_BC_BUILD_SET:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_BUILD_SET;
+            instruction->arg = unum;
+            break;
+
+        #if MICROPY_PY_BUILTINS_SLICE
+        case MP_BC_BUILD_SLICE:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_BUILD_SLICE;
+            instruction->arg = unum;
+            break;
+        #endif
+
+        case MP_BC_STORE_COMP:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_STORE_COMP;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_UNPACK_SEQUENCE:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_UNPACK_SEQUENCE;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_UNPACK_EX:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_UNPACK_EX;
+            instruction->arg = unum;
+            break;
+
+        case MP_BC_MAKE_FUNCTION:
+            DECODE_PTR;
+            instruction->qstr_opname = MP_QSTR_MAKE_FUNCTION;
+            instruction->arg = unum;
+            instruction->argobj= mp_obj_new_int_from_ull((uint64_t)ptr);
+            break;
+
+        case MP_BC_MAKE_FUNCTION_DEFARGS:
+            DECODE_PTR;
+            instruction->qstr_opname = MP_QSTR_MAKE_FUNCTION_DEFARGS;
+            instruction->arg = unum;
+            instruction->argobj= mp_obj_new_int_from_ull((uint64_t)ptr);
+            break;
+
+        case MP_BC_MAKE_CLOSURE: {
+            DECODE_PTR;
+            mp_uint_t n_closed_over = *ip++;
+            instruction->qstr_opname = MP_QSTR_MAKE_CLOSURE;
+            instruction->arg = unum;
+            instruction->argobj= mp_obj_new_int_from_ull((uint64_t)ptr);
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(n_closed_over);
+            break;
+        }
+
+        case MP_BC_MAKE_CLOSURE_DEFARGS: {
+            DECODE_PTR;
+            mp_uint_t n_closed_over = *ip++;
+            instruction->qstr_opname = MP_QSTR_MAKE_CLOSURE_DEFARGS;
+            instruction->arg = unum;
+            instruction->argobj= mp_obj_new_int_from_ull((uint64_t)ptr);
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT(n_closed_over);
+            break;
+        }
+
+        case MP_BC_CALL_FUNCTION:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_CALL_FUNCTION;
+            instruction->arg = unum & 0xff;
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT((unum >> 8) & 0xff);
+            break;
+
+        case MP_BC_CALL_FUNCTION_VAR_KW:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_CALL_FUNCTION_VAR_KW;
+            instruction->arg = unum & 0xff;
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT((unum >> 8) & 0xff);
+            break;
+
+        case MP_BC_CALL_METHOD:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_CALL_METHOD;
+            instruction->arg = unum & 0xff;
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT((unum >> 8) & 0xff);
+            break;
+
+        case MP_BC_CALL_METHOD_VAR_KW:
+            DECODE_UINT;
+            instruction->qstr_opname = MP_QSTR_CALL_METHOD_VAR_KW;
+            instruction->arg = unum & 0xff;
+            instruction->argobjex_cache = MP_OBJ_NEW_SMALL_INT((unum >> 8) & 0xff);
+            break;
+
+        case MP_BC_RETURN_VALUE:
+            instruction->qstr_opname = MP_QSTR_RETURN_VALUE;
+            break;
+
+        case MP_BC_RAISE_LAST:
+            instruction->qstr_opname = MP_QSTR_RAISE_LAST;
+            break;
+
+        case MP_BC_RAISE_OBJ:
+            instruction->qstr_opname = MP_QSTR_RAISE_OBJ;
+            break;
+
+        case MP_BC_RAISE_FROM:
+            instruction->qstr_opname = MP_QSTR_RAISE_FROM;
+            break;
+
+        case MP_BC_YIELD_VALUE:
+            instruction->qstr_opname = MP_QSTR_YIELD_VALUE;
+            break;
+
+        case MP_BC_YIELD_FROM:
+            instruction->qstr_opname = MP_QSTR_YIELD_FROM;
+            break;
+
+        case MP_BC_IMPORT_NAME:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_IMPORT_NAME;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_IMPORT_FROM:
+            DECODE_QSTR;
+            instruction->qstr_opname = MP_QSTR_IMPORT_FROM;
+            instruction->arg = qst;
+            instruction->argobj= MP_OBJ_NEW_QSTR(qst);
+            break;
+
+        case MP_BC_IMPORT_STAR:
+            instruction->qstr_opname = MP_QSTR_IMPORT_STAR;
+            break;
+
+        default:
+            if (ip[-1] < MP_BC_LOAD_CONST_SMALL_INT_MULTI + 64) {
+                instruction->qstr_opname = MP_QSTR_LOAD_CONST_SMALL_INT;
+                instruction->arg = (mp_int_t)ip[-1] - MP_BC_LOAD_CONST_SMALL_INT_MULTI - 16;
+            } else if (ip[-1] < MP_BC_LOAD_FAST_MULTI + 16) {
+                instruction->qstr_opname = MP_QSTR_LOAD_FAST;
+                instruction->arg = (mp_uint_t)ip[-1] - MP_BC_LOAD_FAST_MULTI;
+            } else if (ip[-1] < MP_BC_STORE_FAST_MULTI + 16) {
+                instruction->qstr_opname = MP_QSTR_STORE_FAST;
+                instruction->arg = (mp_uint_t)ip[-1] - MP_BC_STORE_FAST_MULTI;
+            } else if (ip[-1] < MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NUM_BYTECODE) {
+                instruction->qstr_opname = MP_QSTR_UNARY_OP;
+                instruction->arg = (mp_uint_t)ip[-1] - MP_BC_UNARY_OP_MULTI;
+            } else if (ip[-1] < MP_BC_BINARY_OP_MULTI + MP_BINARY_OP_NUM_BYTECODE) {
+                mp_uint_t op = ip[-1] - MP_BC_BINARY_OP_MULTI;
+                instruction->qstr_opname = MP_QSTR_BINARY_OP;
+                instruction->arg = op;
+            } else {
+                mp_printf(&mp_plat_print, "code %p, opcode 0x%02x not implemented\n", ip-1, ip[-1]);
+                assert(0);
+                return ip;
+            }
+            break;
+    }
+
+    return ip;
+}
+
+void mp_prof_print_instr(const byte* ip, mp_code_state_t *code_state) {
+    mp_dis_instruction_t _instruction, *instruction = &_instruction;
+    mp_prof_opcode_decode(ip, code_state->fun_bc->rc->const_table, instruction);
+    const mp_raw_code_t *rc = code_state->fun_bc->rc;
+    const mp_bytecode_prelude_t *prelude = &rc->prelude;
+
+    mp_uint_t offset = ip - prelude->opcodes;
+    mp_printf(&mp_plat_print, "instr");
+
+    /* long path */ if (1) {
+        mp_printf(&mp_plat_print,
+            "@0x%p:%q:%q+0x%04x:%d",
+            ip,
+            prelude->qstr_source_file,
+            prelude->qstr_block_name,
+            offset,
+            mp_prof_bytecode_lineno(rc, offset)
+        );
+    }
+
+    /* bytecode */ if (0) {
+        mp_printf(&mp_plat_print, " %02x %02x %02x %02x", ip[0], ip[1], ip[2], ip[3]);
+    }
+
+    mp_printf(&mp_plat_print, " 0x%02x %q [%d]", *ip, instruction->qstr_opname, instruction->arg);
+
+    if (instruction->argobj != mp_const_none) {
+        mp_printf(&mp_plat_print, " $");
+        mp_obj_print_helper(&mp_plat_print, instruction->argobj, PRINT_REPR);
+    }
+    if (instruction->argobjex_cache != mp_const_none) {
+        mp_printf(&mp_plat_print, " #");
+        mp_obj_print_helper(&mp_plat_print, instruction->argobjex_cache, PRINT_REPR);
+    }
+
+    mp_printf(&mp_plat_print, "\n");
+}
+
+#endif // MICROPY_PROF_INSTR_DEBUG_PRINT_ENABLE
+
+#endif // MICROPY_PY_SYS_SETTRACE

py/profile.h

@@ -0,0 +1,79 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) SatoshiLabs
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_PY_PROFILING_H
+#define MICROPY_INCLUDED_PY_PROFILING_H
+
+#include "py/emitglue.h"
+
+#if MICROPY_PY_SYS_SETTRACE
+
+#define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing)
+
+typedef struct _mp_obj_code_t {
+    mp_obj_base_t base;
+    const mp_raw_code_t *rc;
+    mp_obj_dict_t *dict_locals;
+    mp_obj_t lnotab;
+} mp_obj_code_t;
+
+typedef struct _mp_obj_frame_t {
+    mp_obj_base_t base;
+    const mp_code_state_t *code_state;
+    struct _mp_obj_frame_t *back;
+    mp_obj_t callback;
+    mp_obj_code_t *code;
+    mp_uint_t lasti;
+    mp_uint_t lineno;
+    bool trace_opcodes;
+} mp_obj_frame_t;
+
+void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude);
+
+mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc);
+mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state);
+
+// This is the implementation for the sys.settrace
+mp_obj_t mp_prof_settrace(mp_obj_t callback);
+
+mp_obj_t mp_prof_frame_enter(mp_code_state_t *code_state);
+mp_obj_t mp_prof_frame_update(const mp_code_state_t *code_state);
+
+// For every VM instruction tick this function deduces events from the state
+mp_obj_t mp_prof_instr_tick(mp_code_state_t *code_state, bool is_exception);
+
+// This section is for debugging the settrace feature itself, and is not intended
+// to be included in production/release builds.
+#define MICROPY_PROF_INSTR_DEBUG_PRINT_ENABLE 0
+#if MICROPY_PROF_INSTR_DEBUG_PRINT_ENABLE
+void mp_prof_print_instr(const byte* ip, mp_code_state_t *code_state);
+#define MP_PROF_INSTR_DEBUG_PRINT(current_ip) mp_prof_print_instr((current_ip), code_state)
+#else
+#define MP_PROF_INSTR_DEBUG_PRINT(current_ip)
+#endif
+
+#endif // MICROPY_PY_SYS_SETTRACE
+#endif // MICROPY_INCLUDED_PY_PROFILING_H

py/py.mk

@@ -13,110 +13,30 @@ ifneq ($(QSTR_AUTOGEN_DISABLE),1)
 QSTR_DEFS_COLLECTED = $(HEADER_BUILD)/qstrdefs.collected.h
 endif
 
-# Any files listed by this variable will cause a full regeneration of qstrs
+# Any files listed by these variables will cause a full regeneration of qstrs
+# DEPENDENCIES: included in qstr processing; REQUIREMENTS: not included
 QSTR_GLOBAL_DEPENDENCIES += $(PY_SRC)/mpconfig.h mpconfigport.h
+QSTR_GLOBAL_REQUIREMENTS += $(HEADER_BUILD)/mpversion.h
 
 # some code is performance bottleneck and compiled with other optimization options
 CSUPEROPT = -O3
 
-# this sets the config file for FatFs
-CFLAGS_MOD += -DFFCONF_H=\"lib/oofatfs/ffconf.h\"
+# External modules written in C.
+ifneq ($(USER_C_MODULES),)
+# pre-define USERMOD variables as expanded so that variables are immediate
+# expanded as they're added to them
+SRC_USERMOD :=
+CFLAGS_USERMOD :=
+LDFLAGS_USERMOD :=
+$(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \
+    $(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\
+    $(info Including User C Module from $(USERMOD_DIR))\
+	$(eval include $(module))\
+)
 
-ifeq ($(MICROPY_PY_USSL),1)
-CFLAGS_MOD += -DMICROPY_PY_USSL=1
-ifeq ($(MICROPY_SSL_AXTLS),1)
-CFLAGS_MOD += -DMICROPY_SSL_AXTLS=1 -I$(TOP)/lib/axtls/ssl -I$(TOP)/lib/axtls/crypto -I$(TOP)/extmod/axtls-include
-AXTLS_DIR = lib/axtls
-$(BUILD)/$(AXTLS_DIR)/%.o: CFLAGS += -Wno-all -Wno-unused-parameter -Wno-uninitialized -Wno-sign-compare -Wno-old-style-definition $(AXTLS_DEFS_EXTRA)
-SRC_MOD += $(addprefix $(AXTLS_DIR)/,\
-	ssl/asn1.c \
-	ssl/loader.c \
-	ssl/tls1.c \
-	ssl/tls1_svr.c \
-	ssl/tls1_clnt.c \
-	ssl/x509.c \
-	crypto/aes.c \
-	crypto/bigint.c \
-	crypto/crypto_misc.c \
-	crypto/hmac.c \
-	crypto/md5.c \
-	crypto/rsa.c \
-	crypto/sha1.c \
-	)
-else ifeq ($(MICROPY_SSL_MBEDTLS),1)
-# Can be overridden by ports which have "builtin" mbedTLS
-MICROPY_SSL_MBEDTLS_INCLUDE ?= $(TOP)/lib/mbedtls/include
-CFLAGS_MOD += -DMICROPY_SSL_MBEDTLS=1 -I$(MICROPY_SSL_MBEDTLS_INCLUDE)
-LDFLAGS_MOD += -L$(TOP)/lib/mbedtls/library -lmbedx509 -lmbedtls -lmbedcrypto
-endif
-endif
-
-#ifeq ($(MICROPY_PY_LWIP),1)
-#CFLAGS_MOD += -DMICROPY_PY_LWIP=1 -I../lib/lwip/src/include -I../lib/lwip/src/include/ipv4 -I../extmod/lwip-include
-#endif
-
-ifeq ($(MICROPY_PY_LWIP),1)
-LWIP_DIR = lib/lwip/src
-INC += -I$(TOP)/lib/lwip/src/include -I$(TOP)/lib/lwip/src/include/ipv4 -I$(TOP)/extmod/lwip-include
-CFLAGS_MOD += -DMICROPY_PY_LWIP=1
-SRC_MOD += extmod/modlwip.c lib/netutils/netutils.c
-SRC_MOD += $(addprefix $(LWIP_DIR)/,\
-	core/def.c \
-	core/dns.c \
-	core/init.c \
-	core/mem.c \
-	core/memp.c \
-	core/netif.c \
-	core/pbuf.c \
-	core/raw.c \
-	core/stats.c \
-	core/sys.c \
-	core/tcp.c \
-	core/tcp_in.c \
-	core/tcp_out.c \
-	core/timers.c \
-	core/udp.c \
-	core/ipv4/autoip.c \
-	core/ipv4/icmp.c \
-	core/ipv4/igmp.c \
-	core/ipv4/inet.c \
-	core/ipv4/inet_chksum.c \
-	core/ipv4/ip_addr.c \
-	core/ipv4/ip.c \
-	core/ipv4/ip_frag.c \
-	)
-ifeq ($(MICROPY_PY_LWIP_SLIP),1)
-CFLAGS_MOD += -DMICROPY_PY_LWIP_SLIP=1
-SRC_MOD += $(LWIP_DIR)/netif/slipif.c
-endif
-endif
-
-ifeq ($(MICROPY_PY_BTREE),1)
-BTREE_DIR = lib/berkeley-db-1.xx
-BTREE_DEFS = -D__DBINTERFACE_PRIVATE=1 -Dmpool_error=printf -Dabort=abort_ "-Dvirt_fd_t=void*" $(BTREE_DEFS_EXTRA)
-INC += -I$(TOP)/$(BTREE_DIR)/PORT/include
-SRC_MOD += extmod/modbtree.c
-SRC_MOD += $(addprefix $(BTREE_DIR)/,\
-btree/bt_close.c \
-btree/bt_conv.c \
-btree/bt_debug.c \
-btree/bt_delete.c \
-btree/bt_get.c \
-btree/bt_open.c \
-btree/bt_overflow.c \
-btree/bt_page.c \
-btree/bt_put.c \
-btree/bt_search.c \
-btree/bt_seq.c \
-btree/bt_split.c \
-btree/bt_utils.c \
-mpool/mpool.c \
-	)
-CFLAGS_MOD += -DMICROPY_PY_BTREE=1
-# we need to suppress certain warnings to get berkeley-db to compile cleanly
-# and we have separate BTREE_DEFS so the definitions don't interfere with other source code
-$(BUILD)/$(BTREE_DIR)/%.o: CFLAGS += -Wno-old-style-definition -Wno-sign-compare -Wno-unused-parameter $(BTREE_DEFS)
-$(BUILD)/extmod/modbtree.o: CFLAGS += $(BTREE_DEFS)
+SRC_MOD += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD))
+CFLAGS_MOD += $(CFLAGS_USERMOD)
+LDFLAGS_MOD += $(LDFLAGS_USERMOD)
 endif
 
 # py object files
@@ -126,6 +46,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\
 	nlrx86.o \
 	nlrx64.o \
 	nlrthumb.o \
+	nlrpowerpc.o \
 	nlrxtensa.o \
 	nlrsetjmp.o \
 	malloc.o \
@@ -156,6 +77,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\
 	asmxtensa.o \
 	emitnxtensa.o \
 	emitinlinextensa.o \
+	emitnxtensawin.o \
 	formatfloat.o \
 	parsenumbase.o \
 	parsenum.o \
@@ -165,9 +87,11 @@ PY_CORE_O_BASENAME = $(addprefix py/,\
 	runtime_utils.o \
 	scheduler.o \
 	nativeglue.o \
+	ringbuf.o \
 	stackctrl.o \
 	argcheck.o \
 	warning.o \
+	profile.o \
 	map.o \
 	obj.o \
 	objarray.o \
@@ -252,6 +176,7 @@ PY_EXTMOD_O_BASENAME = \
 	extmod/machine_pulse.o \
 	extmod/machine_i2c.o \
 	extmod/machine_spi.o \
+	extmod/modbluetooth.o \
 	extmod/modussl_axtls.o \
 	extmod/modussl_mbedtls.o \
 	extmod/modurandom.o \
@@ -260,12 +185,14 @@ PY_EXTMOD_O_BASENAME = \
 	extmod/modwebrepl.o \
 	extmod/modframebuf.o \
 	extmod/vfs.o \
+	extmod/vfs_blockdev.o \
 	extmod/vfs_reader.o \
 	extmod/vfs_posix.o \
 	extmod/vfs_posix_file.o \
 	extmod/vfs_fat.o \
 	extmod/vfs_fat_diskio.o \
 	extmod/vfs_fat_file.o \
+	extmod/vfs_lfs.o \
 	extmod/utime_mphal.o \
 	extmod/uos_dupterm.o \
 	lib/embed/abort_.o \
@@ -278,6 +205,11 @@ PY_EXTMOD_O = $(addprefix $(BUILD)/, $(PY_EXTMOD_O_BASENAME))
 # this is a convenience variable for ports that want core, extmod and frozen code
 PY_O = $(PY_CORE_O) $(PY_EXTMOD_O)
 
+# object file for frozen code specified via a manifest
+ifneq ($(FROZEN_MANIFEST),)
+PY_O += $(BUILD)/$(BUILD)/frozen_content.o
+endif
+
 # object file for frozen files
 ifneq ($(FROZEN_DIR),)
 PY_O += $(BUILD)/$(BUILD)/frozen.o
@@ -290,7 +222,7 @@ endif
 
 # Sources that may contain qstrings
 SRC_QSTR_IGNORE = py/nlr%
-SRC_QSTR = $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) $(PY_EXTMOD_O_BASENAME:.o=.c)
+SRC_QSTR += $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) $(PY_EXTMOD_O_BASENAME:.o=.c)
 
 # Anything that depends on FORCE will be considered out-of-date
 FORCE:
@@ -308,11 +240,19 @@ MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
 # created before we run the script to generate the .h
 # Note: we need to protect the qstr names from the preprocessor, so we wrap
 # the lines in "" and then unwrap after the preprocessor is finished.
+# See more information about this process in docs/develop/qstr.rst.
 $(HEADER_BUILD)/qstrdefs.generated.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) $(PY_SRC)/makeqstrdata.py mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
 	$(ECHO) "GEN $@"
-	$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
+	$(Q)$(CAT) $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^\"\(Q(.*)\)\"/\1/' > $(HEADER_BUILD)/qstrdefs.preprocessed.h
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
 
+# build a list of registered modules for py/objmodule.c.
+$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
+	@$(ECHO) "GEN $@"
+	$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
+
+SRC_QSTR += $(HEADER_BUILD)/moduledefs.h
+
 # Force nlr code to always be compiled with space-saving optimisation so
 # that the function preludes are of a minimal and predictable form.
 $(PY_BUILD)/nlr%.o: CFLAGS += -Os
@@ -330,3 +270,6 @@ $(PY_BUILD)/vm.o: CFLAGS += $(CSUPEROPT)
 # http://hg.python.org/cpython/file/b127046831e2/Python/ceval.c#l828
 # http://www.emulators.com/docs/nx25_nostradamus.htm
 #-fno-crossjumping
+
+# Include rules for extmod related code
+include $(TOP)/extmod/extmod.mk

py/qstr.c

@@ -31,6 +31,7 @@
 #include "py/mpstate.h"
 #include "py/qstr.h"
 #include "py/gc.h"
+#include "py/runtime.h"
 
 // NOTE: we are using linear arrays to store and search for qstr's (unique strings, interned strings)
 // ultimately we will replace this with a static hash table of some kind
@@ -192,12 +193,17 @@ qstr qstr_from_str(const char *str) {
 }
 
 qstr qstr_from_strn(const char *str, size_t len) {
-    assert(len < (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN)));
     QSTR_ENTER();
     qstr q = qstr_find_strn(str, len);
     if (q == 0) {
         // qstr does not exist in interned pool so need to add it
 
+        // check that len is not too big
+        if (len >= (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN))) {
+            QSTR_EXIT();
+            mp_raise_msg(&mp_type_RuntimeError, "name too long");
+        }
+
         // compute number of bytes needed to intern this string
         size_t n_bytes = MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len + 1;
 

py/qstr.h

@@ -35,7 +35,7 @@
 // Note: it would be possible to define MP_QSTR_xxx as qstr_from_str_static("xxx")
 // for qstrs that are referenced this way, but you don't want to have them in ROM.
 
-// first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr
+// first entry in enum will be MP_QSTRnull=0, which indicates invalid/no qstr
 enum {
 #ifndef NO_QSTR
 #define QDEF(id, str) id,
@@ -61,7 +61,7 @@ typedef struct _qstr_pool_t {
 void qstr_init(void);
 
 mp_uint_t qstr_compute_hash(const byte *data, size_t len);
-qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTR_NULL if not found
+qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if not found
 
 qstr qstr_from_str(const char *str);
 qstr qstr_from_strn(const char *str, size_t len);

py/repl.c

@@ -159,7 +159,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print
         if (str < top) {
             // a complete word, lookup in current object
             qstr q = qstr_find_strn(s_start, s_len);
-            if (q == MP_QSTR_NULL) {
+            if (q == MP_QSTRnull) {
                 // lookup will fail
                 return 0;
             }

py/ringbuf.c

@@ -0,0 +1,73 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Jim Mussared
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "ringbuf.h"
+
+int ringbuf_get16(ringbuf_t *r) {
+    int v = ringbuf_peek16(r);
+    if (v == -1) {
+        return v;
+    }
+    r->iget += 2;
+    if (r->iget >= r->size) {
+        r->iget -= r->size;
+    }
+    return v;
+}
+
+int ringbuf_peek16(ringbuf_t *r) {
+    if (r->iget == r->iput) {
+        return -1;
+    }
+    uint32_t iget_a = r->iget + 1;
+    if (iget_a == r->size) {
+        iget_a = 0;
+    }
+    if (iget_a == r->iput) {
+        return -1;
+    }
+    return (r->buf[r->iget] << 8) | (r->buf[iget_a]);
+}
+
+int ringbuf_put16(ringbuf_t *r, uint16_t v) {
+    uint32_t iput_a = r->iput + 1;
+    if (iput_a == r->size) {
+        iput_a = 0;
+    }
+    if (iput_a == r->iget) {
+        return -1;
+    }
+    uint32_t iput_b = iput_a + 1;
+    if (iput_b == r->size) {
+        iput_b = 0;
+    }
+    if (iput_b == r->iget) {
+        return -1;
+    }
+    r->buf[r->iput] = (v >> 8) & 0xff;
+    r->buf[iput_a] = v & 0xff;
+    r->iput = iput_b;
+    return 0;
+}

py/ringbuf.h

@@ -26,6 +26,9 @@
 #ifndef MICROPY_INCLUDED_PY_RINGBUF_H
 #define MICROPY_INCLUDED_PY_RINGBUF_H
 
+#include <stddef.h>
+#include <stdint.h>
+
 typedef struct _ringbuf_t {
     uint8_t *buf;
     uint16_t size;
@@ -37,7 +40,7 @@ typedef struct _ringbuf_t {
 // byte buf_array[N];
 // ringbuf_t buf = {buf_array, sizeof(buf_array)};
 
-// Dynamic initialization. This creates root pointer!
+// Dynamic initialization. This needs to become findable as a root pointer!
 #define ringbuf_alloc(r, sz) \
 { \
     (r)->buf = m_new(uint8_t, sz); \
@@ -69,4 +72,17 @@ static inline int ringbuf_put(ringbuf_t *r, uint8_t v) {
     return 0;
 }
 
+static inline size_t ringbuf_free(ringbuf_t *r) {
+    return (r->size + r->iget - r->iput - 1) % r->size;
+}
+
+static inline size_t ringbuf_avail(ringbuf_t *r) {
+    return (r->size + r->iput - r->iget) % r->size;
+}
+
+// Note: big-endian. No-op if not enough room available for both bytes.
+int ringbuf_get16(ringbuf_t *r);
+int ringbuf_peek16(ringbuf_t *r);
+int ringbuf_put16(ringbuf_t *r, uint16_t v);
+
 #endif // MICROPY_INCLUDED_PY_RINGBUF_H

py/runtime.c

@@ -63,7 +63,8 @@ void mp_init(void) {
     MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL;
     #if MICROPY_ENABLE_SCHEDULER
     MP_STATE_VM(sched_state) = MP_SCHED_IDLE;
-    MP_STATE_VM(sched_sp) = 0;
+    MP_STATE_VM(sched_idx) = 0;
+    MP_STATE_VM(sched_len) = 0;
     #endif
 
 #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF
@@ -87,6 +88,9 @@ void mp_init(void) {
     #if MICROPY_ENABLE_COMPILER
     // optimization disabled by default
     MP_STATE_VM(mp_optimise_value) = 0;
+    #if MICROPY_EMIT_NATIVE
+    MP_STATE_VM(default_emit_opt) = MP_EMIT_OPT_NONE;
+    #endif
     #endif
 
     // init global module dict
@@ -111,17 +115,26 @@ void mp_init(void) {
     }
     #endif
 
-    #if MICROPY_FSUSERMOUNT
-    // zero out the pointers to the user-mounted devices
-    memset(MP_STATE_VM(fs_user_mount), 0, sizeof(MP_STATE_VM(fs_user_mount)));
-    #endif
-
     #if MICROPY_VFS
     // initialise the VFS sub-system
     MP_STATE_VM(vfs_cur) = NULL;
     MP_STATE_VM(vfs_mount_table) = NULL;
     #endif
 
+    #if MICROPY_PY_SYS_ATEXIT
+    MP_STATE_VM(sys_exitfunc) = mp_const_none;
+    #endif
+
+    #if MICROPY_PY_SYS_SETTRACE
+    MP_STATE_THREAD(prof_trace_callback) = MP_OBJ_NULL;
+    MP_STATE_THREAD(prof_callback_is_executing) = false;
+    MP_STATE_THREAD(current_code_state) = NULL;
+    #endif
+
+    #if MICROPY_PY_BLUETOOTH
+    MP_STATE_VM(bluetooth) = MP_OBJ_NULL;
+    #endif
+
     #if MICROPY_PY_THREAD_GIL
     mp_thread_mutex_init(&MP_STATE_VM(gil_mutex));
     #endif
@@ -460,8 +473,7 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) {
                 case MP_BINARY_OP_INPLACE_POWER:
                     if (rhs_val < 0) {
                         #if MICROPY_PY_BUILTINS_FLOAT
-                        lhs = mp_obj_new_float(lhs_val);
-                        goto generic_binary_op;
+                        return mp_obj_float_binary_op(op, lhs_val, rhs);
                         #else
                         mp_raise_ValueError("negative power with no float support");
                         #endif
@@ -558,16 +570,17 @@ generic_binary_op:
     }
 
 #if MICROPY_PY_REVERSE_SPECIAL_METHODS
-    if (op >= MP_BINARY_OP_OR && op <= MP_BINARY_OP_REVERSE_POWER) {
+    if (op >= MP_BINARY_OP_OR && op <= MP_BINARY_OP_POWER) {
         mp_obj_t t = rhs;
         rhs = lhs;
         lhs = t;
-        if (op <= MP_BINARY_OP_POWER) {
-            op += MP_BINARY_OP_REVERSE_OR - MP_BINARY_OP_OR;
-            goto generic_binary_op;
-        }
-
+        op += MP_BINARY_OP_REVERSE_OR - MP_BINARY_OP_OR;
+        goto generic_binary_op;
+    } else if (op >= MP_BINARY_OP_REVERSE_OR) {
         // Convert __rop__ back to __op__ for error message
+        mp_obj_t t = rhs;
+        rhs = lhs;
+        lhs = t;
         op -= MP_BINARY_OP_REVERSE_OR - MP_BINARY_OP_OR;
     }
 #endif
@@ -880,8 +893,12 @@ void mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) {
     DEBUG_OP_printf("unpack ex " UINT_FMT " " UINT_FMT "\n", num_left, num_right);
     size_t seq_len;
     if (mp_obj_is_type(seq_in, &mp_type_tuple) || mp_obj_is_type(seq_in, &mp_type_list)) {
+        // Make the seq variable volatile so the compiler keeps a reference to it,
+        // since if it's a tuple then seq_items points to the interior of the GC cell
+        // and mp_obj_new_list may trigger a GC which doesn't trace this and reclaims seq.
+        volatile mp_obj_t seq = seq_in;
         mp_obj_t *seq_items;
-        mp_obj_get_array(seq_in, &seq_len, &seq_items);
+        mp_obj_get_array(seq, &seq_len, &seq_items);
         if (seq_len < num_left + num_right) {
             goto too_short;
         }
@@ -892,6 +909,7 @@ void mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) {
         for (size_t i = 0; i < num_left; i++) {
             items[num_right + 1 + i] = seq_items[num_left - 1 - i];
         }
+        seq = MP_OBJ_NULL;
     } else {
         // Generic iterable; this gets a bit messy: we unpack known left length to the
         // items destination array, then the rest to a dynamically created list.  Once the
@@ -1049,14 +1067,13 @@ void mp_load_method_maybe(mp_obj_t obj, qstr attr, mp_obj_t *dest) {
     mp_obj_type_t *type = mp_obj_get_type(obj);
 
     // look for built-in names
-    if (0) {
 #if MICROPY_CPYTHON_COMPAT
-    } else if (attr == MP_QSTR___class__) {
+    if (attr == MP_QSTR___class__) {
         // a.__class__ is equivalent to type(a)
         dest[0] = MP_OBJ_FROM_PTR(type);
+    } else
 #endif
-
-    } else if (attr == MP_QSTR___next__ && type->iternext != NULL) {
+    if (attr == MP_QSTR___next__ && type->iternext != NULL) {
         dest[0] = MP_OBJ_FROM_PTR(&mp_builtin_next_obj);
         dest[1] = obj;
 
@@ -1303,7 +1320,12 @@ mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t th
         // will be propagated up. This behavior is approved by test_pep380.py
         // test_delegation_of_close_to_non_generator(),
         //  test_delegating_throw_to_non_generator()
-        *ret_val = mp_make_raise_obj(throw_value);
+        if (mp_obj_exception_match(throw_value, MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
+            // PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError
+            *ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, "generator raised StopIteration");
+        } else {
+            *ret_val = mp_make_raise_obj(throw_value);
+        }
         return MP_VM_RETURN_EXCEPTION;
     }
 }
@@ -1336,7 +1358,17 @@ mp_obj_t mp_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level) {
     args[3] = fromlist;
     args[4] = level;
 
-    // TODO lookup __import__ and call that instead of going straight to builtin implementation
+    #if MICROPY_CAN_OVERRIDE_BUILTINS
+    // Lookup __import__ and call that if it exists
+    mp_obj_dict_t *bo_dict = MP_STATE_VM(mp_module_builtins_override_dict);
+    if (bo_dict != NULL) {
+        mp_map_elem_t *import = mp_map_lookup(&bo_dict->map, MP_OBJ_NEW_QSTR(MP_QSTR___import__), MP_MAP_LOOKUP);
+        if (import != NULL) {
+            return mp_call_function_n_kw(import->value, 5, 0, args);
+        }
+    }
+    #endif
+
     return mp_builtin___import__(5, args);
 }
 
@@ -1408,7 +1440,6 @@ void mp_import_all(mp_obj_t module) {
 
 #if MICROPY_ENABLE_COMPILER
 
-// this is implemented in this file so it can optimise access to locals/globals
 mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_input_kind, mp_obj_dict_t *globals, mp_obj_dict_t *locals) {
     // save context
     mp_obj_dict_t *volatile old_globals = mp_globals_get();
@@ -1422,7 +1453,7 @@ mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_i
     if (nlr_push(&nlr) == 0) {
         qstr source_name = lex->source_name;
         mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind);
-        mp_obj_t module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, false);
+        mp_obj_t module_fun = mp_compile(&parse_tree, source_name, false);
 
         mp_obj_t ret;
         if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) {

py/runtime.h

@@ -70,7 +70,7 @@ void mp_handle_pending_tail(mp_uint_t atomic_state);
 #if MICROPY_ENABLE_SCHEDULER
 void mp_sched_lock(void);
 void mp_sched_unlock(void);
-static inline unsigned int mp_sched_num_pending(void) { return MP_STATE_VM(sched_sp); }
+static inline unsigned int mp_sched_num_pending(void) { return MP_STATE_VM(sched_len); }
 bool mp_sched_schedule(mp_obj_t function, mp_obj_t arg);
 #endif
 
@@ -151,7 +151,6 @@ mp_obj_t mp_import_from(mp_obj_t module, qstr name);
 void mp_import_all(mp_obj_t module);
 
 NORETURN void mp_raise_msg(const mp_obj_type_t *exc_type, const char *msg);
-//NORETURN void nlr_raise_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...);
 NORETURN void mp_raise_ValueError(const char *msg);
 NORETURN void mp_raise_TypeError(const char *msg);
 NORETURN void mp_raise_NotImplementedError(const char *msg);
@@ -169,11 +168,9 @@ NORETURN void mp_raise_recursion_depth(void);
 #endif
 
 // helper functions for native/viper code
-mp_uint_t mp_convert_obj_to_native(mp_obj_t obj, mp_uint_t type);
-mp_obj_t mp_convert_native_to_obj(mp_uint_t val, mp_uint_t type);
-mp_obj_dict_t *mp_native_swap_globals(mp_obj_dict_t *new_globals);
-mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args);
-void mp_native_raise(mp_obj_t o);
+int mp_native_type_from_qstr(qstr qst);
+mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type);
+mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type);
 
 #define mp_sys_path (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_path_obj)))
 #define mp_sys_argv (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_argv_obj)))

py/runtime0.h

@@ -28,13 +28,17 @@
 
 // The first four must fit in 8 bits, see emitbc.c
 // The remaining must fit in 16 bits, see scope.h
-#define MP_SCOPE_FLAG_VARARGS      (0x01)
+#define MP_SCOPE_FLAG_ALL_SIG      (0x0f)
+#define MP_SCOPE_FLAG_GENERATOR    (0x01)
 #define MP_SCOPE_FLAG_VARKEYWORDS  (0x02)
-#define MP_SCOPE_FLAG_GENERATOR    (0x04)
+#define MP_SCOPE_FLAG_VARARGS      (0x04)
 #define MP_SCOPE_FLAG_DEFKWARGS    (0x08)
 #define MP_SCOPE_FLAG_REFGLOBALS   (0x10) // used only if native emitter enabled
 #define MP_SCOPE_FLAG_HASCONSTS    (0x20) // used only if native emitter enabled
-#define MP_SCOPE_FLAG_VIPERRET_POS    (6) // 3 bits used for viper return type
+#define MP_SCOPE_FLAG_VIPERRET_POS    (6) // 3 bits used for viper return type, to pass from compiler to native emitter
+#define MP_SCOPE_FLAG_VIPERRELOC   (0x10) // used only when loading viper from .mpy
+#define MP_SCOPE_FLAG_VIPERRODATA  (0x20) // used only when loading viper from .mpy
+#define MP_SCOPE_FLAG_VIPERBSS     (0x40) // used only when loading viper from .mpy
 
 // types for native (viper) function signature
 #define MP_NATIVE_TYPE_OBJ  (0x00)
@@ -46,6 +50,18 @@
 #define MP_NATIVE_TYPE_PTR16 (0x06)
 #define MP_NATIVE_TYPE_PTR32 (0x07)
 
+// Bytecode and runtime boundaries for unary ops
+#define MP_UNARY_OP_NUM_BYTECODE    (MP_UNARY_OP_NOT + 1)
+#define MP_UNARY_OP_NUM_RUNTIME     (MP_UNARY_OP_SIZEOF + 1)
+
+// Bytecode and runtime boundaries for binary ops
+#define MP_BINARY_OP_NUM_BYTECODE   (MP_BINARY_OP_POWER + 1)
+#if MICROPY_PY_REVERSE_SPECIAL_METHODS
+#define MP_BINARY_OP_NUM_RUNTIME    (MP_BINARY_OP_REVERSE_POWER + 1)
+#else
+#define MP_BINARY_OP_NUM_RUNTIME    (MP_BINARY_OP_CONTAINS + 1)
+#endif
+
 typedef enum {
     // These ops may appear in the bytecode. Changing this group
     // in any way requires changing the bytecode version.
@@ -55,22 +71,19 @@ typedef enum {
     MP_UNARY_OP_NOT,
 
     // Following ops cannot appear in the bytecode
-    MP_UNARY_OP_NUM_BYTECODE,
-
-    MP_UNARY_OP_BOOL = MP_UNARY_OP_NUM_BYTECODE, // __bool__
+    MP_UNARY_OP_BOOL, // __bool__
     MP_UNARY_OP_LEN, // __len__
     MP_UNARY_OP_HASH, // __hash__; must return a small int
     MP_UNARY_OP_ABS, // __abs__
     MP_UNARY_OP_INT, // __int__
     MP_UNARY_OP_SIZEOF, // for sys.getsizeof()
-
-    MP_UNARY_OP_NUM_RUNTIME,
 } mp_unary_op_t;
 
-// Note: the first 9+12+12 of these are used in bytecode and changing
-// them requires changing the bytecode version.
 typedef enum {
-    // 9 relational operations, should return a bool
+    // The following 9+13+13 ops are used in bytecode and changing
+    // them requires changing the bytecode version.
+
+    // 9 relational operations, should return a bool; order of first 6 matches corresponding mp_token_kind_t
     MP_BINARY_OP_LESS,
     MP_BINARY_OP_MORE,
     MP_BINARY_OP_EQUAL,
@@ -81,7 +94,7 @@ typedef enum {
     MP_BINARY_OP_IS,
     MP_BINARY_OP_EXCEPTION_MATCH,
 
-    // 12 inplace arithmetic operations
+    // 13 inplace arithmetic operations; order matches corresponding mp_token_kind_t
     MP_BINARY_OP_INPLACE_OR,
     MP_BINARY_OP_INPLACE_XOR,
     MP_BINARY_OP_INPLACE_AND,
@@ -90,12 +103,13 @@ typedef enum {
     MP_BINARY_OP_INPLACE_ADD,
     MP_BINARY_OP_INPLACE_SUBTRACT,
     MP_BINARY_OP_INPLACE_MULTIPLY,
+    MP_BINARY_OP_INPLACE_MAT_MULTIPLY,
     MP_BINARY_OP_INPLACE_FLOOR_DIVIDE,
     MP_BINARY_OP_INPLACE_TRUE_DIVIDE,
     MP_BINARY_OP_INPLACE_MODULO,
     MP_BINARY_OP_INPLACE_POWER,
 
-    // 12 normal arithmetic operations
+    // 13 normal arithmetic operations; order matches corresponding mp_token_kind_t
     MP_BINARY_OP_OR,
     MP_BINARY_OP_XOR,
     MP_BINARY_OP_AND,
@@ -104,6 +118,7 @@ typedef enum {
     MP_BINARY_OP_ADD,
     MP_BINARY_OP_SUBTRACT,
     MP_BINARY_OP_MULTIPLY,
+    MP_BINARY_OP_MAT_MULTIPLY,
     MP_BINARY_OP_FLOOR_DIVIDE,
     MP_BINARY_OP_TRUE_DIVIDE,
     MP_BINARY_OP_MODULO,
@@ -111,11 +126,18 @@ typedef enum {
 
     // Operations below this line don't appear in bytecode, they
     // just identify special methods.
-    MP_BINARY_OP_NUM_BYTECODE,
 
-    // MP_BINARY_OP_REVERSE_* must follow immediately after MP_BINARY_OP_*
-#if MICROPY_PY_REVERSE_SPECIAL_METHODS
-    MP_BINARY_OP_REVERSE_OR = MP_BINARY_OP_NUM_BYTECODE,
+    // This is not emitted by the compiler but is supported by the runtime.
+    // It must follow immediately after MP_BINARY_OP_POWER.
+    MP_BINARY_OP_DIVMOD,
+
+    // The runtime will convert MP_BINARY_OP_IN to this operator with swapped args.
+    // A type should implement this containment operator instead of MP_BINARY_OP_IN.
+    MP_BINARY_OP_CONTAINS,
+
+    // 13 MP_BINARY_OP_REVERSE_* operations must be in the same order as MP_BINARY_OP_*,
+    // and be the last ones supported by the runtime.
+    MP_BINARY_OP_REVERSE_OR,
     MP_BINARY_OP_REVERSE_XOR,
     MP_BINARY_OP_REVERSE_AND,
     MP_BINARY_OP_REVERSE_LSHIFT,
@@ -123,88 +145,15 @@ typedef enum {
     MP_BINARY_OP_REVERSE_ADD,
     MP_BINARY_OP_REVERSE_SUBTRACT,
     MP_BINARY_OP_REVERSE_MULTIPLY,
+    MP_BINARY_OP_REVERSE_MAT_MULTIPLY,
     MP_BINARY_OP_REVERSE_FLOOR_DIVIDE,
     MP_BINARY_OP_REVERSE_TRUE_DIVIDE,
     MP_BINARY_OP_REVERSE_MODULO,
     MP_BINARY_OP_REVERSE_POWER,
-#endif
-
-    // This is not emitted by the compiler but is supported by the runtime
-    MP_BINARY_OP_DIVMOD
-        #if !MICROPY_PY_REVERSE_SPECIAL_METHODS
-        = MP_BINARY_OP_NUM_BYTECODE
-        #endif
-    ,
-
-    // The runtime will convert MP_BINARY_OP_IN to this operator with swapped args.
-    // A type should implement this containment operator instead of MP_BINARY_OP_IN.
-    MP_BINARY_OP_CONTAINS,
-
-    MP_BINARY_OP_NUM_RUNTIME,
 
     // These 2 are not supported by the runtime and must be synthesised by the emitter
     MP_BINARY_OP_NOT_IN,
     MP_BINARY_OP_IS_NOT,
 } mp_binary_op_t;
 
-typedef enum {
-    MP_F_CONST_NONE_OBJ = 0,
-    MP_F_CONST_FALSE_OBJ,
-    MP_F_CONST_TRUE_OBJ,
-    MP_F_CONVERT_OBJ_TO_NATIVE,
-    MP_F_CONVERT_NATIVE_TO_OBJ,
-    MP_F_NATIVE_SWAP_GLOBALS,
-    MP_F_LOAD_NAME,
-    MP_F_LOAD_GLOBAL,
-    MP_F_LOAD_BUILD_CLASS,
-    MP_F_LOAD_ATTR,
-    MP_F_LOAD_METHOD,
-    MP_F_LOAD_SUPER_METHOD,
-    MP_F_STORE_NAME,
-    MP_F_STORE_GLOBAL,
-    MP_F_STORE_ATTR,
-    MP_F_OBJ_SUBSCR,
-    MP_F_OBJ_IS_TRUE,
-    MP_F_UNARY_OP,
-    MP_F_BINARY_OP,
-    MP_F_BUILD_TUPLE,
-    MP_F_BUILD_LIST,
-    MP_F_LIST_APPEND,
-    MP_F_BUILD_MAP,
-    MP_F_STORE_MAP,
-#if MICROPY_PY_BUILTINS_SET
-    MP_F_STORE_SET,
-    MP_F_BUILD_SET,
-#endif
-    MP_F_MAKE_FUNCTION_FROM_RAW_CODE,
-    MP_F_NATIVE_CALL_FUNCTION_N_KW,
-    MP_F_CALL_METHOD_N_KW,
-    MP_F_CALL_METHOD_N_KW_VAR,
-    MP_F_NATIVE_GETITER,
-    MP_F_NATIVE_ITERNEXT,
-    MP_F_NLR_PUSH,
-    MP_F_NLR_POP,
-    MP_F_NATIVE_RAISE,
-    MP_F_IMPORT_NAME,
-    MP_F_IMPORT_FROM,
-    MP_F_IMPORT_ALL,
-#if MICROPY_PY_BUILTINS_SLICE
-    MP_F_NEW_SLICE,
-#endif
-    MP_F_UNPACK_SEQUENCE,
-    MP_F_UNPACK_EX,
-    MP_F_DELETE_NAME,
-    MP_F_DELETE_GLOBAL,
-    MP_F_NEW_CELL,
-    MP_F_MAKE_CLOSURE_FROM_RAW_CODE,
-    MP_F_ARG_CHECK_NUM_SIG,
-    MP_F_SETUP_CODE_STATE,
-    MP_F_SMALL_INT_FLOOR_DIVIDE,
-    MP_F_SMALL_INT_MODULO,
-    MP_F_NATIVE_YIELD_FROM,
-    MP_F_NUMBER_OF,
-} mp_fun_kind_t;
-
-extern const void *const mp_fun_table[MP_F_NUMBER_OF];
-
 #endif // MICROPY_INCLUDED_PY_RUNTIME0_H

py/sequence.c

@@ -165,7 +165,7 @@ bool mp_seq_cmp_bytes(mp_uint_t op, const byte *data1, size_t len1, const byte *
     size_t min_len = len1 < len2 ? len1 : len2;
     int res = memcmp(data1, data2, min_len);
     if (op == MP_BINARY_OP_EQUAL) {
-        // If we are checking for equality, here're the answer
+        // If we are checking for equality, here's the answer
         return res == 0;
     }
     if (res < 0) {

py/showbc.c

@@ -83,17 +83,10 @@ const mp_uint_t *mp_showbc_const_table;
 void mp_bytecode_print(const void *descr, const byte *ip, mp_uint_t len, const mp_uint_t *const_table) {
     mp_showbc_code_start = ip;
 
-    // get bytecode parameters
-    mp_uint_t n_state = mp_decode_uint(&ip);
-    mp_uint_t n_exc_stack = mp_decode_uint(&ip);
-    /*mp_uint_t scope_flags =*/ ip++;
-    mp_uint_t n_pos_args = *ip++;
-    mp_uint_t n_kwonly_args = *ip++;
-    /*mp_uint_t n_def_pos_args =*/ ip++;
-
+    // Decode prelude
+    MP_BC_PRELUDE_SIG_DECODE(ip);
+    MP_BC_PRELUDE_SIZE_DECODE(ip);
     const byte *code_info = ip;
-    mp_uint_t code_info_size = mp_decode_uint(&code_info);
-    ip += code_info_size;
 
     #if MICROPY_PERSISTENT_CODE
     qstr block_name = code_info[0] | (code_info[1] << 8);
@@ -107,7 +100,9 @@ void mp_bytecode_print(const void *descr, const byte *ip, mp_uint_t len, const m
         qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len);
 
     // raw bytecode dump
-    printf("Raw bytecode (code_info_size=" UINT_FMT ", bytecode_size=" UINT_FMT "):\n", code_info_size, len - code_info_size);
+    size_t prelude_size = ip - mp_showbc_code_start + n_info + n_cell;
+    printf("Raw bytecode (code_info_size=" UINT_FMT ", bytecode_size=" UINT_FMT "):\n",
+        prelude_size, len - prelude_size);
     for (mp_uint_t i = 0; i < len; i++) {
         if (i > 0 && i % 16 == 0) {
             printf("\n");
@@ -123,24 +118,21 @@ void mp_bytecode_print(const void *descr, const byte *ip, mp_uint_t len, const m
     }
     printf("\n");
 
-    printf("(N_STATE " UINT_FMT ")\n", n_state);
-    printf("(N_EXC_STACK " UINT_FMT ")\n", n_exc_stack);
+    printf("(N_STATE %u)\n", (unsigned)n_state);
+    printf("(N_EXC_STACK %u)\n", (unsigned)n_exc_stack);
 
-    // for printing line number info
-    const byte *bytecode_start = ip;
+    // skip over code_info
+    ip += n_info;
 
     // bytecode prelude: initialise closed over variables
-    {
-        uint local_num;
-        while ((local_num = *ip++) != 255) {
-            printf("(INIT_CELL %u)\n", local_num);
-        }
-        len -= ip - mp_showbc_code_start;
+    for (size_t i = 0; i < n_cell; ++i) {
+        uint local_num = *ip++;
+        printf("(INIT_CELL %u)\n", local_num);
     }
 
     // print out line number info
     {
-        mp_int_t bc = bytecode_start - ip;
+        mp_int_t bc = 0;
         mp_uint_t source_line = 1;
         printf("  bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line);
         for (const byte* ci = code_info; *ci;) {
@@ -158,7 +150,7 @@ void mp_bytecode_print(const void *descr, const byte *ip, mp_uint_t len, const m
             printf("  bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line);
         }
     }
-    mp_bytecode_print2(ip, len - 0, const_table);
+    mp_bytecode_print2(ip, len - prelude_size, const_table);
 }
 
 const byte *mp_bytecode_print_str(const byte *ip) {
@@ -401,14 +393,9 @@ const byte *mp_bytecode_print_str(const byte *ip) {
             printf("FOR_ITER " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start));
             break;
 
-        case MP_BC_POP_BLOCK:
-            // pops block and restores the stack
-            printf("POP_BLOCK");
-            break;
-
-        case MP_BC_POP_EXCEPT:
-            // pops block, checks it's an exception block, and restores the stack, saving the 3 exception values to local threadstate
-            printf("POP_EXCEPT");
+        case MP_BC_POP_EXCEPT_JUMP:
+            DECODE_ULABEL; // these labels are always forward
+            printf("POP_EXCEPT_JUMP " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start));
             break;
 
         case MP_BC_BUILD_TUPLE:
@@ -505,9 +492,16 @@ const byte *mp_bytecode_print_str(const byte *ip) {
             printf("RETURN_VALUE");
             break;
 
-        case MP_BC_RAISE_VARARGS:
-            unum = *ip++;
-            printf("RAISE_VARARGS " UINT_FMT, unum);
+        case MP_BC_RAISE_LAST:
+            printf("RAISE_LAST");
+            break;
+
+        case MP_BC_RAISE_OBJ:
+            printf("RAISE_OBJ");
+            break;
+
+        case MP_BC_RAISE_FROM:
+            printf("RAISE_FROM");
             break;
 
         case MP_BC_YIELD_VALUE:
@@ -545,7 +539,7 @@ const byte *mp_bytecode_print_str(const byte *ip) {
                 mp_uint_t op = ip[-1] - MP_BC_BINARY_OP_MULTI;
                 printf("BINARY_OP " UINT_FMT " %s", op, qstr_str(mp_binary_op_method_name[op]));
             } else {
-                printf("code %p, byte code 0x%02x not implemented\n", ip, ip[-1]);
+                printf("code %p, byte code 0x%02x not implemented\n", ip - 1, ip[-1]);
                 assert(0);
                 return ip;
             }