Merge pull request #155 from SummerLife/update_micropython_to_1_12

[upgrade] micropython to 1.12 version

.gitignore

@@ -0,0 +1 @@
+/.vscode/settings.json

extmod/crypto-algorithms/sha256.c

@@ -1,7 +1,8 @@
 /*********************************************************************
+* Source:     https://github.com/B-Con/crypto-algorithms
 * Filename:   sha256.c
 * Author:     Brad Conte (brad AT bradconte.com)
-* Copyright:
+* Copyright:  This code is released into the public domain.
 * Disclaimer: This code is presented "as is" without any guarantees.
 * Details:    Implementation of the SHA-256 hashing algorithm.
               SHA-256 is one of the three algorithms in the SHA2

extmod/crypto-algorithms/sha256.h

@@ -1,7 +1,8 @@
 /*********************************************************************
+* Source:     https://github.com/B-Con/crypto-algorithms
 * Filename:   sha256.h
 * Author:     Brad Conte (brad AT bradconte.com)
-* Copyright:
+* Copyright:  This code is released into the public domain.
 * Disclaimer: This code is presented "as is" without any guarantees.
 * Details:    Defines the API for the corresponding SHA1 implementation.
 *********************************************************************/

extmod/machine_i2c.c

@@ -180,9 +180,9 @@ STATIC int mp_hal_i2c_read_byte(machine_i2c_obj_t *self, uint8_t *val, int nack)
 }
 
 // return value:
-//  >=0 - number of acks received
+//  >=0 - success; for read it's 0, for write it's number of acks received
 //   <0 - error, with errno being the negative of the return value
-int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uint8_t *src, size_t len, bool stop) {
+int mp_machine_soft_i2c_transfer(mp_obj_base_t *self_in, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags) {
     machine_i2c_obj_t *self = (machine_i2c_obj_t*)self_in;
 
     // start the I2C transaction
@@ -192,7 +192,7 @@ int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uin
     }
 
     // write the slave address
-    ret = mp_hal_i2c_write_byte(self, addr << 1);
+    ret = mp_hal_i2c_write_byte(self, (addr << 1) | (flags & MP_MACHINE_I2C_FLAG_READ));
     if (ret < 0) {
         return ret;
     } else if (ret != 0) {
@@ -201,69 +201,102 @@ int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uin
         return -MP_ENODEV;
     }
 
-    // write the buffer to the I2C memory
-    int num_acks = 0;
-    while (len--) {
-        ret = mp_hal_i2c_write_byte(self, *src++);
-        if (ret < 0) {
-            return ret;
-        } else if (ret != 0) {
-            // nack received, stop sending
-            break;
+    int transfer_ret = 0;
+    for (; n--; ++bufs) {
+        size_t len = bufs->len;
+        uint8_t *buf = bufs->buf;
+        if (flags & MP_MACHINE_I2C_FLAG_READ) {
+            // read bytes from the slave into the given buffer(s)
+            while (len--) {
+                ret = mp_hal_i2c_read_byte(self, buf++, (n | len) == 0);
+                if (ret != 0) {
+                    return ret;
+                }
+            }
+        } else {
+            // write bytes from the given buffer(s) to the slave
+            while (len--) {
+                ret = mp_hal_i2c_write_byte(self, *buf++);
+                if (ret < 0) {
+                    return ret;
+                } else if (ret != 0) {
+                    // nack received, stop sending
+                    n = 0;
+                    break;
+                }
+                ++transfer_ret; // count the number of acks
+            }
         }
-        ++num_acks;
     }
 
     // finish the I2C transaction
-    if (stop) {
+    if (flags & MP_MACHINE_I2C_FLAG_STOP) {
         ret = mp_hal_i2c_stop(self);
         if (ret != 0) {
             return ret;
         }
     }
 
-    return num_acks;
+    return transfer_ret;
 }
 
-// return value:
-//    0 - success
-//   <0 - error, with errno being the negative of the return value
-int mp_machine_soft_i2c_readfrom(mp_obj_base_t *self_in, uint16_t addr, uint8_t *dest, size_t len, bool stop) {
-    machine_i2c_obj_t *self = (machine_i2c_obj_t*)self_in;
-
-    // start the I2C transaction
-    int ret = mp_hal_i2c_start(self);
-    if (ret != 0) {
-        return ret;
-    }
-
-    // write the slave address
-    ret = mp_hal_i2c_write_byte(self, (addr << 1) | 1);
-    if (ret < 0) {
-        return ret;
-    } else if (ret != 0) {
-        // nack received, release the bus cleanly
-        mp_hal_i2c_stop(self);
-        return -MP_ENODEV;
-    }
-
-    // read the bytes from the slave
-    while (len--) {
-        ret = mp_hal_i2c_read_byte(self, dest++, len == 0);
-        if (ret != 0) {
-            return ret;
+/******************************************************************************/
+// Generic helper functions
+
+// For use by ports that require a single buffer of data for a read/write transfer
+int mp_machine_i2c_transfer_adaptor(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags) {
+    size_t len;
+    uint8_t *buf;
+    if (n == 1) {
+        // Use given single buffer
+        len = bufs[0].len;
+        buf = bufs[0].buf;
+    } else {
+        // Combine buffers into a single one
+        len = 0;
+        for (size_t i = 0; i < n; ++i) {
+            len += bufs[i].len;
+        }
+        buf = m_new(uint8_t, len);
+        if (!(flags & MP_MACHINE_I2C_FLAG_READ)) {
+            len = 0;
+            for (size_t i = 0; i < n; ++i) {
+                memcpy(buf + len, bufs[i].buf, bufs[i].len);
+                len += bufs[i].len;
+            }
         }
     }
 
-    // finish the I2C transaction
-    if (stop) {
-        ret = mp_hal_i2c_stop(self);
-        if (ret != 0) {
-            return ret;
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    int ret = i2c_p->transfer_single(self, addr, len, buf, flags);
+
+    if (n > 1) {
+        if (flags & MP_MACHINE_I2C_FLAG_READ) {
+            // Copy data from single buffer to individual ones
+            len = 0;
+            for (size_t i = 0; i < n; ++i) {
+                memcpy(bufs[i].buf, buf + len, bufs[i].len);
+                len += bufs[i].len;
+            }
         }
+        m_del(uint8_t, buf, len);
     }
 
-    return 0; // success
+    return ret;
+}
+
+STATIC int mp_machine_i2c_readfrom(mp_obj_base_t *self, uint16_t addr, uint8_t *dest, size_t len, bool stop) {
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    mp_machine_i2c_buf_t buf = {.len = len, .buf = dest};
+    unsigned int flags = MP_MACHINE_I2C_FLAG_READ | (stop ? MP_MACHINE_I2C_FLAG_STOP : 0);
+    return i2c_p->transfer(self, addr, 1, &buf, flags);
+}
+
+STATIC int mp_machine_i2c_writeto(mp_obj_base_t *self, uint16_t addr, const uint8_t *src, size_t len, bool stop) {
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    mp_machine_i2c_buf_t buf = {.len = len, .buf = (uint8_t*)src};
+    unsigned int flags = stop ? MP_MACHINE_I2C_FLAG_STOP : 0;
+    return i2c_p->transfer(self, addr, 1, &buf, flags);
 }
 
 /******************************************************************************/
@@ -318,11 +351,10 @@ MP_DEFINE_CONST_FUN_OBJ_KW(machine_i2c_init_obj, 1, machine_i2c_obj_init);
 
 STATIC mp_obj_t machine_i2c_scan(mp_obj_t self_in) {
     mp_obj_base_t *self = MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_obj_t list = mp_obj_new_list(0, NULL);
     // 7-bit addresses 0b0000xxx and 0b1111xxx are reserved
     for (int addr = 0x08; addr < 0x78; ++addr) {
-        int ret = i2c_p->writeto(self, addr, NULL, 0, true);
+        int ret = mp_machine_i2c_writeto(self, addr, NULL, 0, true);
         if (ret == 0) {
             mp_obj_list_append(list, MP_OBJ_NEW_SMALL_INT(addr));
         }
@@ -407,12 +439,11 @@ MP_DEFINE_CONST_FUN_OBJ_2(machine_i2c_write_obj, machine_i2c_write);
 
 STATIC mp_obj_t machine_i2c_readfrom(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     vstr_t vstr;
     vstr_init_len(&vstr, mp_obj_get_int(args[2]));
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->readfrom(self, addr, (uint8_t*)vstr.buf, vstr.len, stop);
+    int ret = mp_machine_i2c_readfrom(self, addr, (uint8_t*)vstr.buf, vstr.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -422,12 +453,11 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_readfrom_obj, 3, 4, machine_i2c_
 
 STATIC mp_obj_t machine_i2c_readfrom_into(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_WRITE);
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->readfrom(self, addr, bufinfo.buf, bufinfo.len, stop);
+    int ret = mp_machine_i2c_readfrom(self, addr, bufinfo.buf, bufinfo.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -437,12 +467,11 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_readfrom_into_obj, 3, 4, machine
 
 STATIC mp_obj_t machine_i2c_writeto(size_t n_args, const mp_obj_t *args) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     mp_int_t addr = mp_obj_get_int(args[1]);
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[2], &bufinfo, MP_BUFFER_READ);
     bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
-    int ret = i2c_p->writeto(self, addr,  bufinfo.buf, bufinfo.len, stop);
+    int ret = mp_machine_i2c_writeto(self, addr, bufinfo.buf, bufinfo.len, stop);
     if (ret < 0) {
         mp_raise_OSError(-ret);
     }
@@ -451,53 +480,87 @@ STATIC mp_obj_t machine_i2c_writeto(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_writeto_obj, 3, 4, machine_i2c_writeto);
 
+STATIC mp_obj_t machine_i2c_writevto(size_t n_args, const mp_obj_t *args) {
+    mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
+    mp_int_t addr = mp_obj_get_int(args[1]);
+
+    // Get the list of data buffer(s) to write
+    size_t nitems;
+    const mp_obj_t *items;
+    mp_obj_get_array(args[2], &nitems, (mp_obj_t**)&items);
+
+    // Get the stop argument
+    bool stop = (n_args == 3) ? true : mp_obj_is_true(args[3]);
+
+    // Extract all buffer data, skipping zero-length buffers
+    size_t alloc = nitems == 0 ? 1 : nitems;
+    size_t nbufs = 0;
+    mp_machine_i2c_buf_t *bufs = mp_local_alloc(alloc * sizeof(mp_machine_i2c_buf_t));
+    for (; nitems--; ++items) {
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(*items, &bufinfo, MP_BUFFER_READ);
+        if (bufinfo.len > 0) {
+            bufs[nbufs].len = bufinfo.len;
+            bufs[nbufs++].buf = bufinfo.buf;
+        }
+    }
+
+    // Make sure there is at least one buffer, empty if needed
+    if (nbufs == 0) {
+        bufs[0].len = 0;
+        bufs[0].buf = NULL;
+        nbufs = 1;
+    }
+
+    // Do the I2C transfer
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    int ret = i2c_p->transfer(self, addr, nbufs, bufs, stop ? MP_MACHINE_I2C_FLAG_STOP : 0);
+    mp_local_free(bufs);
+
+    if (ret < 0) {
+        mp_raise_OSError(-ret);
+    }
+
+    // Return number of acks received
+    return MP_OBJ_NEW_SMALL_INT(ret);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(machine_i2c_writevto_obj, 3, 4, machine_i2c_writevto);
+
 STATIC int read_mem(mp_obj_t self_in, uint16_t addr, uint32_t memaddr, uint8_t addrsize, uint8_t *buf, size_t len) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
     uint8_t memaddr_buf[4];
     size_t memaddr_len = 0;
     for (int16_t i = addrsize - 8; i >= 0; i -= 8) {
         memaddr_buf[memaddr_len++] = memaddr >> i;
     }
-    int ret = i2c_p->writeto(self, addr, memaddr_buf, memaddr_len, false);
+    int ret = mp_machine_i2c_writeto(self, addr, memaddr_buf, memaddr_len, false);
     if (ret != memaddr_len) {
         // must generate STOP
-        i2c_p->writeto(self, addr, NULL, 0, true);
+        mp_machine_i2c_writeto(self, addr, NULL, 0, true);
         return ret;
     }
-    return i2c_p->readfrom(self, addr, buf, len, true);
+    return mp_machine_i2c_readfrom(self, addr, buf, len, true);
 }
 
-#define MAX_MEMADDR_SIZE (4)
-#define BUF_STACK_SIZE (12)
-
 STATIC int write_mem(mp_obj_t self_in, uint16_t addr, uint32_t memaddr, uint8_t addrsize, const uint8_t *buf, size_t len) {
     mp_obj_base_t *self = (mp_obj_base_t*)MP_OBJ_TO_PTR(self_in);
-    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
 
-    // need some memory to create the buffer to send; try to use stack if possible
-    uint8_t buf2_stack[MAX_MEMADDR_SIZE + BUF_STACK_SIZE];
-    uint8_t *buf2;
-    size_t buf2_alloc = 0;
-    if (len <= BUF_STACK_SIZE) {
-        buf2 = buf2_stack;
-    } else {
-        buf2_alloc = MAX_MEMADDR_SIZE + len;
-        buf2 = m_new(uint8_t, buf2_alloc);
-    }
-
-    // create the buffer to send
+    // Create buffer with memory address
     size_t memaddr_len = 0;
+    uint8_t memaddr_buf[4];
     for (int16_t i = addrsize - 8; i >= 0; i -= 8) {
-        buf2[memaddr_len++] = memaddr >> i;
+        memaddr_buf[memaddr_len++] = memaddr >> i;
     }
-    memcpy(buf2 + memaddr_len, buf, len);
 
-    int ret = i2c_p->writeto(self, addr, buf2, memaddr_len + len, true);
-    if (buf2_alloc != 0) {
-        m_del(uint8_t, buf2, buf2_alloc);
-    }
-    return ret;
+    // Create partial write buffers
+    mp_machine_i2c_buf_t bufs[2] = {
+        {.len = memaddr_len, .buf = memaddr_buf},
+        {.len = len, .buf = (uint8_t*)buf},
+    };
+
+    // Do I2C transfer
+    mp_machine_i2c_p_t *i2c_p = (mp_machine_i2c_p_t*)self->type->protocol;
+    return i2c_p->transfer(self, addr, 2, bufs, MP_MACHINE_I2C_FLAG_STOP);
 }
 
 STATIC const mp_arg_t machine_i2c_mem_allowed_args[] = {
@@ -584,6 +647,7 @@ STATIC const mp_rom_map_elem_t machine_i2c_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_readfrom), MP_ROM_PTR(&machine_i2c_readfrom_obj) },
     { MP_ROM_QSTR(MP_QSTR_readfrom_into), MP_ROM_PTR(&machine_i2c_readfrom_into_obj) },
     { MP_ROM_QSTR(MP_QSTR_writeto), MP_ROM_PTR(&machine_i2c_writeto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_writevto), MP_ROM_PTR(&machine_i2c_writevto_obj) },
 
     // memory operations
     { MP_ROM_QSTR(MP_QSTR_readfrom_mem), MP_ROM_PTR(&machine_i2c_readfrom_mem_obj) },
@@ -625,8 +689,7 @@ STATIC const mp_machine_i2c_p_t mp_machine_soft_i2c_p = {
     .stop = (int(*)(mp_obj_base_t*))mp_hal_i2c_stop,
     .read = mp_machine_soft_i2c_read,
     .write = mp_machine_soft_i2c_write,
-    .readfrom = mp_machine_soft_i2c_readfrom,
-    .writeto = mp_machine_soft_i2c_writeto,
+    .transfer = mp_machine_soft_i2c_transfer,
 };
 
 const mp_obj_type_t machine_i2c_type = {

extmod/machine_i2c.h

@@ -28,15 +28,24 @@
 
 #include "py/obj.h"
 
+#define MP_MACHINE_I2C_FLAG_READ (0x01) // if not set then it's a write
+#define MP_MACHINE_I2C_FLAG_STOP (0x02)
+
+typedef struct _mp_machine_i2c_buf_t {
+    size_t len;
+    uint8_t *buf;
+} mp_machine_i2c_buf_t;
+
 // I2C protocol
 // the first 4 methods can be NULL, meaning operation is not supported
+// transfer_single only needs to be set if transfer=mp_machine_i2c_transfer_adaptor
 typedef struct _mp_machine_i2c_p_t {
     int (*start)(mp_obj_base_t *obj);
     int (*stop)(mp_obj_base_t *obj);
     int (*read)(mp_obj_base_t *obj, uint8_t *dest, size_t len, bool nack);
     int (*write)(mp_obj_base_t *obj, const uint8_t *src, size_t len);
-    int (*readfrom)(mp_obj_base_t *obj, uint16_t addr, uint8_t *dest, size_t len, bool stop);
-    int (*writeto)(mp_obj_base_t *obj, uint16_t addr, const uint8_t *src, size_t len, bool stop);
+    int (*transfer)(mp_obj_base_t *obj, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+    int (*transfer_single)(mp_obj_base_t *obj, uint16_t addr, size_t len, uint8_t *buf, unsigned int flags);
 } mp_machine_i2c_p_t;
 
 typedef struct _mp_machine_soft_i2c_obj_t {
@@ -50,7 +59,7 @@ typedef struct _mp_machine_soft_i2c_obj_t {
 extern const mp_obj_type_t machine_i2c_type;
 extern const mp_obj_dict_t mp_machine_soft_i2c_locals_dict;
 
-int mp_machine_soft_i2c_readfrom(mp_obj_base_t *self_in, uint16_t addr, uint8_t *dest, size_t len, bool stop);
-int mp_machine_soft_i2c_writeto(mp_obj_base_t *self_in, uint16_t addr, const uint8_t *src, size_t len, bool stop);
+int mp_machine_i2c_transfer_adaptor(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
+int mp_machine_soft_i2c_transfer(mp_obj_base_t *self, uint16_t addr, size_t n, mp_machine_i2c_buf_t *bufs, unsigned int flags);
 
 #endif // MICROPY_INCLUDED_EXTMOD_MACHINE_I2C_H

extmod/machine_pulse.c

@@ -30,7 +30,7 @@
 
 #if MICROPY_PY_MACHINE_PULSE
 
-mp_uint_t machine_time_pulse_us(mp_hal_pin_obj_t pin, int pulse_level, mp_uint_t timeout_us) {
+MP_WEAK mp_uint_t machine_time_pulse_us(mp_hal_pin_obj_t pin, int pulse_level, mp_uint_t timeout_us) {
     mp_uint_t start = mp_hal_ticks_us();
     while (mp_hal_pin_read(pin) != pulse_level) {
         if ((mp_uint_t)(mp_hal_ticks_us() - start) >= timeout_us) {

extmod/machine_signal.c

@@ -43,13 +43,13 @@ typedef struct _machine_signal_t {
 } machine_signal_t;
 
 STATIC mp_obj_t signal_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
-    mp_obj_t pin = args[0];
+    mp_obj_t pin;
     bool invert = false;
 
     #if defined(MICROPY_PY_MACHINE_PIN_MAKE_NEW)
     mp_pin_p_t *pin_p = NULL;
 
-    if (mp_obj_is_obj(pin)) {
+    if (n_args > 0 && mp_obj_is_obj(args[0])) {
         mp_obj_base_t *pin_base = (mp_obj_base_t*)MP_OBJ_TO_PTR(args[0]);
         pin_p = (mp_pin_p_t*)pin_base->type->protocol;
     }
@@ -96,6 +96,7 @@ STATIC mp_obj_t signal_make_new(const mp_obj_type_t *type, size_t n_args, size_t
     // Otherwise there should be 1 or 2 args
     {
         if (n_args == 1) {
+            pin = args[0];
             if (n_kw == 0) {
             } else if (n_kw == 1 && args[1] == MP_OBJ_NEW_QSTR(MP_QSTR_invert)) {
                 invert = mp_obj_is_true(args[2]);

extmod/misc.h

@@ -35,6 +35,8 @@
 MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_uos_dupterm_obj);
 
 #if MICROPY_PY_OS_DUPTERM
+bool mp_uos_dupterm_is_builtin_stream(mp_const_obj_t stream);
+uintptr_t mp_uos_dupterm_poll(uintptr_t poll_flags);
 int mp_uos_dupterm_rx_chr(void);
 void mp_uos_dupterm_tx_strn(const char *str, size_t len);
 void mp_uos_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc);

extmod/modbtree.c

@@ -52,7 +52,9 @@ typedef struct _mp_obj_btree_t {
     byte next_flags;
 } mp_obj_btree_t;
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_obj_type_t btree_type;
+#endif
 
 #define CHECK_ERROR(res) \
         if (res == RET_ERROR) { \
@@ -60,7 +62,7 @@ STATIC const mp_obj_type_t btree_type;
         }
 
 void __dbpanic(DB *db) {
-    printf("__dbpanic(%p)\n", db);
+    mp_printf(&mp_plat_print, "__dbpanic(%p)\n", db);
 }
 
 STATIC mp_obj_btree_t *btree_new(DB *db) {
@@ -295,6 +297,7 @@ STATIC mp_obj_t btree_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs
     }
 }
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t btree_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&btree_close_obj) },
     { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&btree_flush_obj) },
@@ -319,14 +322,16 @@ STATIC const mp_obj_type_t btree_type = {
     .subscr = btree_subscr,
     .locals_dict = (void*)&btree_locals_dict,
 };
+#endif
 
-STATIC FILEVTABLE btree_stream_fvtable = {
+STATIC const FILEVTABLE btree_stream_fvtable = {
     mp_stream_posix_read,
     mp_stream_posix_write,
     mp_stream_posix_lseek,
     mp_stream_posix_fsync
 };
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC mp_obj_t mod_btree_open(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     static const mp_arg_t allowed_args[] = {
         { MP_QSTR_flags, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} },
@@ -373,5 +378,6 @@ const mp_obj_module_t mp_module_btree = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_btree_globals,
 };
+#endif
 
 #endif // MICROPY_PY_BTREE

extmod/modframebuf.c

@@ -580,6 +580,7 @@ STATIC mp_obj_t framebuf_text(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(framebuf_text_obj, 4, 5, framebuf_text);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t framebuf_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_fill), MP_ROM_PTR(&framebuf_fill_obj) },
     { MP_ROM_QSTR(MP_QSTR_fill_rect), MP_ROM_PTR(&framebuf_fill_rect_obj) },
@@ -601,6 +602,7 @@ STATIC const mp_obj_type_t mp_type_framebuf = {
     .buffer_p = { .get_buffer = framebuf_get_buffer },
     .locals_dict = (mp_obj_dict_t*)&framebuf_locals_dict,
 };
+#endif
 
 // this factory function is provided for backwards compatibility with old FrameBuffer1 class
 STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) {
@@ -624,6 +626,7 @@ STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) {
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(legacy_framebuffer1_obj, 3, 4, legacy_framebuffer1);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t framebuf_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_framebuf) },
     { MP_ROM_QSTR(MP_QSTR_FrameBuffer), MP_ROM_PTR(&mp_type_framebuf) },
@@ -644,5 +647,6 @@ const mp_obj_module_t mp_module_framebuf = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&framebuf_module_globals,
 };
+#endif
 
 #endif // MICROPY_PY_FRAMEBUF

extmod/moducryptolib.c

@@ -41,10 +41,17 @@
 
 // values follow PEP 272
 enum {
-    UCRYPTOLIB_MODE_MIN = 0,
-    UCRYPTOLIB_MODE_ECB,
-    UCRYPTOLIB_MODE_CBC,
-    UCRYPTOLIB_MODE_MAX,
+    UCRYPTOLIB_MODE_ECB = 1,
+    UCRYPTOLIB_MODE_CBC = 2,
+    UCRYPTOLIB_MODE_CTR = 6,
+};
+
+struct ctr_params {
+    // counter is the IV of the AES context.
+
+    size_t offset; // in encrypted_counter
+    // encrypted counter
+    uint8_t encrypted_counter[16];
 };
 
 #if MICROPY_SSL_AXTLS
@@ -82,6 +89,19 @@ typedef struct _mp_obj_aes_t {
     uint8_t key_type: 2;
 } mp_obj_aes_t;
 
+static inline bool is_ctr_mode(int block_mode) {
+    #if MICROPY_PY_UCRYPTOLIB_CTR
+    return block_mode == UCRYPTOLIB_MODE_CTR;
+    #else
+    return false;
+    #endif
+}
+
+static inline struct ctr_params *ctr_params_from_aes(mp_obj_aes_t *o) {
+    // ctr_params follows aes object struct
+    return (struct ctr_params*)&o[1];
+}
+
 #if MICROPY_SSL_AXTLS
 STATIC void aes_initial_set_key_impl(AES_CTX_IMPL *ctx, const uint8_t *key, size_t keysize, const uint8_t iv[16]) {
     assert(16 == keysize || 32 == keysize);
@@ -119,6 +139,33 @@ STATIC void aes_process_cbc_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *
         AES_cbc_decrypt(ctx, in, out, in_len);
     }
 }
+
+#if MICROPY_PY_UCRYPTOLIB_CTR
+// axTLS doesn't have CTR support out of the box. This implements the counter part using the ECB primitive.
+STATIC void aes_process_ctr_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, struct ctr_params *ctr_params) {
+    size_t n = ctr_params->offset;
+    uint8_t *const counter = ctx->iv;
+
+    while (in_len--) {
+        if (n == 0) {
+            aes_process_ecb_impl(ctx, counter, ctr_params->encrypted_counter, true);
+
+            // increment the 128-bit counter
+            for (int i = 15; i >= 0; --i) {
+                if (++counter[i] != 0) {
+                    break;
+                }
+            }
+        }
+
+        *out++ = *in++ ^ ctr_params->encrypted_counter[n];
+        n = (n + 1) & 0xf;
+    }
+
+    ctr_params->offset = n;
+}
+#endif
+
 #endif
 
 #if MICROPY_SSL_MBEDTLS
@@ -155,20 +202,38 @@ STATIC void aes_process_ecb_impl(AES_CTX_IMPL *ctx, const uint8_t in[16], uint8_
 STATIC void aes_process_cbc_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, bool encrypt) {
     mbedtls_aes_crypt_cbc(&ctx->u.mbedtls_ctx, encrypt ? MBEDTLS_AES_ENCRYPT : MBEDTLS_AES_DECRYPT, in_len, ctx->iv, in, out);
 }
+
+#if MICROPY_PY_UCRYPTOLIB_CTR
+STATIC void aes_process_ctr_impl(AES_CTX_IMPL *ctx, const uint8_t *in, uint8_t *out, size_t in_len, struct ctr_params *ctr_params) {
+    mbedtls_aes_crypt_ctr(&ctx->u.mbedtls_ctx, in_len, &ctr_params->offset, ctx->iv, ctr_params->encrypted_counter, in, out);
+}
+#endif
+
 #endif
 
 STATIC mp_obj_t ucryptolib_aes_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_arg_check_num(n_args, n_kw, 2, 3, false);
-    mp_obj_aes_t *o = m_new_obj(mp_obj_aes_t);
-    o->base.type = type;
 
-    o->block_mode = mp_obj_get_int(args[1]);
-    o->key_type = AES_KEYTYPE_NONE;
+    const mp_int_t block_mode = mp_obj_get_int(args[1]);
+
+    switch (block_mode) {
+        case UCRYPTOLIB_MODE_ECB:
+        case UCRYPTOLIB_MODE_CBC:
+        #if MICROPY_PY_UCRYPTOLIB_CTR
+        case UCRYPTOLIB_MODE_CTR:
+        #endif
+            break;
 
-    if (o->block_mode <= UCRYPTOLIB_MODE_MIN || o->block_mode >= UCRYPTOLIB_MODE_MAX) {
-        mp_raise_ValueError("mode");
+        default:
+            mp_raise_ValueError("mode");
     }
 
+    mp_obj_aes_t *o = m_new_obj_var(mp_obj_aes_t, struct ctr_params, !!is_ctr_mode(block_mode));
+    o->base.type = type;
+
+    o->block_mode = block_mode;
+    o->key_type = AES_KEYTYPE_NONE;
+
     mp_buffer_info_t keyinfo;
     mp_get_buffer_raise(args[0], &keyinfo, MP_BUFFER_READ);
     if (32 != keyinfo.len && 16 != keyinfo.len) {
@@ -183,10 +248,14 @@ STATIC mp_obj_t ucryptolib_aes_make_new(const mp_obj_type_t *type, size_t n_args
         if (16 != ivinfo.len) {
             mp_raise_ValueError("IV");
         }
-    } else if (o->block_mode == UCRYPTOLIB_MODE_CBC) {
+    } else if (o->block_mode == UCRYPTOLIB_MODE_CBC || is_ctr_mode(o->block_mode)) {
         mp_raise_ValueError("IV");
     }
 
+    if (is_ctr_mode(block_mode)) {
+        ctr_params_from_aes(o)->offset = 0;
+    }
+
     aes_initial_set_key_impl(&o->ctx, keyinfo.buf, keyinfo.len, ivinfo.buf);
 
     return MP_OBJ_FROM_PTR(o);
@@ -204,7 +273,7 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
     mp_buffer_info_t in_bufinfo;
     mp_get_buffer_raise(in_buf, &in_bufinfo, MP_BUFFER_READ);
 
-    if (in_bufinfo.len % 16 != 0) {
+    if (!is_ctr_mode(self->block_mode) && in_bufinfo.len % 16 != 0) {
         mp_raise_ValueError("blksize % 16");
     }
 
@@ -224,7 +293,9 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
     }
 
     if (AES_KEYTYPE_NONE == self->key_type) {
-        aes_final_set_key_impl(&self->ctx, encrypt);
+        // always set key for encryption if CTR mode.
+        const bool encrypt_mode = encrypt || is_ctr_mode(self->block_mode);
+        aes_final_set_key_impl(&self->ctx, encrypt_mode);
         self->key_type = encrypt ? AES_KEYTYPE_ENC : AES_KEYTYPE_DEC;
     } else {
         if ((encrypt && self->key_type == AES_KEYTYPE_DEC) ||
@@ -234,14 +305,26 @@ STATIC mp_obj_t aes_process(size_t n_args, const mp_obj_t *args, bool encrypt) {
         }
     }
 
-    if (self->block_mode == UCRYPTOLIB_MODE_ECB) {
-        uint8_t *in = in_bufinfo.buf, *out = out_buf_ptr;
-        uint8_t *top = in + in_bufinfo.len;
-        for (; in < top; in += 16, out += 16) {
-            aes_process_ecb_impl(&self->ctx, in, out, encrypt);
+    switch (self->block_mode) {
+        case UCRYPTOLIB_MODE_ECB: {
+            uint8_t *in = in_bufinfo.buf, *out = out_buf_ptr;
+            uint8_t *top = in + in_bufinfo.len;
+            for (; in < top; in += 16, out += 16) {
+                aes_process_ecb_impl(&self->ctx, in, out, encrypt);
+            }
+            break;
         }
-    } else {
-        aes_process_cbc_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len, encrypt);
+
+        case UCRYPTOLIB_MODE_CBC:
+            aes_process_cbc_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len, encrypt);
+            break;
+
+        #if MICROPY_PY_UCRYPTOLIB_CTR
+        case UCRYPTOLIB_MODE_CTR:
+            aes_process_ctr_impl(&self->ctx, in_bufinfo.buf, out_buf_ptr, in_bufinfo.len,
+                ctr_params_from_aes(self));
+            break;
+        #endif
     }
 
     if (out_buf != MP_OBJ_NULL) {
@@ -279,6 +362,9 @@ STATIC const mp_rom_map_elem_t mp_module_ucryptolib_globals_table[] = {
 #if MICROPY_PY_UCRYPTOLIB_CONSTS
     { MP_ROM_QSTR(MP_QSTR_MODE_ECB), MP_ROM_INT(UCRYPTOLIB_MODE_ECB) },
     { MP_ROM_QSTR(MP_QSTR_MODE_CBC), MP_ROM_INT(UCRYPTOLIB_MODE_CBC) },
+    #if MICROPY_PY_UCRYPTOLIB_CTR
+    { MP_ROM_QSTR(MP_QSTR_MODE_CTR), MP_ROM_INT(UCRYPTOLIB_MODE_CTR) },
+    #endif
 #endif
 };
 

extmod/moductypes.c

@@ -299,13 +299,13 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(uctypes_struct_sizeof_obj, 1, 2, ucty
 static inline mp_obj_t get_unaligned(uint val_type, byte *p, int big_endian) {
     char struct_type = big_endian ? '>' : '<';
     static const char type2char[16] = "BbHhIiQq------fd";
-    return mp_binary_get_val(struct_type, type2char[val_type], &p);
+    return mp_binary_get_val(struct_type, type2char[val_type], p, &p);
 }
 
 static inline void set_unaligned(uint val_type, byte *p, int big_endian, mp_obj_t val) {
     char struct_type = big_endian ? '>' : '<';
     static const char type2char[16] = "BbHhIiQq------fd";
-    mp_binary_set_val(struct_type, type2char[val_type], val, &p);
+    mp_binary_set_val(struct_type, type2char[val_type], val, p, &p);
 }
 
 static inline mp_uint_t get_aligned_basic(uint val_type, void *p) {
@@ -558,7 +558,7 @@ STATIC mp_obj_t uctypes_struct_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_ob
             uint val_type = GET_TYPE(arr_sz, VAL_TYPE_BITS);
             arr_sz &= VALUE_MASK(VAL_TYPE_BITS);
             if (index >= arr_sz) {
-                nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "struct: index out of range"));
+                mp_raise_msg(&mp_type_IndexError, "struct: index out of range");
             }
 
             if (t->len == 2) {

extmod/moduheapq.c

@@ -31,14 +31,14 @@
 
 // the algorithm here is modelled on CPython's heapq.py
 
-STATIC mp_obj_list_t *get_heap(mp_obj_t heap_in) {
+STATIC mp_obj_list_t *uheapq_get_heap(mp_obj_t heap_in) {
     if (!mp_obj_is_type(heap_in, &mp_type_list)) {
         mp_raise_TypeError("heap must be a list");
     }
     return MP_OBJ_TO_PTR(heap_in);
 }
 
-STATIC void heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_uint_t pos) {
+STATIC void uheapq_heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_uint_t pos) {
     mp_obj_t item = heap->items[pos];
     while (pos > start_pos) {
         mp_uint_t parent_pos = (pos - 1) >> 1;
@@ -53,7 +53,7 @@ STATIC void heap_siftdown(mp_obj_list_t *heap, mp_uint_t start_pos, mp_uint_t po
     heap->items[pos] = item;
 }
 
-STATIC void heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) {
+STATIC void uheapq_heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) {
     mp_uint_t start_pos = pos;
     mp_uint_t end_pos = heap->len;
     mp_obj_t item = heap->items[pos];
@@ -67,42 +67,43 @@ STATIC void heap_siftup(mp_obj_list_t *heap, mp_uint_t pos) {
         pos = child_pos;
     }
     heap->items[pos] = item;
-    heap_siftdown(heap, start_pos, pos);
+    uheapq_heap_siftdown(heap, start_pos, pos);
 }
 
 STATIC mp_obj_t mod_uheapq_heappush(mp_obj_t heap_in, mp_obj_t item) {
-    mp_obj_list_t *heap = get_heap(heap_in);
+    mp_obj_list_t *heap = uheapq_get_heap(heap_in);
     mp_obj_list_append(heap_in, item);
-    heap_siftdown(heap, 0, heap->len - 1);
+    uheapq_heap_siftdown(heap, 0, heap->len - 1);
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_uheapq_heappush_obj, mod_uheapq_heappush);
 
 STATIC mp_obj_t mod_uheapq_heappop(mp_obj_t heap_in) {
-    mp_obj_list_t *heap = get_heap(heap_in);
+    mp_obj_list_t *heap = uheapq_get_heap(heap_in);
     if (heap->len == 0) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "empty heap"));
+        mp_raise_msg(&mp_type_IndexError, "empty heap");
     }
     mp_obj_t item = heap->items[0];
     heap->len -= 1;
     heap->items[0] = heap->items[heap->len];
     heap->items[heap->len] = MP_OBJ_NULL; // so we don't retain a pointer
     if (heap->len) {
-        heap_siftup(heap, 0);
+        uheapq_heap_siftup(heap, 0);
     }
     return item;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_uheapq_heappop_obj, mod_uheapq_heappop);
 
 STATIC mp_obj_t mod_uheapq_heapify(mp_obj_t heap_in) {
-    mp_obj_list_t *heap = get_heap(heap_in);
+    mp_obj_list_t *heap = uheapq_get_heap(heap_in);
     for (mp_uint_t i = heap->len / 2; i > 0;) {
-        heap_siftup(heap, --i);
+        uheapq_heap_siftup(heap, --i);
     }
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_uheapq_heapify_obj, mod_uheapq_heapify);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_uheapq_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uheapq) },
     { MP_ROM_QSTR(MP_QSTR_heappush), MP_ROM_PTR(&mod_uheapq_heappush_obj) },
@@ -116,5 +117,6 @@ const mp_obj_module_t mp_module_uheapq = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_uheapq_globals,
 };
+#endif
 
 #endif //MICROPY_PY_UHEAPQ

extmod/modujson.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2014-2016 Damien P. George
+ * Copyright (c) 2014-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -185,7 +185,7 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) {
                     cur = S_CUR(s);
                     if (cur == '.' || cur == 'E' || cur == 'e') {
                         flt = true;
-                    } else if (cur == '-' || unichar_isdigit(cur)) {
+                    } else if (cur == '+' || cur == '-' || unichar_isdigit(cur)) {
                         // pass
                     } else {
                         break;
@@ -281,9 +281,9 @@ STATIC mp_obj_t mod_ujson_load(mp_obj_t stream_obj) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_ujson_load_obj, mod_ujson_load);
 
 STATIC mp_obj_t mod_ujson_loads(mp_obj_t obj) {
-    size_t len;
-    const char *buf = mp_obj_str_get_data(obj, &len);
-    vstr_t vstr = {len, len, (char*)buf, true};
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ);
+    vstr_t vstr = {bufinfo.len, bufinfo.len, (char*)bufinfo.buf, true};
     mp_obj_stringio_t sio = {{&mp_type_stringio}, &vstr, 0, MP_OBJ_NULL};
     return mod_ujson_load(MP_OBJ_FROM_PTR(&sio));
 }

extmod/modurandom.c

@@ -36,8 +36,10 @@
 // http://www.literatecode.com/yasmarang
 // Public Domain
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC uint32_t yasmarang_pad = 0xeda4baba, yasmarang_n = 69, yasmarang_d = 233;
 STATIC uint8_t yasmarang_dat = 0;
+#endif
 
 STATIC uint32_t yasmarang(void)
 {
@@ -200,8 +202,20 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_urandom_uniform_obj, mod_urandom_uniform);
 
 #endif // MICROPY_PY_URANDOM_EXTRA_FUNCS
 
+#ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC
+STATIC mp_obj_t mod_urandom___init__() {
+    mod_urandom_seed(MP_OBJ_NEW_SMALL_INT(MICROPY_PY_URANDOM_SEED_INIT_FUNC));
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(mod_urandom___init___obj, mod_urandom___init__);
+#endif
+
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_urandom_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_urandom) },
+    #ifdef MICROPY_PY_URANDOM_SEED_INIT_FUNC
+    { MP_ROM_QSTR(MP_QSTR___init__), MP_ROM_PTR(&mod_urandom___init___obj) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_getrandbits), MP_ROM_PTR(&mod_urandom_getrandbits_obj) },
     { MP_ROM_QSTR(MP_QSTR_seed), MP_ROM_PTR(&mod_urandom_seed_obj) },
     #if MICROPY_PY_URANDOM_EXTRA_FUNCS
@@ -221,5 +235,6 @@ const mp_obj_module_t mp_module_urandom = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_urandom_globals,
 };
+#endif
 
 #endif //MICROPY_PY_URANDOM

extmod/modure.c

@@ -144,6 +144,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(match_end_obj, 1, 2, match_end);
 
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t match_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_group), MP_ROM_PTR(&match_group_obj) },
     #if MICROPY_PY_URE_MATCH_GROUPS
@@ -164,6 +165,7 @@ STATIC const mp_obj_type_t match_type = {
     .print = match_print,
     .locals_dict = (void*)&match_locals_dict,
 };
+#endif
 
 STATIC void re_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
@@ -363,6 +365,7 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(re_sub_obj, 3, 5, re_sub);
 
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t re_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_match), MP_ROM_PTR(&re_match_obj) },
     { MP_ROM_QSTR(MP_QSTR_search), MP_ROM_PTR(&re_search_obj) },
@@ -380,8 +383,10 @@ STATIC const mp_obj_type_t re_type = {
     .print = re_print,
     .locals_dict = (void*)&re_locals_dict,
 };
+#endif
 
 STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) {
+    (void)n_args;
     const char *re_str = mp_obj_str_get_str(args[0]);
     int size = re1_5_sizecode(re_str);
     if (size == -1) {
@@ -389,18 +394,22 @@ STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) {
     }
     mp_obj_re_t *o = m_new_obj_var(mp_obj_re_t, char, size);
     o->base.type = &re_type;
+    #if MICROPY_PY_URE_DEBUG
     int flags = 0;
     if (n_args > 1) {
         flags = mp_obj_get_int(args[1]);
     }
+    #endif
     int error = re1_5_compilecode(&o->re, re_str);
     if (error != 0) {
 error:
         mp_raise_ValueError("Error in regex");
     }
+    #if MICROPY_PY_URE_DEBUG
     if (flags & FLAG_DEBUG) {
         re1_5_dumpcode(&o->re);
     }
+    #endif
     return MP_OBJ_FROM_PTR(o);
 }
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_compile_obj, 1, 2, mod_re_compile);
@@ -432,6 +441,7 @@ STATIC mp_obj_t mod_re_sub(size_t n_args, const mp_obj_t *args) {
 MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_re_sub_obj, 3, 5, mod_re_sub);
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ure) },
     { MP_ROM_QSTR(MP_QSTR_compile), MP_ROM_PTR(&mod_re_compile_obj) },
@@ -440,7 +450,9 @@ STATIC const mp_rom_map_elem_t mp_module_re_globals_table[] = {
     #if MICROPY_PY_URE_SUB
     { MP_ROM_QSTR(MP_QSTR_sub), MP_ROM_PTR(&mod_re_sub_obj) },
     #endif
+    #if MICROPY_PY_URE_DEBUG
     { MP_ROM_QSTR(MP_QSTR_DEBUG), MP_ROM_INT(FLAG_DEBUG) },
+    #endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_re_globals, mp_module_re_globals_table);
@@ -449,13 +461,16 @@ const mp_obj_module_t mp_module_ure = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_re_globals,
 };
+#endif
 
 // Source files #include'd here to make sure they're compiled in
 // only if module is enabled by config setting.
 
 #define re1_5_fatal(x) assert(!x)
 #include "re1.5/compilecode.c"
+#if MICROPY_PY_URE_DEBUG
 #include "re1.5/dumpcode.c"
+#endif
 #include "re1.5/recursiveloop.c"
 #include "re1.5/charclass.c"
 

extmod/moduselect.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2015-2017 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -111,7 +112,7 @@ STATIC mp_uint_t poll_map_poll(mp_map_t *poll_map, size_t *rwx_num) {
 }
 
 /// \function select(rlist, wlist, xlist[, timeout])
-STATIC mp_obj_t select_select(uint n_args, const mp_obj_t *args) {
+STATIC mp_obj_t select_select(size_t n_args, const mp_obj_t *args) {
     // get array data from tuple/list arguments
     size_t rwx_len[3];
     mp_obj_t *r_array, *w_array, *x_array;
@@ -190,7 +191,7 @@ typedef struct _mp_obj_poll_t {
 } mp_obj_poll_t;
 
 /// \method register(obj[, eventmask])
-STATIC mp_obj_t poll_register(uint n_args, const mp_obj_t *args) {
+STATIC mp_obj_t poll_register(size_t n_args, const mp_obj_t *args) {
     mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]);
     mp_uint_t flags;
     if (n_args == 3) {

extmod/modussl_axtls.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2015-2017 Paul Sokolovsky
+ * Copyright (c) 2015-2019 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -41,6 +41,7 @@ typedef struct _mp_obj_ssl_socket_t {
     SSL *ssl_sock;
     byte *buf;
     uint32_t bytes_left;
+    bool blocking;
 } mp_obj_ssl_socket_t;
 
 struct ssl_args {
@@ -48,11 +49,12 @@ struct ssl_args {
     mp_arg_val_t cert;
     mp_arg_val_t server_side;
     mp_arg_val_t server_hostname;
+    mp_arg_val_t do_handshake;
 };
 
 STATIC const mp_obj_type_t ussl_socket_type;
 
-STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
+STATIC mp_obj_ssl_socket_t *ussl_socket_new(mp_obj_t sock, struct ssl_args *args) {
 #if MICROPY_PY_USSL_FINALISER
     mp_obj_ssl_socket_t *o = m_new_obj_with_finaliser(mp_obj_ssl_socket_t);
 #else
@@ -62,8 +64,12 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
     o->buf = NULL;
     o->bytes_left = 0;
     o->sock = sock;
+    o->blocking = true;
 
     uint32_t options = SSL_SERVER_VERIFY_LATER;
+    if (!args->do_handshake.u_bool) {
+        options |= SSL_CONNECT_IN_PARTS;
+    }
     if (args->key.u_obj != mp_const_none) {
         options |= SSL_NO_DEFAULT_KEY;
     }
@@ -97,17 +103,14 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
 
         o->ssl_sock = ssl_client_new(o->ssl_ctx, (long)sock, NULL, 0, ext);
 
-        int res = ssl_handshake_status(o->ssl_sock);
-        // Pointer to SSL_EXTENSIONS as being passed to ssl_client_new()
-        // is saved in ssl_sock->extensions.
-        // As of axTLS 2.1.3, extensions aren't used beyond the initial
-        // handshake, and that's pretty much how it's expected to be. So
-        // we allocate them on stack and reset the pointer after handshake.
+        if (args->do_handshake.u_bool) {
+            int res = ssl_handshake_status(o->ssl_sock);
 
-        if (res != SSL_OK) {
-            printf("ssl_handshake_status: %d\n", res);
-            ssl_display_error(res);
-            mp_raise_OSError(MP_EIO);
+            if (res != SSL_OK) {
+                printf("ssl_handshake_status: %d\n", res);
+                ssl_display_error(res);
+                mp_raise_OSError(MP_EIO);
+            }
         }
 
     }
@@ -115,13 +118,13 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
     return o;
 }
 
-STATIC void socket_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
+STATIC void ussl_socket_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) {
     (void)kind;
     mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(self_in);
     mp_printf(print, "<_SSLSocket %p>", self->ssl_sock);
 }
 
-STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
+STATIC mp_uint_t ussl_socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
     mp_obj_ssl_socket_t *o = MP_OBJ_TO_PTR(o_in);
 
     if (o->ssl_sock == NULL) {
@@ -133,8 +136,18 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
         mp_int_t r = ssl_read(o->ssl_sock, &o->buf);
         if (r == SSL_OK) {
             // SSL_OK from ssl_read() means "everything is ok, but there's
-            // no user data yet". So, we just keep reading.
-            continue;
+            // no user data yet". It may happen e.g. if handshake is not
+            // finished yet. The best way we can treat it is by returning
+            // EAGAIN. This may be a bit unexpected in blocking mode, but
+            // default is to perform complete handshake in constructor, so
+            // this should not happen in blocking mode. On the other hand,
+            // in nonblocking mode EAGAIN (comparing to the alternative of
+            // looping) is really preferrable.
+            if (o->blocking) {
+                continue;
+            } else {
+                goto eagain;
+            }
         }
         if (r < 0) {
             if (r == SSL_CLOSE_NOTIFY || r == SSL_ERROR_CONN_LOST) {
@@ -142,6 +155,7 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
                 return 0;
             }
             if (r == SSL_EAGAIN) {
+eagain:
                 r = MP_EAGAIN;
             }
             *errcode = r;
@@ -159,7 +173,7 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
     return size;
 }
 
-STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) {
+STATIC mp_uint_t ussl_socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, int *errcode) {
     mp_obj_ssl_socket_t *o = MP_OBJ_TO_PTR(o_in);
 
     if (o->ssl_sock == NULL) {
@@ -175,7 +189,7 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in
     return r;
 }
 
-STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+STATIC mp_uint_t ussl_socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
     mp_obj_ssl_socket_t *self = MP_OBJ_TO_PTR(o_in);
     if (request == MP_STREAM_CLOSE && self->ssl_sock != NULL) {
         ssl_free(self->ssl_sock);
@@ -186,22 +200,24 @@ STATIC mp_uint_t socket_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, i
     return mp_get_stream(self->sock)->ioctl(self->sock, request, arg, errcode);
 }
 
-STATIC mp_obj_t socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) {
-    // Currently supports only blocking mode
-    (void)self_in;
-    if (!mp_obj_is_true(flag_in)) {
-        mp_raise_NotImplementedError(NULL);
-    }
-    return mp_const_none;
+STATIC mp_obj_t ussl_socket_setblocking(mp_obj_t self_in, mp_obj_t flag_in) {
+    mp_obj_ssl_socket_t *o = MP_OBJ_TO_PTR(self_in);
+    mp_obj_t sock = o->sock;
+    mp_obj_t dest[3];
+    mp_load_method(sock, MP_QSTR_setblocking, dest);
+    dest[2] = flag_in;
+    mp_obj_t res = mp_call_method_n_kw(1, 0, dest);
+    o->blocking = mp_obj_is_true(flag_in);
+    return res;
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_2(socket_setblocking_obj, socket_setblocking);
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(ussl_socket_setblocking_obj, ussl_socket_setblocking);
 
 STATIC const mp_rom_map_elem_t ussl_socket_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
     { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
     { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
     { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
-    { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&socket_setblocking_obj) },
+    { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&ussl_socket_setblocking_obj) },
     { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
 #if MICROPY_PY_USSL_FINALISER
     { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) },
@@ -211,16 +227,16 @@ STATIC const mp_rom_map_elem_t ussl_socket_locals_dict_table[] = {
 STATIC MP_DEFINE_CONST_DICT(ussl_socket_locals_dict, ussl_socket_locals_dict_table);
 
 STATIC const mp_stream_p_t ussl_socket_stream_p = {
-    .read = socket_read,
-    .write = socket_write,
-    .ioctl = socket_ioctl,
+    .read = ussl_socket_read,
+    .write = ussl_socket_write,
+    .ioctl = ussl_socket_ioctl,
 };
 
 STATIC const mp_obj_type_t ussl_socket_type = {
     { &mp_type_type },
     // Save on qstr's, reuse same as for module
     .name = MP_QSTR_ussl,
-    .print = socket_print,
+    .print = ussl_socket_print,
     .getiter = NULL,
     .iternext = NULL,
     .protocol = &ussl_socket_stream_p,
@@ -234,6 +250,7 @@ STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_
         { MP_QSTR_cert, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
         { MP_QSTR_server_side, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
         { MP_QSTR_server_hostname, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
+        { MP_QSTR_do_handshake, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
     };
 
     // TODO: Check that sock implements stream protocol
@@ -243,7 +260,7 @@ STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_
     mp_arg_parse_all(n_args - 1, pos_args + 1, kw_args,
         MP_ARRAY_SIZE(allowed_args), allowed_args, (mp_arg_val_t*)&args);
 
-    return MP_OBJ_FROM_PTR(socket_new(sock, &args));
+    return MP_OBJ_FROM_PTR(ussl_socket_new(sock, &args));
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_ssl_wrap_socket_obj, 1, mod_ssl_wrap_socket);
 

extmod/modussl_mbedtls.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2016 Linaro Ltd.
+ * Copyright (c) 2019 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -60,6 +61,7 @@ struct ssl_args {
     mp_arg_val_t cert;
     mp_arg_val_t server_side;
     mp_arg_val_t server_hostname;
+    mp_arg_val_t do_handshake;
 };
 
 STATIC const mp_obj_type_t ussl_socket_type;
@@ -167,27 +169,37 @@ STATIC mp_obj_ssl_socket_t *socket_new(mp_obj_t sock, struct ssl_args *args) {
 
     mbedtls_ssl_set_bio(&o->ssl, &o->sock, _mbedtls_ssl_send, _mbedtls_ssl_recv, NULL);
 
-    if (args->key.u_obj != MP_OBJ_NULL) {
+    if (args->key.u_obj != mp_const_none) {
         size_t key_len;
         const byte *key = (const byte*)mp_obj_str_get_data(args->key.u_obj, &key_len);
         // len should include terminating null
         ret = mbedtls_pk_parse_key(&o->pkey, key, key_len + 1, NULL, 0);
-        assert(ret == 0);
+        if (ret != 0) {
+            ret = MBEDTLS_ERR_PK_BAD_INPUT_DATA; // use general error for all key errors
+            goto cleanup;
+        }
 
         size_t cert_len;
         const byte *cert = (const byte*)mp_obj_str_get_data(args->cert.u_obj, &cert_len);
         // len should include terminating null
         ret = mbedtls_x509_crt_parse(&o->cert, cert, cert_len + 1);
-        assert(ret == 0);
+        if (ret != 0) {
+            ret = MBEDTLS_ERR_X509_BAD_INPUT_DATA; // use general error for all cert errors
+            goto cleanup;
+        }
 
         ret = mbedtls_ssl_conf_own_cert(&o->conf, &o->cert, &o->pkey);
-        assert(ret == 0);
+        if (ret != 0) {
+            goto cleanup;
+        }
     }
 
-    while ((ret = mbedtls_ssl_handshake(&o->ssl)) != 0) {
-        if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
-            printf("mbedtls_ssl_handshake error: -%x\n", -ret);
-            goto cleanup;
+    if (args->do_handshake.u_bool) {
+        while ((ret = mbedtls_ssl_handshake(&o->ssl)) != 0) {
+            if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+                printf("mbedtls_ssl_handshake error: -%x\n", -ret);
+                goto cleanup;
+            }
         }
     }
 
@@ -204,6 +216,10 @@ cleanup:
 
     if (ret == MBEDTLS_ERR_SSL_ALLOC_FAILED) {
         mp_raise_OSError(MP_ENOMEM);
+    } else if (ret == MBEDTLS_ERR_PK_BAD_INPUT_DATA) {
+        mp_raise_ValueError("invalid key");
+    } else if (ret == MBEDTLS_ERR_X509_BAD_INPUT_DATA) {
+        mp_raise_ValueError("invalid cert");
     } else {
         mp_raise_OSError(MP_EIO);
     }
@@ -215,6 +231,9 @@ STATIC mp_obj_t mod_ssl_getpeercert(mp_obj_t o_in, mp_obj_t binary_form) {
         mp_raise_NotImplementedError(NULL);
     }
     const mbedtls_x509_crt* peer_cert = mbedtls_ssl_get_peer_cert(&o->ssl);
+    if (peer_cert == NULL) {
+        return mp_const_none;
+    }
     return mp_obj_new_bytes(peer_cert->raw.p, peer_cert->raw.len);
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_ssl_getpeercert_obj, mod_ssl_getpeercert);
@@ -238,6 +257,11 @@ STATIC mp_uint_t socket_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errc
     }
     if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
         ret = MP_EWOULDBLOCK;
+    } else if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
+        // If handshake is not finished, read attempt may end up in protocol
+        // wanting to write next handshake message. The same may happen with
+        // renegotation.
+        ret = MP_EWOULDBLOCK;
     }
     *errcode = ret;
     return MP_STREAM_ERROR;
@@ -252,6 +276,11 @@ STATIC mp_uint_t socket_write(mp_obj_t o_in, const void *buf, mp_uint_t size, in
     }
     if (ret == MBEDTLS_ERR_SSL_WANT_WRITE) {
         ret = MP_EWOULDBLOCK;
+    } else if (ret == MBEDTLS_ERR_SSL_WANT_READ) {
+        // If handshake is not finished, write attempt may end up in protocol
+        // wanting to read next handshake message. The same may happen with
+        // renegotation.
+        ret = MP_EWOULDBLOCK;
     }
     *errcode = ret;
     return MP_STREAM_ERROR;
@@ -317,10 +346,11 @@ STATIC const mp_obj_type_t ussl_socket_type = {
 STATIC mp_obj_t mod_ssl_wrap_socket(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     // TODO: Implement more args
     static const mp_arg_t allowed_args[] = {
-        { MP_QSTR_key, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} },
-        { MP_QSTR_cert, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} },
+        { MP_QSTR_key, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
+        { MP_QSTR_cert, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
         { MP_QSTR_server_side, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} },
-        { MP_QSTR_server_hostname, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} },
+        { MP_QSTR_server_hostname, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_const_none_obj)} },
+        { MP_QSTR_do_handshake, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
     };
 
     // TODO: Check that sock implements stream protocol

extmod/modutimeq.c

@@ -55,7 +55,7 @@ typedef struct _mp_obj_utimeq_t {
 
 STATIC mp_uint_t utimeq_id;
 
-STATIC mp_obj_utimeq_t *get_heap(mp_obj_t heap_in) {
+STATIC mp_obj_utimeq_t *utimeq_get_heap(mp_obj_t heap_in) {
     return MP_OBJ_TO_PTR(heap_in);
 }
 
@@ -85,7 +85,7 @@ STATIC mp_obj_t utimeq_make_new(const mp_obj_type_t *type, size_t n_args, size_t
     return MP_OBJ_FROM_PTR(o);
 }
 
-STATIC void heap_siftdown(mp_obj_utimeq_t *heap, mp_uint_t start_pos, mp_uint_t pos) {
+STATIC void utimeq_heap_siftdown(mp_obj_utimeq_t *heap, mp_uint_t start_pos, mp_uint_t pos) {
     struct qentry item = heap->items[pos];
     while (pos > start_pos) {
         mp_uint_t parent_pos = (pos - 1) >> 1;
@@ -101,7 +101,7 @@ STATIC void heap_siftdown(mp_obj_utimeq_t *heap, mp_uint_t start_pos, mp_uint_t
     heap->items[pos] = item;
 }
 
-STATIC void heap_siftup(mp_obj_utimeq_t *heap, mp_uint_t pos) {
+STATIC void utimeq_heap_siftup(mp_obj_utimeq_t *heap, mp_uint_t pos) {
     mp_uint_t start_pos = pos;
     mp_uint_t end_pos = heap->len;
     struct qentry item = heap->items[pos];
@@ -118,13 +118,13 @@ STATIC void heap_siftup(mp_obj_utimeq_t *heap, mp_uint_t pos) {
         pos = child_pos;
     }
     heap->items[pos] = item;
-    heap_siftdown(heap, start_pos, pos);
+    utimeq_heap_siftdown(heap, start_pos, pos);
 }
 
 STATIC mp_obj_t mod_utimeq_heappush(size_t n_args, const mp_obj_t *args) {
     (void)n_args;
     mp_obj_t heap_in = args[0];
-    mp_obj_utimeq_t *heap = get_heap(heap_in);
+    mp_obj_utimeq_t *heap = utimeq_get_heap(heap_in);
     if (heap->len == heap->alloc) {
         mp_raise_msg(&mp_type_IndexError, "queue overflow");
     }
@@ -133,16 +133,16 @@ STATIC mp_obj_t mod_utimeq_heappush(size_t n_args, const mp_obj_t *args) {
     heap->items[l].id = utimeq_id++;
     heap->items[l].callback = args[2];
     heap->items[l].args = args[3];
-    heap_siftdown(heap, 0, heap->len);
+    utimeq_heap_siftdown(heap, 0, heap->len);
     heap->len++;
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_utimeq_heappush_obj, 4, 4, mod_utimeq_heappush);
 
 STATIC mp_obj_t mod_utimeq_heappop(mp_obj_t heap_in, mp_obj_t list_ref) {
-    mp_obj_utimeq_t *heap = get_heap(heap_in);
+    mp_obj_utimeq_t *heap = utimeq_get_heap(heap_in);
     if (heap->len == 0) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "empty heap"));
+        mp_raise_msg(&mp_type_IndexError, "empty heap");
     }
     mp_obj_list_t *ret = MP_OBJ_TO_PTR(list_ref);
     if (!mp_obj_is_type(list_ref, &mp_type_list) || ret->len < 3) {
@@ -158,16 +158,16 @@ STATIC mp_obj_t mod_utimeq_heappop(mp_obj_t heap_in, mp_obj_t list_ref) {
     heap->items[heap->len].callback = MP_OBJ_NULL; // so we don't retain a pointer
     heap->items[heap->len].args = MP_OBJ_NULL;
     if (heap->len) {
-        heap_siftup(heap, 0);
+        utimeq_heap_siftup(heap, 0);
     }
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_utimeq_heappop_obj, mod_utimeq_heappop);
 
 STATIC mp_obj_t mod_utimeq_peektime(mp_obj_t heap_in) {
-    mp_obj_utimeq_t *heap = get_heap(heap_in);
+    mp_obj_utimeq_t *heap = utimeq_get_heap(heap_in);
     if (heap->len == 0) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_IndexError, "empty heap"));
+        mp_raise_msg(&mp_type_IndexError, "empty heap");
     }
 
     struct qentry *item = &heap->items[0];
@@ -177,7 +177,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_utimeq_peektime_obj, mod_utimeq_peektime);
 
 #if DEBUG
 STATIC mp_obj_t mod_utimeq_dump(mp_obj_t heap_in) {
-    mp_obj_utimeq_t *heap = get_heap(heap_in);
+    mp_obj_utimeq_t *heap = utimeq_get_heap(heap_in);
     for (int i = 0; i < heap->len; i++) {
         printf(UINT_FMT "\t%p\t%p\n", heap->items[i].time,
             MP_OBJ_TO_PTR(heap->items[i].callback), MP_OBJ_TO_PTR(heap->items[i].args));

extmod/moduwebsocket.c

@@ -194,7 +194,7 @@ no_payload:
                     if (last_state == CONTROL) {
                         byte frame_type = self->last_flags & FRAME_OPCODE_MASK;
                         if (frame_type == FRAME_CLOSE) {
-                            static char close_resp[2] = {0x88, 0};
+                            static const char close_resp[2] = {0x88, 0};
                             int err;
                             websocket_write(self_in, close_resp, sizeof(close_resp), &err);
                             return 0;

extmod/moduzlib.c

@@ -122,6 +122,7 @@ STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *er
     return o->decomp.dest - (byte*)buf;
 }
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
     { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
@@ -129,11 +130,13 @@ STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = {
 };
 
 STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
+#endif
 
 STATIC const mp_stream_p_t decompio_stream_p = {
     .read = decompio_read,
 };
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_obj_type_t decompio_type = {
     { &mp_type_type },
     .name = MP_QSTR_DecompIO,
@@ -141,6 +144,7 @@ STATIC const mp_obj_type_t decompio_type = {
     .protocol = &decompio_stream_p,
     .locals_dict = (void*)&decompio_locals_dict,
 };
+#endif
 
 STATIC mp_obj_t mod_uzlib_decompress(size_t n_args, const mp_obj_t *args) {
     mp_obj_t data = args[0];
@@ -201,6 +205,7 @@ error:
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_uzlib_decompress_obj, 1, 3, mod_uzlib_decompress);
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_rom_map_elem_t mp_module_uzlib_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uzlib) },
     { MP_ROM_QSTR(MP_QSTR_decompress), MP_ROM_PTR(&mod_uzlib_decompress_obj) },
@@ -213,6 +218,7 @@ const mp_obj_module_t mp_module_uzlib = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_uzlib_globals,
 };
+#endif
 
 // Source files #include'd here to make sure they're compiled in
 // only if module is enabled by config setting.

extmod/modwebrepl.c

@@ -67,10 +67,9 @@ typedef struct _mp_obj_webrepl_t {
     mp_obj_t cur_file;
 } mp_obj_webrepl_t;
 
-// These get passed to functions which aren't force-l32, so can't be const
-STATIC char passwd_prompt[] = "Password: ";
-STATIC char connected_prompt[] = "\r\nWebREPL connected\r\n>>> ";
-STATIC char denied_prompt[] = "\r\nAccess denied\r\n";
+STATIC const char passwd_prompt[] = "Password: ";
+STATIC const char connected_prompt[] = "\r\nWebREPL connected\r\n>>> ";
+STATIC const char denied_prompt[] = "\r\nAccess denied\r\n";
 
 STATIC char webrepl_passwd[10];
 
@@ -108,6 +107,15 @@ STATIC mp_obj_t webrepl_make_new(const mp_obj_type_t *type, size_t n_args, size_
     return o;
 }
 
+STATIC void check_file_op_finished(mp_obj_webrepl_t *self) {
+    if (self->data_to_recv == 0) {
+        mp_stream_close(self->cur_file);
+        self->hdr_to_recv = sizeof(struct webrepl_file);
+        DEBUG_printf("webrepl: Finished file operation %d\n", self->hdr.type);
+        write_webrepl_resp(self->sock, 0);
+    }
+}
+
 STATIC int write_file_chunk(mp_obj_webrepl_t *self) {
     const mp_stream_p_t *file_stream = mp_get_stream(self->cur_file);
     byte readbuf[2 + 256];
@@ -129,7 +137,7 @@ STATIC void handle_op(mp_obj_webrepl_t *self) {
 
     switch (self->hdr.type) {
         case GET_VER: {
-            static char ver[] = {MICROPY_VERSION_MAJOR, MICROPY_VERSION_MINOR, MICROPY_VERSION_MICRO};
+            static const char ver[] = {MICROPY_VERSION_MAJOR, MICROPY_VERSION_MINOR, MICROPY_VERSION_MICRO};
             write_webrepl(self->sock, ver, sizeof(ver));
             self->hdr_to_recv = sizeof(struct webrepl_file);
             return;
@@ -160,6 +168,7 @@ STATIC void handle_op(mp_obj_webrepl_t *self) {
 
     if (self->hdr.type == PUT_FILE) {
         self->data_to_recv = self->hdr.size;
+        check_file_op_finished(self);
     } else if (self->hdr.type == GET_FILE) {
         self->data_to_recv = 1;
     }
@@ -236,7 +245,11 @@ STATIC mp_uint_t _webrepl_read(mp_obj_t self_in, void *buf, mp_uint_t size, int
     }
 
     if (self->data_to_recv != 0) {
-        static byte filebuf[512];
+        // Ports that don't have much available stack can make this filebuf static
+        #if MICROPY_PY_WEBREPL_STATIC_FILEBUF
+        static
+        #endif
+        byte filebuf[512];
         filebuf[0] = *(byte*)buf;
         mp_uint_t buf_sz = 1;
         if (--self->data_to_recv != 0) {
@@ -266,12 +279,7 @@ STATIC mp_uint_t _webrepl_read(mp_obj_t self_in, void *buf, mp_uint_t size, int
             }
         }
 
-        if (self->data_to_recv == 0) {
-            mp_stream_close(self->cur_file);
-            self->hdr_to_recv = sizeof(struct webrepl_file);
-            DEBUG_printf("webrepl: Finished file operation %d\n", self->hdr.type);
-            write_webrepl_resp(self->sock, 0);
-        }
+        check_file_op_finished(self);
 
         #ifdef MICROPY_PY_WEBREPL_DELAY
         // Some platforms may have broken drivers and easily gets

extmod/re1.5/compilecode.c

@@ -53,6 +53,9 @@ static const char *_compilecode(const char *re, ByteProg *prog, int sizecode)
             PC++; // Skip # of pair byte
             prog->len++;
             for (cnt = 0; *re != ']'; re++, cnt++) {
+                if (*re == '\\') {
+                    ++re;
+                }
                 if (!*re) return NULL;
                 EMIT(PC++, *re);
                 if (re[1] == '-' && re[2] != ']') {

extmod/uos_dupterm.c

@@ -4,7 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2016 Paul Sokolovsky
- * Copyright (c) 2017 Damien P. George
+ * Copyright (c) 2017-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -32,6 +32,7 @@
 #include "py/objtuple.h"
 #include "py/objarray.h"
 #include "py/stream.h"
+#include "extmod/misc.h"
 #include "lib/utils/interrupt_char.h"
 
 #if MICROPY_PY_OS_DUPTERM
@@ -52,12 +53,65 @@ void mp_uos_deactivate(size_t dupterm_idx, const char *msg, mp_obj_t exc) {
     }
 }
 
+uintptr_t mp_uos_dupterm_poll(uintptr_t poll_flags) {
+    uintptr_t poll_flags_out = 0;
+
+    for (size_t idx = 0; idx < MICROPY_PY_OS_DUPTERM; ++idx) {
+        mp_obj_t s = MP_STATE_VM(dupterm_objs[idx]);
+        if (s == MP_OBJ_NULL) {
+            continue;
+        }
+
+        int errcode = 0;
+        mp_uint_t ret = 0;
+        const mp_stream_p_t *stream_p = mp_get_stream(s);
+        #if MICROPY_PY_UOS_DUPTERM_BUILTIN_STREAM
+        if (mp_uos_dupterm_is_builtin_stream(s)) {
+            ret = stream_p->ioctl(s, MP_STREAM_POLL, poll_flags, &errcode);
+        } else
+        #endif
+        {
+            nlr_buf_t nlr;
+            if (nlr_push(&nlr) == 0) {
+                ret = stream_p->ioctl(s, MP_STREAM_POLL, poll_flags, &errcode);
+                nlr_pop();
+            } else {
+                // Ignore error with ioctl
+            }
+        }
+
+        if (ret != MP_STREAM_ERROR) {
+            poll_flags_out |= ret;
+            if (poll_flags_out == poll_flags) {
+                // Finish early if all requested flags are set
+                break;
+            }
+        }
+    }
+
+    return poll_flags_out;
+}
+
 int mp_uos_dupterm_rx_chr(void) {
     for (size_t idx = 0; idx < MICROPY_PY_OS_DUPTERM; ++idx) {
         if (MP_STATE_VM(dupterm_objs[idx]) == MP_OBJ_NULL) {
             continue;
         }
 
+        #if MICROPY_PY_UOS_DUPTERM_BUILTIN_STREAM
+        if (mp_uos_dupterm_is_builtin_stream(MP_STATE_VM(dupterm_objs[idx]))) {
+            byte buf[1];
+            int errcode = 0;
+            const mp_stream_p_t *stream_p = mp_get_stream(MP_STATE_VM(dupterm_objs[idx]));
+            mp_uint_t out_sz = stream_p->read(MP_STATE_VM(dupterm_objs[idx]), buf, 1, &errcode);
+            if (errcode == 0 && out_sz != 0) {
+                return buf[0];
+             } else {
+                continue;
+             }
+        }
+        #endif
+
         nlr_buf_t nlr;
         if (nlr_push(&nlr) == 0) {
             byte buf[1];
@@ -98,6 +152,16 @@ void mp_uos_dupterm_tx_strn(const char *str, size_t len) {
         if (MP_STATE_VM(dupterm_objs[idx]) == MP_OBJ_NULL) {
             continue;
         }
+
+        #if MICROPY_PY_UOS_DUPTERM_BUILTIN_STREAM
+        if (mp_uos_dupterm_is_builtin_stream(MP_STATE_VM(dupterm_objs[idx]))) {
+            int errcode = 0;
+            const mp_stream_p_t *stream_p = mp_get_stream(MP_STATE_VM(dupterm_objs[idx]));
+            stream_p->write(MP_STATE_VM(dupterm_objs[idx]), str, len, &errcode);
+            continue;
+        }
+        #endif
+
         nlr_buf_t nlr;
         if (nlr_push(&nlr) == 0) {
             mp_stream_write(MP_STATE_VM(dupterm_objs[idx]), str, len, MP_STREAM_RW_WRITE);

extmod/vfs.c

@@ -38,6 +38,10 @@
 #include "extmod/vfs_fat.h"
 #endif
 
+#if MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2
+#include "extmod/vfs_lfs.h"
+#endif
+
 #if MICROPY_VFS_POSIX
 #include "extmod/vfs_posix.h"
 #endif
@@ -156,6 +160,44 @@ mp_import_stat_t mp_vfs_import_stat(const char *path) {
     }
 }
 
+STATIC mp_obj_t mp_vfs_autodetect(mp_obj_t bdev_obj) {
+    #if MICROPY_VFS_LFS1 || MICROPY_VFS_LFS2
+    nlr_buf_t nlr;
+    if (nlr_push(&nlr) == 0) {
+        mp_obj_t vfs = MP_OBJ_NULL;
+        mp_vfs_blockdev_t blockdev;
+        mp_vfs_blockdev_init(&blockdev, bdev_obj);
+        uint8_t buf[44];
+        mp_vfs_blockdev_read_ext(&blockdev, 0, 8, sizeof(buf), buf);
+        #if MICROPY_VFS_LFS1
+        if (memcmp(&buf[32], "littlefs", 8) == 0) {
+            // LFS1
+            vfs = mp_type_vfs_lfs1.make_new(&mp_type_vfs_lfs1, 1, 0, &bdev_obj);
+            nlr_pop();
+            return vfs;
+        }
+        #endif
+        #if MICROPY_VFS_LFS2
+        if (memcmp(&buf[0], "littlefs", 8) == 0) {
+            // LFS2
+            vfs = mp_type_vfs_lfs2.make_new(&mp_type_vfs_lfs2, 1, 0, &bdev_obj);
+            nlr_pop();
+            return vfs;
+        }
+        #endif
+        nlr_pop();
+    } else {
+        // Ignore exception (eg block device doesn't support extended readblocks)
+    }
+    #endif
+
+    #if MICROPY_VFS_FAT
+    return mp_fat_vfs_type.make_new(&mp_fat_vfs_type, 1, 0, &bdev_obj);
+    #endif
+
+    return bdev_obj;
+}
+
 mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_readonly, ARG_mkfs };
     static const mp_arg_t allowed_args[] = {
@@ -178,10 +220,7 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args
     if (dest[0] == MP_OBJ_NULL) {
         // Input object has no mount method, assume it's a block device and try to
         // auto-detect the filesystem and create the corresponding VFS entity.
-        // (At the moment we only support FAT filesystems.)
-        #if MICROPY_VFS_FAT
-        vfs_obj = mp_fat_vfs_type.make_new(&mp_fat_vfs_type, 1, 0, &vfs_obj);
-        #endif
+        vfs_obj = mp_vfs_autodetect(vfs_obj);
     }
 
     // create new object

extmod/vfs.h

@@ -38,18 +38,40 @@
 #define MP_S_IFDIR (0x4000)
 #define MP_S_IFREG (0x8000)
 
+// these are the values for mp_vfs_blockdev_t.flags
+#define MP_BLOCKDEV_FLAG_NATIVE         (0x0001) // readblocks[2]/writeblocks[2] contain native func
+#define MP_BLOCKDEV_FLAG_FREE_OBJ       (0x0002) // fs_user_mount_t obj should be freed on umount
+#define MP_BLOCKDEV_FLAG_HAVE_IOCTL     (0x0004) // new protocol with ioctl
+#define MP_BLOCKDEV_FLAG_NO_FILESYSTEM  (0x0008) // the block device has no filesystem on it
+
 // constants for block protocol ioctl
-#define BP_IOCTL_INIT           (1)
-#define BP_IOCTL_DEINIT         (2)
-#define BP_IOCTL_SYNC           (3)
-#define BP_IOCTL_SEC_COUNT      (4)
-#define BP_IOCTL_SEC_SIZE       (5)
+#define MP_BLOCKDEV_IOCTL_INIT          (1)
+#define MP_BLOCKDEV_IOCTL_DEINIT        (2)
+#define MP_BLOCKDEV_IOCTL_SYNC          (3)
+#define MP_BLOCKDEV_IOCTL_BLOCK_COUNT   (4)
+#define MP_BLOCKDEV_IOCTL_BLOCK_SIZE    (5)
+#define MP_BLOCKDEV_IOCTL_BLOCK_ERASE   (6)
 
 // At the moment the VFS protocol just has import_stat, but could be extended to other methods
 typedef struct _mp_vfs_proto_t {
     mp_import_stat_t (*import_stat)(void *self, const char *path);
 } mp_vfs_proto_t;
 
+typedef struct _mp_vfs_blockdev_t {
+    uint16_t flags;
+    size_t block_size;
+    mp_obj_t readblocks[5];
+    mp_obj_t writeblocks[5];
+    // new protocol uses just ioctl, old uses sync (optional) and count
+    union {
+        mp_obj_t ioctl[4];
+        struct {
+            mp_obj_t sync[2];
+            mp_obj_t count[2];
+        } old;
+    } u;
+} mp_vfs_blockdev_t;
+
 typedef struct _mp_vfs_mount_t {
     const char *str; // mount point with leading /
     size_t len;
@@ -57,6 +79,13 @@ typedef struct _mp_vfs_mount_t {
     struct _mp_vfs_mount_t *next;
 } mp_vfs_mount_t;
 
+void mp_vfs_blockdev_init(mp_vfs_blockdev_t *self, mp_obj_t bdev);
+int mp_vfs_blockdev_read(mp_vfs_blockdev_t *self, size_t block_num, size_t num_blocks, uint8_t *buf);
+int mp_vfs_blockdev_read_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t block_off, size_t len, uint8_t *buf);
+int mp_vfs_blockdev_write(mp_vfs_blockdev_t *self, size_t block_num, size_t num_blocks, const uint8_t *buf);
+int mp_vfs_blockdev_write_ext(mp_vfs_blockdev_t *self, size_t block_num, size_t block_off, size_t len, const uint8_t *buf);
+mp_obj_t mp_vfs_blockdev_ioctl(mp_vfs_blockdev_t *self, uintptr_t cmd, uintptr_t arg);
+
 mp_vfs_mount_t *mp_vfs_lookup_path(const char *path, const char **path_out);
 mp_import_stat_t mp_vfs_import_stat(const char *path);
 mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args);

extmod/vfs_fat.c

@@ -39,8 +39,8 @@
 #include "extmod/vfs_fat.h"
 #include "lib/timeutils/timeutils.h"
 
-#if _MAX_SS == _MIN_SS
-#define SECSIZE(fs) (_MIN_SS)
+#if FF_MAX_SS == FF_MIN_SS
+#define SECSIZE(fs) (FF_MIN_SS)
 #else
 #define SECSIZE(fs) ((fs)->ssize)
 #endif
@@ -68,27 +68,18 @@ STATIC mp_obj_t fat_vfs_make_new(const mp_obj_type_t *type, size_t n_args, size_
     // create new object
     fs_user_mount_t *vfs = m_new_obj(fs_user_mount_t);
     vfs->base.type = type;
-    vfs->flags = FSUSER_FREE_OBJ;
     vfs->fatfs.drv = vfs;
 
-    // load block protocol methods
-    mp_load_method(args[0], MP_QSTR_readblocks, vfs->readblocks);
-    mp_load_method_maybe(args[0], MP_QSTR_writeblocks, vfs->writeblocks);
-    mp_load_method_maybe(args[0], MP_QSTR_ioctl, vfs->u.ioctl);
-    if (vfs->u.ioctl[0] != MP_OBJ_NULL) {
-        // device supports new block protocol, so indicate it
-        vfs->flags |= FSUSER_HAVE_IOCTL;
-    } else {
-        // no ioctl method, so assume the device uses the old block protocol
-        mp_load_method_maybe(args[0], MP_QSTR_sync, vfs->u.old.sync);
-        mp_load_method(args[0], MP_QSTR_count, vfs->u.old.count);
-    }
+    // Initialise underlying block device
+    vfs->blockdev.flags = MP_BLOCKDEV_FLAG_FREE_OBJ;
+    vfs->blockdev.block_size = FF_MIN_SS; // default, will be populated by call to MP_BLOCKDEV_IOCTL_BLOCK_SIZE
+    mp_vfs_blockdev_init(&vfs->blockdev, args[0]);
 
     // mount the block device so the VFS methods can be used
     FRESULT res = f_mount(&vfs->fatfs);
     if (res == FR_NO_FILESYSTEM) {
         // don't error out if no filesystem, to let mkfs()/mount() create one if wanted
-        vfs->flags |= FSUSER_NO_FILESYSTEM;
+        vfs->blockdev.flags |= MP_BLOCKDEV_FLAG_NO_FILESYSTEM;
     } else if (res != FR_OK) {
         mp_raise_OSError(fresult_to_errno_table[res]);
     }
@@ -111,8 +102,11 @@ STATIC mp_obj_t fat_vfs_mkfs(mp_obj_t bdev_in) {
     fs_user_mount_t *vfs = MP_OBJ_TO_PTR(fat_vfs_make_new(&mp_fat_vfs_type, 1, 0, &bdev_in));
 
     // make the filesystem
-    uint8_t working_buf[_MAX_SS];
+    uint8_t working_buf[FF_MAX_SS];
     FRESULT res = f_mkfs(&vfs->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
+    if (res == FR_MKFS_ABORTED) { // Probably doesn't support FAT16
+        res = f_mkfs(&vfs->fatfs, FM_FAT32, 0, working_buf, sizeof(working_buf));
+    }
     if (res != FR_OK) {
         mp_raise_OSError(fresult_to_errno_table[res]);
     }
@@ -363,7 +357,7 @@ STATIC mp_obj_t fat_vfs_statvfs(mp_obj_t vfs_in, mp_obj_t path_in) {
     t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree
     t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail
     t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags
-    t->items[9] = MP_OBJ_NEW_SMALL_INT(_MAX_LFN); // f_namemax
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(FF_MAX_LFN); // f_namemax
 
     return MP_OBJ_FROM_PTR(t);
 }
@@ -377,19 +371,19 @@ STATIC mp_obj_t vfs_fat_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs
     //  1. readonly=True keyword argument
     //  2. nonexistent writeblocks method (then writeblocks[0] == MP_OBJ_NULL already)
     if (mp_obj_is_true(readonly)) {
-        self->writeblocks[0] = MP_OBJ_NULL;
+        self->blockdev.writeblocks[0] = MP_OBJ_NULL;
     }
 
     // check if we need to make the filesystem
-    FRESULT res = (self->flags & FSUSER_NO_FILESYSTEM) ? FR_NO_FILESYSTEM : FR_OK;
+    FRESULT res = (self->blockdev.flags & MP_BLOCKDEV_FLAG_NO_FILESYSTEM) ? FR_NO_FILESYSTEM : FR_OK;
     if (res == FR_NO_FILESYSTEM && mp_obj_is_true(mkfs)) {
-        uint8_t working_buf[_MAX_SS];
+        uint8_t working_buf[FF_MAX_SS];
         res = f_mkfs(&self->fatfs, FM_FAT | FM_SFD, 0, working_buf, sizeof(working_buf));
     }
     if (res != FR_OK) {
         mp_raise_OSError(fresult_to_errno_table[res]);
     }
-    self->flags &= ~FSUSER_NO_FILESYSTEM;
+    self->blockdev.flags &= ~MP_BLOCKDEV_FLAG_NO_FILESYSTEM;
 
     return mp_const_none;
 }

extmod/vfs_fat.h

@@ -26,30 +26,13 @@
 #ifndef MICROPY_INCLUDED_EXTMOD_VFS_FAT_H
 #define MICROPY_INCLUDED_EXTMOD_VFS_FAT_H
 
-#include "py/lexer.h"
 #include "py/obj.h"
 #include "lib/oofatfs/ff.h"
 #include "extmod/vfs.h"
 
-// these are the values for fs_user_mount_t.flags
-#define FSUSER_NATIVE       (0x0001) // readblocks[2]/writeblocks[2] contain native func
-#define FSUSER_FREE_OBJ     (0x0002) // fs_user_mount_t obj should be freed on umount
-#define FSUSER_HAVE_IOCTL   (0x0004) // new protocol with ioctl
-#define FSUSER_NO_FILESYSTEM (0x0008) // the block device has no filesystem on it
-
 typedef struct _fs_user_mount_t {
     mp_obj_base_t base;
-    uint16_t flags;
-    mp_obj_t readblocks[4];
-    mp_obj_t writeblocks[4];
-    // new protocol uses just ioctl, old uses sync (optional) and count
-    union {
-        mp_obj_t ioctl[4];
-        struct {
-            mp_obj_t sync[2];
-            mp_obj_t count[2];
-        } old;
-    } u;
+    mp_vfs_blockdev_t blockdev;
     FATFS fatfs;
 } fs_user_mount_t;
 

extmod/vfs_fat_diskio.c

@@ -38,16 +38,11 @@
 #include "py/runtime.h"
 #include "py/binary.h"
 #include "py/objarray.h"
+#include "py/mperrno.h"
 #include "lib/oofatfs/ff.h"
 #include "lib/oofatfs/diskio.h"
 #include "extmod/vfs_fat.h"
 
-#if _MAX_SS == _MIN_SS
-#define SECSIZE(fs) (_MIN_SS)
-#else
-#define SECSIZE(fs) ((fs)->ssize)
-#endif
-
 typedef void *bdev_t;
 STATIC fs_user_mount_t *disk_get_device(void *bdev) {
     return (fs_user_mount_t*)bdev;
@@ -69,20 +64,9 @@ DRESULT disk_read (
         return RES_PARERR;
     }
 
-    if (vfs->flags & FSUSER_NATIVE) {
-        mp_uint_t (*f)(uint8_t*, uint32_t, uint32_t) = (void*)(uintptr_t)vfs->readblocks[2];
-        if (f(buff, sector, count) != 0) {
-            return RES_ERROR;
-        }
-    } else {
-        mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, count * SECSIZE(&vfs->fatfs), buff};
-        vfs->readblocks[2] = MP_OBJ_NEW_SMALL_INT(sector);
-        vfs->readblocks[3] = MP_OBJ_FROM_PTR(&ar);
-        mp_call_method_n_kw(2, 0, vfs->readblocks);
-        // TODO handle error return
-    }
+    int ret = mp_vfs_blockdev_read(&vfs->blockdev, sector, count, buff);
 
-    return RES_OK;
+    return ret == 0 ? RES_OK : RES_ERROR;
 }
 
 /*-----------------------------------------------------------------------*/
@@ -101,25 +85,14 @@ DRESULT disk_write (
         return RES_PARERR;
     }
 
-    if (vfs->writeblocks[0] == MP_OBJ_NULL) {
+    int ret = mp_vfs_blockdev_write(&vfs->blockdev, sector, count, buff);
+
+    if (ret == -MP_EROFS) {
         // read-only block device
         return RES_WRPRT;
     }
 
-    if (vfs->flags & FSUSER_NATIVE) {
-        mp_uint_t (*f)(const uint8_t*, uint32_t, uint32_t) = (void*)(uintptr_t)vfs->writeblocks[2];
-        if (f(buff, sector, count) != 0) {
-            return RES_ERROR;
-        }
-    } else {
-        mp_obj_array_t ar = {{&mp_type_bytearray}, BYTEARRAY_TYPECODE, 0, count * SECSIZE(&vfs->fatfs), (void*)buff};
-        vfs->writeblocks[2] = MP_OBJ_NEW_SMALL_INT(sector);
-        vfs->writeblocks[3] = MP_OBJ_FROM_PTR(&ar);
-        mp_call_method_n_kw(2, 0, vfs->writeblocks);
-        // TODO handle error return
-    }
-
-    return RES_OK;
+    return ret == 0 ? RES_OK : RES_ERROR;
 }
 
 
@@ -139,42 +112,16 @@ DRESULT disk_ioctl (
     }
 
     // First part: call the relevant method of the underlying block device
+    static const uint8_t op_map[8] = {
+        [CTRL_SYNC] = MP_BLOCKDEV_IOCTL_SYNC,
+        [GET_SECTOR_COUNT] = MP_BLOCKDEV_IOCTL_BLOCK_COUNT,
+        [GET_SECTOR_SIZE] = MP_BLOCKDEV_IOCTL_BLOCK_SIZE,
+        [IOCTL_INIT] = MP_BLOCKDEV_IOCTL_INIT,
+    };
+    uint8_t bp_op = op_map[cmd & 7];
     mp_obj_t ret = mp_const_none;
-    if (vfs->flags & FSUSER_HAVE_IOCTL) {
-        // new protocol with ioctl
-        static const uint8_t op_map[8] = {
-            [CTRL_SYNC] = BP_IOCTL_SYNC,
-            [GET_SECTOR_COUNT] = BP_IOCTL_SEC_COUNT,
-            [GET_SECTOR_SIZE] = BP_IOCTL_SEC_SIZE,
-            [IOCTL_INIT] = BP_IOCTL_INIT,
-        };
-        uint8_t bp_op = op_map[cmd & 7];
-        if (bp_op != 0) {
-            vfs->u.ioctl[2] = MP_OBJ_NEW_SMALL_INT(bp_op);
-            vfs->u.ioctl[3] = MP_OBJ_NEW_SMALL_INT(0); // unused
-            ret = mp_call_method_n_kw(2, 0, vfs->u.ioctl);
-        }
-    } else {
-        // old protocol with sync and count
-        switch (cmd) {
-            case CTRL_SYNC:
-                if (vfs->u.old.sync[0] != MP_OBJ_NULL) {
-                    mp_call_method_n_kw(0, 0, vfs->u.old.sync);
-                }
-                break;
-
-            case GET_SECTOR_COUNT:
-                ret = mp_call_method_n_kw(0, 0, vfs->u.old.count);
-                break;
-
-            case GET_SECTOR_SIZE:
-                // old protocol has fixed sector size of 512 bytes
-                break;
-
-            case IOCTL_INIT:
-                // old protocol doesn't have init
-                break;
-        }
+    if (bp_op != 0) {
+        ret = mp_vfs_blockdev_ioctl(&vfs->blockdev, bp_op, 0);
     }
 
     // Second part: convert the result for return
@@ -194,10 +141,8 @@ DRESULT disk_ioctl (
             } else {
                 *((WORD*)buff) = mp_obj_get_int(ret);
             }
-            #if _MAX_SS != _MIN_SS
             // need to store ssize because we use it in disk_read/disk_write
-            vfs->fatfs.ssize = *((WORD*)buff);
-            #endif
+            vfs->blockdev.block_size = *((WORD*)buff);
             return RES_OK;
         }
 
@@ -211,7 +156,7 @@ DRESULT disk_ioctl (
             if (ret != mp_const_none && MP_OBJ_SMALL_INT_VALUE(ret) != 0) {
                 // error initialising
                 stat = STA_NOINIT;
-            } else if (vfs->writeblocks[0] == MP_OBJ_NULL) {
+            } else if (vfs->blockdev.writeblocks[0] == MP_OBJ_NULL) {
                 stat = STA_PROTECT;
             } else {
                 stat = 0;

extmod/vfs_fat_file.c

@@ -219,7 +219,7 @@ STATIC mp_obj_t file_obj_make_new(const mp_obj_type_t *type, size_t n_args, size
 
 // TODO gc hook to close the file if not already closed
 
-STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = {
+STATIC const mp_rom_map_elem_t vfs_fat_rawfile_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
     { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
     { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
@@ -234,10 +234,10 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&file_obj___exit___obj) },
 };
 
-STATIC MP_DEFINE_CONST_DICT(rawfile_locals_dict, rawfile_locals_dict_table);
+STATIC MP_DEFINE_CONST_DICT(vfs_fat_rawfile_locals_dict, vfs_fat_rawfile_locals_dict_table);
 
 #if MICROPY_PY_IO_FILEIO
-STATIC const mp_stream_p_t fileio_stream_p = {
+STATIC const mp_stream_p_t vfs_fat_fileio_stream_p = {
     .read = file_obj_read,
     .write = file_obj_write,
     .ioctl = file_obj_ioctl,
@@ -250,12 +250,12 @@ const mp_obj_type_t mp_type_vfs_fat_fileio = {
     .make_new = file_obj_make_new,
     .getiter = mp_identity_getiter,
     .iternext = mp_stream_unbuffered_iter,
-    .protocol = &fileio_stream_p,
-    .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict,
+    .protocol = &vfs_fat_fileio_stream_p,
+    .locals_dict = (mp_obj_dict_t*)&vfs_fat_rawfile_locals_dict,
 };
 #endif
 
-STATIC const mp_stream_p_t textio_stream_p = {
+STATIC const mp_stream_p_t vfs_fat_textio_stream_p = {
     .read = file_obj_read,
     .write = file_obj_write,
     .ioctl = file_obj_ioctl,
@@ -269,8 +269,8 @@ const mp_obj_type_t mp_type_vfs_fat_textio = {
     .make_new = file_obj_make_new,
     .getiter = mp_identity_getiter,
     .iternext = mp_stream_unbuffered_iter,
-    .protocol = &textio_stream_p,
-    .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict,
+    .protocol = &vfs_fat_textio_stream_p,
+    .locals_dict = (mp_obj_dict_t*)&vfs_fat_rawfile_locals_dict,
 };
 
 // Factory function for I/O stream classes

extmod/vfs_posix.c

@@ -31,6 +31,7 @@
 
 #if MICROPY_VFS_POSIX
 
+#include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <dirent.h>

extmod/vfs_posix_file.c

@@ -44,7 +44,7 @@ typedef struct _mp_obj_vfs_posix_file_t {
 #ifdef MICROPY_CPYTHON_COMPAT
 STATIC void check_fd_is_open(const mp_obj_vfs_posix_file_t *o) {
     if (o->fd < 0) {
-        nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "I/O operation on closed file"));
+        mp_raise_msg(&mp_type_ValueError, "I/O operation on closed file");
     }
 }
 #else
@@ -200,7 +200,7 @@ STATIC mp_uint_t vfs_posix_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_
     }
 }
 
-STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = {
+STATIC const mp_rom_map_elem_t vfs_posix_rawfile_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_fileno), MP_ROM_PTR(&vfs_posix_file_fileno_obj) },
     { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
     { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
@@ -215,10 +215,10 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_posix_file___exit___obj) },
 };
 
-STATIC MP_DEFINE_CONST_DICT(rawfile_locals_dict, rawfile_locals_dict_table);
+STATIC MP_DEFINE_CONST_DICT(vfs_posix_rawfile_locals_dict, vfs_posix_rawfile_locals_dict_table);
 
 #if MICROPY_PY_IO_FILEIO
-STATIC const mp_stream_p_t fileio_stream_p = {
+STATIC const mp_stream_p_t vfs_posix_fileio_stream_p = {
     .read = vfs_posix_file_read,
     .write = vfs_posix_file_write,
     .ioctl = vfs_posix_file_ioctl,
@@ -231,12 +231,12 @@ const mp_obj_type_t mp_type_vfs_posix_fileio = {
     .make_new = vfs_posix_file_make_new,
     .getiter = mp_identity_getiter,
     .iternext = mp_stream_unbuffered_iter,
-    .protocol = &fileio_stream_p,
-    .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict,
+    .protocol = &vfs_posix_fileio_stream_p,
+    .locals_dict = (mp_obj_dict_t*)&vfs_posix_rawfile_locals_dict,
 };
 #endif
 
-STATIC const mp_stream_p_t textio_stream_p = {
+STATIC const mp_stream_p_t vfs_posix_textio_stream_p = {
     .read = vfs_posix_file_read,
     .write = vfs_posix_file_write,
     .ioctl = vfs_posix_file_ioctl,
@@ -250,8 +250,8 @@ const mp_obj_type_t mp_type_vfs_posix_textio = {
     .make_new = vfs_posix_file_make_new,
     .getiter = mp_identity_getiter,
     .iternext = mp_stream_unbuffered_iter,
-    .protocol = &textio_stream_p,
-    .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict,
+    .protocol = &vfs_posix_textio_stream_p,
+    .locals_dict = (mp_obj_dict_t*)&vfs_posix_rawfile_locals_dict,
 };
 
 const mp_obj_vfs_posix_file_t mp_sys_stdin_obj  = {{&mp_type_textio}, STDIN_FILENO};

extmod/webrepl/manifest.py

@@ -0,0 +1 @@
+freeze('.', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',))

extmod/webrepl/webrepl.py

@@ -0,0 +1,80 @@
+# This module should be imported from REPL, not run from command line.
+import socket
+import uos
+import network
+import uwebsocket
+import websocket_helper
+import _webrepl
+
+listen_s = None
+client_s = None
+
+def setup_conn(port, accept_handler):
+    global listen_s
+    listen_s = socket.socket()
+    listen_s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+    ai = socket.getaddrinfo("0.0.0.0", port)
+    addr = ai[0][4]
+
+    listen_s.bind(addr)
+    listen_s.listen(1)
+    if accept_handler:
+        listen_s.setsockopt(socket.SOL_SOCKET, 20, accept_handler)
+    for i in (network.AP_IF, network.STA_IF):
+        iface = network.WLAN(i)
+        if iface.active():
+            print("WebREPL daemon started on ws://%s:%d" % (iface.ifconfig()[0], port))
+    return listen_s
+
+
+def accept_conn(listen_sock):
+    global client_s
+    cl, remote_addr = listen_sock.accept()
+    prev = uos.dupterm(None)
+    uos.dupterm(prev)
+    if prev:
+        print("\nConcurrent WebREPL connection from", remote_addr, "rejected")
+        cl.close()
+        return
+    print("\nWebREPL connection from:", remote_addr)
+    client_s = cl
+    websocket_helper.server_handshake(cl)
+    ws = uwebsocket.websocket(cl, True)
+    ws = _webrepl._webrepl(ws)
+    cl.setblocking(False)
+    # notify REPL on socket incoming data (ESP32/ESP8266-only)
+    if hasattr(uos, 'dupterm_notify'):
+        cl.setsockopt(socket.SOL_SOCKET, 20, uos.dupterm_notify)
+    uos.dupterm(ws)
+
+
+def stop():
+    global listen_s, client_s
+    uos.dupterm(None)
+    if client_s:
+        client_s.close()
+    if listen_s:
+        listen_s.close()
+
+
+def start(port=8266, password=None):
+    stop()
+    if password is None:
+        try:
+            import webrepl_cfg
+            _webrepl.password(webrepl_cfg.PASS)
+            setup_conn(port, accept_conn)
+            print("Started webrepl in normal mode")
+        except:
+            print("WebREPL is not configured, run 'import webrepl_setup'")
+    else:
+        _webrepl.password(password)
+        setup_conn(port, accept_conn)
+        print("Started webrepl in manual override mode")
+
+
+def start_foreground(port=8266):
+    stop()
+    s = setup_conn(port, None)
+    accept_conn(s)

extmod/webrepl/webrepl_setup.py

@@ -0,0 +1,102 @@
+import sys
+#import uos as os
+import os
+import machine
+
+RC = "./boot.py"
+CONFIG = "./webrepl_cfg.py"
+
+def input_choice(prompt, choices):
+    while 1:
+        resp = input(prompt)
+        if resp in choices:
+            return resp
+
+def getpass(prompt):
+    return input(prompt)
+
+def input_pass():
+    while 1:
+        passwd1 = getpass("New password (4-9 chars): ")
+        if len(passwd1) < 4 or len(passwd1) > 9:
+            print("Invalid password length")
+            continue
+        passwd2 = getpass("Confirm password: ")
+        if passwd1 == passwd2:
+            return passwd1
+        print("Passwords do not match")
+
+
+def exists(fname):
+    try:
+        with open(fname):
+            pass
+        return True
+    except OSError:
+        return False
+
+
+def get_daemon_status():
+    with open(RC) as f:
+        for l in f:
+            if "webrepl" in l:
+                if l.startswith("#"):
+                    return False
+                return True
+        return None
+
+
+def change_daemon(action):
+    LINES = ("import webrepl", "webrepl.start()")
+    with open(RC) as old_f, open(RC + ".tmp", "w") as new_f:
+        found = False
+        for l in old_f:
+            for patt in LINES:
+                if patt in l:
+                    found = True
+                    if action and l.startswith("#"):
+                        l = l[1:]
+                    elif not action and not l.startswith("#"):
+                        l = "#" + l
+            new_f.write(l)
+        if not found:
+            new_f.write("import webrepl\nwebrepl.start()\n")
+    # FatFs rename() is not POSIX compliant, will raise OSError if
+    # dest file exists.
+    os.remove(RC)
+    os.rename(RC + ".tmp", RC)
+
+
+def main():
+    status = get_daemon_status()
+
+    print("WebREPL daemon auto-start status:", "enabled" if status else "disabled")
+    print("\nWould you like to (E)nable or (D)isable it running on boot?")
+    print("(Empty line to quit)")
+    resp = input("> ").upper()
+
+    if resp == "E":
+        if exists(CONFIG):
+            resp2 = input_choice("Would you like to change WebREPL password? (y/n) ", ("y", "n", ""))
+        else:
+            print("To enable WebREPL, you must set password for it")
+            resp2 = "y"
+
+        if resp2 == "y":
+            passwd = input_pass()
+            with open(CONFIG, "w") as f:
+                f.write("PASS = %r\n" % passwd)
+
+
+    if resp not in ("D", "E") or (resp == "D" and not status) or (resp == "E" and status):
+        print("No further action required")
+        sys.exit()
+
+    change_daemon(resp == "E")
+
+    print("Changes will be activated after reboot")
+    resp = input_choice("Would you like to reboot now? (y/n) ", ("y", "n", ""))
+    if resp == "y":
+        machine.reset()
+
+main()

extmod/webrepl/websocket_helper.py

@@ -0,0 +1,74 @@
+import sys
+try:
+    import ubinascii as binascii
+except:
+    import binascii
+try:
+    import uhashlib as hashlib
+except:
+    import hashlib
+
+DEBUG = 0
+
+def server_handshake(sock):
+    clr = sock.makefile("rwb", 0)
+    l = clr.readline()
+    #sys.stdout.write(repr(l))
+
+    webkey = None
+
+    while 1:
+        l = clr.readline()
+        if not l:
+            raise OSError("EOF in headers")
+        if l == b"\r\n":
+            break
+    #    sys.stdout.write(l)
+        h, v = [x.strip() for x in l.split(b":", 1)]
+        if DEBUG:
+            print((h, v))
+        if h == b'Sec-WebSocket-Key':
+            webkey = v
+
+    if not webkey:
+        raise OSError("Not a websocket request")
+
+    if DEBUG:
+        print("Sec-WebSocket-Key:", webkey, len(webkey))
+
+    d = hashlib.sha1(webkey)
+    d.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11")
+    respkey = d.digest()
+    respkey = binascii.b2a_base64(respkey)[:-1]
+    if DEBUG:
+        print("respkey:", respkey)
+
+    sock.send(b"""\
+HTTP/1.1 101 Switching Protocols\r
+Upgrade: websocket\r
+Connection: Upgrade\r
+Sec-WebSocket-Accept: """)
+    sock.send(respkey)
+    sock.send("\r\n\r\n")
+
+
+# Very simplified client handshake, works for MicroPython's
+# websocket server implementation, but probably not for other
+# servers.
+def client_handshake(sock):
+    cl = sock.makefile("rwb", 0)
+    cl.write(b"""\
+GET / HTTP/1.1\r
+Host: echo.websocket.org\r
+Connection: Upgrade\r
+Upgrade: websocket\r
+Sec-WebSocket-Key: foo\r
+\r
+""")
+    l = cl.readline()
+#    print(l)
+    while 1:
+        l = cl.readline()
+        if l == b"\r\n":
+            break
+#        sys.stdout.write(l)

lib/utils/gchelper.h

@@ -0,0 +1,34 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H
+#define MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H
+
+#include <stdint.h>
+
+uintptr_t gc_helper_get_sp(void);
+uintptr_t gc_helper_get_regs_and_sp(uintptr_t *regs);
+
+#endif // MICROPY_INCLUDED_LIB_UTILS_GCHELPER_H

lib/utils/gchelper_m0.s

@@ -0,0 +1,61 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2018 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+    .syntax unified
+    .cpu cortex-m0
+    .thumb
+
+    .section .text
+    .align 2
+
+    .global gc_helper_get_regs_and_sp
+    .type gc_helper_get_regs_and_sp, %function
+
+@ uint gc_helper_get_regs_and_sp(r0=uint regs[10])
+gc_helper_get_regs_and_sp:
+    @ store registers into given array
+    str    r4, [r0, #0]
+    str    r5, [r0, #4]
+    str    r6, [r0, #8]
+    str    r7, [r0, #12]
+    mov    r1, r8
+    str    r1, [r0, #16]
+    mov    r1, r9
+    str    r1, [r0, #20]
+    mov    r1, r10
+    str    r1, [r0, #24]
+    mov    r1, r11
+    str    r1, [r0, #28]
+    mov    r1, r12
+    str    r1, [r0, #32]
+    mov    r1, r13
+    str    r1, [r0, #36]
+
+    @ return the sp
+    mov    r0, sp
+    bx     lr
+
+    .size gc_helper_get_regs_and_sp, .-gc_helper_get_regs_and_sp

lib/utils/gchelper_m3.s

@@ -0,0 +1,67 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2013-2014 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+    .syntax unified
+    .cpu cortex-m3
+    .thumb
+
+    .section .text
+    .align  2
+
+    .global gc_helper_get_sp
+    .type gc_helper_get_sp, %function
+
+@ uint gc_helper_get_sp(void)
+gc_helper_get_sp:
+    @ return the sp
+    mov     r0, sp
+    bx      lr
+
+    .size gc_helper_get_sp, .-gc_helper_get_sp
+
+
+    .global gc_helper_get_regs_and_sp
+    .type gc_helper_get_regs_and_sp, %function
+
+@ uint gc_helper_get_regs_and_sp(r0=uint regs[10])
+gc_helper_get_regs_and_sp:
+    @ store registers into given array
+    str     r4, [r0], #4
+    str     r5, [r0], #4
+    str     r6, [r0], #4
+    str     r7, [r0], #4
+    str     r8, [r0], #4
+    str     r9, [r0], #4
+    str     r10, [r0], #4
+    str     r11, [r0], #4
+    str     r12, [r0], #4
+    str     r13, [r0], #4
+
+    @ return the sp
+    mov     r0, sp
+    bx      lr
+
+    .size gc_helper_get_regs_and_sp, .-gc_helper_get_regs_and_sp

lib/utils/interrupt_char.c

@@ -29,7 +29,7 @@
 
 #if MICROPY_KBD_EXCEPTION
 
-int mp_interrupt_char;
+int mp_interrupt_char = -1;
 
 void mp_hal_set_interrupt_char(int c) {
     if (c != -1) {

lib/utils/mpirq.c

@@ -78,10 +78,8 @@ void mp_irq_handler(mp_irq_obj_t *self) {
             }
             gc_unlock();
         } else {
-#if MICROPY_ENABLE_SCHEDULER
             // Schedule call to user function
             mp_sched_schedule(self->handler, self->parent);
-#endif
         }
     }
 }

lib/utils/pyexec.c

@@ -89,7 +89,7 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input
             // source is a lexer, parse and compile the script
             qstr source_name = lex->source_name;
             mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
-            module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
+            module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL);
             #else
             mp_raise_msg(&mp_type_RuntimeError, "script compilation not supported");
             #endif
@@ -482,8 +482,8 @@ friendly_repl_reset:
         } else if (ret == CHAR_CTRL_D) {
             // exit for a soft reset
             mp_hal_stdout_tx_str("\r\n");
-            //TODO it will occur assert on RT-Thread platform, so comment it
-//            vstr_clear(&line);
+            // TODO it will occur assert on RT-Thread platform, so comment it
+            // vstr_clear(&line);
             return PYEXEC_FORCED_EXIT;
         } else if (ret == CHAR_CTRL_E) {
             // paste mode
@@ -542,6 +542,18 @@ int pyexec_file(const char *filename) {
     return parse_compile_execute(filename, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_FILENAME);
 }
 
+int pyexec_file_if_exists(const char *filename) {
+    #if MICROPY_MODULE_FROZEN
+    if (mp_frozen_stat(filename) == MP_IMPORT_STAT_FILE) {
+        return pyexec_frozen_module(filename);
+    }
+    #endif
+    if (mp_import_stat(filename) != MP_IMPORT_STAT_FILE) {
+        return 1; // success (no file is the same as an empty file executing without fail)
+    }
+    return pyexec_file(filename);
+}
+
 #if MICROPY_MODULE_FROZEN
 int pyexec_frozen_module(const char *name) {
     void *frozen_data;

lib/utils/pyexec.h

@@ -46,6 +46,7 @@ extern int pyexec_system_exit;
 int pyexec_raw_repl(void);
 int pyexec_friendly_repl(void);
 int pyexec_file(const char *filename);
+int pyexec_file_if_exists(const char *filename);
 int pyexec_frozen_module(const char *name);
 void pyexec_event_repl_init(void);
 int pyexec_event_repl_process_char(int c);

lib/utils/sys_stdio_mphal.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013-2017 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -85,6 +85,17 @@ STATIC mp_uint_t stdio_write(mp_obj_t self_in, const void *buf, mp_uint_t size,
     }
 }
 
+// TODO NOT IMPLEMENT STDIO_IOCTL ON RT-THREAD YET
+// STATIC mp_uint_t stdio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+//     (void)self_in;
+//     if (request == MP_STREAM_POLL) {
+//         return mp_hal_stdio_poll(arg);
+//     } else {
+//         *errcode = MP_EINVAL;
+//         return MP_STREAM_ERROR;
+//     }
+// }
+
 STATIC mp_obj_t stdio_obj___exit__(size_t n_args, const mp_obj_t *args) {
     return mp_const_none;
 }
@@ -112,6 +123,7 @@ STATIC MP_DEFINE_CONST_DICT(stdio_locals_dict, stdio_locals_dict_table);
 STATIC const mp_stream_p_t stdio_obj_stream_p = {
     .read = stdio_read,
     .write = stdio_write,
+    // .ioctl = stdio_ioctl,
     .is_text = true,
 };
 
@@ -146,6 +158,7 @@ STATIC mp_uint_t stdio_buffer_write(mp_obj_t self_in, const void *buf, mp_uint_t
 STATIC const mp_stream_p_t stdio_buffer_obj_stream_p = {
     .read = stdio_buffer_read,
     .write = stdio_buffer_write,
+    // .ioctl = stdio_ioctl,
     .is_text = false,
 };
 

port/frozen_mpy.c

@@ -2,6 +2,7 @@
 #include "py/objint.h"
 #include "py/objstr.h"
 #include "py/emitglue.h"
+#include "py/nativeglue.h"
 
 #if MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE != 0
 #error "incompatible MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE"
@@ -46,48 +47,48 @@ const qstr_pool_t mp_qstr_frozen_const_pool = {
 };
 
 // frozen bytecode for file frozentest.py, scope frozentest_<module>
-STATIC const byte bytecode_data_frozentest__lt_module_gt_[92] = {
-    0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d,
+STATIC const byte fun_data_frozentest__lt_module_gt_[85] = {
+    0x10, 0x16,
     MP_QSTR__lt_module_gt_ & 0xff, MP_QSTR__lt_module_gt_ >> 8,
     MP_QSTR_frozentest_dot_py & 0xff, MP_QSTR_frozentest_dot_py >> 8,
-    0x2a, 0x28, 0x28, 0x28, 0x2b, 0x28, 0x00, 0x00, 0xff,
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x16, MP_QSTR_uPy & 0xff, MP_QSTR_uPy >> 8,
-    0x64, 0x01, 
-    0x32, 
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x17, 0x00, 
-    0x64, 0x01, 
-    0x32, 
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x17, 0x01, 
-    0x64, 0x01, 
-    0x32, 
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x17, 0x02, 
-    0x64, 0x01, 
-    0x32, 
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x14, 0xba, 0xef, 0x9a, 0x15, 
-    0x64, 0x01, 
-    0x32, 
+    0x29, 0x28, 0x28, 0x28, 0x2b, 0x28, 0x00,
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x10, MP_QSTR_uPy & 0xff, MP_QSTR_uPy >> 8, 
+    0x34, 0x01, 
+    0x59, 
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x23, 0x00, 
+    0x34, 0x01, 
+    0x59, 
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x23, 0x01, 
+    0x34, 0x01, 
+    0x59, 
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x23, 0x02, 
+    0x34, 0x01, 
+    0x59, 
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x22, 0xba, 0xef, 0x9a, 0x15, 
+    0x34, 0x01, 
+    0x59, 
     0x80, 
-    0x35, 0x0f, 0x80, 
-    0x30, 
-    0x24, MP_QSTR_i & 0xff, MP_QSTR_i >> 8,
-    0x1b, MP_QSTR_print & 0xff, MP_QSTR_print >> 8,
-    0x1b, MP_QSTR_i & 0xff, MP_QSTR_i >> 8,
-    0x64, 0x01, 
-    0x32, 
+    0x42, 0x0f, 0x80, 
+    0x57, 
+    0x16, MP_QSTR_i & 0xff, MP_QSTR_i >> 8, 
+    0x11, MP_QSTR_print & 0xff, MP_QSTR_print >> 8, 
+    0x11, MP_QSTR_i & 0xff, MP_QSTR_i >> 8, 
+    0x34, 0x01, 
+    0x59, 
     0x81, 
     0xe5, 
-    0x30, 
+    0x57, 
     0x84, 
     0xd7, 
-    0x36, 0xeb, 0x7f, 
-    0x32, 
-    0x11, 
-    0x5b, 
+    0x43, 0xeb, 0x7f, 
+    0x59, 
+    0x51, 
+    0x63, 
 };
 STATIC const mp_obj_str_t const_obj_frozentest__lt_module_gt__0 = {{&mp_type_str}, 246, 34, (const byte*)"\x61\x20\x6c\x6f\x6e\x67\x20\x73\x74\x72\x69\x6e\x67\x20\x74\x68\x61\x74\x20\x69\x73\x20\x6e\x6f\x74\x20\x69\x6e\x74\x65\x72\x6e\x65\x64"};
 STATIC const mp_obj_str_t const_obj_frozentest__lt_module_gt__1 = {{&mp_type_str}, 200, 38, (const byte*)"\x61\x20\x73\x74\x72\x69\x6e\x67\x20\x74\x68\x61\x74\x20\x68\x61\x73\x20\x75\x6e\x69\x63\x6f\x64\x65\x20\xce\xb1\xce\xb2\xce\xb3\x20\x63\x68\x61\x72\x73"};
@@ -101,15 +102,36 @@ const mp_raw_code_t raw_code_frozentest__lt_module_gt_ = {
     .kind = MP_CODE_BYTECODE,
     .scope_flags = 0x00,
     .n_pos_args = 0,
-    .data.u_byte = {
-        .bytecode = bytecode_data_frozentest__lt_module_gt_,
-        .const_table = (mp_uint_t*)const_table_data_frozentest__lt_module_gt_,
-        #if MICROPY_PERSISTENT_CODE_SAVE
-        .bc_len = 92,
-        .n_obj = 3,
-        .n_raw_code = 0,
-        #endif
+    .fun_data = fun_data_frozentest__lt_module_gt_,
+    .const_table = (mp_uint_t*)const_table_data_frozentest__lt_module_gt_,
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    .fun_data_len = 85,
+    .n_obj = 3,
+    .n_raw_code = 0,
+    #if MICROPY_PY_SYS_SETTRACE
+    .prelude = {
+        .n_state = 3,
+        .n_exc_stack = 0,
+        .scope_flags = 0,
+        .n_pos_args = 0,
+        .n_kwonly_args = 0,
+        .n_def_pos_args = 0,
+        .qstr_block_name = MP_QSTR__lt_module_gt_,
+        .qstr_source_file = MP_QSTR_frozentest_dot_py,
+        .line_info = fun_data_frozentest__lt_module_gt_ + 0,
+        .opcodes = fun_data_frozentest__lt_module_gt_ + 13,
     },
+    .line_of_definition = 0,
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    .prelude_offset = 0,
+    .n_qstr = 0,
+    .qstr_link = NULL,
+    #endif
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    .type_sig = 0,
+    #endif
 };
 
 const char mp_frozen_mpy_names[] = {
@@ -117,4 +139,4 @@ const char mp_frozen_mpy_names[] = {
 "\0"};
 const mp_raw_code_t *const mp_frozen_mpy_content[] = {
     &raw_code_frozentest__lt_module_gt_,
-};
+};

port/genhdr/mpversion.h

@@ -1,4 +1,4 @@
 // This file was generated by py/makeversionhdr.py
-#define MICROPY_GIT_TAG "v1.10.3.99"
+#define MICROPY_GIT_TAG "v1.12.99.99"
 #define MICROPY_GIT_HASH "ed1a88e-dirty"
-#define MICROPY_BUILD_DATE "2019-09-24"
+#define MICROPY_BUILD_DATE "2020-02-20"

port/genhdr/qstrdefs.generated.h

@@ -1,6 +1,6 @@
 // This file was automatically generated by makeqstrdata.py
 
-QDEF(MP_QSTR_NULL, (const byte*)"\x00\x00" "")
+QDEF(MP_QSTRnull, (const byte*)"\x00\x00" "")
 QDEF(MP_QSTR_, (const byte*)"\x05\x00" "")
 QDEF(MP_QSTR___abs__, (const byte*)"\x95\x07" "__abs__")
 QDEF(MP_QSTR___add__, (const byte*)"\xc4\x07" "__add__")
@@ -770,3 +770,7 @@ QDEF(MP_QSTR_IRQ_HIGH_LEVEL, (const byte*)"\x57\x0e" "IRQ_HIGH_LEVEL")
 QDEF(MP_QSTR_IRQ_LOW_LEVEL, (const byte*)"\x8d\x0d" "IRQ_LOW_LEVEL")
 QDEF(MP_QSTR_show_bmp, (const byte*)"\xe6\x08" "show_bmp")
 QDEF(MP_QSTR_userfunc, (const byte*)"\x4a\x08" "userfunc")
+QDEF(MP_QSTR_uarray, (const byte*)"\x89\x06" "uarray")
+QDEF(MP_QSTR_mpy, (const byte*)"\xc1\x03" "mpy")
+QDEF(MP_QSTR___matmul__, (const byte*)"\x49\x0a" "__matmul__")
+QDEF(MP_QSTR___bases__, (const byte*)"\x03\x09" "__bases__")

port/mpconfigport.h

@@ -112,6 +112,7 @@
 #define MICROPY_PY_STRUCT           (1)
 #define MICROPY_PY_SYS              (1)
 #define MICROPY_MODULE_FROZEN_MPY   (1)
+#define MICROPY_MODULE_FROZEN_STR   (0)
 #define MICROPY_CPYTHON_COMPAT      (1)
 #define MICROPY_LONGINT_IMPL        (MICROPY_LONGINT_IMPL_MPZ)
 #define MICROPY_FLOAT_IMPL          (MICROPY_FLOAT_IMPL_DOUBLE)

port/mpy_main.c

@@ -53,7 +53,7 @@ void do_str(const char *src, mp_parse_input_kind_t input_kind) {
         mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR__lt_stdin_gt_, src, strlen(src), 0);
         qstr source_name = lex->source_name;
         mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
-        mp_obj_t module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, true);
+        mp_obj_t module_fun = mp_compile(&parse_tree, source_name, true);
         mp_call_function_0(module_fun);
         nlr_pop();
     } else {

py/asmarm.c

@@ -40,7 +40,11 @@
 
 void asm_arm_end_pass(asm_arm_t *as) {
     if (as->base.pass == MP_ASM_PASS_EMIT) {
-#ifdef __arm__
+#if defined(__linux__) && defined(__GNUC__)
+        char *start = mp_asm_base_get_code(&as->base);
+        char *end = start + mp_asm_base_get_code_size(&as->base);
+        __builtin___clear_cache(start, end);
+#elif defined(__arm__)
         // flush I- and D-cache
         asm volatile(
                 "0:"
@@ -197,7 +201,16 @@ void asm_arm_mov_reg_reg(asm_arm_t *as, uint reg_dest, uint reg_src) {
     emit_al(as, asm_arm_op_mov_reg(reg_dest, reg_src));
 }
 
-void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
+size_t asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
+    // Insert immediate into code and jump over it
+    emit_al(as, 0x59f0000 | (rd << 12)); // ldr rd, [pc]
+    emit_al(as, 0xa000000); // b pc
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
+    emit(as, imm);
+    return loc;
+}
+
+void asm_arm_mov_reg_i32_optimised(asm_arm_t *as, uint rd, int imm) {
     // TODO: There are more variants of immediate values
     if ((imm & 0xFF) == imm) {
         emit_al(as, asm_arm_op_mov_imm(rd, imm));
@@ -205,10 +218,7 @@ void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm) {
         // mvn is "move not", not "move negative"
         emit_al(as, asm_arm_op_mvn_imm(rd, ~imm));
     } else {
-        //Insert immediate into code and jump over it
-        emit_al(as, 0x59f0000 | (rd << 12)); // ldr rd, [pc]
-        emit_al(as, 0xa000000); // b pc
-        emit(as, imm);
+        asm_arm_mov_reg_i32(as, rd, imm);
     }
 }
 

py/asmarm.h

@@ -81,7 +81,8 @@ void asm_arm_bkpt(asm_arm_t *as);
 
 // mov
 void asm_arm_mov_reg_reg(asm_arm_t *as, uint reg_dest, uint reg_src);
-void asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm);
+size_t asm_arm_mov_reg_i32(asm_arm_t *as, uint rd, int imm);
+void asm_arm_mov_reg_i32_optimised(asm_arm_t *as, uint rd, int imm);
 void asm_arm_mov_local_reg(asm_arm_t *as, int local_num, uint rd);
 void asm_arm_mov_reg_local(asm_arm_t *as, uint rd, int local_num);
 void asm_arm_setcc_reg(asm_arm_t *as, uint rd, uint cond);
@@ -177,7 +178,9 @@ void asm_arm_bx_reg(asm_arm_t *as, uint reg_src);
 #define ASM_CALL_IND(as, idx) asm_arm_bl_ind(as, idx, ASM_ARM_REG_R3)
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_arm_mov_local_reg((as), (local_num), (reg_src))
-#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_arm_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_arm_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_arm_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmbase.c

@@ -31,7 +31,7 @@
 #include "py/misc.h"
 #include "py/asmbase.h"
 
-#if MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
+#if MICROPY_EMIT_MACHINE_CODE
 
 void mp_asm_base_init(mp_asm_base_t *as, size_t max_num_labels) {
     as->max_num_labels = max_num_labels;
@@ -99,4 +99,4 @@ void mp_asm_base_data(mp_asm_base_t* as, unsigned int bytesize, uintptr_t val) {
     }
 }
 
-#endif // MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
+#endif // MICROPY_EMIT_MACHINE_CODE

py/asmbase.h

@@ -60,7 +60,7 @@ static inline size_t mp_asm_base_get_code_size(mp_asm_base_t *as) {
 
 static inline void *mp_asm_base_get_code(mp_asm_base_t *as) {
     #if defined(MP_PLAT_COMMIT_EXEC)
-    return MP_PLAT_COMMIT_EXEC(as->code_base, as->code_size);
+    return MP_PLAT_COMMIT_EXEC(as->code_base, as->code_size, NULL);
     #else
     return as->code_base;
     #endif

py/asmthumb.c

@@ -33,6 +33,8 @@
 // wrapper around everything in this file
 #if MICROPY_EMIT_THUMB || MICROPY_EMIT_INLINE_THUMB
 
+#include "py/mpstate.h"
+#include "py/persistentcode.h"
 #include "py/mphal.h"
 #include "py/asmthumb.h"
 
@@ -118,6 +120,21 @@ STATIC void asm_thumb_write_word32(asm_thumb_t *as, int w32) {
 void asm_thumb_entry(asm_thumb_t *as, int num_locals) {
     assert(num_locals >= 0);
 
+    // If this Thumb machine code is run from ARM state then add a prelude
+    // to switch to Thumb state for the duration of the function.
+    #if MICROPY_DYNAMIC_COMPILER || MICROPY_EMIT_ARM || (defined(__arm__) && !defined(__thumb2__))
+    #if MICROPY_DYNAMIC_COMPILER
+    if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_ARMV6)
+    #endif
+    {
+        asm_thumb_op32(as, 0x4010, 0xe92d); // push {r4, lr}
+        asm_thumb_op32(as, 0xe009, 0xe28f); // add lr, pc, 8 + 1
+        asm_thumb_op32(as, 0xff3e, 0xe12f); // blx lr
+        asm_thumb_op32(as, 0x4010, 0xe8bd); // pop {r4, lr}
+        asm_thumb_op32(as, 0xff1e, 0xe12f); // bx lr
+    }
+    #endif
+
     // work out what to push and how many extra spaces to reserve on stack
     // so that we have enough for all locals and it's aligned an 8-byte boundary
     // we push extra regs (r1, r2, r3) to help do the stack adjustment
@@ -225,10 +242,12 @@ void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src) {
 }
 
 // if loading lo half with movw, the i16 value will be zero extended into the r32 register!
-void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) {
+size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src) {
     assert(reg_dest < ASM_THUMB_REG_R15);
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     // mov[wt] reg_dest, #i16_src
     asm_thumb_op32(as, mov_op | ((i16_src >> 1) & 0x0400) | ((i16_src >> 12) & 0xf), ((i16_src << 4) & 0x7000) | (reg_dest << 8) | (i16_src & 0xff));
+    return loc;
 }
 
 #define OP_B_N(byte_offset) (0xe000 | (((byte_offset) >> 1) & 0x07ff))
@@ -271,12 +290,16 @@ bool asm_thumb_bl_label(asm_thumb_t *as, uint label) {
     return as->base.pass != MP_ASM_PASS_EMIT || SIGNED_FIT23(rel);
 }
 
-void asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) {
+size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32) {
     // movw, movt does it in 8 bytes
     // ldr [pc, #], dw does it in 6 bytes, but we might not reach to end of code for dw
 
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
+
     asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVW, reg_dest, i32);
     asm_thumb_mov_reg_i16(as, ASM_THUMB_OP_MOVT, reg_dest, i32 >> 16);
+
+    return loc;
 }
 
 void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32) {

py/asmthumb.h

@@ -241,14 +241,14 @@ static inline void asm_thumb_ldrh_rlo_rlo_i5(asm_thumb_t *as, uint rlo_dest, uin
 #define ASM_THUMB_OP_MOVT (0xf2c0)
 
 void asm_thumb_mov_reg_reg(asm_thumb_t *as, uint reg_dest, uint reg_src);
-void asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src);
+size_t asm_thumb_mov_reg_i16(asm_thumb_t *as, uint mov_op, uint reg_dest, int i16_src);
 
 // these return true if the destination is in range, false otherwise
 bool asm_thumb_b_n_label(asm_thumb_t *as, uint label);
 bool asm_thumb_bcc_nw_label(asm_thumb_t *as, int cond, uint label, bool wide);
 bool asm_thumb_bl_label(asm_thumb_t *as, uint label);
 
-void asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32_src); // convenience
+size_t asm_thumb_mov_reg_i32(asm_thumb_t *as, uint reg_dest, mp_uint_t i32_src); // convenience
 void asm_thumb_mov_reg_i32_optimised(asm_thumb_t *as, uint reg_dest, int i32_src); // convenience
 void asm_thumb_mov_local_reg(asm_thumb_t *as, int local_num_dest, uint rlo_src); // convenience
 void asm_thumb_mov_reg_local(asm_thumb_t *as, uint rlo_dest, int local_num); // convenience
@@ -315,6 +315,8 @@ void asm_thumb_bl_ind(asm_thumb_t *as, uint fun_id, uint reg_temp); // convenien
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_thumb_mov_reg_i16((as), ASM_THUMB_OP_MOVW, (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_thumb_mov_reg_i32((as), (reg_dest), (imm))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_thumb_mov_reg_local_addr((as), (reg_dest), (local_num))

py/asmx64.c

@@ -334,14 +334,16 @@ void asm_x64_mov_i8_to_r8(asm_x64_t *as, int src_i8, int dest_r64) {
 }
 */
 
-STATIC void asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64) {
+size_t asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64) {
     // cpu defaults to i32 to r64, with zero extension
     if (dest_r64 < 8) {
         asm_x64_write_byte_1(as, OPCODE_MOV_I64_TO_R64 | dest_r64);
     } else {
         asm_x64_write_byte_2(as, REX_PREFIX | REX_B, OPCODE_MOV_I64_TO_R64 | (dest_r64 & 7));
     }
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     asm_x64_write_word32(as, src_i32);
+    return loc;
 }
 
 void asm_x64_mov_i64_to_r64(asm_x64_t *as, int64_t src_i64, int dest_r64) {

py/asmx64.h

@@ -83,6 +83,7 @@ void asm_x64_nop(asm_x64_t* as);
 void asm_x64_push_r64(asm_x64_t* as, int src_r64);
 void asm_x64_pop_r64(asm_x64_t* as, int dest_r64);
 void asm_x64_mov_r64_r64(asm_x64_t* as, int dest_r64, int src_r64);
+size_t asm_x64_mov_i32_to_r64(asm_x64_t *as, int src_i32, int dest_r64);
 void asm_x64_mov_i64_to_r64(asm_x64_t* as, int64_t src_i64, int dest_r64);
 void asm_x64_mov_i64_to_r64_optimised(asm_x64_t *as, int64_t src_i64, int dest_r64);
 void asm_x64_mov_r8_to_mem8(asm_x64_t *as, int src_r64, int dest_r64, int dest_disp);
@@ -181,6 +182,8 @@ void asm_x64_call_ind(asm_x64_t* as, size_t fun_id, int temp_r32);
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x64_mov_r64_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_optimised((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x64_mov_i32_to_r64((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x64_mov_local_to_r64((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x64_mov_r64_r64((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x64_mov_local_addr_to_r64((as), (local_num), (reg_dest))

py/asmx86.c

@@ -236,9 +236,11 @@ void asm_x86_mov_i8_to_r8(asm_x86_t *as, int src_i8, int dest_r32) {
 }
 #endif
 
-void asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32) {
+size_t asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32) {
     asm_x86_write_byte_1(as, OPCODE_MOV_I32_TO_R32 | dest_r32);
+    size_t loc = mp_asm_base_get_code_pos(&as->base);
     asm_x86_write_word32(as, src_i32);
+    return loc;
 }
 
 void asm_x86_and_r32_r32(asm_x86_t *as, int dest_r32, int src_r32) {

py/asmx86.h

@@ -83,7 +83,7 @@ static inline void asm_x86_end_pass(asm_x86_t *as) {
 }
 
 void asm_x86_mov_r32_r32(asm_x86_t* as, int dest_r32, int src_r32);
-void asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32);
+size_t asm_x86_mov_i32_to_r32(asm_x86_t *as, int32_t src_i32, int dest_r32);
 void asm_x86_mov_r8_to_mem8(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
 void asm_x86_mov_r16_to_mem16(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
 void asm_x86_mov_r32_to_mem32(asm_x86_t *as, int src_r32, int dest_r32, int dest_disp);
@@ -179,6 +179,8 @@ void asm_x86_call_ind(asm_x86_t* as, size_t fun_id, mp_uint_t n_args, int temp_r
 
 #define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x86_mov_r32_to_local((as), (reg_src), (local_num))
 #define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest))
 #define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x86_mov_local_to_r32((as), (local_num), (reg_dest))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x86_mov_r32_r32((as), (reg_dest), (reg_src))
 #define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x86_mov_local_addr_to_r32((as), (local_num), (reg_dest))

py/asmxtensa.c

@@ -30,14 +30,13 @@
 #include "py/mpconfig.h"
 
 // wrapper around everything in this file
-#if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA
+#if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN
 
 #include "py/asmxtensa.h"
 
 #define WORD_SIZE (4)
 #define SIGNED_FIT8(x) ((((x) & 0xffffff80) == 0) || (((x) & 0xffffff80) == 0xffffff80))
 #define SIGNED_FIT12(x) ((((x) & 0xfffff800) == 0) || (((x) & 0xfffff800) == 0xfffff800))
-#define NUM_REGS_SAVED (5)
 
 void asm_xtensa_end_pass(asm_xtensa_t *as) {
     as->num_const = as->cur_const;
@@ -69,7 +68,7 @@ void asm_xtensa_entry(asm_xtensa_t *as, int num_locals) {
     as->const_table = (uint32_t*)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4);
 
     // adjust the stack-pointer to store a0, a12, a13, a14, a15 and locals, 16-byte aligned
-    as->stack_adjust = (((NUM_REGS_SAVED + num_locals) * WORD_SIZE) + 15) & ~15;
+    as->stack_adjust = (((ASM_XTENSA_NUM_REGS_SAVED + num_locals) * WORD_SIZE) + 15) & ~15;
     if (SIGNED_FIT8(-as->stack_adjust)) {
         asm_xtensa_op_addi(as, ASM_XTENSA_REG_A1, ASM_XTENSA_REG_A1, -as->stack_adjust);
     } else {
@@ -79,14 +78,14 @@ void asm_xtensa_entry(asm_xtensa_t *as, int num_locals) {
 
     // save return value (a0) and callee-save registers (a12, a13, a14, a15)
     asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
-    for (int i = 1; i < NUM_REGS_SAVED; ++i) {
+    for (int i = 1; i < ASM_XTENSA_NUM_REGS_SAVED; ++i) {
         asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A11 + i, ASM_XTENSA_REG_A1, i);
     }
 }
 
 void asm_xtensa_exit(asm_xtensa_t *as) {
     // restore registers
-    for (int i = NUM_REGS_SAVED - 1; i >= 1; --i) {
+    for (int i = ASM_XTENSA_NUM_REGS_SAVED - 1; i >= 1; --i) {
         asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A11 + i, ASM_XTENSA_REG_A1, i);
     }
     asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
@@ -102,6 +101,22 @@ void asm_xtensa_exit(asm_xtensa_t *as) {
     asm_xtensa_op_ret_n(as);
 }
 
+void asm_xtensa_entry_win(asm_xtensa_t *as, int num_locals) {
+    // jump over the constants
+    asm_xtensa_op_j(as, as->num_const * WORD_SIZE + 4 - 4);
+    mp_asm_base_get_cur_to_write_bytes(&as->base, 1); // padding/alignment byte
+    as->const_table = (uint32_t*)mp_asm_base_get_cur_to_write_bytes(&as->base, as->num_const * 4);
+
+    as->stack_adjust = 32 + ((((ASM_XTENSA_NUM_REGS_SAVED_WIN + num_locals) * WORD_SIZE) + 15) & ~15);
+    asm_xtensa_op_entry(as, ASM_XTENSA_REG_A1, as->stack_adjust);
+    asm_xtensa_op_s32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
+}
+
+void asm_xtensa_exit_win(asm_xtensa_t *as) {
+    asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A0, ASM_XTENSA_REG_A1, 0);
+    asm_xtensa_op_retw_n(as);
+}
+
 STATIC uint32_t get_label_dest(asm_xtensa_t *as, uint label) {
     assert(label < as->base.max_num_labels);
     return as->base.label_offsets[label];
@@ -156,31 +171,37 @@ void asm_xtensa_setcc_reg_reg_reg(asm_xtensa_t *as, uint cond, uint reg_dest, ui
     asm_xtensa_op_movi_n(as, reg_dest, 0);
 }
 
-void asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
+size_t asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
+    // load the constant
+    uint32_t const_table_offset = (uint8_t*)as->const_table - as->base.code_base;
+    size_t loc = const_table_offset + as->cur_const * WORD_SIZE;
+    asm_xtensa_op_l32r(as, reg_dest, as->base.code_offset, loc);
+    // store the constant in the table
+    if (as->const_table != NULL) {
+        as->const_table[as->cur_const] = i32;
+    }
+    ++as->cur_const;
+    return loc;
+}
+
+void asm_xtensa_mov_reg_i32_optimised(asm_xtensa_t *as, uint reg_dest, uint32_t i32) {
     if (SIGNED_FIT12(i32)) {
         asm_xtensa_op_movi(as, reg_dest, i32);
     } else {
-        // load the constant
-        uint32_t const_table_offset = (uint8_t*)as->const_table - as->base.code_base;
-        asm_xtensa_op_l32r(as, reg_dest, as->base.code_offset, const_table_offset + as->cur_const * WORD_SIZE);
-        // store the constant in the table
-        if (as->const_table != NULL) {
-            as->const_table[as->cur_const] = i32;
-        }
-        ++as->cur_const;
+        asm_xtensa_mov_reg_i32(as, reg_dest, i32);
     }
 }
 
 void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src) {
-    asm_xtensa_op_s32i(as, reg_src, ASM_XTENSA_REG_A1, NUM_REGS_SAVED + local_num);
+    asm_xtensa_op_s32i(as, reg_src, ASM_XTENSA_REG_A1, local_num);
 }
 
 void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num) {
-    asm_xtensa_op_l32i(as, reg_dest, ASM_XTENSA_REG_A1, NUM_REGS_SAVED + local_num);
+    asm_xtensa_op_l32i(as, reg_dest, ASM_XTENSA_REG_A1, local_num);
 }
 
 void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num) {
-    uint off = (NUM_REGS_SAVED + local_num) * WORD_SIZE;
+    uint off = local_num * WORD_SIZE;
     if (SIGNED_FIT8(off)) {
         asm_xtensa_op_addi(as, reg_dest, ASM_XTENSA_REG_A1, off);
     } else {
@@ -220,4 +241,13 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx) {
     asm_xtensa_op_callx0(as, ASM_XTENSA_REG_A0);
 }
 
-#endif // MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA
+void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx) {
+    if (idx < 16) {
+        asm_xtensa_op_l32i_n(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx);
+    } else {
+        asm_xtensa_op_l32i(as, ASM_XTENSA_REG_A8, ASM_XTENSA_REG_FUN_TABLE_WIN, idx);
+    }
+    asm_xtensa_op_callx8(as, ASM_XTENSA_REG_A8);
+}
+
+#endif // MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA || MICROPY_EMIT_XTENSAWIN

py/asmxtensa.h

@@ -26,6 +26,7 @@
 #ifndef MICROPY_INCLUDED_PY_ASMXTENSA_H
 #define MICROPY_INCLUDED_PY_ASMXTENSA_H
 
+#include "py/misc.h"
 #include "py/asmbase.h"
 
 // calling conventions:
@@ -36,6 +37,16 @@
 // callee save: a1, a12, a13, a14, a15
 // caller save: a3
 
+// With windowed registers, size 8:
+// - a0: return PC
+// - a1: stack pointer, full descending, aligned to 16 bytes
+// - a2-a7: incoming args, and essentially callee save
+// - a2: return value
+// - a8-a15: caller save temporaries
+// - a10-a15: input args to called function
+// - a10: return value of called function
+// note: a0-a7 are saved automatically via window shift of called function
+
 #define ASM_XTENSA_REG_A0  (0)
 #define ASM_XTENSA_REG_A1  (1)
 #define ASM_XTENSA_REG_A2  (2)
@@ -95,6 +106,10 @@
 #define ASM_XTENSA_ENCODE_RI7(op0, s, imm7) \
     ((((imm7) & 0xf) << 12) | ((s) << 8) | ((imm7) & 0x70) | (op0))
 
+// Number of registers saved on the stack upon entry to function
+#define ASM_XTENSA_NUM_REGS_SAVED (5)
+#define ASM_XTENSA_NUM_REGS_SAVED_WIN (1)
+
 typedef struct _asm_xtensa_t {
     mp_asm_base_t base;
     uint32_t cur_const;
@@ -108,11 +123,18 @@ void asm_xtensa_end_pass(asm_xtensa_t *as);
 void asm_xtensa_entry(asm_xtensa_t *as, int num_locals);
 void asm_xtensa_exit(asm_xtensa_t *as);
 
+void asm_xtensa_entry_win(asm_xtensa_t *as, int num_locals);
+void asm_xtensa_exit_win(asm_xtensa_t *as);
+
 void asm_xtensa_op16(asm_xtensa_t *as, uint16_t op);
 void asm_xtensa_op24(asm_xtensa_t *as, uint32_t op);
 
 // raw instructions
 
+static inline void asm_xtensa_op_entry(asm_xtensa_t *as, uint reg_src, int32_t num_bytes) {
+    asm_xtensa_op24(as, ASM_XTENSA_ENCODE_BRI12(6, reg_src, 0, 3, (num_bytes / 8) & 0xfff));
+}
+
 static inline void asm_xtensa_op_add_n(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) {
     asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(10, reg_dest, reg_src_a, reg_src_b));
 }
@@ -141,6 +163,10 @@ static inline void asm_xtensa_op_callx0(asm_xtensa_t *as, uint reg) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALLX(0, 0, 0, 0, reg, 3, 0));
 }
 
+static inline void asm_xtensa_op_callx8(asm_xtensa_t *as, uint reg) {
+    asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALLX(0, 0, 0, 0, reg, 3, 2));
+}
+
 static inline void asm_xtensa_op_j(asm_xtensa_t *as, int32_t rel18) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_CALL(6, 0, rel18 & 0x3ffff));
 }
@@ -193,6 +219,10 @@ static inline void asm_xtensa_op_ret_n(asm_xtensa_t *as) {
     asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(13, 15, 0, 0));
 }
 
+static inline void asm_xtensa_op_retw_n(asm_xtensa_t *as) {
+    asm_xtensa_op16(as, ASM_XTENSA_ENCODE_RRRN(13, 15, 0, 1));
+}
+
 static inline void asm_xtensa_op_s8i(asm_xtensa_t *as, uint reg_src, uint reg_base, uint byte_offset) {
     asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 4, reg_base, reg_src, byte_offset & 0xff));
 }
@@ -238,15 +268,18 @@ void asm_xtensa_j_label(asm_xtensa_t *as, uint label);
 void asm_xtensa_bccz_reg_label(asm_xtensa_t *as, uint cond, uint reg, uint label);
 void asm_xtensa_bcc_reg_reg_label(asm_xtensa_t *as, uint cond, uint reg1, uint reg2, uint label);
 void asm_xtensa_setcc_reg_reg_reg(asm_xtensa_t *as, uint cond, uint reg_dest, uint reg_src1, uint reg_src2);
-void asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
+size_t asm_xtensa_mov_reg_i32(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
+void asm_xtensa_mov_reg_i32_optimised(asm_xtensa_t *as, uint reg_dest, uint32_t i32);
 void asm_xtensa_mov_local_reg(asm_xtensa_t *as, int local_num, uint reg_src);
 void asm_xtensa_mov_reg_local(asm_xtensa_t *as, uint reg_dest, int local_num);
 void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_num);
 void asm_xtensa_mov_reg_pcrel(asm_xtensa_t *as, uint reg_dest, uint label);
 void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
+void asm_xtensa_call_ind_win(asm_xtensa_t *as, uint idx);
 
 // Holds a pointer to mp_fun_table
 #define ASM_XTENSA_REG_FUN_TABLE ASM_XTENSA_REG_A15
+#define ASM_XTENSA_REG_FUN_TABLE_WIN ASM_XTENSA_REG_A7
 
 #if GENERIC_ASM_API
 
@@ -255,6 +288,9 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 
 #define ASM_WORD_SIZE (4)
 
+#if !GENERIC_ASM_API_WIN
+// Configuration for non-windowed calls
+
 #define REG_RET ASM_XTENSA_REG_A2
 #define REG_ARG_1 ASM_XTENSA_REG_A2
 #define REG_ARG_2 ASM_XTENSA_REG_A3
@@ -271,12 +307,47 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 #define REG_LOCAL_3 ASM_XTENSA_REG_A14
 #define REG_LOCAL_NUM (3)
 
+#define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED
 #define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE
 
+#define ASM_ENTRY(as, nlocal)   asm_xtensa_entry((as), (nlocal))
+#define ASM_EXIT(as)            asm_xtensa_exit((as))
+#define ASM_CALL_IND(as, idx)   asm_xtensa_call_ind((as), (idx))
+
+#else
+// Configuration for windowed calls with window size 8
+
+#define REG_PARENT_RET ASM_XTENSA_REG_A2
+#define REG_PARENT_ARG_1 ASM_XTENSA_REG_A2
+#define REG_PARENT_ARG_2 ASM_XTENSA_REG_A3
+#define REG_PARENT_ARG_3 ASM_XTENSA_REG_A4
+#define REG_PARENT_ARG_4 ASM_XTENSA_REG_A5
+#define REG_RET ASM_XTENSA_REG_A10
+#define REG_ARG_1 ASM_XTENSA_REG_A10
+#define REG_ARG_2 ASM_XTENSA_REG_A11
+#define REG_ARG_3 ASM_XTENSA_REG_A12
+#define REG_ARG_4 ASM_XTENSA_REG_A13
+
+#define REG_TEMP0 ASM_XTENSA_REG_A10
+#define REG_TEMP1 ASM_XTENSA_REG_A11
+#define REG_TEMP2 ASM_XTENSA_REG_A12
+
+#define REG_LOCAL_1 ASM_XTENSA_REG_A4
+#define REG_LOCAL_2 ASM_XTENSA_REG_A5
+#define REG_LOCAL_3 ASM_XTENSA_REG_A6
+#define REG_LOCAL_NUM (3)
+
+#define ASM_NUM_REGS_SAVED ASM_XTENSA_NUM_REGS_SAVED_WIN
+#define REG_FUN_TABLE ASM_XTENSA_REG_FUN_TABLE_WIN
+
+#define ASM_ENTRY(as, nlocal)   asm_xtensa_entry_win((as), (nlocal))
+#define ASM_EXIT(as)            asm_xtensa_exit_win((as))
+#define ASM_CALL_IND(as, idx)   asm_xtensa_call_ind_win((as), (idx))
+
+#endif
+
 #define ASM_T               asm_xtensa_t
 #define ASM_END_PASS        asm_xtensa_end_pass
-#define ASM_ENTRY           asm_xtensa_entry
-#define ASM_EXIT            asm_xtensa_exit
 
 #define ASM_JUMP            asm_xtensa_j_label
 #define ASM_JUMP_IF_REG_ZERO(as, reg, label, bool_test) \
@@ -286,13 +357,14 @@ void asm_xtensa_call_ind(asm_xtensa_t *as, uint idx);
 #define ASM_JUMP_IF_REG_EQ(as, reg1, reg2, label) \
     asm_xtensa_bcc_reg_reg_label(as, ASM_XTENSA_CC_EQ, reg1, reg2, label)
 #define ASM_JUMP_REG(as, reg) asm_xtensa_op_jx((as), (reg))
-#define ASM_CALL_IND(as, idx) asm_xtensa_call_ind((as), (idx))
 
-#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), (local_num), (reg_src))
-#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
-#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), (local_num))
+#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), ASM_NUM_REGS_SAVED + (local_num), (reg_src))
+#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32_optimised((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_U16(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_IMM_FIX_WORD(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm))
+#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))
 #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mov_n((as), (reg_dest), (reg_src))
-#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), (local_num))
+#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), ASM_NUM_REGS_SAVED + (local_num))
 #define ASM_MOV_REG_PCREL(as, reg_dest, label) asm_xtensa_mov_reg_pcrel((as), (reg_dest), (label))
 
 #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) \

py/bc.c

@@ -40,6 +40,8 @@
 #define DEBUG_printf(...) (void)0
 #endif
 
+#if !MICROPY_PERSISTENT_CODE
+
 mp_uint_t mp_decode_uint(const byte **ptr) {
     mp_uint_t unum = 0;
     byte val;
@@ -70,6 +72,8 @@ const byte *mp_decode_uint_skip(const byte *ptr) {
     return ptr;
 }
 
+#endif
+
 STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) {
 #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE
     // generic message, used also for other argument issues
@@ -119,16 +123,22 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
     code_state->prev = NULL;
     #endif
 
-    // get params
-    size_t n_state = mp_decode_uint(&code_state->ip);
-    code_state->ip = mp_decode_uint_skip(code_state->ip); // skip n_exc_stack
-    size_t scope_flags = *code_state->ip++;
-    size_t n_pos_args = *code_state->ip++;
-    size_t n_kwonly_args = *code_state->ip++;
-    size_t n_def_pos_args = *code_state->ip++;
+    #if MICROPY_PY_SYS_SETTRACE
+    code_state->prev_state = NULL;
+    code_state->frame = NULL;
+    #endif
+
+    // Get cached n_state (rather than decode it again)
+    size_t n_state = code_state->n_state;
+
+    // Decode prelude
+    size_t n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args;
+    MP_BC_PRELUDE_SIG_DECODE_INTO(code_state->ip, n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args);
+    (void)n_state_unused;
+    (void)n_exc_stack_unused;
 
     code_state->sp = &code_state->state[0] - 1;
-    code_state->exc_sp = (mp_exc_stack_t*)(code_state->state + n_state) - 1;
+    code_state->exc_sp_idx = 0;
 
     // zero out the local stack to begin with
     memset(code_state->state, 0, n_state * sizeof(*code_state->state));
@@ -263,19 +273,25 @@ continue2:;
         }
     }
 
-    // get the ip and skip argument names
+    // read the size part of the prelude
     const byte *ip = code_state->ip;
+    MP_BC_PRELUDE_SIZE_DECODE(ip);
 
     // jump over code info (source file and line-number mapping)
-    ip += mp_decode_uint_value(ip);
+    ip += n_info;
 
     // bytecode prelude: initialise closed over variables
-    size_t local_num;
-    while ((local_num = *ip++) != 255) {
+    for (; n_cell; --n_cell) {
+        size_t local_num = *ip++;
         code_state->state[n_state - 1 - local_num] =
             mp_obj_new_cell(code_state->state[n_state - 1 - local_num]);
     }
 
+    #if !MICROPY_PERSISTENT_CODE
+    // so bytecode is aligned
+    ip = MP_ALIGN(ip, sizeof(mp_uint_t));
+    #endif
+
     // now that we skipped over the prelude, set the ip for the VM
     code_state->ip = ip;
 
@@ -287,105 +303,17 @@ continue2:;
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
 // The following table encodes the number of bytes that a specific opcode
-// takes up.  There are 3 special opcodes that always have an extra byte:
-//     MP_BC_MAKE_CLOSURE
-//     MP_BC_MAKE_CLOSURE_DEFARGS
-//     MP_BC_RAISE_VARARGS
+// takes up.  Some opcodes have an extra byte, defined by MP_BC_MASK_EXTRA_BYTE.
 // There are 4 special opcodes that have an extra byte only when
 // MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE is enabled (and they take a qstr):
 //     MP_BC_LOAD_NAME
 //     MP_BC_LOAD_GLOBAL
 //     MP_BC_LOAD_ATTR
 //     MP_BC_STORE_ATTR
-#define OC4(a, b, c, d) (a | (b << 2) | (c << 4) | (d << 6))
-#define U (0) // undefined opcode
-#define B (MP_OPCODE_BYTE) // single byte
-#define Q (MP_OPCODE_QSTR) // single byte plus 2-byte qstr
-#define V (MP_OPCODE_VAR_UINT) // single byte plus variable encoded unsigned int
-#define O (MP_OPCODE_OFFSET) // single byte plus 2-byte bytecode offset
-STATIC const byte opcode_format_table[64] = {
-    OC4(U, U, U, U), // 0x00-0x03
-    OC4(U, U, U, U), // 0x04-0x07
-    OC4(U, U, U, U), // 0x08-0x0b
-    OC4(U, U, U, U), // 0x0c-0x0f
-    OC4(B, B, B, U), // 0x10-0x13
-    OC4(V, U, Q, V), // 0x14-0x17
-    OC4(B, V, V, Q), // 0x18-0x1b
-    OC4(Q, Q, Q, Q), // 0x1c-0x1f
-    OC4(B, B, V, V), // 0x20-0x23
-    OC4(Q, Q, Q, B), // 0x24-0x27
-    OC4(V, V, Q, Q), // 0x28-0x2b
-    OC4(U, U, U, U), // 0x2c-0x2f
-    OC4(B, B, B, B), // 0x30-0x33
-    OC4(B, O, O, O), // 0x34-0x37
-    OC4(O, O, U, U), // 0x38-0x3b
-    OC4(U, O, B, O), // 0x3c-0x3f
-    OC4(O, B, B, O), // 0x40-0x43
-    OC4(B, B, O, B), // 0x44-0x47
-    OC4(U, U, U, U), // 0x48-0x4b
-    OC4(U, U, U, U), // 0x4c-0x4f
-    OC4(V, V, U, V), // 0x50-0x53
-    OC4(B, U, V, V), // 0x54-0x57
-    OC4(V, V, V, B), // 0x58-0x5b
-    OC4(B, B, B, U), // 0x5c-0x5f
-    OC4(V, V, V, V), // 0x60-0x63
-    OC4(V, V, V, V), // 0x64-0x67
-    OC4(Q, Q, B, U), // 0x68-0x6b
-    OC4(U, U, U, U), // 0x6c-0x6f
-
-    OC4(B, B, B, B), // 0x70-0x73
-    OC4(B, B, B, B), // 0x74-0x77
-    OC4(B, B, B, B), // 0x78-0x7b
-    OC4(B, B, B, B), // 0x7c-0x7f
-    OC4(B, B, B, B), // 0x80-0x83
-    OC4(B, B, B, B), // 0x84-0x87
-    OC4(B, B, B, B), // 0x88-0x8b
-    OC4(B, B, B, B), // 0x8c-0x8f
-    OC4(B, B, B, B), // 0x90-0x93
-    OC4(B, B, B, B), // 0x94-0x97
-    OC4(B, B, B, B), // 0x98-0x9b
-    OC4(B, B, B, B), // 0x9c-0x9f
-    OC4(B, B, B, B), // 0xa0-0xa3
-    OC4(B, B, B, B), // 0xa4-0xa7
-    OC4(B, B, B, B), // 0xa8-0xab
-    OC4(B, B, B, B), // 0xac-0xaf
-
-    OC4(B, B, B, B), // 0xb0-0xb3
-    OC4(B, B, B, B), // 0xb4-0xb7
-    OC4(B, B, B, B), // 0xb8-0xbb
-    OC4(B, B, B, B), // 0xbc-0xbf
-
-    OC4(B, B, B, B), // 0xc0-0xc3
-    OC4(B, B, B, B), // 0xc4-0xc7
-    OC4(B, B, B, B), // 0xc8-0xcb
-    OC4(B, B, B, B), // 0xcc-0xcf
-
-    OC4(B, B, B, B), // 0xd0-0xd3
-    OC4(U, U, U, B), // 0xd4-0xd7
-    OC4(B, B, B, B), // 0xd8-0xdb
-    OC4(B, B, B, B), // 0xdc-0xdf
-
-    OC4(B, B, B, B), // 0xe0-0xe3
-    OC4(B, B, B, B), // 0xe4-0xe7
-    OC4(B, B, B, B), // 0xe8-0xeb
-    OC4(B, B, B, B), // 0xec-0xef
-
-    OC4(B, B, B, B), // 0xf0-0xf3
-    OC4(B, B, B, B), // 0xf4-0xf7
-    OC4(U, U, U, U), // 0xf8-0xfb
-    OC4(U, U, U, U), // 0xfc-0xff
-};
-#undef OC4
-#undef U
-#undef B
-#undef Q
-#undef V
-#undef O
-
-uint mp_opcode_format(const byte *ip, size_t *opcode_size) {
-    uint f = (opcode_format_table[*ip >> 2] >> (2 * (*ip & 3))) & 3;
+uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint) {
+    uint f = MP_BC_FORMAT(*ip);
     const byte *ip_start = ip;
-    if (f == MP_OPCODE_QSTR) {
+    if (f == MP_BC_FORMAT_QSTR) {
         if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
             if (*ip == MP_BC_LOAD_NAME
                 || *ip == MP_BC_LOAD_GLOBAL
@@ -396,16 +324,14 @@ uint mp_opcode_format(const byte *ip, size_t *opcode_size) {
         }
         ip += 3;
     } else {
-        int extra_byte = (
-            *ip == MP_BC_RAISE_VARARGS
-            || *ip == MP_BC_MAKE_CLOSURE
-            || *ip == MP_BC_MAKE_CLOSURE_DEFARGS
-        );
+        int extra_byte = (*ip & MP_BC_MASK_EXTRA_BYTE) == 0;
         ip += 1;
-        if (f == MP_OPCODE_VAR_UINT) {
-            while ((*ip++ & 0x80) != 0) {
+        if (f == MP_BC_FORMAT_VAR_UINT) {
+            if (count_var_uint) {
+                while ((*ip++ & 0x80) != 0) {
+                }
             }
-        } else if (f == MP_OPCODE_OFFSET) {
+        } else if (f == MP_BC_FORMAT_OFFSET) {
             ip += 2;
         }
         ip += extra_byte;

py/bc.h

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -31,24 +32,35 @@
 
 // bytecode layout:
 //
-//  n_state         : var uint
-//  n_exc_stack     : var uint
-//  scope_flags     : byte
-//  n_pos_args      : byte          number of arguments this function takes
-//  n_kwonly_args   : byte          number of keyword-only arguments this function takes
-//  n_def_pos_args  : byte          number of default positional arguments
+//  func signature  : var uint
+//      contains six values interleaved bit-wise as: xSSSSEAA [xFSSKAED repeated]
+//          x = extension           another byte follows
+//          S = n_state - 1         number of entries in Python value stack
+//          E = n_exc_stack         number of entries in exception stack
+//          F = scope_flags         four bits of flags, MP_SCOPE_FLAG_xxx
+//          A = n_pos_args          number of arguments this function takes
+//          K = n_kwonly_args       number of keyword-only arguments this function takes
+//          D = n_def_pos_args      number of default positional arguments
 //
-//  code_info_size  : var uint |    code_info_size counts bytes in this chunk
-//  simple_name     : var qstr |
-//  source_file     : var qstr |
-//  <line number info>         |
-//  <word alignment padding>   |    only needed if bytecode contains pointers
+//  prelude size    : var uint
+//      contains two values interleaved bit-wise as: xIIIIIIC repeated
+//          x = extension           another byte follows
+//          I = n_info              number of bytes in source info section
+//          C = n_cells             number of bytes/cells in closure section
 //
-//  local_num0      : byte     |
-//  ...             : byte     |
-//  local_numN      : byte     |    N = num_cells
-//  255             : byte     |    end of list sentinel
-//  <bytecode>                 |
+//  source info section:
+//      simple_name : var qstr
+//      source_file : var qstr
+//      <line number info>
+//
+//  closure section:
+//      local_num0  : byte
+//      ...         : byte
+//      local_numN  : byte          N = n_cells-1
+//
+//  <word alignment padding>        only needed if bytecode contains pointers
+//
+//  <bytecode>
 //
 //
 // constant table layout:
@@ -59,13 +71,127 @@
 //  const0          : obj
 //  constN          : obj
 
+#define MP_BC_PRELUDE_SIG_ENCODE(S, E, scope, out_byte, out_env) \
+do {                                                            \
+    /*// Get values to store in prelude */                      \
+    size_t F = scope->scope_flags & MP_SCOPE_FLAG_ALL_SIG;      \
+    size_t A = scope->num_pos_args;                             \
+    size_t K = scope->num_kwonly_args;                          \
+    size_t D = scope->num_def_pos_args;                         \
+                                                                \
+    /* Adjust S to shrink range, to compress better */          \
+    S -= 1;                                                     \
+                                                                \
+    /* Encode prelude */                                        \
+    /* xSSSSEAA */                                              \
+    uint8_t z = (S & 0xf) << 3 | (E & 1) << 2 | (A & 3);        \
+    S >>= 4;                                                    \
+    E >>= 1;                                                    \
+    A >>= 2;                                                    \
+    while (S | E | F | A | K | D) {                             \
+        out_byte(out_env, 0x80 | z);                            \
+        /* xFSSKAED */                                          \
+        z = (F & 1) << 6 | (S & 3) << 4 | (K & 1) << 3          \
+            | (A & 1) << 2 | (E & 1) << 1 | (D & 1);            \
+        S >>= 2;                                                \
+        E >>= 1;                                                \
+        F >>= 1;                                                \
+        A >>= 1;                                                \
+        K >>= 1;                                                \
+        D >>= 1;                                                \
+    }                                                           \
+    out_byte(out_env, z);                                       \
+} while (0)
+
+#define MP_BC_PRELUDE_SIG_DECODE_INTO(ip, S, E, F, A, K, D)     \
+do {                                                            \
+    uint8_t z = *(ip)++;                                        \
+    /* xSSSSEAA */                                              \
+    S = (z >> 3) & 0xf;                                         \
+    E = (z >> 2) & 0x1;                                         \
+    F = 0;                                                      \
+    A = z & 0x3;                                                \
+    K = 0;                                                      \
+    D = 0;                                                      \
+    for (unsigned n = 0; z & 0x80; ++n) {                       \
+        z = *(ip)++;                                            \
+        /* xFSSKAED */                                          \
+        S |= (z & 0x30) << (2 * n);                             \
+        E |= (z & 0x02) << n;                                   \
+        F |= ((z & 0x40) >> 6) << n;                            \
+        A |= (z & 0x4) << n;                                    \
+        K |= ((z & 0x08) >> 3) << n;                            \
+        D |= (z & 0x1) << n;                                    \
+    }                                                           \
+    S += 1;                                                     \
+} while (0)
+
+#define MP_BC_PRELUDE_SIG_DECODE(ip) \
+    size_t n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args; \
+    MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args)
+
+#define MP_BC_PRELUDE_SIZE_ENCODE(I, C, out_byte, out_env)      \
+do {                                                            \
+    /* Encode bit-wise as: xIIIIIIC */                          \
+    uint8_t z = 0;                                              \
+    do {                                                        \
+        z = (I & 0x3f) << 1 | (C & 1);                          \
+        C >>= 1;                                                \
+        I >>= 6;                                                \
+        if (C | I) {                                            \
+            z |= 0x80;                                          \
+        }                                                       \
+        out_byte(out_env, z);                                   \
+    } while (C | I);                                            \
+} while (0)
+
+#define MP_BC_PRELUDE_SIZE_DECODE_INTO(ip, I, C)                \
+do {                                                            \
+    uint8_t z;                                                  \
+    C = 0;                                                      \
+    I = 0;                                                      \
+    for (unsigned n = 0;; ++n) {                                \
+        z = *(ip)++;                                            \
+        /* xIIIIIIC */                                          \
+        C |= (z & 1) << n;                                      \
+        I |= ((z & 0x7e) >> 1) << (6 * n);                      \
+        if (!(z & 0x80)) {                                      \
+            break;                                              \
+        }                                                       \
+    }                                                           \
+} while (0)
+
+#define MP_BC_PRELUDE_SIZE_DECODE(ip) \
+    size_t n_info, n_cell; \
+    MP_BC_PRELUDE_SIZE_DECODE_INTO(ip, n_info, n_cell)
+
+// Sentinel value for mp_code_state_t.exc_sp_idx
+#define MP_CODE_STATE_EXC_SP_IDX_SENTINEL ((uint16_t)-1)
+
+// To convert mp_code_state_t.exc_sp_idx to/from a pointer to mp_exc_stack_t
+#define MP_CODE_STATE_EXC_SP_IDX_FROM_PTR(exc_stack, exc_sp) ((exc_sp) + 1 - (exc_stack))
+#define MP_CODE_STATE_EXC_SP_IDX_TO_PTR(exc_stack, exc_sp_idx) ((exc_stack) + (exc_sp_idx) - 1)
+
+typedef struct _mp_bytecode_prelude_t {
+    uint n_state;
+    uint n_exc_stack;
+    uint scope_flags;
+    uint n_pos_args;
+    uint n_kwonly_args;
+    uint n_def_pos_args;
+    qstr qstr_block_name;
+    qstr qstr_source_file;
+    const byte *line_info;
+    const byte *opcodes;
+} mp_bytecode_prelude_t;
+
 // Exception stack entry
 typedef struct _mp_exc_stack_t {
     const byte *handler;
-    // bit 0 is saved currently_in_except_block value
+    // bit 0 is currently unused
     // bit 1 is whether the opcode was SETUP_WITH or SETUP_FINALLY
     mp_obj_t *val_sp;
-    // Saved exception, valid if currently_in_except_block bit is 1
+    // Saved exception
     mp_obj_base_t *prev_exc;
 } mp_exc_stack_t;
 
@@ -77,12 +203,16 @@ typedef struct _mp_code_state_t {
     mp_obj_fun_bc_t *fun_bc;
     const byte *ip;
     mp_obj_t *sp;
-    // bit 0 is saved currently_in_except_block value
-    mp_exc_stack_t *exc_sp;
+    uint16_t n_state;
+    uint16_t exc_sp_idx;
     mp_obj_dict_t *old_globals;
     #if MICROPY_STACKLESS
     struct _mp_code_state_t *prev;
     #endif
+    #if MICROPY_PY_SYS_SETTRACE
+    struct _mp_code_state_t *prev_state;
+    struct _mp_obj_frame_t *frame;
+    #endif
     // Variable-length
     mp_obj_t state[0];
     // Variable-length, never accessed by name, only as (void*)(state + n_state)
@@ -109,13 +239,35 @@ const byte *mp_bytecode_print_str(const byte *ip);
 
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
-#define MP_OPCODE_BYTE (0)
-#define MP_OPCODE_QSTR (1)
-#define MP_OPCODE_VAR_UINT (2)
-#define MP_OPCODE_OFFSET (3)
-
-uint mp_opcode_format(const byte *ip, size_t *opcode_size);
+uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint);
 
 #endif
 
+static inline size_t mp_bytecode_get_source_line(const byte *line_info, size_t bc_offset) {
+    size_t source_line = 1;
+    size_t c;
+    while ((c = *line_info)) {
+        size_t b, l;
+        if ((c & 0x80) == 0) {
+            // 0b0LLBBBBB encoding
+            b = c & 0x1f;
+            l = c >> 5;
+            line_info += 1;
+        } else {
+            // 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte)
+            b = c & 0xf;
+            l = ((c << 4) & 0x700) | line_info[1];
+            line_info += 2;
+        }
+        if (bc_offset >= b) {
+            bc_offset -= b;
+            source_line += l;
+        } else {
+            // found source line corresponding to bytecode offset
+            break;
+        }
+    }
+    return source_line;
+}
+
 #endif // MICROPY_INCLUDED_PY_BC_H

py/bc0.h

@@ -26,94 +26,125 @@
 #ifndef MICROPY_INCLUDED_PY_BC0_H
 #define MICROPY_INCLUDED_PY_BC0_H
 
-// MicroPython byte-codes.
-// The comment at the end of the line (if it exists) tells the arguments to the byte-code.
-
-#define MP_BC_LOAD_CONST_FALSE   (0x10)
-#define MP_BC_LOAD_CONST_NONE    (0x11)
-#define MP_BC_LOAD_CONST_TRUE    (0x12)
-#define MP_BC_LOAD_CONST_SMALL_INT   (0x14) // signed var-int
-#define MP_BC_LOAD_CONST_STRING  (0x16) // qstr
-#define MP_BC_LOAD_CONST_OBJ     (0x17) // ptr
-#define MP_BC_LOAD_NULL          (0x18)
-
-#define MP_BC_LOAD_FAST_N        (0x19) // uint
-#define MP_BC_LOAD_DEREF         (0x1a) // uint
-#define MP_BC_LOAD_NAME          (0x1b) // qstr
-#define MP_BC_LOAD_GLOBAL        (0x1c) // qstr
-#define MP_BC_LOAD_ATTR          (0x1d) // qstr
-#define MP_BC_LOAD_METHOD        (0x1e) // qstr
-#define MP_BC_LOAD_SUPER_METHOD  (0x1f) // qstr
-#define MP_BC_LOAD_BUILD_CLASS   (0x20)
-#define MP_BC_LOAD_SUBSCR        (0x21)
-
-#define MP_BC_STORE_FAST_N       (0x22) // uint
-#define MP_BC_STORE_DEREF        (0x23) // uint
-#define MP_BC_STORE_NAME         (0x24) // qstr
-#define MP_BC_STORE_GLOBAL       (0x25) // qstr
-#define MP_BC_STORE_ATTR         (0x26) // qstr
-#define MP_BC_STORE_SUBSCR       (0x27)
-
-#define MP_BC_DELETE_FAST        (0x28) // uint
-#define MP_BC_DELETE_DEREF       (0x29) // uint
-#define MP_BC_DELETE_NAME        (0x2a) // qstr
-#define MP_BC_DELETE_GLOBAL      (0x2b) // qstr
-
-#define MP_BC_DUP_TOP            (0x30)
-#define MP_BC_DUP_TOP_TWO        (0x31)
-#define MP_BC_POP_TOP            (0x32)
-#define MP_BC_ROT_TWO            (0x33)
-#define MP_BC_ROT_THREE          (0x34)
-
-#define MP_BC_JUMP               (0x35) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_POP_JUMP_IF_TRUE   (0x36) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_POP_JUMP_IF_FALSE  (0x37) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_JUMP_IF_TRUE_OR_POP    (0x38) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_JUMP_IF_FALSE_OR_POP   (0x39) // rel byte code offset, 16-bit signed, in excess
-#define MP_BC_SETUP_WITH         (0x3d) // rel byte code offset, 16-bit unsigned
-#define MP_BC_WITH_CLEANUP       (0x3e)
-#define MP_BC_SETUP_EXCEPT       (0x3f) // rel byte code offset, 16-bit unsigned
-#define MP_BC_SETUP_FINALLY      (0x40) // rel byte code offset, 16-bit unsigned
-#define MP_BC_END_FINALLY        (0x41)
-#define MP_BC_GET_ITER           (0x42)
-#define MP_BC_FOR_ITER           (0x43) // rel byte code offset, 16-bit unsigned
-#define MP_BC_POP_BLOCK          (0x44)
-#define MP_BC_POP_EXCEPT         (0x45)
-#define MP_BC_UNWIND_JUMP        (0x46) // rel byte code offset, 16-bit signed, in excess; then a byte
-#define MP_BC_GET_ITER_STACK     (0x47)
-
-#define MP_BC_BUILD_TUPLE        (0x50) // uint
-#define MP_BC_BUILD_LIST         (0x51) // uint
-#define MP_BC_BUILD_MAP          (0x53) // uint
-#define MP_BC_STORE_MAP          (0x54)
-#define MP_BC_BUILD_SET          (0x56) // uint
-#define MP_BC_BUILD_SLICE        (0x58) // uint
-#define MP_BC_STORE_COMP         (0x57) // uint
-#define MP_BC_UNPACK_SEQUENCE    (0x59) // uint
-#define MP_BC_UNPACK_EX          (0x5a) // uint
-
-#define MP_BC_RETURN_VALUE       (0x5b)
-#define MP_BC_RAISE_VARARGS      (0x5c) // byte
-#define MP_BC_YIELD_VALUE        (0x5d)
-#define MP_BC_YIELD_FROM         (0x5e)
-
-#define MP_BC_MAKE_FUNCTION         (0x60) // uint
-#define MP_BC_MAKE_FUNCTION_DEFARGS (0x61) // uint
-#define MP_BC_MAKE_CLOSURE          (0x62) // uint
-#define MP_BC_MAKE_CLOSURE_DEFARGS  (0x63) // uint
-#define MP_BC_CALL_FUNCTION         (0x64) // uint
-#define MP_BC_CALL_FUNCTION_VAR_KW  (0x65) // uint
-#define MP_BC_CALL_METHOD           (0x66) // uint
-#define MP_BC_CALL_METHOD_VAR_KW    (0x67) // uint
-
-#define MP_BC_IMPORT_NAME        (0x68) // qstr
-#define MP_BC_IMPORT_FROM        (0x69) // qstr
-#define MP_BC_IMPORT_STAR        (0x6a)
-
-#define MP_BC_LOAD_CONST_SMALL_INT_MULTI (0x70) // + N(64)
-#define MP_BC_LOAD_FAST_MULTI            (0xb0) // + N(16)
-#define MP_BC_STORE_FAST_MULTI           (0xc0) // + N(16)
-#define MP_BC_UNARY_OP_MULTI             (0xd0) // + op(<MP_UNARY_OP_NUM_BYTECODE)
-#define MP_BC_BINARY_OP_MULTI            (0xd7) // + op(<MP_BINARY_OP_NUM_BYTECODE)
+// MicroPython bytecode opcodes, grouped based on the format of the opcode
+
+#define MP_BC_MASK_FORMAT                   (0xf0)
+#define MP_BC_MASK_EXTRA_BYTE               (0x9e)
+
+#define MP_BC_FORMAT_BYTE                   (0)
+#define MP_BC_FORMAT_QSTR                   (1)
+#define MP_BC_FORMAT_VAR_UINT               (2)
+#define MP_BC_FORMAT_OFFSET                 (3)
+
+// Nibbles in magic number are: BB BB BB BB BB BO VV QU
+#define MP_BC_FORMAT(op) ((0x000003a4 >> (2 * ((op) >> 4))) & 3)
+
+// Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op
+#define MP_BC_BASE_RESERVED                 (0x00) // ----------------
+#define MP_BC_BASE_QSTR_O                   (0x10) // LLLLLLSSSDDII---
+#define MP_BC_BASE_VINT_E                   (0x20) // MMLLLLSSDDBBBBBB
+#define MP_BC_BASE_VINT_O                   (0x30) // UUMMCCCC--------
+#define MP_BC_BASE_JUMP_E                   (0x40) // J-JJJJJEEEEF----
+#define MP_BC_BASE_BYTE_O                   (0x50) // LLLLSSDTTTTTEEFF
+#define MP_BC_BASE_BYTE_E                   (0x60) // --BREEEYYI------
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI    (0x70) // LLLLLLLLLLLLLLLL
+                                         // (0x80) // LLLLLLLLLLLLLLLL
+                                         // (0x90) // LLLLLLLLLLLLLLLL
+                                         // (0xa0) // LLLLLLLLLLLLLLLL
+#define MP_BC_LOAD_FAST_MULTI               (0xb0) // LLLLLLLLLLLLLLLL
+#define MP_BC_STORE_FAST_MULTI              (0xc0) // SSSSSSSSSSSSSSSS
+#define MP_BC_UNARY_OP_MULTI                (0xd0) // OOOOOOO
+#define MP_BC_BINARY_OP_MULTI               (0xd7) //        OOOOOOOOO
+                                         // (0xe0) // OOOOOOOOOOOOOOOO
+                                         // (0xf0) // OOOOOOOOOO------
+
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM (64)
+#define MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS (16)
+#define MP_BC_LOAD_FAST_MULTI_NUM           (16)
+#define MP_BC_STORE_FAST_MULTI_NUM          (16)
+#define MP_BC_UNARY_OP_MULTI_NUM            (MP_UNARY_OP_NUM_BYTECODE)
+#define MP_BC_BINARY_OP_MULTI_NUM           (MP_BINARY_OP_NUM_BYTECODE)
+
+#define MP_BC_LOAD_CONST_FALSE              (MP_BC_BASE_BYTE_O + 0x00)
+#define MP_BC_LOAD_CONST_NONE               (MP_BC_BASE_BYTE_O + 0x01)
+#define MP_BC_LOAD_CONST_TRUE               (MP_BC_BASE_BYTE_O + 0x02)
+#define MP_BC_LOAD_CONST_SMALL_INT          (MP_BC_BASE_VINT_E + 0x02) // signed var-int
+#define MP_BC_LOAD_CONST_STRING             (MP_BC_BASE_QSTR_O + 0x00) // qstr
+#define MP_BC_LOAD_CONST_OBJ                (MP_BC_BASE_VINT_E + 0x03) // ptr
+#define MP_BC_LOAD_NULL                     (MP_BC_BASE_BYTE_O + 0x03)
+
+#define MP_BC_LOAD_FAST_N                   (MP_BC_BASE_VINT_E + 0x04) // uint
+#define MP_BC_LOAD_DEREF                    (MP_BC_BASE_VINT_E + 0x05) // uint
+#define MP_BC_LOAD_NAME                     (MP_BC_BASE_QSTR_O + 0x01) // qstr
+#define MP_BC_LOAD_GLOBAL                   (MP_BC_BASE_QSTR_O + 0x02) // qstr
+#define MP_BC_LOAD_ATTR                     (MP_BC_BASE_QSTR_O + 0x03) // qstr
+#define MP_BC_LOAD_METHOD                   (MP_BC_BASE_QSTR_O + 0x04) // qstr
+#define MP_BC_LOAD_SUPER_METHOD             (MP_BC_BASE_QSTR_O + 0x05) // qstr
+#define MP_BC_LOAD_BUILD_CLASS              (MP_BC_BASE_BYTE_O + 0x04)
+#define MP_BC_LOAD_SUBSCR                   (MP_BC_BASE_BYTE_O + 0x05)
+
+#define MP_BC_STORE_FAST_N                  (MP_BC_BASE_VINT_E + 0x06) // uint
+#define MP_BC_STORE_DEREF                   (MP_BC_BASE_VINT_E + 0x07) // uint
+#define MP_BC_STORE_NAME                    (MP_BC_BASE_QSTR_O + 0x06) // qstr
+#define MP_BC_STORE_GLOBAL                  (MP_BC_BASE_QSTR_O + 0x07) // qstr
+#define MP_BC_STORE_ATTR                    (MP_BC_BASE_QSTR_O + 0x08) // qstr
+#define MP_BC_STORE_SUBSCR                  (MP_BC_BASE_BYTE_O + 0x06)
+
+#define MP_BC_DELETE_FAST                   (MP_BC_BASE_VINT_E + 0x08) // uint
+#define MP_BC_DELETE_DEREF                  (MP_BC_BASE_VINT_E + 0x09) // uint
+#define MP_BC_DELETE_NAME                   (MP_BC_BASE_QSTR_O + 0x09) // qstr
+#define MP_BC_DELETE_GLOBAL                 (MP_BC_BASE_QSTR_O + 0x0a) // qstr
+
+#define MP_BC_DUP_TOP                       (MP_BC_BASE_BYTE_O + 0x07)
+#define MP_BC_DUP_TOP_TWO                   (MP_BC_BASE_BYTE_O + 0x08)
+#define MP_BC_POP_TOP                       (MP_BC_BASE_BYTE_O + 0x09)
+#define MP_BC_ROT_TWO                       (MP_BC_BASE_BYTE_O + 0x0a)
+#define MP_BC_ROT_THREE                     (MP_BC_BASE_BYTE_O + 0x0b)
+
+#define MP_BC_JUMP                          (MP_BC_BASE_JUMP_E + 0x02) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_POP_JUMP_IF_TRUE              (MP_BC_BASE_JUMP_E + 0x03) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_POP_JUMP_IF_FALSE             (MP_BC_BASE_JUMP_E + 0x04) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_JUMP_IF_TRUE_OR_POP           (MP_BC_BASE_JUMP_E + 0x05) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_JUMP_IF_FALSE_OR_POP          (MP_BC_BASE_JUMP_E + 0x06) // rel byte code offset, 16-bit signed, in excess
+#define MP_BC_UNWIND_JUMP                   (MP_BC_BASE_JUMP_E + 0x00) // rel byte code offset, 16-bit signed, in excess; then a byte
+#define MP_BC_SETUP_WITH                    (MP_BC_BASE_JUMP_E + 0x07) // rel byte code offset, 16-bit unsigned
+#define MP_BC_SETUP_EXCEPT                  (MP_BC_BASE_JUMP_E + 0x08) // rel byte code offset, 16-bit unsigned
+#define MP_BC_SETUP_FINALLY                 (MP_BC_BASE_JUMP_E + 0x09) // rel byte code offset, 16-bit unsigned
+#define MP_BC_POP_EXCEPT_JUMP               (MP_BC_BASE_JUMP_E + 0x0a) // rel byte code offset, 16-bit unsigned
+#define MP_BC_FOR_ITER                      (MP_BC_BASE_JUMP_E + 0x0b) // rel byte code offset, 16-bit unsigned
+#define MP_BC_WITH_CLEANUP                  (MP_BC_BASE_BYTE_O + 0x0c)
+#define MP_BC_END_FINALLY                   (MP_BC_BASE_BYTE_O + 0x0d)
+#define MP_BC_GET_ITER                      (MP_BC_BASE_BYTE_O + 0x0e)
+#define MP_BC_GET_ITER_STACK                (MP_BC_BASE_BYTE_O + 0x0f)
+
+#define MP_BC_BUILD_TUPLE                   (MP_BC_BASE_VINT_E + 0x0a) // uint
+#define MP_BC_BUILD_LIST                    (MP_BC_BASE_VINT_E + 0x0b) // uint
+#define MP_BC_BUILD_MAP                     (MP_BC_BASE_VINT_E + 0x0c) // uint
+#define MP_BC_STORE_MAP                     (MP_BC_BASE_BYTE_E + 0x02)
+#define MP_BC_BUILD_SET                     (MP_BC_BASE_VINT_E + 0x0d) // uint
+#define MP_BC_BUILD_SLICE                   (MP_BC_BASE_VINT_E + 0x0e) // uint
+#define MP_BC_STORE_COMP                    (MP_BC_BASE_VINT_E + 0x0f) // uint
+#define MP_BC_UNPACK_SEQUENCE               (MP_BC_BASE_VINT_O + 0x00) // uint
+#define MP_BC_UNPACK_EX                     (MP_BC_BASE_VINT_O + 0x01) // uint
+
+#define MP_BC_RETURN_VALUE                  (MP_BC_BASE_BYTE_E + 0x03)
+#define MP_BC_RAISE_LAST                    (MP_BC_BASE_BYTE_E + 0x04)
+#define MP_BC_RAISE_OBJ                     (MP_BC_BASE_BYTE_E + 0x05)
+#define MP_BC_RAISE_FROM                    (MP_BC_BASE_BYTE_E + 0x06)
+#define MP_BC_YIELD_VALUE                   (MP_BC_BASE_BYTE_E + 0x07)
+#define MP_BC_YIELD_FROM                    (MP_BC_BASE_BYTE_E + 0x08)
+
+#define MP_BC_MAKE_FUNCTION                 (MP_BC_BASE_VINT_O + 0x02) // uint
+#define MP_BC_MAKE_FUNCTION_DEFARGS         (MP_BC_BASE_VINT_O + 0x03) // uint
+#define MP_BC_MAKE_CLOSURE                  (MP_BC_BASE_VINT_E + 0x00) // uint; extra byte
+#define MP_BC_MAKE_CLOSURE_DEFARGS          (MP_BC_BASE_VINT_E + 0x01) // uint; extra byte
+#define MP_BC_CALL_FUNCTION                 (MP_BC_BASE_VINT_O + 0x04) // uint
+#define MP_BC_CALL_FUNCTION_VAR_KW          (MP_BC_BASE_VINT_O + 0x05) // uint
+#define MP_BC_CALL_METHOD                   (MP_BC_BASE_VINT_O + 0x06) // uint
+#define MP_BC_CALL_METHOD_VAR_KW            (MP_BC_BASE_VINT_O + 0x07) // uint
+
+#define MP_BC_IMPORT_NAME                   (MP_BC_BASE_QSTR_O + 0x0b) // qstr
+#define MP_BC_IMPORT_FROM                   (MP_BC_BASE_QSTR_O + 0x0c) // qstr
+#define MP_BC_IMPORT_STAR                   (MP_BC_BASE_BYTE_E + 0x09)
 
 #endif // MICROPY_INCLUDED_PY_BC0_H

py/binary.c

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014-2017 Paul Sokolovsky
+ * Copyright (c) 2014-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -41,7 +42,7 @@
 #define alignof(type) offsetof(struct { char c; type t; }, t)
 #endif
 
-size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign) {
+size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign) {
     size_t size = 0;
     int align = 1;
     switch (struct_type) {
@@ -112,7 +113,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign) {
     return size;
 }
 
-mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index) {
+mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index) {
     mp_int_t val = 0;
     switch (typecode) {
         case 'b':
@@ -161,7 +162,7 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index) {
 // The long long type is guaranteed to hold at least 64 bits, and size is at
 // most 8 (for q and Q), so we will always be able to parse the given data
 // and fit it into a long long.
-long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src) {
+long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const byte *src) {
     int delta;
     if (!big_endian) {
         delta = -1;
@@ -184,14 +185,14 @@ long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, con
 }
 
 #define is_signed(typecode) (typecode > 'Z')
-mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) {
+mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr) {
     byte *p = *ptr;
-    mp_uint_t align;
+    size_t align;
 
     size_t size = mp_binary_get_size(struct_type, val_type, &align);
     if (struct_type == '@') {
-        // Make pointer aligned
-        p = (byte*)MP_ALIGN(p, (size_t)align);
+        // Align p relative to p_base
+        p = p_base + (uintptr_t)MP_ALIGN(p - p_base, align);
         #if MP_ENDIANNESS_LITTLE
         struct_type = '<';
         #else
@@ -230,7 +231,7 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) {
     }
 }
 
-void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val) {
+void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val) {
     if (MP_ENDIANNESS_LITTLE && !big_endian) {
         memcpy(dest, &val, val_sz);
     } else if (MP_ENDIANNESS_BIG && big_endian) {
@@ -249,14 +250,14 @@ void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t
     }
 }
 
-void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr) {
+void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p_base, byte **ptr) {
     byte *p = *ptr;
-    mp_uint_t align;
+    size_t align;
 
     size_t size = mp_binary_get_size(struct_type, val_type, &align);
     if (struct_type == '@') {
-        // Make pointer aligned
-        p = (byte*)MP_ALIGN(p, (size_t)align);
+        // Align p relative to p_base
+        p = p_base + (uintptr_t)MP_ALIGN(p - p_base, align);
         if (MP_ENDIANNESS_LITTLE) {
             struct_type = '<';
         } else {
@@ -314,7 +315,7 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **
     mp_binary_set_int(MIN((size_t)size, sizeof(val)), struct_type == '>', p, val);
 }
 
-void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in) {
+void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in) {
     switch (typecode) {
 #if MICROPY_PY_BUILTINS_FLOAT
         case 'f':
@@ -341,7 +342,7 @@ void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t v
     }
 }
 
-void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, mp_int_t val) {
+void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val) {
     switch (typecode) {
         case 'b':
             ((signed char*)p)[index] = val;

py/binary.h

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
+ * Copyright (c) 2014-2017 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -33,13 +34,13 @@
 // type-specification errors due to end-of-string.
 #define BYTEARRAY_TYPECODE 1
 
-size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign);
-mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index);
-void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in);
-void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, mp_int_t val);
-mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr);
-void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr);
-long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src);
-void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val);
+size_t mp_binary_get_size(char struct_type, char val_type, size_t *palign);
+mp_obj_t mp_binary_get_val_array(char typecode, void *p, size_t index);
+void mp_binary_set_val_array(char typecode, void *p, size_t index, mp_obj_t val_in);
+void mp_binary_set_val_array_from_int(char typecode, void *p, size_t index, mp_int_t val);
+mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr);
+void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte *p_base, byte **ptr);
+long long mp_binary_get_int(size_t size, bool is_signed, bool big_endian, const byte *src);
+void mp_binary_set_int(size_t val_sz, bool big_endian, byte *dest, mp_uint_t val);
 
 #endif // MICROPY_INCLUDED_PY_BINARY_H

py/builtin.h

@@ -89,7 +89,7 @@ MP_DECLARE_CONST_FUN_OBJ_2(mp_op_delitem_obj);
 
 extern const mp_obj_module_t mp_module___main__;
 extern const mp_obj_module_t mp_module_builtins;
-extern const mp_obj_module_t mp_module_array;
+extern const mp_obj_module_t mp_module_uarray;
 extern const mp_obj_module_t mp_module_collections;
 extern const mp_obj_module_t mp_module_io;
 extern const mp_obj_module_t mp_module_math;
@@ -122,6 +122,7 @@ extern const mp_obj_module_t mp_module_uwebsocket;
 extern const mp_obj_module_t mp_module_webrepl;
 extern const mp_obj_module_t mp_module_framebuf;
 extern const mp_obj_module_t mp_module_btree;
+extern const mp_obj_module_t mp_module_ubluetooth;
 
 extern const char MICROPY_PY_BUILTINS_HELP_TEXT[];
 

py/builtinhelp.c

@@ -80,10 +80,6 @@ STATIC void mp_help_print_modules(void) {
 
     mp_help_add_from_map(list, &mp_builtin_module_map);
 
-    #if MICROPY_MODULE_WEAK_LINKS
-    mp_help_add_from_map(list, &mp_builtin_module_weak_links_map);
-    #endif
-
     #if MICROPY_MODULE_FROZEN_STR
     extern const char mp_frozen_str_names[];
     mp_help_add_from_names(list, mp_frozen_str_names);

py/builtinimport.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -145,11 +145,11 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) {
 #endif
 
 #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY
-STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
+STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, const char* source_name) {
+    (void)source_name;
+
     #if MICROPY_PY___FILE__
-    // TODO
-    //qstr source_name = lex->source_name;
-    //mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
+    mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name)));
     #endif
 
     // execute the module in its context
@@ -206,7 +206,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     // its data) in the list of frozen files, execute it.
     #if MICROPY_MODULE_FROZEN_MPY
     if (frozen_type == MP_FROZEN_MPY) {
-        do_execute_raw_code(module_obj, modref);
+        do_execute_raw_code(module_obj, modref, file_str);
         return;
     }
     #endif
@@ -216,7 +216,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD
     if (file_str[file->len - 3] == 'm') {
         mp_raw_code_t *raw_code = mp_raw_code_load_file(file_str);
-        do_execute_raw_code(module_obj, raw_code);
+        do_execute_raw_code(module_obj, raw_code, file_str);
         return;
     }
     #endif
@@ -334,6 +334,10 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
         mod_len = new_mod_l;
     }
 
+    if (mod_len == 0) {
+        mp_raise_ValueError(NULL);
+    }
+
     // check if module already exists
     qstr module_name_qstr = mp_obj_str_get_qstr(module_name);
     mp_obj_t module_obj = mp_module_get(module_name_qstr);
@@ -381,21 +385,18 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
             DEBUG_printf("Current path: %.*s\n", vstr_len(&path), vstr_str(&path));
 
             if (stat == MP_IMPORT_STAT_NO_EXIST) {
+                module_obj = MP_OBJ_NULL;
                 #if MICROPY_MODULE_WEAK_LINKS
                 // check if there is a weak link to this module
                 if (i == mod_len) {
-                    mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(mod_name), MP_MAP_LOOKUP);
-                    if (el == NULL) {
-                        goto no_exist;
+                    module_obj = mp_module_search_umodule(mod_str);
+                    if (module_obj != MP_OBJ_NULL) {
+                        // found weak linked module
+                        mp_module_call_init(mod_name, module_obj);
                     }
-                    // found weak linked module
-                    module_obj = el->value;
-                    mp_module_call_init(mod_name, module_obj);
-                } else {
-                    no_exist:
-                #else
-                {
+                }
                 #endif
+                if (module_obj == MP_OBJ_NULL) {
                     // couldn't find the file, so fail
                     if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) {
                         mp_raise_msg(&mp_type_ImportError, "module not found");
@@ -492,11 +493,11 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
 
     #if MICROPY_MODULE_WEAK_LINKS
     // Check if there is a weak link to this module
-    mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(module_name_qstr), MP_MAP_LOOKUP);
-    if (el != NULL) {
+    module_obj = mp_module_search_umodule(qstr_str(module_name_qstr));
+    if (module_obj != MP_OBJ_NULL) {
         // Found weak-linked module
-        mp_module_call_init(module_name_qstr, el->value);
-        return el->value;
+        mp_module_call_init(module_name_qstr, module_obj);
+        return module_obj;
     }
     #endif
 

py/compile.c

@@ -35,6 +35,7 @@
 #include "py/compile.h"
 #include "py/runtime.h"
 #include "py/asmbase.h"
+#include "py/persistentcode.h"
 
 #if MICROPY_ENABLE_COMPILER
 
@@ -78,7 +79,26 @@ typedef enum {
 
 #endif
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER
+
+#define NATIVE_EMITTER(f) emit_native_table[mp_dynamic_compiler.native_arch]->emit_##f
+#define NATIVE_EMITTER_TABLE emit_native_table[mp_dynamic_compiler.native_arch]
+
+STATIC const emit_method_table_t *emit_native_table[] = {
+    NULL,
+    &emit_native_x86_method_table,
+    &emit_native_x64_method_table,
+    &emit_native_arm_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_thumb_method_table,
+    &emit_native_xtensa_method_table,
+    &emit_native_xtensawin_method_table,
+};
+
+#elif MICROPY_EMIT_NATIVE
 // define a macro to access external native emitter
 #if MICROPY_EMIT_X64
 #define NATIVE_EMITTER(f) emit_native_x64_##f
@@ -90,12 +110,34 @@ typedef enum {
 #define NATIVE_EMITTER(f) emit_native_arm_##f
 #elif MICROPY_EMIT_XTENSA
 #define NATIVE_EMITTER(f) emit_native_xtensa_##f
+#elif MICROPY_EMIT_XTENSAWIN
+#define NATIVE_EMITTER(f) emit_native_xtensawin_##f
 #else
 #error "unknown native emitter"
 #endif
+#define NATIVE_EMITTER_TABLE &NATIVE_EMITTER(method_table)
 #endif
 
-#if MICROPY_EMIT_INLINE_ASM
+#if MICROPY_EMIT_INLINE_ASM && MICROPY_DYNAMIC_COMPILER
+
+#define ASM_EMITTER(f) emit_asm_table[mp_dynamic_compiler.native_arch]->asm_##f
+#define ASM_EMITTER_TABLE emit_asm_table[mp_dynamic_compiler.native_arch]
+
+STATIC const emit_inline_asm_method_table_t *emit_asm_table[] = {
+    NULL,
+    NULL,
+    NULL,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_thumb_method_table,
+    &emit_inline_xtensa_method_table,
+    NULL,
+};
+
+#elif MICROPY_EMIT_INLINE_ASM
 // define macros for inline assembler
 #if MICROPY_EMIT_INLINE_THUMB
 #define ASM_DECORATOR_QSTR MP_QSTR_asm_thumb
@@ -106,6 +148,7 @@ typedef enum {
 #else
 #error "unknown asm emitter"
 #endif
+#define ASM_EMITTER_TABLE &ASM_EMITTER(method_table)
 #endif
 
 #define EMIT_INLINE_ASM(fun) (comp->emit_inline_asm_method_table->fun(comp->emit_inline_asm))
@@ -164,6 +207,7 @@ STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const cha
 
 STATIC void compile_trailer_paren_helper(compiler_t *comp, mp_parse_node_t pn_arglist, bool is_method_call, int n_positional_extra);
 STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, scope_kind_t kind);
+STATIC void compile_atom_brace_helper(compiler_t *comp, mp_parse_node_struct_t *pns, bool create_map);
 STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn);
 
 STATIC uint comp_next_label(compiler_t *comp) {
@@ -798,13 +842,32 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, int name_len, mp_parse_
         *emit_options = MP_EMIT_OPT_VIPER;
 #endif
     #if MICROPY_EMIT_INLINE_ASM
+    #if MICROPY_DYNAMIC_COMPILER
+    } else if (attr == MP_QSTR_asm_thumb) {
+        *emit_options = MP_EMIT_OPT_ASM;
+    } else if (attr == MP_QSTR_asm_xtensa) {
+        *emit_options = MP_EMIT_OPT_ASM;
+    #else
     } else if (attr == ASM_DECORATOR_QSTR) {
         *emit_options = MP_EMIT_OPT_ASM;
     #endif
+    #endif
     } else {
         compile_syntax_error(comp, name_nodes[1], "invalid micropython decorator");
     }
 
+    #if MICROPY_DYNAMIC_COMPILER
+    if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) {
+        if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) {
+            compile_syntax_error(comp, name_nodes[1], "invalid arch");
+        }
+    } else if (*emit_options == MP_EMIT_OPT_ASM) {
+        if (emit_asm_table[mp_dynamic_compiler.native_arch] == NULL) {
+            compile_syntax_error(comp, name_nodes[1], "invalid arch");
+        }
+    }
+    #endif
+
     return true;
 }
 
@@ -965,16 +1028,13 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
 
 STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
     uint16_t label;
-    const char *error_msg;
     if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) {
         label = comp->break_label;
-        error_msg = "'break' outside loop";
     } else {
         label = comp->continue_label;
-        error_msg = "'continue' outside loop";
     }
     if (label == INVALID_LABEL) {
-        compile_syntax_error(comp, (mp_parse_node_t)pns, error_msg);
+        compile_syntax_error(comp, (mp_parse_node_t)pns, "'break'/'continue' outside loop");
     }
     assert(comp->cur_except_level >= comp->break_continue_except_level);
     EMIT_ARG(unwind_jump, label, comp->cur_except_level - comp->break_continue_except_level);
@@ -1137,6 +1197,13 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) {
     } while (0);
 
     if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[1], MP_TOKEN_OP_STAR)) {
+        #if MICROPY_CPYTHON_COMPAT
+        if (comp->scope_cur->kind != SCOPE_MODULE) {
+            compile_syntax_error(comp, (mp_parse_node_t)pns, "import * not at module level");
+            return;
+        }
+        #endif
+
         EMIT_ARG(load_const_small_int, import_level);
 
         // build the "fromlist" tuple
@@ -1146,7 +1213,7 @@ STATIC void compile_import_from(compiler_t *comp, mp_parse_node_struct_t *pns) {
         // do the import
         qstr dummy_q;
         do_import_name(comp, pn_import_source, &dummy_q);
-        EMIT_ARG(import, MP_QSTR_NULL, MP_EMIT_IMPORT_STAR);
+        EMIT_ARG(import, MP_QSTRnull, MP_EMIT_IMPORT_STAR);
 
     } else {
         EMIT_ARG(load_const_small_int, import_level);
@@ -1534,8 +1601,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
     compile_increase_except_level(comp, l1, MP_EMIT_SETUP_BLOCK_EXCEPT);
 
     compile_node(comp, pn_body); // body
-    EMIT(pop_block);
-    EMIT_ARG(jump, success_label); // jump over exception handler
+    EMIT_ARG(pop_except_jump, success_label, false); // jump over exception handler
 
     EMIT_ARG(label_assign, l1); // start of exception handler
     EMIT(start_except_handler);
@@ -1550,6 +1616,9 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
 
         qstr qstr_exception_local = 0;
         uint end_finally_label = comp_next_label(comp);
+        #if MICROPY_PY_SYS_SETTRACE
+        EMIT_ARG(set_source_line, pns_except->source_line);
+        #endif
 
         if (MP_PARSE_NODE_IS_NULL(pns_except->nodes[0])) {
             // this is a catch all exception handler
@@ -1598,7 +1667,6 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
         }
         compile_node(comp, pns_except->nodes[1]); // the <body>
         if (qstr_exception_local != 0) {
-            EMIT(pop_block);
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
             EMIT_ARG(label_assign, l3);
             EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
@@ -1607,8 +1675,7 @@ STATIC void compile_try_except(compiler_t *comp, mp_parse_node_t pn_body, int n_
             compile_decrease_except_level(comp);
         }
 
-        EMIT(pop_except);
-        EMIT_ARG(jump, l2);
+        EMIT_ARG(pop_except_jump, l2, true);
         EMIT_ARG(label_assign, end_finally_label);
         EMIT_ARG(adjust_stack_size, 1); // stack adjust for the exception instance
     }
@@ -1634,7 +1701,6 @@ STATIC void compile_try_finally(compiler_t *comp, mp_parse_node_t pn_body, int n
     } else {
         compile_try_except(comp, pn_body, n_except, pn_except, pn_else);
     }
-    EMIT(pop_block);
     EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
     EMIT_ARG(label_assign, l_finally_block);
     compile_node(comp, pn_finally);
@@ -1742,8 +1808,7 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns
     compile_load_id(comp, context);
     compile_await_object_method(comp, MP_QSTR___anext__);
     c_assign(comp, pns->nodes[0], ASSIGN_STORE); // variable
-    EMIT(pop_block);
-    EMIT_ARG(jump, try_else_label);
+    EMIT_ARG(pop_except_jump, try_else_label, false);
 
     EMIT_ARG(label_assign, try_exception_label);
     EMIT(start_except_handler);
@@ -1752,8 +1817,7 @@ STATIC void compile_async_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns
     EMIT_ARG(binary_op, MP_BINARY_OP_EXCEPTION_MATCH);
     EMIT_ARG(pop_jump_if, false, try_finally_label);
     EMIT(pop_top); // pop exception instance
-    EMIT(pop_except);
-    EMIT_ARG(jump, while_else_label);
+    EMIT_ARG(pop_except_jump, while_else_label, true);
 
     EMIT_ARG(label_assign, try_finally_label);
     EMIT_ARG(adjust_stack_size, 1); // if we jump here, the exc is on the stack
@@ -1810,8 +1874,7 @@ STATIC void compile_async_with_stmt_helper(compiler_t *comp, int n, mp_parse_nod
         compile_async_with_stmt_helper(comp, n - 1, nodes + 1, body);
         EMIT_ARG(adjust_stack_size, -3);
 
-        // Finish the "try" block
-        EMIT(pop_block);
+        // We have now finished the "try" block and fall through to the "finally"
 
         // At this point, after the with body has executed, we have 3 cases:
         // 1. no exception, we just fall through to this point; stack: (..., ctx_mgr)
@@ -1941,21 +2004,8 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
             c_assign(comp, pns->nodes[0], ASSIGN_AUG_LOAD); // lhs load for aug assign
             compile_node(comp, pns1->nodes[1]); // rhs
             assert(MP_PARSE_NODE_IS_TOKEN(pns1->nodes[0]));
-            mp_binary_op_t op;
-            switch (MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])) {
-                case MP_TOKEN_DEL_PIPE_EQUAL: op = MP_BINARY_OP_INPLACE_OR; break;
-                case MP_TOKEN_DEL_CARET_EQUAL: op = MP_BINARY_OP_INPLACE_XOR; break;
-                case MP_TOKEN_DEL_AMPERSAND_EQUAL: op = MP_BINARY_OP_INPLACE_AND; break;
-                case MP_TOKEN_DEL_DBL_LESS_EQUAL: op = MP_BINARY_OP_INPLACE_LSHIFT; break;
-                case MP_TOKEN_DEL_DBL_MORE_EQUAL: op = MP_BINARY_OP_INPLACE_RSHIFT; break;
-                case MP_TOKEN_DEL_PLUS_EQUAL: op = MP_BINARY_OP_INPLACE_ADD; break;
-                case MP_TOKEN_DEL_MINUS_EQUAL: op = MP_BINARY_OP_INPLACE_SUBTRACT; break;
-                case MP_TOKEN_DEL_STAR_EQUAL: op = MP_BINARY_OP_INPLACE_MULTIPLY; break;
-                case MP_TOKEN_DEL_DBL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_FLOOR_DIVIDE; break;
-                case MP_TOKEN_DEL_SLASH_EQUAL: op = MP_BINARY_OP_INPLACE_TRUE_DIVIDE; break;
-                case MP_TOKEN_DEL_PERCENT_EQUAL: op = MP_BINARY_OP_INPLACE_MODULO; break;
-                case MP_TOKEN_DEL_DBL_STAR_EQUAL: default: op = MP_BINARY_OP_INPLACE_POWER; break;
-            }
+            mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0]);
+            mp_binary_op_t op = MP_BINARY_OP_INPLACE_OR + (tok - MP_TOKEN_DEL_PIPE_EQUAL);
             EMIT_ARG(binary_op, op);
             c_assign(comp, pns->nodes[0], ASSIGN_AUG_STORE); // lhs store for aug assign
         } else if (kind == PN_expr_stmt_assign_list) {
@@ -2089,15 +2139,12 @@ STATIC void compile_comparison(compiler_t *comp, mp_parse_node_struct_t *pns) {
             EMIT(rot_three);
         }
         if (MP_PARSE_NODE_IS_TOKEN(pns->nodes[i])) {
+            mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]);
             mp_binary_op_t op;
-            switch (MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])) {
-                case MP_TOKEN_OP_LESS: op = MP_BINARY_OP_LESS; break;
-                case MP_TOKEN_OP_MORE: op = MP_BINARY_OP_MORE; break;
-                case MP_TOKEN_OP_DBL_EQUAL: op = MP_BINARY_OP_EQUAL; break;
-                case MP_TOKEN_OP_LESS_EQUAL: op = MP_BINARY_OP_LESS_EQUAL; break;
-                case MP_TOKEN_OP_MORE_EQUAL: op = MP_BINARY_OP_MORE_EQUAL; break;
-                case MP_TOKEN_OP_NOT_EQUAL: op = MP_BINARY_OP_NOT_EQUAL; break;
-                case MP_TOKEN_KW_IN: default: op = MP_BINARY_OP_IN; break;
+            if (tok == MP_TOKEN_KW_IN) {
+                op = MP_BINARY_OP_IN;
+            } else {
+                op = MP_BINARY_OP_LESS + (tok - MP_TOKEN_OP_LESS);
             }
             EMIT_ARG(binary_op, op);
         } else {
@@ -2151,36 +2198,21 @@ STATIC void compile_term(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_node(comp, pns->nodes[0]);
     for (int i = 1; i + 1 < num_nodes; i += 2) {
         compile_node(comp, pns->nodes[i + 1]);
-        mp_binary_op_t op;
         mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]);
-        switch (tok) {
-            case MP_TOKEN_OP_PLUS:      op = MP_BINARY_OP_ADD; break;
-            case MP_TOKEN_OP_MINUS:     op = MP_BINARY_OP_SUBTRACT; break;
-            case MP_TOKEN_OP_STAR:      op = MP_BINARY_OP_MULTIPLY; break;
-            case MP_TOKEN_OP_DBL_SLASH: op = MP_BINARY_OP_FLOOR_DIVIDE; break;
-            case MP_TOKEN_OP_SLASH:     op = MP_BINARY_OP_TRUE_DIVIDE; break;
-            case MP_TOKEN_OP_PERCENT:   op = MP_BINARY_OP_MODULO; break;
-            case MP_TOKEN_OP_DBL_LESS:  op = MP_BINARY_OP_LSHIFT; break;
-            default:
-                assert(tok == MP_TOKEN_OP_DBL_MORE);
-                op = MP_BINARY_OP_RSHIFT;
-                break;
-        }
+        mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS);
         EMIT_ARG(binary_op, op);
     }
 }
 
 STATIC void compile_factor_2(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_node(comp, pns->nodes[1]);
-    mp_unary_op_t op;
     mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
-    switch (tok) {
-        case MP_TOKEN_OP_PLUS:  op = MP_UNARY_OP_POSITIVE; break;
-        case MP_TOKEN_OP_MINUS: op = MP_UNARY_OP_NEGATIVE; break;
-        default:
-            assert(tok == MP_TOKEN_OP_TILDE);
-            op = MP_UNARY_OP_INVERT;
-            break;
+    mp_unary_op_t op;
+    if (tok == MP_TOKEN_OP_TILDE) {
+        op = MP_UNARY_OP_INVERT;
+    } else {
+        assert(tok == MP_TOKEN_OP_PLUS || tok == MP_TOKEN_OP_MINUS);
+        op = MP_UNARY_OP_POSITIVE + (tok - MP_TOKEN_OP_PLUS);
     }
     EMIT_ARG(unary_op, op);
 }
@@ -2247,6 +2279,20 @@ STATIC void compile_atom_expr_normal(compiler_t *comp, mp_parse_node_struct_t *p
             EMIT_ARG(call_function, 2, 0, 0);
             i = 1;
         }
+
+    #if MICROPY_COMP_CONST_LITERAL && MICROPY_PY_COLLECTIONS_ORDEREDDICT
+    // handle special OrderedDict constructor
+    } else if (MP_PARSE_NODE_IS_ID(pns->nodes[0])
+        && MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]) == MP_QSTR_OrderedDict
+        && MP_PARSE_NODE_STRUCT_KIND(pns_trail[0]) == PN_trailer_paren
+        && MP_PARSE_NODE_IS_STRUCT_KIND(pns_trail[0]->nodes[0], PN_atom_brace)) {
+        // at this point we have matched "OrderedDict({...})"
+
+        EMIT_ARG(call_function, 0, 0, 0);
+        mp_parse_node_struct_t *pns_dict = (mp_parse_node_struct_t*)pns_trail[0]->nodes[0];
+        compile_atom_brace_helper(comp, pns_dict, false);
+        i = 1;
+    #endif
     }
 
     // compile the remaining trailers
@@ -2455,16 +2501,20 @@ STATIC void compile_atom_bracket(compiler_t *comp, mp_parse_node_struct_t *pns)
     }
 }
 
-STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
+STATIC void compile_atom_brace_helper(compiler_t *comp, mp_parse_node_struct_t *pns, bool create_map) {
     mp_parse_node_t pn = pns->nodes[0];
     if (MP_PARSE_NODE_IS_NULL(pn)) {
         // empty dict
-        EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP);
+        if (create_map) {
+            EMIT_ARG(build, 0, MP_EMIT_BUILD_MAP);
+        }
     } else if (MP_PARSE_NODE_IS_STRUCT(pn)) {
         pns = (mp_parse_node_struct_t*)pn;
         if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker_item) {
             // dict with one element
-            EMIT_ARG(build, 1, MP_EMIT_BUILD_MAP);
+            if (create_map) {
+                EMIT_ARG(build, 1, MP_EMIT_BUILD_MAP);
+            }
             compile_node(comp, pn);
             EMIT(store_map);
         } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_dictorsetmaker) {
@@ -2481,7 +2531,9 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
                 bool is_dict;
                 if (!MICROPY_PY_BUILTINS_SET || MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_dictorsetmaker_item)) {
                     // a dictionary
-                    EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_MAP);
+                    if (create_map) {
+                        EMIT_ARG(build, 1 + n, MP_EMIT_BUILD_MAP);
+                    }
                     compile_node(comp, pns->nodes[0]);
                     EMIT(store_map);
                     is_dict = true;
@@ -2551,6 +2603,10 @@ STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
     }
 }
 
+STATIC void compile_atom_brace(compiler_t *comp, mp_parse_node_struct_t *pns) {
+    compile_atom_brace_helper(comp, pns, true);
+}
+
 STATIC void compile_trailer_paren(compiler_t *comp, mp_parse_node_struct_t *pns) {
     compile_trailer_paren_helper(comp, pns->nodes[0], false, 0);
 }
@@ -2675,9 +2731,6 @@ STATIC mp_obj_t get_const_object(mp_parse_node_struct_t *pns) {
 }
 
 STATIC void compile_const_object(compiler_t *comp, mp_parse_node_struct_t *pns) {
-    #if MICROPY_EMIT_NATIVE
-    comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-    #endif
     EMIT_ARG(load_const_obj, get_const_object(pns));
 }
 
@@ -2712,9 +2765,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
             } else {
                 EMIT_ARG(load_const_obj, mp_obj_new_int_from_ll(arg));
             }
-            #if MICROPY_EMIT_NATIVE
-            comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-            #endif
         }
         #else
         EMIT_ARG(load_const_small_int, arg);
@@ -2733,9 +2783,6 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) {
                     const byte *data = qstr_data(arg, &len);
                     EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len));
                 }
-                #if MICROPY_EMIT_NATIVE
-                comp->scope_cur->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
-                #endif
                 break;
             case MP_PARSE_NODE_TOKEN: default:
                 if (arg == MP_TOKEN_NEWLINE) {
@@ -2784,7 +2831,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn
         return;
     }
 
-    qstr param_name = MP_QSTR_NULL;
+    qstr param_name = MP_QSTRnull;
     uint param_flag = ID_FLAG_IS_PARAM;
     mp_parse_node_struct_t *pns = NULL;
     if (MP_PARSE_NODE_IS_ID(pn)) {
@@ -2843,7 +2890,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn
         }
     }
 
-    if (param_name != MP_QSTR_NULL) {
+    if (param_name != MP_QSTRnull) {
         id_info_t *id_info = scope_find_or_add_id(comp->scope_cur, param_name, ID_INFO_KIND_UNDECIDED);
         if (id_info->kind != ID_INFO_KIND_UNDECIDED) {
             compile_syntax_error(comp, pn, "argument name reused");
@@ -3011,6 +3058,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
         mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)scope->pn;
         assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 3);
 
+        // Set the source line number for the start of the lambda
+        EMIT_ARG(set_source_line, pns->source_line);
+
         // work out number of parameters, keywords and default parameters, and add them to the id_info array
         // must be done before compiling the body so that arguments are numbered first (for LOAD_FAST etc)
         if (comp->pass == MP_PASS_SCOPE) {
@@ -3045,6 +3095,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
             scope->num_pos_args = 1;
         }
 
+        // Set the source line number for the start of the comprehension
+        EMIT_ARG(set_source_line, pns->source_line);
+
         if (scope->kind == SCOPE_LIST_COMP) {
             EMIT_ARG(build, 0, MP_EMIT_BUILD_LIST);
         } else if (scope->kind == SCOPE_DICT_COMP) {
@@ -3084,6 +3137,9 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
             scope_find_or_add_id(scope, MP_QSTR___class__, ID_INFO_KIND_LOCAL);
         }
 
+        #if MICROPY_PY_SYS_SETTRACE
+        EMIT_ARG(set_source_line, pns->source_line);
+        #endif
         compile_load_id(comp, MP_QSTR___name__);
         compile_store_id(comp, MP_QSTR___module__);
         EMIT_ARG(load_const_str, MP_PARSE_NODE_LEAF_ARG(pns->nodes[0])); // 0 is class name
@@ -3257,7 +3313,11 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind
             void *f = mp_asm_base_get_code((mp_asm_base_t*)comp->emit_inline_asm);
             mp_emit_glue_assign_native(comp->scope_cur->raw_code, MP_CODE_NATIVE_ASM,
                 f, mp_asm_base_get_code_size((mp_asm_base_t*)comp->emit_inline_asm),
-                NULL, comp->scope_cur->num_pos_args, 0, type_sig);
+                NULL,
+                #if MICROPY_PERSISTENT_CODE_SAVE
+                0, 0, 0, 0, NULL,
+                #endif
+                comp->scope_cur->num_pos_args, 0, type_sig);
         }
     }
 
@@ -3363,7 +3423,7 @@ STATIC void scope_compute_things(scope_t *scope) {
 #if !MICROPY_PERSISTENT_CODE_SAVE
 STATIC
 #endif
-mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
+mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) {
     // put compiler state on the stack, it's relatively small
     compiler_t comp_state = {0};
     compiler_t *comp = &comp_state;
@@ -3374,6 +3434,11 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     comp->continue_label = INVALID_LABEL;
 
     // create the module scope
+    #if MICROPY_EMIT_NATIVE
+    const uint emit_opt = MP_STATE_VM(default_emit_opt);
+    #else
+    const uint emit_opt = MP_EMIT_OPT_NONE;
+    #endif
     scope_t *module_scope = scope_new_and_link(comp, SCOPE_MODULE, parse_tree->root, emit_opt);
 
     // create standard emitter; it's used at least for MP_PASS_SCOPE
@@ -3386,12 +3451,12 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     #endif
     uint max_num_labels = 0;
     for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
-        if (false) {
         #if MICROPY_EMIT_INLINE_ASM
-        } else if (s->emit_options == MP_EMIT_OPT_ASM) {
+        if (s->emit_options == MP_EMIT_OPT_ASM) {
             compile_scope_inline_asm(comp, s, MP_PASS_SCOPE);
+        } else
         #endif
-        } else {
+        {
             compile_scope(comp, s, MP_PASS_SCOPE);
 
             // Check if any implicitly declared variables should be closed over
@@ -3422,30 +3487,32 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     emit_t *emit_native = NULL;
 #endif
     for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
-        if (false) {
-            // dummy
-
         #if MICROPY_EMIT_INLINE_ASM
-        } else if (s->emit_options == MP_EMIT_OPT_ASM) {
+        if (s->emit_options == MP_EMIT_OPT_ASM) {
             // inline assembly
             if (comp->emit_inline_asm == NULL) {
                 comp->emit_inline_asm = ASM_EMITTER(new)(max_num_labels);
             }
             comp->emit = NULL;
-            comp->emit_inline_asm_method_table = &ASM_EMITTER(method_table);
+            comp->emit_inline_asm_method_table = ASM_EMITTER_TABLE;
             compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
             #if MICROPY_EMIT_INLINE_XTENSA
             // Xtensa requires an extra pass to compute size of l32r const table
             // TODO this can be improved by calculating it during SCOPE pass
             // but that requires some other structural changes to the asm emitters
-            compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
+            #if MICROPY_DYNAMIC_COMPILER
+            if (mp_dynamic_compiler.native_arch == MP_NATIVE_ARCH_XTENSA)
+            #endif
+            {
+                compile_scope_inline_asm(comp, s, MP_PASS_CODE_SIZE);
+            }
             #endif
             if (comp->compile_error == MP_OBJ_NULL) {
                 compile_scope_inline_asm(comp, s, MP_PASS_EMIT);
             }
+        } else
         #endif
-
-        } else {
+        {
 
             // choose the emit type
 
@@ -3457,7 +3524,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
                     if (emit_native == NULL) {
                         emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels);
                     }
-                    comp->emit_method_table = &NATIVE_EMITTER(method_table);
+                    comp->emit_method_table = NATIVE_EMITTER_TABLE;
                     comp->emit = emit_native;
                     break;
 #endif // MICROPY_EMIT_NATIVE
@@ -3526,8 +3593,8 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f
     }
 }
 
-mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
-    mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl);
+mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) {
+    mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl);
     // return function that executes the outer module
     return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
 }

py/compile.h

@@ -32,11 +32,11 @@
 
 // the compiler will raise an exception if an error occurred
 // the compiler will clear the parse tree before it returns
-mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl);
+mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl);
 
 #if MICROPY_PERSISTENT_CODE_SAVE
 // this has the same semantics as mp_compile
-mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl);
+mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl);
 #endif
 
 // this is implemented in runtime.c

py/dynruntime.h

@@ -0,0 +1,218 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_PY_DYNRUNTIME_H
+#define MICROPY_INCLUDED_PY_DYNRUNTIME_H
+
+// This header file contains definitions to dynamically implement the static
+// MicroPython runtime API defined in py/obj.h and py/runtime.h.
+
+#include "py/nativeglue.h"
+#include "py/objstr.h"
+
+#undef MP_ROM_QSTR
+#undef MP_OBJ_QSTR_VALUE
+#undef MP_OBJ_NEW_QSTR
+#undef mp_const_none
+#undef mp_const_false
+#undef mp_const_true
+#undef mp_const_empty_tuple
+#undef nlr_raise
+
+/******************************************************************************/
+// Memory allocation
+
+#define m_malloc(n)                     (m_malloc_dyn((n)))
+#define m_free(ptr)                     (m_free_dyn((ptr)))
+#define m_realloc(ptr, new_num_bytes)   (m_realloc_dyn((ptr), (new_num_bytes)))
+
+static inline void *m_malloc_dyn(size_t n) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(NULL, n, false);
+}
+
+static inline void m_free_dyn(void *ptr) {
+    mp_fun_table.realloc_(ptr, 0, false);
+}
+
+static inline void *m_realloc_dyn(void *ptr, size_t new_num_bytes) {
+    // TODO won't raise on OOM
+    return mp_fun_table.realloc_(ptr, new_num_bytes, true);
+}
+
+/******************************************************************************/
+// Printing
+
+#define mp_plat_print               (*mp_fun_table.plat_print)
+#define mp_printf(p, ...)           (mp_fun_table.printf_((p), __VA_ARGS__))
+#define mp_vprintf(p, fmt, args)    (mp_fun_table.vprintf_((p), (fmt), (args)))
+
+/******************************************************************************/
+// Types and objects
+
+#define MP_OBJ_NEW_QSTR(x) MP_OBJ_NEW_QSTR_ ## x
+
+#define mp_type_type                        (*mp_fun_table.type_type)
+#define mp_type_str                         (*mp_fun_table.type_str)
+#define mp_type_list                        (*mp_fun_table.type_list)
+#define mp_type_EOFError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_EOFError)))
+#define mp_type_IndexError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_IndexError)))
+#define mp_type_KeyError                    (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_KeyError)))
+#define mp_type_NotImplementedError         (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_NotImplementedError)))
+#define mp_type_RuntimeError                (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_RuntimeError)))
+#define mp_type_TypeError                   (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_TypeError)))
+#define mp_type_ValueError                  (*(mp_obj_type_t*)(mp_load_global(MP_QSTR_ValueError)))
+
+#define mp_stream_read_obj                  (*mp_fun_table.stream_read_obj)
+#define mp_stream_readinto_obj              (*mp_fun_table.stream_readinto_obj)
+#define mp_stream_unbuffered_readline_obj   (*mp_fun_table.stream_unbuffered_readline_obj)
+#define mp_stream_write_obj                 (*mp_fun_table.stream_write_obj)
+
+#define mp_const_none                       ((mp_obj_t)mp_fun_table.const_none)
+#define mp_const_false                      ((mp_obj_t)mp_fun_table.const_false)
+#define mp_const_true                       ((mp_obj_t)mp_fun_table.const_true)
+#define mp_const_empty_tuple                (mp_fun_table.new_tuple(0, NULL))
+
+#define mp_obj_new_bool(b)                  ((b) ? (mp_obj_t)mp_fun_table.const_true : (mp_obj_t)mp_fun_table.const_false)
+#define mp_obj_new_int(i)                   (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_INT))
+#define mp_obj_new_int_from_uint(i)         (mp_fun_table.native_to_obj(i, MP_NATIVE_TYPE_UINT))
+#define mp_obj_new_str(data, len)           (mp_fun_table.obj_new_str((data), (len)))
+#define mp_obj_new_str_of_type(t, d, l)     (mp_obj_new_str_of_type_dyn((t), (d), (l)))
+#define mp_obj_new_bytes(data, len)         (mp_fun_table.obj_new_bytes((data), (len)))
+#define mp_obj_new_bytearray_by_ref(n, i)   (mp_fun_table.obj_new_bytearray_by_ref((n), (i)))
+#define mp_obj_new_tuple(n, items)          (mp_fun_table.new_tuple((n), (items)))
+#define mp_obj_new_list(n, items)           (mp_fun_table.new_list((n), (items)))
+
+#define mp_obj_get_type(o)                  (mp_fun_table.obj_get_type((o)))
+#define mp_obj_get_int(o)                   (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_INT))
+#define mp_obj_get_int_truncated(o)         (mp_fun_table.native_from_obj(o, MP_NATIVE_TYPE_UINT))
+#define mp_obj_str_get_str(s)               ((void*)mp_fun_table.native_from_obj(s, MP_NATIVE_TYPE_PTR))
+#define mp_obj_str_get_data(o, len)         (mp_obj_str_get_data_dyn((o), (len)))
+#define mp_get_buffer_raise(o, bufinfo, fl) (mp_fun_table.get_buffer_raise((o), (bufinfo), (fl)))
+#define mp_get_stream_raise(s, flags)       (mp_fun_table.get_stream_raise((s), (flags)))
+
+#define mp_obj_len(o)                       (mp_obj_len_dyn(o))
+#define mp_obj_subscr(base, index, val)     (mp_fun_table.obj_subscr((base), (index), (val)))
+#define mp_obj_list_append(list, item)      (mp_fun_table.list_append((list), (item)))
+
+static inline mp_obj_t mp_obj_new_str_of_type_dyn(const mp_obj_type_t *type, const byte* data, size_t len) {
+    if (type == &mp_type_str) {
+        return mp_obj_new_str((const char*)data, len);
+    } else {
+        return mp_obj_new_bytes(data, len);
+    }
+}
+
+static inline void *mp_obj_str_get_data_dyn(mp_obj_t o, size_t *l) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(o, &bufinfo, MP_BUFFER_READ);
+    *l = bufinfo.len;
+    return bufinfo.buf;
+}
+
+static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) {
+    // If bytes implemented MP_UNARY_OP_LEN could use: mp_unary_op(MP_UNARY_OP_LEN, o)
+    return mp_fun_table.call_function_n_kw(mp_fun_table.load_name(MP_QSTR_len), 1, &o);
+}
+
+/******************************************************************************/
+// General runtime functions
+
+#define mp_load_name(qst)           (mp_fun_table.load_name(qst))
+#define mp_load_global(qst)         (mp_fun_table.load_global(qst))
+#define mp_store_global(qst, obj)   (mp_fun_table.store_global((qst), (obj)))
+#define mp_unary_op(op, obj)        (mp_fun_table.unary_op((op), (obj)))
+#define mp_binary_op(op, lhs, rhs)  (mp_fun_table.binary_op((op), (lhs), (rhs)))
+
+#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \
+    (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args)))
+
+#define mp_call_function_n_kw(fun, n_args, n_kw, args) \
+    (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args))
+
+#define mp_arg_check_num(n_args, n_kw, n_args_min, n_args_max, takes_kw) \
+    (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw))))
+
+#define MP_DYNRUNTIME_INIT_ENTRY \
+    mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \
+    mp_raw_code_t rc; \
+    rc.kind = MP_CODE_NATIVE_VIPER; \
+    rc.scope_flags = 0; \
+    rc.const_table = (void*)self->const_table; \
+    (void)rc;
+
+#define MP_DYNRUNTIME_INIT_EXIT \
+    mp_fun_table.swap_globals(old_globals); \
+    return mp_const_none;
+
+#define MP_DYNRUNTIME_MAKE_FUNCTION(f) \
+    (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL))
+
+/******************************************************************************/
+// Exceptions
+
+#define mp_obj_new_exception(o)                 ((mp_obj_t)(o)) // Assumes returned object will be raised, will create instance then
+#define mp_obj_new_exception_arg1(e_type, arg)  (mp_obj_new_exception_arg1_dyn((e_type), (arg)))
+
+#define nlr_raise(o)                            (mp_raise_dyn(o))
+#define mp_raise_msg(type, msg)                 (mp_fun_table.raise_msg((type), (msg)))
+#define mp_raise_OSError(er)                    (mp_raise_OSError_dyn(er))
+#define mp_raise_NotImplementedError(msg)       (mp_raise_msg(&mp_type_NotImplementedError, (msg)))
+#define mp_raise_TypeError(msg)                 (mp_raise_msg(&mp_type_TypeError, (msg)))
+#define mp_raise_ValueError(msg)                (mp_raise_msg(&mp_type_ValueError, (msg)))
+
+static inline mp_obj_t mp_obj_new_exception_arg1_dyn(const mp_obj_type_t *exc_type, mp_obj_t arg) {
+    mp_obj_t args[1] = { arg };
+    return mp_call_function_n_kw(MP_OBJ_FROM_PTR(exc_type), 1, 0, &args[0]);
+}
+
+static NORETURN inline void mp_raise_dyn(mp_obj_t o) {
+    mp_fun_table.raise(o);
+    for (;;) {
+    }
+}
+
+static inline void mp_raise_OSError_dyn(int er) {
+    mp_obj_t args[1] = { MP_OBJ_NEW_SMALL_INT(er) };
+    nlr_raise(mp_call_function_n_kw(mp_load_global(MP_QSTR_OSError), 1, 0, &args[0]));
+}
+
+/******************************************************************************/
+// Floating point
+
+#define mp_obj_new_float_from_f(f)  (mp_fun_table.obj_new_float_from_f((f)))
+#define mp_obj_new_float_from_d(d)  (mp_fun_table.obj_new_float_from_d((d)))
+#define mp_obj_get_float_to_f(o)    (mp_fun_table.obj_get_float_to_f((o)))
+#define mp_obj_get_float_to_d(o)    (mp_fun_table.obj_get_float_to_d((o)))
+
+#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT
+#define mp_obj_new_float(f)         (mp_obj_new_float_from_f((f)))
+#define mp_obj_get_float(o)         (mp_obj_get_float_to_f((o)))
+#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE
+#define mp_obj_new_float(f)         (mp_obj_new_float_from_d((f)))
+#define mp_obj_get_float(o)         (mp_obj_get_float_to_d((o)))
+#endif
+
+#endif // MICROPY_INCLUDED_PY_DYNRUNTIME_H

py/dynruntime.mk

@@ -0,0 +1,144 @@
+# Makefile fragment for generating native .mpy files from C source
+# MPY_DIR must be set to the top of the MicroPython source tree
+
+BUILD ?= build
+
+ECHO = @echo
+RM = /bin/rm
+MKDIR = /bin/mkdir
+PYTHON = python3
+MPY_CROSS = $(MPY_DIR)/mpy-cross/mpy-cross
+MPY_TOOL = $(PYTHON) $(MPY_DIR)/tools/mpy-tool.py
+MPY_LD = $(PYTHON) $(MPY_DIR)/tools/mpy_ld.py
+
+Q = @
+ifeq ("$(origin V)", "command line")
+ifeq ($(V),1)
+Q =
+MPY_LD += '-vvv'
+endif
+endif
+
+ARCH_UPPER = $(shell echo $(ARCH) | tr '[:lower:]' '[:upper:]')
+CONFIG_H = $(BUILD)/$(MOD).config.h
+
+CFLAGS += -I. -I$(MPY_DIR)
+CFLAGS += -std=c99
+CFLAGS += -Os
+CFLAGS += -Wall -Werror -DNDEBUG
+CFLAGS += -DNO_QSTR
+CFLAGS += -DMP_CONFIGFILE='<$(CONFIG_H)>'
+CFLAGS += -fpic -fno-common
+CFLAGS += -U _FORTIFY_SOURCE # prevent use of __*_chk libc functions
+#CFLAGS += -fdata-sections -ffunction-sections
+
+MPY_CROSS_FLAGS += -march=$(ARCH)
+
+SRC_O += $(addprefix $(BUILD)/, $(patsubst %.c,%.o,$(filter %.c,$(SRC))))
+SRC_MPY += $(addprefix $(BUILD)/, $(patsubst %.py,%.mpy,$(filter %.py,$(SRC))))
+
+################################################################################
+# Architecture configuration
+
+ifeq ($(ARCH),x86)
+
+# x86
+CROSS =
+CFLAGS += -m32 -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),x64)
+
+# x64
+CROSS =
+CFLAGS += -fno-stack-protector
+MPY_CROSS_FLAGS += -mcache-lookup-bc
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),armv7m)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m3
+MICROPY_FLOAT_IMPL ?= none
+
+else ifeq ($(ARCH),armv7emsp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m4
+CFLAGS += -mfpu=fpv4-sp-d16 -mfloat-abi=hard
+MICROPY_FLOAT_IMPL ?= float
+
+else ifeq ($(ARCH),armv7emdp)
+
+# thumb
+CROSS = arm-none-eabi-
+CFLAGS += -mthumb -mcpu=cortex-m7
+CFLAGS += -mfpu=fpv5-d16 -mfloat-abi=hard
+MICROPY_FLOAT_IMPL ?= double
+
+else ifeq ($(ARCH),xtensa)
+
+# xtensa
+CROSS = xtensa-lx106-elf-
+CFLAGS += -mforce-l32
+MICROPY_FLOAT_IMPL ?= none
+
+else ifeq ($(ARCH),xtensawin)
+
+# xtensawin
+CROSS = xtensa-esp32-elf-
+CFLAGS +=
+MICROPY_FLOAT_IMPL ?= float
+
+else
+$(error architecture '$(ARCH)' not supported)
+endif
+
+MICROPY_FLOAT_IMPL_UPPER = $(shell echo $(MICROPY_FLOAT_IMPL) | tr '[:lower:]' '[:upper:]')
+CFLAGS += -DMICROPY_FLOAT_IMPL=MICROPY_FLOAT_IMPL_$(MICROPY_FLOAT_IMPL_UPPER)
+
+CFLAGS += $(CFLAGS_EXTRA)
+
+################################################################################
+# Build rules
+
+.PHONY: all clean
+
+all: $(MOD).mpy
+
+clean:
+	$(RM) -rf $(BUILD) $(CLEAN_EXTRA)
+
+# Create build destination directories first
+BUILD_DIRS = $(sort $(dir $(CONFIG_H) $(SRC_O) $(SRC_MPY)))
+$(CONFIG_H) $(SRC_O) $(SRC_MPY): | $(BUILD_DIRS)
+$(BUILD_DIRS):
+	$(Q)$(MKDIR) -p $@
+
+# Preprocess all source files to generate $(CONFIG_H)
+$(CONFIG_H): $(SRC)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_LD) --arch $(ARCH) --preprocess -o $@ $^
+
+# Build .o from .c source files
+$(BUILD)/%.o: %.c $(CONFIG_H) Makefile
+	$(ECHO) "CC $<"
+	$(Q)$(CROSS)gcc $(CFLAGS) -o $@ -c $<
+
+# Build .mpy from .py source files
+$(BUILD)/%.mpy: %.py
+	$(ECHO) "MPY $<"
+	$(Q)$(MPY_CROSS) $(MPY_CROSS_FLAGS) -o $@ $<
+
+# Build native .mpy from object files
+$(BUILD)/$(MOD).native.mpy: $(SRC_O)
+	$(ECHO) "LINK $<"
+	$(Q)$(MPY_LD) --arch $(ARCH) --qstrs $(CONFIG_H) -o $@ $^
+
+# Build final .mpy from all intermediate .mpy files
+$(MOD).mpy: $(BUILD)/$(MOD).native.mpy $(SRC_MPY)
+	$(ECHO) "GEN $@"
+	$(Q)$(MPY_TOOL) --merge -o $@ $^

py/emit.h

@@ -76,15 +76,15 @@ typedef enum {
 
 // Kind for emit->setup_block()
 #define MP_EMIT_SETUP_BLOCK_WITH (0)
-#define MP_EMIT_SETUP_BLOCK_EXCEPT (2)
-#define MP_EMIT_SETUP_BLOCK_FINALLY (3)
+#define MP_EMIT_SETUP_BLOCK_EXCEPT (1)
+#define MP_EMIT_SETUP_BLOCK_FINALLY (2)
 
 // Kind for emit->build()
 #define MP_EMIT_BUILD_TUPLE (0)
 #define MP_EMIT_BUILD_LIST (1)
-#define MP_EMIT_BUILD_MAP (3)
-#define MP_EMIT_BUILD_SET (6)
-#define MP_EMIT_BUILD_SLICE (8)
+#define MP_EMIT_BUILD_MAP (2)
+#define MP_EMIT_BUILD_SET (3)
+#define MP_EMIT_BUILD_SLICE (4)
 
 // Kind for emit->yield()
 #define MP_EMIT_YIELD_VALUE (0)
@@ -98,6 +98,11 @@ typedef struct _mp_emit_method_table_id_ops_t {
 } mp_emit_method_table_id_ops_t;
 
 typedef struct _emit_method_table_t {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_t *(*emit_new)(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
+    void (*emit_free)(emit_t *emit);
+    #endif
+
     void (*start_pass)(emit_t *emit, pass_kind_t pass, scope_t *scope);
     void (*end_pass)(emit_t *emit);
     bool (*last_emit_was_return_value)(emit_t *emit);
@@ -134,8 +139,7 @@ typedef struct _emit_method_table_t {
     void (*get_iter)(emit_t *emit, bool use_stack);
     void (*for_iter)(emit_t *emit, mp_uint_t label);
     void (*for_iter_end)(emit_t *emit);
-    void (*pop_block)(emit_t *emit);
-    void (*pop_except)(emit_t *emit);
+    void (*pop_except_jump)(emit_t *emit, mp_uint_t label, bool within_exc_handler);
     void (*unary_op)(emit_t *emit, mp_unary_op_t op);
     void (*binary_op)(emit_t *emit, mp_binary_op_t op);
     void (*build)(emit_t *emit, mp_uint_t n_args, int kind);
@@ -157,8 +161,6 @@ typedef struct _emit_method_table_t {
     void (*end_except_handler)(emit_t *emit);
 } emit_method_table_t;
 
-int mp_native_type_from_qstr(qstr qst);
-
 static inline void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) {
     scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT);
 }
@@ -172,6 +174,7 @@ extern const emit_method_table_t emit_native_x86_method_table;
 extern const emit_method_table_t emit_native_thumb_method_table;
 extern const emit_method_table_t emit_native_arm_method_table;
 extern const emit_method_table_t emit_native_xtensa_method_table;
+extern const emit_method_table_t emit_native_xtensawin_method_table;
 
 extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops;
 extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops;
@@ -183,6 +186,7 @@ emit_t *emit_native_x86_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t ma
 emit_t *emit_native_thumb_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 emit_t *emit_native_arm_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 emit_t *emit_native_xtensa_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
+emit_t *emit_native_xtensawin_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels);
 
 void emit_bc_set_max_num_labels(emit_t* emit, mp_uint_t max_num_labels);
 
@@ -192,6 +196,7 @@ void emit_native_x86_free(emit_t *emit);
 void emit_native_thumb_free(emit_t *emit);
 void emit_native_arm_free(emit_t *emit);
 void emit_native_xtensa_free(emit_t *emit);
+void emit_native_xtensawin_free(emit_t *emit);
 
 void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope);
 void mp_emit_bc_end_pass(emit_t *emit);
@@ -232,8 +237,7 @@ void mp_emit_bc_end_finally(emit_t *emit);
 void mp_emit_bc_get_iter(emit_t *emit, bool use_stack);
 void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label);
 void mp_emit_bc_for_iter_end(emit_t *emit);
-void mp_emit_bc_pop_block(emit_t *emit);
-void mp_emit_bc_pop_except(emit_t *emit);
+void mp_emit_bc_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler);
 void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op);
 void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op);
 void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind);
@@ -254,6 +258,11 @@ void mp_emit_bc_end_except_handler(emit_t *emit);
 typedef struct _emit_inline_asm_t emit_inline_asm_t;
 
 typedef struct _emit_inline_asm_method_table_t {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_asm_t *(*asm_new)(mp_uint_t max_num_labels);
+    void (*asm_free)(emit_inline_asm_t *emit);
+    #endif
+
     void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, mp_obj_t *error_slot);
     void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
     mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params);

py/emitbc.c

@@ -3,7 +3,7 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2013-2019 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -64,6 +64,9 @@ struct _emit_t {
     size_t bytecode_size;
     byte *code_base; // stores both byte code and code info
 
+    size_t n_info;
+    size_t n_cell;
+
     #if MICROPY_PERSISTENT_CODE
     uint16_t ct_cur_obj;
     uint16_t ct_num_obj;
@@ -123,10 +126,6 @@ STATIC void emit_write_code_info_byte(emit_t* emit, byte val) {
     *emit_get_cur_to_write_code_info(emit, 1) = val;
 }
 
-STATIC void emit_write_code_info_uint(emit_t* emit, mp_uint_t val) {
-    emit_write_uint(emit, emit_get_cur_to_write_code_info, val);
-}
-
 STATIC void emit_write_code_info_qstr(emit_t *emit, qstr qst) {
     #if MICROPY_PERSISTENT_CODE
     assert((qst >> 16) == 0);
@@ -182,20 +181,20 @@ STATIC byte *emit_get_cur_to_write_bytecode(emit_t *emit, int num_bytes_to_write
     }
 }
 
-STATIC void emit_write_bytecode_byte(emit_t *emit, byte b1) {
+STATIC void emit_write_bytecode_raw_byte(emit_t *emit, byte b1) {
     byte *c = emit_get_cur_to_write_bytecode(emit, 1);
     c[0] = b1;
 }
 
-STATIC void emit_write_bytecode_byte_byte(emit_t* emit, byte b1, byte b2) {
-    byte *c = emit_get_cur_to_write_bytecode(emit, 2);
+STATIC void emit_write_bytecode_byte(emit_t *emit, int stack_adj, byte b1) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
+    byte *c = emit_get_cur_to_write_bytecode(emit, 1);
     c[0] = b1;
-    c[1] = b2;
 }
 
 // Similar to emit_write_bytecode_uint(), just some extra handling to encode sign
-STATIC void emit_write_bytecode_byte_int(emit_t *emit, byte b1, mp_int_t num) {
-    emit_write_bytecode_byte(emit, b1);
+STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, mp_int_t num) {
+    emit_write_bytecode_byte(emit, stack_adj, b1);
 
     // We store each 7 bits in a separate byte, and that's how many bytes needed
     byte buf[BYTES_FOR_INT];
@@ -220,40 +219,41 @@ STATIC void emit_write_bytecode_byte_int(emit_t *emit, byte b1, mp_int_t num) {
     *c = *p;
 }
 
-STATIC void emit_write_bytecode_byte_uint(emit_t *emit, byte b, mp_uint_t val) {
-    emit_write_bytecode_byte(emit, b);
+STATIC void emit_write_bytecode_byte_uint(emit_t *emit, int stack_adj, byte b, mp_uint_t val) {
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit_write_uint(emit, emit_get_cur_to_write_bytecode, val);
 }
 
 #if MICROPY_PERSISTENT_CODE
-STATIC void emit_write_bytecode_byte_const(emit_t *emit, byte b, mp_uint_t n, mp_uint_t c) {
+STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n, mp_uint_t c) {
     if (emit->pass == MP_PASS_EMIT) {
         emit->const_table[n] = c;
     }
-    emit_write_bytecode_byte_uint(emit, b, n);
+    emit_write_bytecode_byte_uint(emit, stack_adj, b, n);
 }
 #endif
 
-STATIC void emit_write_bytecode_byte_qstr(emit_t* emit, byte b, qstr qst) {
+STATIC void emit_write_bytecode_byte_qstr(emit_t* emit, int stack_adj, byte b, qstr qst) {
     #if MICROPY_PERSISTENT_CODE
     assert((qst >> 16) == 0);
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     byte *c = emit_get_cur_to_write_bytecode(emit, 3);
     c[0] = b;
     c[1] = qst;
     c[2] = qst >> 8;
     #else
-    emit_write_bytecode_byte_uint(emit, b, qst);
+    emit_write_bytecode_byte_uint(emit, stack_adj, b, qst);
     #endif
 }
 
-STATIC void emit_write_bytecode_byte_obj(emit_t *emit, byte b, mp_obj_t obj) {
+STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) {
     #if MICROPY_PERSISTENT_CODE
-    emit_write_bytecode_byte_const(emit, b,
+    emit_write_bytecode_byte_const(emit, stack_adj, b,
         emit->scope->num_pos_args + emit->scope->num_kwonly_args
         + emit->ct_cur_obj++, (mp_uint_t)obj);
     #else
     // aligns the pointer so it is friendly to GC
-    emit_write_bytecode_byte(emit, b);
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(mp_obj_t));
     mp_obj_t *c = (mp_obj_t*)emit_get_cur_to_write_bytecode(emit, sizeof(mp_obj_t));
     // Verify thar c is already uint-aligned
@@ -262,24 +262,28 @@ STATIC void emit_write_bytecode_byte_obj(emit_t *emit, byte b, mp_obj_t obj) {
     #endif
 }
 
-STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, byte b, mp_raw_code_t *rc) {
+STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) {
     #if MICROPY_PERSISTENT_CODE
-    emit_write_bytecode_byte_const(emit, b,
+    emit_write_bytecode_byte_const(emit, stack_adj, b,
         emit->scope->num_pos_args + emit->scope->num_kwonly_args
         + emit->ct_num_obj + emit->ct_cur_raw_code++, (mp_uint_t)(uintptr_t)rc);
     #else
     // aligns the pointer so it is friendly to GC
-    emit_write_bytecode_byte(emit, b);
+    emit_write_bytecode_byte(emit, stack_adj, b);
     emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(void*));
     void **c = (void**)emit_get_cur_to_write_bytecode(emit, sizeof(void*));
     // Verify thar c is already uint-aligned
     assert(c == MP_ALIGN(c, sizeof(void*)));
     *c = rc;
     #endif
+    #if MICROPY_PY_SYS_SETTRACE
+    rc->line_of_definition = emit->last_source_line;
+    #endif
 }
 
 // unsigned labels are relative to ip following this instruction, stored as 16 bits
-STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, byte b1, mp_uint_t label) {
+STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     mp_uint_t bytecode_offset;
     if (emit->pass < MP_PASS_EMIT) {
         bytecode_offset = 0;
@@ -293,7 +297,8 @@ STATIC void emit_write_bytecode_byte_unsigned_label(emit_t *emit, byte b1, mp_ui
 }
 
 // signed labels are relative to ip following this instruction, stored as 16 bits, in excess
-STATIC void emit_write_bytecode_byte_signed_label(emit_t *emit, byte b1, mp_uint_t label) {
+STATIC void emit_write_bytecode_byte_signed_label(emit_t *emit, int stack_adj, byte b1, mp_uint_t label) {
+    mp_emit_bc_adjust_stack_size(emit, stack_adj);
     int bytecode_offset;
     if (emit->pass < MP_PASS_EMIT) {
         bytecode_offset = 0;
@@ -322,7 +327,7 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
     emit->bytecode_offset = 0;
     emit->code_info_offset = 0;
 
-    // Write local state size and exception stack size.
+    // Write local state size, exception stack size, scope flags and number of arguments
     {
         mp_uint_t n_state = scope->num_locals + scope->stack_size;
         if (n_state == 0) {
@@ -335,40 +340,22 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) {
         // An extra slot in the stack is needed to detect VM stack overflow
         n_state += 1;
         #endif
-        emit_write_code_info_uint(emit, n_state);
-        emit_write_code_info_uint(emit, scope->exc_stack_size);
-    }
 
-    // Write scope flags and number of arguments.
-    // TODO check that num args all fit in a byte
-    emit_write_code_info_byte(emit, emit->scope->scope_flags);
-    emit_write_code_info_byte(emit, emit->scope->num_pos_args);
-    emit_write_code_info_byte(emit, emit->scope->num_kwonly_args);
-    emit_write_code_info_byte(emit, emit->scope->num_def_pos_args);
+        size_t n_exc_stack = scope->exc_stack_size;
+        MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, scope, emit_write_code_info_byte, emit);
+    }
 
-    // Write size of the rest of the code info.  We don't know how big this
-    // variable uint will be on the MP_PASS_CODE_SIZE pass so we reserve 2 bytes
-    // for it and hope that is enough!  TODO assert this or something.
-    if (pass == MP_PASS_EMIT) {
-        emit_write_code_info_uint(emit, emit->code_info_size - emit->code_info_offset);
-    } else  {
-        emit_get_cur_to_write_code_info(emit, 2);
+    // Write number of cells and size of the source code info
+    if (pass >= MP_PASS_CODE_SIZE) {
+        MP_BC_PRELUDE_SIZE_ENCODE(emit->n_info, emit->n_cell, emit_write_code_info_byte, emit);
     }
 
+    emit->n_info = emit->code_info_offset;
+
     // Write the name and source file of this function.
     emit_write_code_info_qstr(emit, scope->simple_name);
     emit_write_code_info_qstr(emit, scope->source_file);
 
-    // bytecode prelude: initialise closed over variables
-    for (int i = 0; i < scope->id_info_len; i++) {
-        id_info_t *id = &scope->id_info[i];
-        if (id->kind == ID_INFO_KIND_CELL) {
-            assert(id->local_num < 255);
-            emit_write_bytecode_byte(emit, id->local_num); // write the local which should be converted to a cell
-        }
-    }
-    emit_write_bytecode_byte(emit, 255); // end of list sentinel
-
     #if MICROPY_PERSISTENT_CODE
     emit->ct_cur_obj = 0;
     emit->ct_cur_raw_code = 0;
@@ -414,6 +401,20 @@ void mp_emit_bc_end_pass(emit_t *emit) {
 
     emit_write_code_info_byte(emit, 0); // end of line number info
 
+    // Calculate size of source code info section
+    emit->n_info = emit->code_info_offset - emit->n_info;
+
+    // Emit closure section of prelude
+    emit->n_cell = 0;
+    for (size_t i = 0; i < emit->scope->id_info_len; ++i) {
+        id_info_t *id = &emit->scope->id_info[i];
+        if (id->kind == ID_INFO_KIND_CELL) {
+            assert(id->local_num <= 255);
+            emit_write_code_info_byte(emit, id->local_num); // write the local which should be converted to a cell
+            ++emit->n_cell;
+        }
+    }
+
     #if MICROPY_PERSISTENT_CODE
     assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->ct_num_obj == emit->ct_cur_obj));
     emit->ct_num_obj = emit->ct_cur_obj;
@@ -468,10 +469,6 @@ void mp_emit_bc_adjust_stack_size(emit_t *emit, mp_int_t delta) {
     emit->last_emit_was_return_value = false;
 }
 
-static inline void emit_bc_pre(emit_t *emit, mp_int_t stack_size_delta) {
-    mp_emit_bc_adjust_stack_size(emit, stack_size_delta);
-}
-
 void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t source_line) {
     //printf("source: line %d -> %d  offset %d -> %d\n", emit->last_source_line, source_line, emit->last_source_line_offset, emit->bytecode_offset);
 #if MICROPY_ENABLE_SOURCE_LINE
@@ -493,7 +490,7 @@ void mp_emit_bc_set_source_line(emit_t *emit, mp_uint_t source_line) {
 }
 
 void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) {
-    emit_bc_pre(emit, 0);
+    mp_emit_bc_adjust_stack_size(emit, 0);
     if (emit->pass == MP_PASS_SCOPE) {
         return;
     }
@@ -511,64 +508,54 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) {
 void mp_emit_bc_import(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_NAME == MP_BC_IMPORT_NAME);
     MP_STATIC_ASSERT(MP_BC_IMPORT_NAME + MP_EMIT_IMPORT_FROM == MP_BC_IMPORT_FROM);
-    if (kind == MP_EMIT_IMPORT_FROM) {
-        emit_bc_pre(emit, 1);
-    } else {
-        emit_bc_pre(emit, -1);
-    }
+    int stack_adj = kind == MP_EMIT_IMPORT_FROM ? 1 : -1;
     if (kind == MP_EMIT_IMPORT_STAR) {
-        emit_write_bytecode_byte(emit, MP_BC_IMPORT_STAR);
+        emit_write_bytecode_byte(emit, stack_adj, MP_BC_IMPORT_STAR);
     } else {
-        emit_write_bytecode_byte_qstr(emit, MP_BC_IMPORT_NAME + kind, qst);
+        emit_write_bytecode_byte_qstr(emit, stack_adj, MP_BC_IMPORT_NAME + kind, qst);
     }
 }
 
 void mp_emit_bc_load_const_tok(emit_t *emit, mp_token_kind_t tok) {
-    emit_bc_pre(emit, 1);
-    switch (tok) {
-        case MP_TOKEN_KW_FALSE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_FALSE); break;
-        case MP_TOKEN_KW_NONE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_NONE); break;
-        case MP_TOKEN_KW_TRUE: emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_TRUE); break;
-        default:
-            assert(tok == MP_TOKEN_ELLIPSIS);
-            emit_write_bytecode_byte_obj(emit, MP_BC_LOAD_CONST_OBJ, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
-            break;
+    MP_STATIC_ASSERT(MP_BC_LOAD_CONST_FALSE + (MP_TOKEN_KW_NONE - MP_TOKEN_KW_FALSE) == MP_BC_LOAD_CONST_NONE);
+    MP_STATIC_ASSERT(MP_BC_LOAD_CONST_FALSE + (MP_TOKEN_KW_TRUE - MP_TOKEN_KW_FALSE) == MP_BC_LOAD_CONST_TRUE);
+    if (tok == MP_TOKEN_ELLIPSIS) {
+        emit_write_bytecode_byte_obj(emit, 1, MP_BC_LOAD_CONST_OBJ, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+    } else {
+        emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_CONST_FALSE + (tok - MP_TOKEN_KW_FALSE));
     }
 }
 
 void mp_emit_bc_load_const_small_int(emit_t *emit, mp_int_t arg) {
-    emit_bc_pre(emit, 1);
-    if (-16 <= arg && arg <= 47) {
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_CONST_SMALL_INT_MULTI + 16 + arg);
+    if (-MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS <= arg
+        && arg < MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) {
+        emit_write_bytecode_byte(emit, 1,
+            MP_BC_LOAD_CONST_SMALL_INT_MULTI + MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS + arg);
     } else {
-        emit_write_bytecode_byte_int(emit, MP_BC_LOAD_CONST_SMALL_INT, arg);
+        emit_write_bytecode_byte_int(emit, 1, MP_BC_LOAD_CONST_SMALL_INT, arg);
     }
 }
 
 void mp_emit_bc_load_const_str(emit_t *emit, qstr qst) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_CONST_STRING, qst);
+    emit_write_bytecode_byte_qstr(emit, 1, MP_BC_LOAD_CONST_STRING, qst);
 }
 
 void mp_emit_bc_load_const_obj(emit_t *emit, mp_obj_t obj) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_obj(emit, MP_BC_LOAD_CONST_OBJ, obj);
+    emit_write_bytecode_byte_obj(emit, 1, MP_BC_LOAD_CONST_OBJ, obj);
 }
 
 void mp_emit_bc_load_null(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_LOAD_NULL);
+    emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_NULL);
 }
 
 void mp_emit_bc_load_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) {
     MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_LOAD_FAST_N);
     MP_STATIC_ASSERT(MP_BC_LOAD_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_LOAD_DEREF);
     (void)qst;
-    emit_bc_pre(emit, 1);
     if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) {
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_FAST_MULTI + local_num);
+        emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_FAST_MULTI + local_num);
     } else {
-        emit_write_bytecode_byte_uint(emit, MP_BC_LOAD_FAST_N + kind, local_num);
+        emit_write_bytecode_byte_uint(emit, 1, MP_BC_LOAD_FAST_N + kind, local_num);
     }
 }
 
@@ -576,51 +563,45 @@ void mp_emit_bc_load_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_LOAD_NAME);
     MP_STATIC_ASSERT(MP_BC_LOAD_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_LOAD_GLOBAL);
     (void)qst;
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, 1, MP_BC_LOAD_NAME + kind, qst);
     if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
-        emit_write_bytecode_byte(emit, 0);
+        emit_write_bytecode_raw_byte(emit, 0);
     }
 }
 
 void mp_emit_bc_load_method(emit_t *emit, qstr qst, bool is_super) {
-    emit_bc_pre(emit, 1 - 2 * is_super);
-    emit_write_bytecode_byte_qstr(emit, is_super ? MP_BC_LOAD_SUPER_METHOD : MP_BC_LOAD_METHOD, qst);
+    int stack_adj = 1 - 2 * is_super;
+    emit_write_bytecode_byte_qstr(emit, stack_adj, is_super ? MP_BC_LOAD_SUPER_METHOD : MP_BC_LOAD_METHOD, qst);
 }
 
 void mp_emit_bc_load_build_class(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_LOAD_BUILD_CLASS);
+    emit_write_bytecode_byte(emit, 1, MP_BC_LOAD_BUILD_CLASS);
 }
 
 void mp_emit_bc_subscr(emit_t *emit, int kind) {
     if (kind == MP_EMIT_SUBSCR_LOAD) {
-        emit_bc_pre(emit, -1);
-        emit_write_bytecode_byte(emit, MP_BC_LOAD_SUBSCR);
+        emit_write_bytecode_byte(emit, -1, MP_BC_LOAD_SUBSCR);
     } else {
         if (kind == MP_EMIT_SUBSCR_DELETE) {
             mp_emit_bc_load_null(emit);
             mp_emit_bc_rot_three(emit);
         }
-        emit_bc_pre(emit, -3);
-        emit_write_bytecode_byte(emit, MP_BC_STORE_SUBSCR);
+        emit_write_bytecode_byte(emit, -3, MP_BC_STORE_SUBSCR);
     }
 }
 
 void mp_emit_bc_attr(emit_t *emit, qstr qst, int kind) {
     if (kind == MP_EMIT_ATTR_LOAD) {
-        emit_bc_pre(emit, 0);
-        emit_write_bytecode_byte_qstr(emit, MP_BC_LOAD_ATTR, qst);
+        emit_write_bytecode_byte_qstr(emit, 0, MP_BC_LOAD_ATTR, qst);
     } else {
         if (kind == MP_EMIT_ATTR_DELETE) {
             mp_emit_bc_load_null(emit);
             mp_emit_bc_rot_two(emit);
         }
-        emit_bc_pre(emit, -2);
-        emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_ATTR, qst);
+        emit_write_bytecode_byte_qstr(emit, -2, MP_BC_STORE_ATTR, qst);
     }
     if (MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE_DYNAMIC) {
-        emit_write_bytecode_byte(emit, 0);
+        emit_write_bytecode_raw_byte(emit, 0);
     }
 }
 
@@ -628,98 +609,86 @@ void mp_emit_bc_store_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kin
     MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_STORE_FAST_N);
     MP_STATIC_ASSERT(MP_BC_STORE_FAST_N + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_STORE_DEREF);
     (void)qst;
-    emit_bc_pre(emit, -1);
     if (kind == MP_EMIT_IDOP_LOCAL_FAST && local_num <= 15) {
-        emit_write_bytecode_byte(emit, MP_BC_STORE_FAST_MULTI + local_num);
+        emit_write_bytecode_byte(emit, -1, MP_BC_STORE_FAST_MULTI + local_num);
     } else {
-        emit_write_bytecode_byte_uint(emit, MP_BC_STORE_FAST_N + kind, local_num);
+        emit_write_bytecode_byte_uint(emit, -1, MP_BC_STORE_FAST_N + kind, local_num);
     }
 }
 
 void mp_emit_bc_store_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_STORE_NAME);
     MP_STATIC_ASSERT(MP_BC_STORE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_STORE_GLOBAL);
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_STORE_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, -1, MP_BC_STORE_NAME + kind, qst);
 }
 
 void mp_emit_bc_delete_local(emit_t *emit, qstr qst, mp_uint_t local_num, int kind) {
     MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_FAST == MP_BC_DELETE_FAST);
     MP_STATIC_ASSERT(MP_BC_DELETE_FAST + MP_EMIT_IDOP_LOCAL_DEREF == MP_BC_DELETE_DEREF);
     (void)qst;
-    emit_write_bytecode_byte_uint(emit, MP_BC_DELETE_FAST + kind, local_num);
+    emit_write_bytecode_byte_uint(emit, 0, MP_BC_DELETE_FAST + kind, local_num);
 }
 
 void mp_emit_bc_delete_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_BC_DELETE_NAME);
     MP_STATIC_ASSERT(MP_BC_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_BC_DELETE_GLOBAL);
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte_qstr(emit, MP_BC_DELETE_NAME + kind, qst);
+    emit_write_bytecode_byte_qstr(emit, 0, MP_BC_DELETE_NAME + kind, qst);
 }
 
 void mp_emit_bc_dup_top(emit_t *emit) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte(emit, MP_BC_DUP_TOP);
+    emit_write_bytecode_byte(emit, 1, MP_BC_DUP_TOP);
 }
 
 void mp_emit_bc_dup_top_two(emit_t *emit) {
-    emit_bc_pre(emit, 2);
-    emit_write_bytecode_byte(emit, MP_BC_DUP_TOP_TWO);
+    emit_write_bytecode_byte(emit, 2, MP_BC_DUP_TOP_TWO);
 }
 
 void mp_emit_bc_pop_top(emit_t *emit) {
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+    emit_write_bytecode_byte(emit, -1, MP_BC_POP_TOP);
 }
 
 void mp_emit_bc_rot_two(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_ROT_TWO);
+    emit_write_bytecode_byte(emit, 0, MP_BC_ROT_TWO);
 }
 
 void mp_emit_bc_rot_three(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_ROT_THREE);
+    emit_write_bytecode_byte(emit, 0, MP_BC_ROT_THREE);
 }
 
 void mp_emit_bc_jump(emit_t *emit, mp_uint_t label) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP, label);
+    emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label);
 }
 
 void mp_emit_bc_pop_jump_if(emit_t *emit, bool cond, mp_uint_t label) {
-    emit_bc_pre(emit, -1);
     if (cond) {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_POP_JUMP_IF_TRUE, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_TRUE, label);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_POP_JUMP_IF_FALSE, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_POP_JUMP_IF_FALSE, label);
     }
 }
 
 void mp_emit_bc_jump_if_or_pop(emit_t *emit, bool cond, mp_uint_t label) {
-    emit_bc_pre(emit, -1);
     if (cond) {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP_IF_TRUE_OR_POP, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_TRUE_OR_POP, label);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP_IF_FALSE_OR_POP, label);
+        emit_write_bytecode_byte_signed_label(emit, -1, MP_BC_JUMP_IF_FALSE_OR_POP, label);
     }
 }
 
 void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_depth) {
     if (except_depth == 0) {
-        emit_bc_pre(emit, 0);
         if (label & MP_EMIT_BREAK_FROM_FOR) {
             // need to pop the iterator if we are breaking out of a for loop
-            emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+            emit_write_bytecode_raw_byte(emit, MP_BC_POP_TOP);
             // also pop the iter_buf
             for (size_t i = 0; i < MP_OBJ_ITER_BUF_NSLOTS - 1; ++i) {
-                emit_write_bytecode_byte(emit, MP_BC_POP_TOP);
+                emit_write_bytecode_raw_byte(emit, MP_BC_POP_TOP);
             }
         }
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
+        emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
     } else {
-        emit_write_bytecode_byte_signed_label(emit, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
-        emit_write_bytecode_byte(emit, ((label & MP_EMIT_BREAK_FROM_FOR) ? 0x80 : 0) | except_depth);
+        emit_write_bytecode_byte_signed_label(emit, 0, MP_BC_UNWIND_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR);
+        emit_write_bytecode_raw_byte(emit, ((label & MP_EMIT_BREAK_FROM_FOR) ? 0x80 : 0) | except_depth);
     }
 }
 
@@ -727,57 +696,45 @@ void mp_emit_bc_setup_block(emit_t *emit, mp_uint_t label, int kind) {
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_WITH == MP_BC_SETUP_WITH);
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_EXCEPT == MP_BC_SETUP_EXCEPT);
     MP_STATIC_ASSERT(MP_BC_SETUP_WITH + MP_EMIT_SETUP_BLOCK_FINALLY == MP_BC_SETUP_FINALLY);
-    if (kind == MP_EMIT_SETUP_BLOCK_WITH) {
     // The SETUP_WITH opcode pops ctx_mgr from the top of the stack
     // and then pushes 3 entries: __exit__, ctx_mgr, as_value.
-        emit_bc_pre(emit, 2);
-    } else {
-        emit_bc_pre(emit, 0);
-    }
-    emit_write_bytecode_byte_unsigned_label(emit, MP_BC_SETUP_WITH + kind, label);
+    int stack_adj = kind == MP_EMIT_SETUP_BLOCK_WITH ? 2 : 0;
+    emit_write_bytecode_byte_unsigned_label(emit, stack_adj, MP_BC_SETUP_WITH + kind, label);
 }
 
 void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label) {
-    mp_emit_bc_pop_block(emit);
     mp_emit_bc_load_const_tok(emit, MP_TOKEN_KW_NONE);
     mp_emit_bc_label_assign(emit, label);
-    emit_bc_pre(emit, 2); // ensure we have enough stack space to call the __exit__ method
-    emit_write_bytecode_byte(emit, MP_BC_WITH_CLEANUP);
-    emit_bc_pre(emit, -4); // cancel the 2 above, plus the 2 from mp_emit_bc_setup_block(MP_EMIT_SETUP_BLOCK_WITH)
+    // The +2 is to ensure we have enough stack space to call the __exit__ method
+    emit_write_bytecode_byte(emit, 2, MP_BC_WITH_CLEANUP);
+    // Cancel the +2 above, plus the +2 from mp_emit_bc_setup_block(MP_EMIT_SETUP_BLOCK_WITH)
+    mp_emit_bc_adjust_stack_size(emit, -4);
 }
 
 void mp_emit_bc_end_finally(emit_t *emit) {
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_END_FINALLY);
+    emit_write_bytecode_byte(emit, -1, MP_BC_END_FINALLY);
 }
 
 void mp_emit_bc_get_iter(emit_t *emit, bool use_stack) {
-    emit_bc_pre(emit, use_stack ? MP_OBJ_ITER_BUF_NSLOTS - 1 : 0);
-    emit_write_bytecode_byte(emit, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER);
+    int stack_adj = use_stack ? MP_OBJ_ITER_BUF_NSLOTS - 1 : 0;
+    emit_write_bytecode_byte(emit, stack_adj, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER);
 }
 
 void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) {
-    emit_bc_pre(emit, 1);
-    emit_write_bytecode_byte_unsigned_label(emit, MP_BC_FOR_ITER, label);
+    emit_write_bytecode_byte_unsigned_label(emit, 1, MP_BC_FOR_ITER, label);
 }
 
 void mp_emit_bc_for_iter_end(emit_t *emit) {
-    emit_bc_pre(emit, -MP_OBJ_ITER_BUF_NSLOTS);
-}
-
-void mp_emit_bc_pop_block(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_POP_BLOCK);
+    mp_emit_bc_adjust_stack_size(emit, -MP_OBJ_ITER_BUF_NSLOTS);
 }
 
-void mp_emit_bc_pop_except(emit_t *emit) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_POP_EXCEPT);
+void mp_emit_bc_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) {
+    (void)within_exc_handler;
+    emit_write_bytecode_byte_unsigned_label(emit, 0, MP_BC_POP_EXCEPT_JUMP, label);
 }
 
 void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op) {
-    emit_bc_pre(emit, 0);
-    emit_write_bytecode_byte(emit, MP_BC_UNARY_OP_MULTI + op);
+    emit_write_bytecode_byte(emit, 0, MP_BC_UNARY_OP_MULTI + op);
 }
 
 void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op) {
@@ -789,11 +746,9 @@ void mp_emit_bc_binary_op(emit_t *emit, mp_binary_op_t op) {
         invert = true;
         op = MP_BINARY_OP_IS;
     }
-    emit_bc_pre(emit, -1);
-    emit_write_bytecode_byte(emit, MP_BC_BINARY_OP_MULTI + op);
+    emit_write_bytecode_byte(emit, -1, MP_BC_BINARY_OP_MULTI + op);
     if (invert) {
-        emit_bc_pre(emit, 0);
-        emit_write_bytecode_byte(emit, MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NOT);
+        emit_write_bytecode_byte(emit, 0, MP_BC_UNARY_OP_MULTI + MP_UNARY_OP_NOT);
     }
 }
 
@@ -803,17 +758,12 @@ void mp_emit_bc_build(emit_t *emit, mp_uint_t n_args, int kind) {
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_MAP == MP_BC_BUILD_MAP);
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SET == MP_BC_BUILD_SET);
     MP_STATIC_ASSERT(MP_BC_BUILD_TUPLE + MP_EMIT_BUILD_SLICE == MP_BC_BUILD_SLICE);
-    if (kind == MP_EMIT_BUILD_MAP) {
-        emit_bc_pre(emit, 1);
-    } else {
-        emit_bc_pre(emit, 1 - n_args);
-    }
-    emit_write_bytecode_byte_uint(emit, MP_BC_BUILD_TUPLE + kind, n_args);
+    int stack_adj = kind == MP_EMIT_BUILD_MAP ? 1 : 1 - n_args;
+    emit_write_bytecode_byte_uint(emit, stack_adj, MP_BC_BUILD_TUPLE + kind, n_args);
 }
 
 void mp_emit_bc_store_map(emit_t *emit) {
-    emit_bc_pre(emit, -2);
-    emit_write_bytecode_byte(emit, MP_BC_STORE_MAP);
+    emit_write_bytecode_byte(emit, -2, MP_BC_STORE_MAP);
 }
 
 void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t collection_stack_index) {
@@ -829,51 +779,46 @@ void mp_emit_bc_store_comp(emit_t *emit, scope_kind_t kind, mp_uint_t collection
         n = 0;
         t = 2;
     }
-    emit_bc_pre(emit, -1 - n);
     // the lower 2 bits of the opcode argument indicate the collection type
-    emit_write_bytecode_byte_uint(emit, MP_BC_STORE_COMP, ((collection_stack_index + n) << 2) | t);
+    emit_write_bytecode_byte_uint(emit, -1 - n, MP_BC_STORE_COMP, ((collection_stack_index + n) << 2) | t);
 }
 
 void mp_emit_bc_unpack_sequence(emit_t *emit, mp_uint_t n_args) {
-    emit_bc_pre(emit, -1 + n_args);
-    emit_write_bytecode_byte_uint(emit, MP_BC_UNPACK_SEQUENCE, n_args);
+    emit_write_bytecode_byte_uint(emit, -1 + n_args, MP_BC_UNPACK_SEQUENCE, n_args);
 }
 
 void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) {
-    emit_bc_pre(emit, -1 + n_left + n_right + 1);
-    emit_write_bytecode_byte_uint(emit, MP_BC_UNPACK_EX, n_left | (n_right << 8));
+    emit_write_bytecode_byte_uint(emit, -1 + n_left + n_right + 1, MP_BC_UNPACK_EX, n_left | (n_right << 8));
 }
 
 void mp_emit_bc_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) {
     if (n_pos_defaults == 0 && n_kw_defaults == 0) {
-        emit_bc_pre(emit, 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_FUNCTION, scope->raw_code);
+        emit_write_bytecode_byte_raw_code(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code);
     } else {
-        emit_bc_pre(emit, -1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code);
+        emit_write_bytecode_byte_raw_code(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code);
     }
 }
 
 void mp_emit_bc_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) {
     if (n_pos_defaults == 0 && n_kw_defaults == 0) {
-        emit_bc_pre(emit, -n_closed_over + 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_CLOSURE, scope->raw_code);
-        emit_write_bytecode_byte(emit, n_closed_over);
+        int stack_adj = -n_closed_over + 1;
+        emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code);
+        emit_write_bytecode_raw_byte(emit, n_closed_over);
     } else {
         assert(n_closed_over <= 255);
-        emit_bc_pre(emit, -2 - (mp_int_t)n_closed_over + 1);
-        emit_write_bytecode_byte_raw_code(emit, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code);
-        emit_write_bytecode_byte(emit, n_closed_over);
+        int stack_adj = -2 - (mp_int_t)n_closed_over + 1;
+        emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code);
+        emit_write_bytecode_raw_byte(emit, n_closed_over);
     }
 }
 
-STATIC void emit_bc_call_function_method_helper(emit_t *emit, mp_int_t stack_adj, mp_uint_t bytecode_base, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) {
+STATIC void emit_bc_call_function_method_helper(emit_t *emit, int stack_adj, mp_uint_t bytecode_base, mp_uint_t n_positional, mp_uint_t n_keyword, mp_uint_t star_flags) {
     if (star_flags) {
-        emit_bc_pre(emit, stack_adj - (mp_int_t)n_positional - 2 * (mp_int_t)n_keyword - 2);
-        emit_write_bytecode_byte_uint(emit, bytecode_base + 1, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
+        stack_adj -= (int)n_positional + 2 * (int)n_keyword + 2;
+        emit_write_bytecode_byte_uint(emit, stack_adj, bytecode_base + 1, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
     } else {
-        emit_bc_pre(emit, stack_adj - (mp_int_t)n_positional - 2 * (mp_int_t)n_keyword);
-        emit_write_bytecode_byte_uint(emit, bytecode_base, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
+        stack_adj -= (int)n_positional + 2 * (int)n_keyword;
+        emit_write_bytecode_byte_uint(emit, stack_adj, bytecode_base, (n_keyword << 8) | n_positional); // TODO make it 2 separate uints?
     }
 }
 
@@ -886,22 +831,21 @@ void mp_emit_bc_call_method(emit_t *emit, mp_uint_t n_positional, mp_uint_t n_ke
 }
 
 void mp_emit_bc_return_value(emit_t *emit) {
-    emit_bc_pre(emit, -1);
+    emit_write_bytecode_byte(emit, -1, MP_BC_RETURN_VALUE);
     emit->last_emit_was_return_value = true;
-    emit_write_bytecode_byte(emit, MP_BC_RETURN_VALUE);
 }
 
 void mp_emit_bc_raise_varargs(emit_t *emit, mp_uint_t n_args) {
+    MP_STATIC_ASSERT(MP_BC_RAISE_LAST + 1 == MP_BC_RAISE_OBJ);
+    MP_STATIC_ASSERT(MP_BC_RAISE_LAST + 2 == MP_BC_RAISE_FROM);
     assert(n_args <= 2);
-    emit_bc_pre(emit, -n_args);
-    emit_write_bytecode_byte_byte(emit, MP_BC_RAISE_VARARGS, n_args);
+    emit_write_bytecode_byte(emit, -n_args, MP_BC_RAISE_LAST + n_args);
 }
 
 void mp_emit_bc_yield(emit_t *emit, int kind) {
     MP_STATIC_ASSERT(MP_BC_YIELD_VALUE + 1 == MP_BC_YIELD_FROM);
-    emit_bc_pre(emit, -kind);
+    emit_write_bytecode_byte(emit, -kind, MP_BC_YIELD_VALUE + kind);
     emit->scope->scope_flags |= MP_SCOPE_FLAG_GENERATOR;
-    emit_write_bytecode_byte(emit, MP_BC_YIELD_VALUE + kind);
 }
 
 void mp_emit_bc_start_except_handler(emit_t *emit) {
@@ -914,6 +858,11 @@ void mp_emit_bc_end_except_handler(emit_t *emit) {
 
 #if MICROPY_EMIT_NATIVE
 const emit_method_table_t emit_bc_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    NULL,
+    NULL,
+    #endif
+
     mp_emit_bc_start_pass,
     mp_emit_bc_end_pass,
     mp_emit_bc_last_emit_was_return_value,
@@ -959,8 +908,7 @@ const emit_method_table_t emit_bc_method_table = {
     mp_emit_bc_get_iter,
     mp_emit_bc_for_iter,
     mp_emit_bc_for_iter_end,
-    mp_emit_bc_pop_block,
-    mp_emit_bc_pop_except,
+    mp_emit_bc_pop_except_jump,
     mp_emit_bc_unary_op,
     mp_emit_bc_binary_op,
     mp_emit_bc_build,

py/emitglue.c

@@ -34,6 +34,7 @@
 #include "py/emitglue.h"
 #include "py/runtime0.h"
 #include "py/bc.h"
+#include "py/profile.h"
 
 #if MICROPY_DEBUG_VERBOSE // print debugging info
 #define DEBUG_PRINT (1)
@@ -52,6 +53,9 @@ mp_uint_t mp_verbose_flag = 0;
 mp_raw_code_t *mp_emit_glue_new_raw_code(void) {
     mp_raw_code_t *rc = m_new0(mp_raw_code_t, 1);
     rc->kind = MP_CODE_RESERVED;
+    #if MICROPY_PY_SYS_SETTRACE
+    rc->line_of_definition = 0;
+    #endif
     return rc;
 }
 
@@ -67,12 +71,17 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
 
     rc->kind = MP_CODE_BYTECODE;
     rc->scope_flags = scope_flags;
-    rc->data.u_byte.bytecode = code;
-    rc->data.u_byte.const_table = const_table;
+    rc->fun_data = code;
+    rc->const_table = const_table;
     #if MICROPY_PERSISTENT_CODE_SAVE
-    rc->data.u_byte.bc_len = len;
-    rc->data.u_byte.n_obj = n_obj;
-    rc->data.u_byte.n_raw_code = n_raw_code;
+    rc->fun_data_len = len;
+    rc->n_obj = n_obj;
+    rc->n_raw_code = n_raw_code;
+    #endif
+
+    #if MICROPY_PY_SYS_SETTRACE
+    mp_bytecode_prelude_t *prelude = &rc->prelude;
+    mp_prof_extract_prelude(code, prelude);
     #endif
 
 #ifdef DEBUG_PRINT
@@ -88,15 +97,32 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
 #endif
 }
 
-#if MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM
-void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) {
+#if MICROPY_EMIT_MACHINE_CODE
+void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table,
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t prelude_offset,
+    uint16_t n_obj, uint16_t n_raw_code,
+    uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link,
+    #endif
+    mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) {
+
     assert(kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER || kind == MP_CODE_NATIVE_ASM);
+
     rc->kind = kind;
     rc->scope_flags = scope_flags;
     rc->n_pos_args = n_pos_args;
-    rc->data.u_native.fun_data = fun_data;
-    rc->data.u_native.const_table = const_table;
-    rc->data.u_native.type_sig = type_sig;
+    rc->fun_data = fun_data;
+    rc->const_table = const_table;
+    rc->type_sig = type_sig;
+
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    rc->fun_data_len = fun_len;
+    rc->prelude_offset = prelude_offset;
+    rc->n_obj = n_obj;
+    rc->n_raw_code = n_raw_code;
+    rc->n_qstr= n_qstr;
+    rc->qstr_link = qstr_link;
+    #endif
 
 #ifdef DEBUG_PRINT
     DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags);
@@ -135,7 +161,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
         #if MICROPY_EMIT_NATIVE
         case MP_CODE_NATIVE_PY:
         case MP_CODE_NATIVE_VIPER:
-            fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->data.u_native.fun_data, rc->data.u_native.const_table);
+            fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->fun_data, rc->const_table);
             // Check for a generator function, and if so change the type of the object
             if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
                 ((mp_obj_base_t*)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap;
@@ -144,17 +170,23 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
         #endif
         #if MICROPY_EMIT_INLINE_ASM
         case MP_CODE_NATIVE_ASM:
-            fun = mp_obj_new_fun_asm(rc->n_pos_args, rc->data.u_native.fun_data, rc->data.u_native.type_sig);
+            fun = mp_obj_new_fun_asm(rc->n_pos_args, rc->fun_data, rc->type_sig);
             break;
         #endif
         default:
             // rc->kind should always be set and BYTECODE is the only remaining case
             assert(rc->kind == MP_CODE_BYTECODE);
-            fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->data.u_byte.bytecode, rc->data.u_byte.const_table);
+            fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->fun_data, rc->const_table);
             // check for generator functions and if so change the type of the object
             if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
                 ((mp_obj_base_t*)MP_OBJ_TO_PTR(fun))->type = &mp_type_gen_wrap;
             }
+
+            #if MICROPY_PY_SYS_SETTRACE
+            mp_obj_fun_bc_t *self_fun = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(fun);
+            self_fun->rc = rc;
+            #endif
+
             break;
     }
 

py/emitglue.h

@@ -27,6 +27,7 @@
 #define MICROPY_INCLUDED_PY_EMITGLUE_H
 
 #include "py/obj.h"
+#include "py/bc.h"
 
 // These variables and functions glue the code emitters to the runtime.
 
@@ -48,26 +49,38 @@ typedef enum {
     MP_CODE_NATIVE_ASM,
 } mp_raw_code_kind_t;
 
+typedef struct _mp_qstr_link_entry_t {
+    uint16_t off;
+    uint16_t qst;
+} mp_qstr_link_entry_t;
+
 typedef struct _mp_raw_code_t {
     mp_uint_t kind : 3; // of type mp_raw_code_kind_t
     mp_uint_t scope_flags : 7;
     mp_uint_t n_pos_args : 11;
-    union {
-        struct {
-            const byte *bytecode;
-            const mp_uint_t *const_table;
-            #if MICROPY_PERSISTENT_CODE_SAVE
-            mp_uint_t bc_len;
-            uint16_t n_obj;
-            uint16_t n_raw_code;
-            #endif
-        } u_byte;
-        struct {
-            void *fun_data;
-            const mp_uint_t *const_table;
-            mp_uint_t type_sig; // for viper, compressed as 2-bit types; ret is MSB, then arg0, arg1, etc
-        } u_native;
-    } data;
+    const void *fun_data;
+    const mp_uint_t *const_table;
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t fun_data_len;
+    uint16_t n_obj;
+    uint16_t n_raw_code;
+    #if MICROPY_PY_SYS_SETTRACE
+    mp_bytecode_prelude_t prelude;
+    // line_of_definition is a Python source line where the raw_code was
+    // created e.g. MP_BC_MAKE_FUNCTION. This is different from lineno info
+    // stored in prelude, which provides line number for first statement of
+    // a function. Required to properly implement "call" trace event.
+    mp_uint_t line_of_definition;
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    uint16_t prelude_offset;
+    uint16_t n_qstr;
+    mp_qstr_link_entry_t *qstr_link;
+    #endif
+    #endif
+    #if MICROPY_EMIT_MACHINE_CODE
+    mp_uint_t type_sig; // for viper, compressed as 2-bit types; ret is MSB, then arg0, arg1, etc
+    #endif
 } mp_raw_code_t;
 
 mp_raw_code_t *mp_emit_glue_new_raw_code(void);
@@ -81,7 +94,15 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
     uint16_t n_obj, uint16_t n_raw_code,
     #endif
     mp_uint_t scope_flags);
-void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig);
+
+void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len,
+    const mp_uint_t *const_table,
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t prelude_offset,
+    uint16_t n_obj, uint16_t n_raw_code,
+    uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link,
+    #endif
+    mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig);
 
 mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args);
 mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args);

py/emitinlinethumb.c

@@ -812,6 +812,11 @@ branch_not_in_range:
 }
 
 const emit_inline_asm_method_table_t emit_inline_thumb_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_thumb_new,
+    emit_inline_thumb_free,
+    #endif
+
     emit_inline_thumb_start_pass,
     emit_inline_thumb_end_pass,
     emit_inline_thumb_count_params,

py/emitinlinextensa.c

@@ -335,6 +335,11 @@ branch_not_in_range:
 }
 
 const emit_inline_asm_method_table_t emit_inline_xtensa_method_table = {
+    #if MICROPY_DYNAMIC_COMPILER
+    emit_inline_xtensa_new,
+    emit_inline_xtensa_free,
+    #endif
+
     emit_inline_xtensa_start_pass,
     emit_inline_xtensa_end_pass,
     emit_inline_xtensa_count_params,

py/emitnative.c

@@ -47,7 +47,8 @@
 #include <assert.h>
 
 #include "py/emit.h"
-#include "py/bc.h"
+#include "py/nativeglue.h"
+#include "py/objstr.h"
 
 #if MICROPY_DEBUG_VERBOSE // print debugging info
 #define DEBUG_PRINT (1)
@@ -57,7 +58,7 @@
 #endif
 
 // wrapper around everything in this file
-#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA
+#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA || N_XTENSAWIN
 
 // C stack layout for native functions:
 //  0:                          nlr_buf_t [optional]
@@ -80,6 +81,30 @@
 //                              locals (reversed, L0 at end)    |
 //                              (L0-L2 may be in regs instead)
 
+// Native emitter needs to know the following sizes and offsets of C structs (on the target):
+#if MICROPY_DYNAMIC_COMPILER
+#define SIZEOF_NLR_BUF (2 + mp_dynamic_compiler.nlr_buf_num_regs + 1) // the +1 is conservative in case MICROPY_ENABLE_PYSTACK enabled
+#else
+#define SIZEOF_NLR_BUF (sizeof(nlr_buf_t) / sizeof(uintptr_t))
+#endif
+#define SIZEOF_CODE_STATE (sizeof(mp_code_state_t) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_STATE (offsetof(mp_code_state_t, state) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
+#define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t))
+#define OFFSETOF_OBJ_FUN_BC_CONST_TABLE (offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t))
+
+// If not already defined, set parent args to same as child call registers
+#ifndef REG_PARENT_RET
+#define REG_PARENT_RET REG_RET
+#define REG_PARENT_ARG_1 REG_ARG_1
+#define REG_PARENT_ARG_2 REG_ARG_2
+#define REG_PARENT_ARG_3 REG_ARG_3
+#define REG_PARENT_ARG_4 REG_ARG_4
+#endif
+
 // Word index of nlr_buf_t.ret_val
 #define NLR_BUF_IDX_RET_VAL (1)
 
@@ -101,9 +126,9 @@
 #define LOCAL_IDX_EXC_HANDLER_PC(emit) (NLR_BUF_IDX_LOCAL_1)
 #define LOCAL_IDX_EXC_HANDLER_UNWIND(emit) (NLR_BUF_IDX_LOCAL_2)
 #define LOCAL_IDX_RET_VAL(emit) (NLR_BUF_IDX_LOCAL_3)
-#define LOCAL_IDX_FUN_OBJ(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t))
-#define LOCAL_IDX_OLD_GLOBALS(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
-#define LOCAL_IDX_GEN_PC(emit) ((emit)->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t))
+#define LOCAL_IDX_FUN_OBJ(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_FUN_BC)
+#define LOCAL_IDX_OLD_GLOBALS(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP)
+#define LOCAL_IDX_GEN_PC(emit) ((emit)->code_state_start + OFFSETOF_CODE_STATE_IP)
 #define LOCAL_IDX_LOCAL_VAR(emit, local_num) ((emit)->stack_start + (emit)->n_state - 1 - (local_num))
 
 #define REG_GENERATOR_STATE (REG_LOCAL_3)
@@ -136,20 +161,6 @@ typedef enum {
     VTYPE_BUILTIN_CAST = 0x70 | MP_NATIVE_TYPE_OBJ,
 } vtype_kind_t;
 
-int mp_native_type_from_qstr(qstr qst) {
-    switch (qst) {
-        case MP_QSTR_object: return MP_NATIVE_TYPE_OBJ;
-        case MP_QSTR_bool: return MP_NATIVE_TYPE_BOOL;
-        case MP_QSTR_int: return MP_NATIVE_TYPE_INT;
-        case MP_QSTR_uint: return MP_NATIVE_TYPE_UINT;
-        case MP_QSTR_ptr: return MP_NATIVE_TYPE_PTR;
-        case MP_QSTR_ptr8: return MP_NATIVE_TYPE_PTR8;
-        case MP_QSTR_ptr16: return MP_NATIVE_TYPE_PTR16;
-        case MP_QSTR_ptr32: return MP_NATIVE_TYPE_PTR32;
-        default: return -1;
-    }
-}
-
 STATIC qstr vtype_to_qstr(vtype_kind_t vtype) {
     switch (vtype) {
         case VTYPE_PYOBJ: return MP_QSTR_object;
@@ -208,12 +219,18 @@ struct _emit_t {
     uint16_t code_state_start;
     uint16_t stack_start;
     int stack_size;
+    uint16_t n_cell;
 
     uint16_t const_table_cur_obj;
     uint16_t const_table_num_obj;
     uint16_t const_table_cur_raw_code;
     mp_uint_t *const_table;
 
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    uint16_t qstr_link_cur;
+    mp_qstr_link_entry_t *qstr_link;
+    #endif
+
     bool last_emit_was_return_value;
 
     scope_t *scope;
@@ -225,6 +242,7 @@ STATIC const uint8_t reg_local_table[REG_LOCAL_NUM] = {REG_LOCAL_1, REG_LOCAL_2,
 
 STATIC void emit_native_global_exc_entry(emit_t *emit);
 STATIC void emit_native_global_exc_exit(emit_t *emit);
+STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj);
 
 emit_t *EXPORT_FUN(new)(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) {
     emit_t *emit = m_new0(emit_t, 1);
@@ -279,6 +297,32 @@ STATIC void emit_native_mov_reg_state_addr(emit_t *emit, int reg_dest, int local
     }
 }
 
+STATIC void emit_native_mov_reg_qstr(emit_t *emit, int arg_reg, qstr qst) {
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t loc = ASM_MOV_REG_IMM_FIX_U16(emit->as, arg_reg, qst);
+    size_t link_idx = emit->qstr_link_cur++;
+    if (emit->pass == MP_PASS_EMIT) {
+        emit->qstr_link[link_idx].off = loc << 2 | 1;
+        emit->qstr_link[link_idx].qst = qst;
+    }
+    #else
+    ASM_MOV_REG_IMM(emit->as, arg_reg, qst);
+    #endif
+}
+
+STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) {
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    size_t loc = ASM_MOV_REG_IMM_FIX_WORD(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+    size_t link_idx = emit->qstr_link_cur++;
+    if (emit->pass == MP_PASS_EMIT) {
+        emit->qstr_link[link_idx].off = loc << 2 | 2;
+        emit->qstr_link[link_idx].qst = qst;
+    }
+    #else
+    ASM_MOV_REG_IMM(emit->as, reg_dest, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+    #endif
+}
+
 #define emit_native_mov_state_imm_via(emit, local_num, imm, reg_temp) \
     do { \
         ASM_MOV_REG_IMM((emit)->as, (reg_temp), (imm)); \
@@ -291,8 +335,15 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
     emit->pass = pass;
     emit->do_viper_types = scope->emit_options == MP_EMIT_OPT_VIPER;
     emit->stack_size = 0;
-    emit->const_table_cur_obj = 1; // first entry is for mp_fun_table
+    #if N_PRELUDE_AS_BYTES_OBJ
+    emit->const_table_cur_obj = emit->do_viper_types ? 0 : 1; // reserve first obj for prelude bytes obj
+    #else
+    emit->const_table_cur_obj = 0;
+    #endif
     emit->const_table_cur_raw_code = 0;
+    #if MICROPY_PERSISTENT_CODE_SAVE
+    emit->qstr_link_cur = 0;
+    #endif
     emit->last_emit_was_return_value = false;
     emit->scope = scope;
 
@@ -343,7 +394,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
     // Work out start of code state (mp_code_state_t or reduced version for viper)
     emit->code_state_start = 0;
     if (NEED_GLOBAL_EXC_HANDLER(emit)) {
-        emit->code_state_start = sizeof(nlr_buf_t) / sizeof(uintptr_t);
+        emit->code_state_start = SIZEOF_NLR_BUF;
     }
 
     if (emit->do_viper_types) {
@@ -377,16 +428,16 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
         ASM_ENTRY(emit->as, emit->stack_start + emit->n_state - num_locals_in_regs);
 
         #if N_X86
-        asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_1);
+        asm_x86_mov_arg_to_r32(emit->as, 0, REG_PARENT_ARG_1);
         #endif
 
         // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
-        ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_ARG_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+        ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
         ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, 0);
 
         // Store function object (passed as first arg) to stack if needed
         if (NEED_FUN_OBJ(emit)) {
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_ARG_1);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1);
         }
 
         // Put n_args in REG_ARG_1, n_kw in REG_ARG_2, args array in REG_LOCAL_3
@@ -395,9 +446,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
         asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_2);
         asm_x86_mov_arg_to_r32(emit->as, 3, REG_LOCAL_3);
         #else
-        ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_ARG_2);
-        ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_ARG_3);
-        ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_ARG_4);
+        ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_PARENT_ARG_2);
+        ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_PARENT_ARG_3);
+        ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_4);
         #endif
 
         // Check number of args matches this function, and call mp_arg_check_num_sig if not
@@ -437,31 +488,36 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
 
         if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
             emit->code_state_start = 0;
-            emit->stack_start = sizeof(mp_code_state_t) / sizeof(mp_uint_t);
+            emit->stack_start = SIZEOF_CODE_STATE;
+            #if N_PRELUDE_AS_BYTES_OBJ
+            // Load index of prelude bytes object in const_table
+            mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)(emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1));
+            #else
             mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_offset);
+            #endif
             mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->start_offset);
-            ASM_ENTRY(emit->as, sizeof(nlr_buf_t) / sizeof(uintptr_t));
+            ASM_ENTRY(emit->as, SIZEOF_NLR_BUF);
 
             // Put address of code_state into REG_GENERATOR_STATE
             #if N_X86
             asm_x86_mov_arg_to_r32(emit->as, 0, REG_GENERATOR_STATE);
             #else
-            ASM_MOV_REG_REG(emit->as, REG_GENERATOR_STATE, REG_ARG_1);
+            ASM_MOV_REG_REG(emit->as, REG_GENERATOR_STATE, REG_PARENT_ARG_1);
             #endif
 
             // Put throw value into LOCAL_IDX_EXC_VAL slot, for yield/yield-from
             #if N_X86
-            asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_2);
+            asm_x86_mov_arg_to_r32(emit->as, 1, REG_PARENT_ARG_2);
             #endif
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_ARG_2);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_PARENT_ARG_2);
 
             // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit));
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
         } else {
             // The locals and stack start after the code_state structure
-            emit->stack_start = emit->code_state_start + sizeof(mp_code_state_t) / sizeof(mp_uint_t);
+            emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE;
 
             // Allocate space on C-stack for code_state structure, which includes state
             ASM_ENTRY(emit->as, emit->stack_start + emit->n_state);
@@ -469,26 +525,49 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
             // Prepare incoming arguments for call to mp_setup_code_state
 
             #if N_X86
-            asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_1);
-            asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_2);
-            asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_3);
-            asm_x86_mov_arg_to_r32(emit->as, 3, REG_ARG_4);
+            asm_x86_mov_arg_to_r32(emit->as, 0, REG_PARENT_ARG_1);
+            asm_x86_mov_arg_to_r32(emit->as, 1, REG_PARENT_ARG_2);
+            asm_x86_mov_arg_to_r32(emit->as, 2, REG_PARENT_ARG_3);
+            asm_x86_mov_arg_to_r32(emit->as, 3, REG_PARENT_ARG_4);
             #endif
 
             // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_ARG_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
             ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
 
             // Set code_state.fun_bc
-            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_ARG_1);
+            ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1);
 
             // Set code_state.ip (offset from start of this function to prelude info)
+            #if N_PRELUDE_AS_BYTES_OBJ
+            // Prelude is a bytes object in const_table; store ip = prelude->data - fun_bc->bytecode
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1);
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, offsetof(mp_obj_str_t, data) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE);
+            ASM_SUB_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1);
+            emit_native_mov_state_reg(emit, emit->code_state_start + OFFSETOF_CODE_STATE_IP, REG_LOCAL_3);
+            #else
             // TODO this encoding may change size in the final pass, need to make it fixed
-            emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), emit->prelude_offset, REG_ARG_1);
+            emit_native_mov_state_imm_via(emit, emit->code_state_start + OFFSETOF_CODE_STATE_IP, emit->prelude_offset, REG_PARENT_ARG_1);
+            #endif
+
+            // Set code_state.n_state (only works on little endian targets due to n_state being uint16_t)
+            emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t), emit->n_state, REG_ARG_1);
 
             // Put address of code_state into first arg
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, emit->code_state_start);
 
+            // Copy next 3 args if needed
+            #if REG_ARG_2 != REG_PARENT_ARG_2
+            ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_PARENT_ARG_2);
+            #endif
+            #if REG_ARG_3 != REG_PARENT_ARG_3
+            ASM_MOV_REG_REG(emit->as, REG_ARG_3, REG_PARENT_ARG_3);
+            #endif
+            #if REG_ARG_4 != REG_PARENT_ARG_4
+            ASM_MOV_REG_REG(emit->as, REG_ARG_4, REG_PARENT_ARG_4);
+            #endif
+
             // Call mp_setup_code_state to prepare code_state structure
             #if N_THUMB
             asm_thumb_bl_ind(emit->as, MP_F_SETUP_CODE_STATE, ASM_THUMB_REG_R4);
@@ -535,22 +614,28 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
 
 }
 
+static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) {
+    mp_asm_base_data(&emit->as->base, 1, val);
+}
+
 STATIC void emit_native_end_pass(emit_t *emit) {
     emit_native_global_exc_exit(emit);
 
     if (!emit->do_viper_types) {
         emit->prelude_offset = mp_asm_base_get_code_pos(&emit->as->base);
-        mp_asm_base_data(&emit->as->base, 1, 0x80 | ((emit->n_state >> 7) & 0x7f));
-        mp_asm_base_data(&emit->as->base, 1, emit->n_state & 0x7f);
-        mp_asm_base_data(&emit->as->base, 1, 0); // n_exc_stack
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->scope_flags);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_pos_args);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_kwonly_args);
-        mp_asm_base_data(&emit->as->base, 1, emit->scope->num_def_pos_args);
-
-        // write code info
+
+        size_t n_state = emit->n_state;
+        size_t n_exc_stack = 0; // exc-stack not needed for native code
+        MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, emit->scope, emit_native_write_code_info_byte, emit);
+
+        #if MICROPY_PERSISTENT_CODE
+        size_t n_info = 4;
+        #else
+        size_t n_info = 1;
+        #endif
+        MP_BC_PRELUDE_SIZE_ENCODE(n_info, emit->n_cell, emit_native_write_code_info_byte, emit);
+
         #if MICROPY_PERSISTENT_CODE
-        mp_asm_base_data(&emit->as->base, 1, 5);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name >> 8);
         mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file);
@@ -560,14 +645,25 @@ STATIC void emit_native_end_pass(emit_t *emit) {
         #endif
 
         // bytecode prelude: initialise closed over variables
+        size_t cell_start = mp_asm_base_get_code_pos(&emit->as->base);
         for (int i = 0; i < emit->scope->id_info_len; i++) {
             id_info_t *id = &emit->scope->id_info[i];
             if (id->kind == ID_INFO_KIND_CELL) {
-                assert(id->local_num < 255);
+                assert(id->local_num <= 255);
                 mp_asm_base_data(&emit->as->base, 1, id->local_num); // write the local which should be converted to a cell
             }
         }
-        mp_asm_base_data(&emit->as->base, 1, 255); // end of list sentinel
+        emit->n_cell = mp_asm_base_get_code_pos(&emit->as->base) - cell_start;
+
+        #if N_PRELUDE_AS_BYTES_OBJ
+        // Prelude bytes object is after qstr arg names and mp_fun_table
+        size_t table_off = emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1;
+        if (emit->pass == MP_PASS_EMIT) {
+            void *buf = emit->as->base.code_base + emit->prelude_offset;
+            size_t n = emit->as->base.code_offset - emit->prelude_offset;
+            emit->const_table[table_off] = (uintptr_t)mp_obj_new_bytes(buf, n);
+        }
+        #endif
     }
 
     ASM_END_PASS(emit->as);
@@ -580,7 +676,7 @@ STATIC void emit_native_end_pass(emit_t *emit) {
     assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->const_table_num_obj == emit->const_table_cur_obj));
     emit->const_table_num_obj = emit->const_table_cur_obj;
     if (emit->pass == MP_PASS_CODE_SIZE) {
-        size_t const_table_alloc = emit->const_table_num_obj + emit->const_table_cur_raw_code;
+        size_t const_table_alloc = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code;
         size_t nqstr = 0;
         if (!emit->do_viper_types) {
             // Add room for qstr names of arguments
@@ -588,8 +684,18 @@ STATIC void emit_native_end_pass(emit_t *emit) {
             const_table_alloc += nqstr;
         }
         emit->const_table = m_new(mp_uint_t, const_table_alloc);
+        #if !MICROPY_DYNAMIC_COMPILER
         // Store mp_fun_table pointer just after qstrs
-        emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)mp_fun_table;
+        // (but in dynamic-compiler mode eliminate dependency on mp_fun_table)
+        emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)&mp_fun_table;
+        #endif
+
+        #if MICROPY_PERSISTENT_CODE_SAVE
+        size_t qstr_link_alloc = emit->qstr_link_cur;
+        if (qstr_link_alloc > 0) {
+            emit->qstr_link = m_new(mp_qstr_link_entry_t, qstr_link_alloc);
+        }
+        #endif
     }
 
     if (emit->pass == MP_PASS_EMIT) {
@@ -599,6 +705,11 @@ STATIC void emit_native_end_pass(emit_t *emit) {
         mp_emit_glue_assign_native(emit->scope->raw_code,
             emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY,
             f, f_len, emit->const_table,
+            #if MICROPY_PERSISTENT_CODE_SAVE
+            emit->prelude_offset,
+            emit->const_table_cur_obj, emit->const_table_cur_raw_code,
+            emit->qstr_link_cur, emit->qstr_link,
+            #endif
             emit->scope->num_pos_args, emit->scope->scope_flags, 0);
     }
 }
@@ -883,6 +994,12 @@ STATIC void emit_call_with_2_imm_args(emit_t *emit, mp_fun_kind_t fun_kind, mp_i
     ASM_CALL_IND(emit->as, fun_kind);
 }
 
+STATIC void emit_call_with_qstr_arg(emit_t *emit, mp_fun_kind_t fun_kind, qstr qst, int arg_reg) {
+    need_reg_all(emit);
+    emit_native_mov_reg_qstr(emit, arg_reg, qst);
+    ASM_CALL_IND(emit->as, fun_kind);
+}
+
 // vtype of all n_pop objects is VTYPE_PYOBJ
 // Will convert any items that are not VTYPE_PYOBJ to this type and put them back on the stack.
 // If any conversions of non-immediate values are needed, then it uses REG_ARG_1, REG_ARG_2 and REG_RET.
@@ -994,17 +1111,19 @@ STATIC void emit_load_reg_with_ptr(emit_t *emit, int reg, mp_uint_t ptr, size_t
         emit->const_table[table_off] = ptr;
     }
     emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit));
-    ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+    ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE);
     ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off);
 }
 
 STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) {
-    size_t table_off = emit->const_table_cur_obj++;
+    // First entry is for mp_fun_table
+    size_t table_off = 1 + emit->const_table_cur_obj++;
     emit_load_reg_with_ptr(emit, reg, (mp_uint_t)obj, table_off);
 }
 
 STATIC void emit_load_reg_with_raw_code(emit_t *emit, int reg, mp_raw_code_t *rc) {
-    size_t table_off = emit->const_table_num_obj + emit->const_table_cur_raw_code++;
+    // First entry is for mp_fun_table, then constant objects
+    size_t table_off = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code++;
     emit_load_reg_with_ptr(emit, reg, (mp_uint_t)rc, table_off);
 }
 
@@ -1049,7 +1168,7 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
         if (!(emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR)) {
             // Set new globals
             emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit));
-            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_GLOBALS);
             emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS);
 
             // Save old globals (or NULL if globals didn't change)
@@ -1065,6 +1184,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
             // Wrap everything in an nlr context
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0);
             emit_call(emit, MP_F_NLR_PUSH);
+            #if N_NLR_SETJMP
+            ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 2);
+            emit_call(emit, MP_F_SETJMP);
+            #endif
             ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, start_label, true);
         } else {
             // Clear the unwind state
@@ -1079,6 +1202,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
             ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_2, LOCAL_IDX_EXC_HANDLER_UNWIND(emit));
             ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0);
             emit_call(emit, MP_F_NLR_PUSH);
+            #if N_NLR_SETJMP
+            ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 2);
+            emit_call(emit, MP_F_SETJMP);
+            #endif
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_LOCAL_2);
             ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true);
 
@@ -1089,6 +1216,12 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
 
             // Global exception handler: check for valid exception handler
             emit_native_label_assign(emit, global_except_label);
+            #if N_NLR_SETJMP
+            // Reload REG_FUN_TABLE, since it may be clobbered by longjmp
+            emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t));
+            ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
+            #endif
             ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit));
             ASM_JUMP_IF_REG_NONZERO(emit->as, REG_LOCAL_1, nlr_label, false);
         }
@@ -1102,10 +1235,10 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
         if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
             // Store return value in state[0]
             ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, LOCAL_IDX_EXC_VAL(emit));
-            ASM_STORE_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, offsetof(mp_code_state_t, state) / sizeof(uintptr_t));
+            ASM_STORE_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, OFFSETOF_CODE_STATE_STATE);
 
             // Load return kind
-            ASM_MOV_REG_IMM(emit->as, REG_RET, MP_VM_RETURN_EXCEPTION);
+            ASM_MOV_REG_IMM(emit->as, REG_PARENT_RET, MP_VM_RETURN_EXCEPTION);
 
             ASM_EXIT(emit->as);
         } else {
@@ -1160,7 +1293,7 @@ STATIC void emit_native_global_exc_exit(emit_t *emit) {
         }
 
         // Load return value
-        ASM_MOV_REG_LOCAL(emit->as, REG_RET, LOCAL_IDX_RET_VAL(emit));
+        ASM_MOV_REG_LOCAL(emit->as, REG_PARENT_RET, LOCAL_IDX_RET_VAL(emit));
     }
 
     ASM_EXIT(emit->as);
@@ -1181,7 +1314,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) {
     assert(vtype_level == VTYPE_PYOBJ);
     emit->do_viper_types = orig_do_viper_types;
 
-    emit_call_with_imm_arg(emit, MP_F_IMPORT_NAME, qst, REG_ARG_1); // arg1 = import name
+    emit_call_with_qstr_arg(emit, MP_F_IMPORT_NAME, qst, REG_ARG_1); // arg1 = import name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1191,7 +1324,7 @@ STATIC void emit_native_import_from(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_module;
     emit_access_stack(emit, 1, &vtype_module, REG_ARG_1); // arg1 = module
     assert(vtype_module == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_IMPORT_FROM, qst, REG_ARG_2); // arg2 = import name
+    emit_call_with_qstr_arg(emit, MP_F_IMPORT_FROM, qst, REG_ARG_2); // arg2 = import name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1217,7 +1350,11 @@ STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) {
 STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) {
     DEBUG_printf("load_const_tok(tok=%u)\n", tok);
     if (tok == MP_TOKEN_ELLIPSIS) {
+        #if MICROPY_PERSISTENT_CODE_SAVE
+        emit_native_load_const_obj(emit, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+        #else
         emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj));
+        #endif
     } else {
         emit_native_pre(emit);
         if (tok == MP_TOKEN_KW_NONE) {
@@ -1245,11 +1382,14 @@ STATIC void emit_native_load_const_str(emit_t *emit, qstr qst) {
     } else
     */
     {
-        emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_NEW_QSTR(qst));
+        need_reg_single(emit, REG_TEMP0, 0);
+        emit_native_mov_reg_qstr_obj(emit, REG_TEMP0, qst);
+        emit_post_push_reg(emit, VTYPE_PYOBJ, REG_TEMP0);
     }
 }
 
 STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj) {
+    emit->scope->scope_flags |= MP_SCOPE_FLAG_HASCONSTS;
     emit_native_pre(emit);
     need_reg_single(emit, REG_RET, 0);
     emit_load_reg_with_object(emit, REG_RET, obj);
@@ -1308,13 +1448,13 @@ STATIC void emit_native_load_global(emit_t *emit, qstr qst, int kind) {
         if (emit->do_viper_types) {
             // check for builtin casting operators
             int native_type = mp_native_type_from_qstr(qst);
-            if (native_type >= MP_NATIVE_TYPE_INT) {
+            if (native_type >= MP_NATIVE_TYPE_BOOL) {
                 emit_post_push_imm(emit, VTYPE_BUILTIN_CAST, native_type);
                 return;
             }
         }
     }
-    emit_call_with_imm_arg(emit, MP_F_LOAD_NAME + kind, qst, REG_ARG_1);
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_NAME + kind, qst, REG_ARG_1);
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1326,7 +1466,7 @@ STATIC void emit_native_load_attr(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_base;
     emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
     assert(vtype_base == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_LOAD_ATTR, qst, REG_ARG_2); // arg2 = attribute name
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
 
@@ -1334,13 +1474,13 @@ STATIC void emit_native_load_method(emit_t *emit, qstr qst, bool is_super) {
     if (is_super) {
         emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_2, 3); // arg2 = dest ptr
         emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_2, 2); // arg2 = dest ptr
-        emit_call_with_imm_arg(emit, MP_F_LOAD_SUPER_METHOD, qst, REG_ARG_1); // arg1 = method name
+        emit_call_with_qstr_arg(emit, MP_F_LOAD_SUPER_METHOD, qst, REG_ARG_1); // arg1 = method name
     } else {
         vtype_kind_t vtype_base;
         emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
         assert(vtype_base == VTYPE_PYOBJ);
         emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-        emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, qst, REG_ARG_2); // arg2 = method name
+        emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, qst, REG_ARG_2); // arg2 = method name
     }
 }
 
@@ -1547,7 +1687,7 @@ STATIC void emit_native_store_global(emit_t *emit, qstr qst, int kind) {
             ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_RET);
         }
     }
-    emit_call_with_imm_arg(emit, MP_F_STORE_NAME + kind, qst, REG_ARG_1); // arg1 = name
+    emit_call_with_qstr_arg(emit, MP_F_STORE_NAME + kind, qst, REG_ARG_1); // arg1 = name
     emit_post(emit);
 }
 
@@ -1556,7 +1696,7 @@ STATIC void emit_native_store_attr(emit_t *emit, qstr qst) {
     emit_pre_pop_reg_reg(emit, &vtype_base, REG_ARG_1, &vtype_val, REG_ARG_3); // arg1 = base, arg3 = value
     assert(vtype_base == VTYPE_PYOBJ);
     assert(vtype_val == VTYPE_PYOBJ);
-    emit_call_with_imm_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
+    emit_call_with_qstr_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post(emit);
 }
 
@@ -1753,7 +1893,7 @@ STATIC void emit_native_delete_global(emit_t *emit, qstr qst, int kind) {
     MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_NAME == MP_F_DELETE_NAME);
     MP_STATIC_ASSERT(MP_F_DELETE_NAME + MP_EMIT_IDOP_GLOBAL_GLOBAL == MP_F_DELETE_GLOBAL);
     emit_native_pre(emit);
-    emit_call_with_imm_arg(emit, MP_F_DELETE_NAME + kind, qst, REG_ARG_1);
+    emit_call_with_qstr_arg(emit, MP_F_DELETE_NAME + kind, qst, REG_ARG_1);
     emit_post(emit);
 }
 
@@ -1761,7 +1901,8 @@ STATIC void emit_native_delete_attr(emit_t *emit, qstr qst) {
     vtype_kind_t vtype_base;
     emit_pre_pop_reg(emit, &vtype_base, REG_ARG_1); // arg1 = base
     assert(vtype_base == VTYPE_PYOBJ);
-    emit_call_with_2_imm_args(emit, MP_F_STORE_ATTR, qst, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL, REG_ARG_3); // arg2 = attribute name, arg3 = value (null for delete)
+    ASM_XOR_REG_REG(emit->as, REG_ARG_3, REG_ARG_3); // arg3 = value (null for delete)
+    emit_call_with_qstr_arg(emit, MP_F_STORE_ATTR, qst, REG_ARG_2); // arg2 = attribute name
     emit_post(emit);
 }
 
@@ -1916,7 +2057,7 @@ STATIC void emit_native_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t exc
             prev_finally->unwind_label = UNWIND_LABEL_DO_FINAL_UNWIND;
             ASM_MOV_REG_PCREL(emit->as, REG_RET, label & ~MP_EMIT_BREAK_FROM_FOR);
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_RET);
-            // Cancel any active exception (see also emit_native_pop_except)
+            // Cancel any active exception (see also emit_native_pop_except_jump)
             emit_native_mov_reg_const(emit, REG_RET, MP_F_CONST_NONE_OBJ);
             ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_RET);
             // Jump to the innermost active finally
@@ -1935,7 +2076,7 @@ STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) {
     emit_access_stack(emit, 1, &vtype, REG_ARG_1); // arg1 = ctx_mgr
     assert(vtype == VTYPE_PYOBJ);
     emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-    emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___exit__, REG_ARG_2);
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___exit__, REG_ARG_2);
     // stack: (..., ctx_mgr, __exit__, self)
 
     emit_pre_pop_reg(emit, &vtype, REG_ARG_3); // self
@@ -1948,7 +2089,7 @@ STATIC void emit_native_setup_with(emit_t *emit, mp_uint_t label) {
 
     // get __enter__ method
     emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_3, 2); // arg3 = dest ptr
-    emit_call_with_imm_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___enter__, REG_ARG_2); // arg2 = method name
+    emit_call_with_qstr_arg(emit, MP_F_LOAD_METHOD, MP_QSTR___enter__, REG_ARG_2); // arg2 = method name
     // stack: (..., __exit__, self, __enter__, self)
 
     // call __enter__ method
@@ -2094,12 +2235,12 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) {
     emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_1, MP_OBJ_ITER_BUF_NSLOTS);
     adjust_stack(emit, MP_OBJ_ITER_BUF_NSLOTS);
     emit_call(emit, MP_F_NATIVE_ITERNEXT);
-    #ifdef NDEBUG
-    MP_STATIC_ASSERT(MP_OBJ_STOP_ITERATION == 0);
-    ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, label, false);
-    #else
+    #if MICROPY_DEBUG_MP_OBJ_SENTINELS
     ASM_MOV_REG_IMM(emit->as, REG_TEMP1, (mp_uint_t)MP_OBJ_STOP_ITERATION);
     ASM_JUMP_IF_REG_EQ(emit->as, REG_RET, REG_TEMP1, label);
+    #else
+    MP_STATIC_ASSERT(MP_OBJ_STOP_ITERATION == 0);
+    ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, label, false);
     #endif
     emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET);
 }
@@ -2111,18 +2252,15 @@ STATIC void emit_native_for_iter_end(emit_t *emit) {
     emit_post(emit);
 }
 
-STATIC void emit_native_pop_block(emit_t *emit) {
-    emit_native_pre(emit);
-    if (!emit->exc_stack[emit->exc_stack_size - 1].is_finally) {
+STATIC void emit_native_pop_except_jump(emit_t *emit, mp_uint_t label, bool within_exc_handler) {
+    if (within_exc_handler) {
+        // Cancel any active exception so subsequent handlers don't see it
+        emit_native_mov_reg_const(emit, REG_TEMP0, MP_F_CONST_NONE_OBJ);
+        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0);
+    } else {
         emit_native_leave_exc_stack(emit, false);
     }
-    emit_post(emit);
-}
-
-STATIC void emit_native_pop_except(emit_t *emit) {
-    // Cancel any active exception so subsequent handlers don't see it
-    emit_native_mov_reg_const(emit, REG_TEMP0, MP_F_CONST_NONE_OBJ);
-    ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0);
+    emit_native_jump(emit, label);
 }
 
 STATIC void emit_native_unary_op(emit_t *emit, mp_unary_op_t op) {
@@ -2181,17 +2319,16 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) {
         int reg_rhs = REG_ARG_3;
         emit_pre_pop_reg_flexible(emit, &vtype_rhs, &reg_rhs, REG_RET, REG_ARG_2);
         emit_pre_pop_reg(emit, &vtype_lhs, REG_ARG_2);
-        if (0) {
-            // dummy
         #if !(N_X64 || N_X86)
-        } else if (op == MP_BINARY_OP_LSHIFT) {
+        if (op == MP_BINARY_OP_LSHIFT) {
             ASM_LSL_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
         } else if (op == MP_BINARY_OP_RSHIFT) {
             ASM_ASR_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
+        } else
         #endif
-        } else if (op == MP_BINARY_OP_OR) {
+        if (op == MP_BINARY_OP_OR) {
             ASM_OR_REG_REG(emit->as, REG_ARG_2, reg_rhs);
             emit_post_push_reg(emit, VTYPE_INT, REG_ARG_2);
         } else if (op == MP_BINARY_OP_XOR) {
@@ -2267,7 +2404,7 @@ STATIC void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) {
                 ASM_ARM_CC_NE,
             };
             asm_arm_setcc_reg(emit->as, REG_RET, ccs[op - MP_BINARY_OP_LESS]);
-            #elif N_XTENSA
+            #elif N_XTENSA || N_XTENSAWIN
             static uint8_t ccs[6] = {
                 ASM_XTENSA_CC_LT,
                 0x80 | ASM_XTENSA_CC_LT, // for GT we'll swap args
@@ -2527,7 +2664,7 @@ STATIC void emit_native_return_value(emit_t *emit) {
     if (emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR) {
         // Save pointer to current stack position for caller to access return value
         emit_get_stack_pointer_to_reg_for_pop(emit, REG_TEMP0, 1);
-        emit_native_mov_state_reg(emit, offsetof(mp_code_state_t, sp) / sizeof(uintptr_t), REG_TEMP0);
+        emit_native_mov_state_reg(emit, OFFSETOF_CODE_STATE_SP, REG_TEMP0);
 
         // Put return type in return value slot
         ASM_MOV_REG_IMM(emit->as, REG_TEMP0, MP_VM_RETURN_NORMAL);
@@ -2544,13 +2681,13 @@ STATIC void emit_native_return_value(emit_t *emit) {
         if (peek_vtype(emit, 0) == VTYPE_PTR_NONE) {
             emit_pre_pop_discard(emit);
             if (return_vtype == VTYPE_PYOBJ) {
-                emit_native_mov_reg_const(emit, REG_RET, MP_F_CONST_NONE_OBJ);
+                emit_native_mov_reg_const(emit, REG_PARENT_RET, MP_F_CONST_NONE_OBJ);
             } else {
                 ASM_MOV_REG_IMM(emit->as, REG_ARG_1, 0);
             }
         } else {
             vtype_kind_t vtype;
-            emit_pre_pop_reg(emit, &vtype, return_vtype == VTYPE_PYOBJ ? REG_RET : REG_ARG_1);
+            emit_pre_pop_reg(emit, &vtype, return_vtype == VTYPE_PYOBJ ? REG_PARENT_RET : REG_ARG_1);
             if (vtype != return_vtype) {
                 EMIT_NATIVE_VIPER_TYPE_ERROR(emit,
                     "return expected '%q' but got '%q'",
@@ -2559,15 +2696,18 @@ STATIC void emit_native_return_value(emit_t *emit) {
         }
         if (return_vtype != VTYPE_PYOBJ) {
             emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, return_vtype, REG_ARG_2);
+            #if REG_RET != REG_PARENT_RET
+            ASM_MOV_REG_REG(emit->as, REG_PARENT_RET, REG_RET);
+            #endif
         }
     } else {
         vtype_kind_t vtype;
-        emit_pre_pop_reg(emit, &vtype, REG_RET);
+        emit_pre_pop_reg(emit, &vtype, REG_PARENT_RET);
         assert(vtype == VTYPE_PYOBJ);
     }
     if (NEED_GLOBAL_EXC_HANDLER(emit)) {
         // Save return value for the global exception handler to use
-        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_RET_VAL(emit), REG_RET);
+        ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_RET_VAL(emit), REG_PARENT_RET);
     }
     emit_native_unwind_jump(emit, emit->exit_label, emit->exc_stack_size);
     emit->last_emit_was_return_value = true;
@@ -2610,7 +2750,7 @@ STATIC void emit_native_yield(emit_t *emit, int kind) {
 
     // Save pointer to current stack position for caller to access yielded value
     emit_get_stack_pointer_to_reg_for_pop(emit, REG_TEMP0, 1);
-    emit_native_mov_state_reg(emit, offsetof(mp_code_state_t, sp) / sizeof(uintptr_t), REG_TEMP0);
+    emit_native_mov_state_reg(emit, OFFSETOF_CODE_STATE_SP, REG_TEMP0);
 
     // Put return type in return value slot
     ASM_MOV_REG_IMM(emit->as, REG_TEMP0, MP_VM_RETURN_YIELD);
@@ -2681,6 +2821,11 @@ STATIC void emit_native_end_except_handler(emit_t *emit) {
 }
 
 const emit_method_table_t EXPORT_FUN(method_table) = {
+    #if MICROPY_DYNAMIC_COMPILER
+    EXPORT_FUN(new),
+    EXPORT_FUN(free),
+    #endif
+
     emit_native_start_pass,
     emit_native_end_pass,
     emit_native_last_emit_was_return_value,
@@ -2726,8 +2871,7 @@ const emit_method_table_t EXPORT_FUN(method_table) = {
     emit_native_get_iter,
     emit_native_for_iter,
     emit_native_for_iter_end,
-    emit_native_pop_block,
-    emit_native_pop_except,
+    emit_native_pop_except_jump,
     emit_native_unary_op,
     emit_native_binary_op,
     emit_native_build,

py/emitnx86.c

@@ -1,7 +1,7 @@
 // x86 specific stuff
 
 #include "py/mpconfig.h"
-#include "py/runtime0.h"
+#include "py/nativeglue.h"
 
 #if MICROPY_EMIT_X86
 
@@ -34,13 +34,11 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = {
     [MP_F_BINARY_OP] = 3,
     [MP_F_BUILD_TUPLE] = 2,
     [MP_F_BUILD_LIST] = 2,
-    [MP_F_LIST_APPEND] = 2,
     [MP_F_BUILD_MAP] = 1,
-    [MP_F_STORE_MAP] = 3,
-    #if MICROPY_PY_BUILTINS_SET
     [MP_F_BUILD_SET] = 2,
     [MP_F_STORE_SET] = 2,
-    #endif
+    [MP_F_LIST_APPEND] = 2,
+    [MP_F_STORE_MAP] = 3,
     [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3,
     [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3,
     [MP_F_CALL_METHOD_N_KW] = 3,
@@ -53,20 +51,18 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = {
     [MP_F_IMPORT_NAME] = 3,
     [MP_F_IMPORT_FROM] = 2,
     [MP_F_IMPORT_ALL] = 1,
-    #if MICROPY_PY_BUILTINS_SLICE
     [MP_F_NEW_SLICE] = 3,
-    #endif
     [MP_F_UNPACK_SEQUENCE] = 3,
     [MP_F_UNPACK_EX] = 3,
     [MP_F_DELETE_NAME] = 1,
     [MP_F_DELETE_GLOBAL] = 1,
-    [MP_F_NEW_CELL] = 1,
     [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3,
     [MP_F_ARG_CHECK_NUM_SIG] = 3,
     [MP_F_SETUP_CODE_STATE] = 4,
     [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2,
     [MP_F_SMALL_INT_MODULO] = 2,
     [MP_F_NATIVE_YIELD_FROM] = 3,
+    [MP_F_SETJMP] = 1,
 };
 
 #define N_X86 (1)

py/emitnxtensawin.c

@@ -0,0 +1,23 @@
+// Xtensa-Windowed specific stuff
+
+#include "py/mpconfig.h"
+
+#if MICROPY_EMIT_XTENSAWIN
+
+// this is defined so that the assembler exports generic assembler API macros
+#define GENERIC_ASM_API (1)
+#define GENERIC_ASM_API_WIN (1)
+#include "py/asmxtensa.h"
+
+// Word indices of REG_LOCAL_x in nlr_buf_t
+#define NLR_BUF_IDX_LOCAL_1 (2 + 4) // a4
+#define NLR_BUF_IDX_LOCAL_2 (2 + 5) // a5
+#define NLR_BUF_IDX_LOCAL_3 (2 + 6) // a6
+
+#define N_NLR_SETJMP (1)
+#define N_PRELUDE_AS_BYTES_OBJ (1)
+#define N_XTENSAWIN (1)
+#define EXPORT_FUN(name) emit_native_xtensawin_##name
+#include "py/emitnative.c"
+
+#endif

py/frozenmod.h

@@ -3,7 +3,8 @@
  *
  * The MIT License (MIT)
  *
- * Copyright (c) 2014 Damien P. George
+ * Copyright (c) 2015 Paul Sokolovsky
+ * Copyright (c) 2016 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

py/gc.c

@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal

py/grammar.h

@@ -55,7 +55,7 @@ DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE))
 // varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
 // vfpdef: NAME
 
-DEF_RULE_NC(decorator, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
+DEF_RULE_NC(decorator, and(4), tok(OP_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
 DEF_RULE_NC(decorators, one_or_more, rule(decorator))
 DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body))
 #if MICROPY_PY_ASYNC_AWAIT
@@ -96,7 +96,7 @@ DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), t
 // small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
 // expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*)
 // testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
-// augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
+// augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
 // # For normal assignments, additional restrictions enforced by the interpreter
 
 DEF_RULE_NC(small_stmt, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt))
@@ -108,7 +108,7 @@ DEF_RULE_NC(expr_stmt_assign, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6))
 DEF_RULE_NC(expr_stmt_6, or(2), rule(yield_expr), rule(testlist_star_expr))
 DEF_RULE(testlist_star_expr, c(generic_tuple), list_with_end, rule(testlist_star_expr_2), tok(DEL_COMMA))
 DEF_RULE_NC(testlist_star_expr_2, or(2), rule(star_expr), rule(test))
-DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
+DEF_RULE_NC(augassign, or(13), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_AT_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
 
 // del_stmt: 'del' exprlist
 // pass_stmt: 'pass'
@@ -226,7 +226,7 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara
 // and_expr: shift_expr ('&' shift_expr)*
 // shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 // arith_expr: term (('+'|'-') term)*
-// term: factor (('*'|'/'|'%'|'//') factor)*
+// term: factor (('*'|'@'|'/'|'%'|'//') factor)*
 // factor: ('+'|'-'|'~') factor | power
 // power: atom_expr ['**' factor]
 // atom_expr: 'await' atom trailer* | atom trailer*
@@ -249,7 +249,7 @@ DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE))
 DEF_RULE(arith_expr, c(term), list, rule(term), rule(arith_op))
 DEF_RULE_NC(arith_op, or(2), tok(OP_PLUS), tok(OP_MINUS))
 DEF_RULE(term, c(term), list, rule(factor), rule(term_op))
-DEF_RULE_NC(term_op, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
+DEF_RULE_NC(term_op, or(5), tok(OP_STAR), tok(OP_AT), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
 DEF_RULE_NC(factor, or(2), rule(factor_2), rule(power))
 DEF_RULE(factor_2, c(factor_2), and_ident(2), rule(factor_op), rule(factor))
 DEF_RULE_NC(factor_op, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE))

py/lexer.c

@@ -174,7 +174,7 @@ STATIC void indent_pop(mp_lexer_t *lex) {
 // this means if the start of two ops are the same then they are equal til the last char
 
 STATIC const char *const tok_enc =
-    "()[]{},:;@~" // singles
+    "()[]{},:;~" // singles
     "<e=c<e="     // < <= << <<=
     ">e=c>e="     // > >= >> >>=
     "*e=c*e="     // * *= ** **=
@@ -185,6 +185,7 @@ STATIC const char *const tok_enc =
     "/e=c/e="     // / /= // //=
     "%e="         // % %=
     "^e="         // ^ ^=
+    "@e="         // @ @=
     "=e="         // = ==
     "!.";         // start of special cases: != . ...
 
@@ -193,7 +194,7 @@ STATIC const uint8_t tok_enc_kind[] = {
     MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
     MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
     MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE,
-    MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE,
+    MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_OP_TILDE,
 
     MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL,
     MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL,
@@ -205,6 +206,7 @@ STATIC const uint8_t tok_enc_kind[] = {
     MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL,
     MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL,
     MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
+    MP_TOKEN_OP_AT, MP_TOKEN_DEL_AT_EQUAL,
     MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
 };
 

py/lexer.h

@@ -96,26 +96,46 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_KW_WITH,
     MP_TOKEN_KW_YIELD,
 
-    MP_TOKEN_OP_PLUS,
-    MP_TOKEN_OP_MINUS,
-    MP_TOKEN_OP_STAR,
-    MP_TOKEN_OP_DBL_STAR,
-    MP_TOKEN_OP_SLASH,
-    MP_TOKEN_OP_DBL_SLASH,
-    MP_TOKEN_OP_PERCENT,
+    MP_TOKEN_OP_TILDE,
+
+    // Order of these 6 matches corresponding mp_binary_op_t operator
     MP_TOKEN_OP_LESS,
-    MP_TOKEN_OP_DBL_LESS,
     MP_TOKEN_OP_MORE,
-    MP_TOKEN_OP_DBL_MORE,
-    MP_TOKEN_OP_AMPERSAND,
-    MP_TOKEN_OP_PIPE,
-    MP_TOKEN_OP_CARET,
-    MP_TOKEN_OP_TILDE,
+    MP_TOKEN_OP_DBL_EQUAL,
     MP_TOKEN_OP_LESS_EQUAL,
     MP_TOKEN_OP_MORE_EQUAL,
-    MP_TOKEN_OP_DBL_EQUAL,
     MP_TOKEN_OP_NOT_EQUAL,
 
+    // Order of these 13 matches corresponding mp_binary_op_t operator
+    MP_TOKEN_OP_PIPE,
+    MP_TOKEN_OP_CARET,
+    MP_TOKEN_OP_AMPERSAND,
+    MP_TOKEN_OP_DBL_LESS,
+    MP_TOKEN_OP_DBL_MORE,
+    MP_TOKEN_OP_PLUS,
+    MP_TOKEN_OP_MINUS,
+    MP_TOKEN_OP_STAR,
+    MP_TOKEN_OP_AT,
+    MP_TOKEN_OP_DBL_SLASH,
+    MP_TOKEN_OP_SLASH,
+    MP_TOKEN_OP_PERCENT,
+    MP_TOKEN_OP_DBL_STAR,
+
+    // Order of these 13 matches corresponding mp_binary_op_t operator
+    MP_TOKEN_DEL_PIPE_EQUAL,
+    MP_TOKEN_DEL_CARET_EQUAL,
+    MP_TOKEN_DEL_AMPERSAND_EQUAL,
+    MP_TOKEN_DEL_DBL_LESS_EQUAL,
+    MP_TOKEN_DEL_DBL_MORE_EQUAL,
+    MP_TOKEN_DEL_PLUS_EQUAL,
+    MP_TOKEN_DEL_MINUS_EQUAL,
+    MP_TOKEN_DEL_STAR_EQUAL,
+    MP_TOKEN_DEL_AT_EQUAL,
+    MP_TOKEN_DEL_DBL_SLASH_EQUAL,
+    MP_TOKEN_DEL_SLASH_EQUAL,
+    MP_TOKEN_DEL_PERCENT_EQUAL,
+    MP_TOKEN_DEL_DBL_STAR_EQUAL,
+
     MP_TOKEN_DEL_PAREN_OPEN,
     MP_TOKEN_DEL_PAREN_CLOSE,
     MP_TOKEN_DEL_BRACKET_OPEN,
@@ -126,20 +146,7 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_DEL_COLON,
     MP_TOKEN_DEL_PERIOD,
     MP_TOKEN_DEL_SEMICOLON,
-    MP_TOKEN_DEL_AT,
     MP_TOKEN_DEL_EQUAL,
-    MP_TOKEN_DEL_PLUS_EQUAL,
-    MP_TOKEN_DEL_MINUS_EQUAL,
-    MP_TOKEN_DEL_STAR_EQUAL,
-    MP_TOKEN_DEL_SLASH_EQUAL,
-    MP_TOKEN_DEL_DBL_SLASH_EQUAL,
-    MP_TOKEN_DEL_PERCENT_EQUAL,
-    MP_TOKEN_DEL_AMPERSAND_EQUAL,
-    MP_TOKEN_DEL_PIPE_EQUAL,
-    MP_TOKEN_DEL_CARET_EQUAL,
-    MP_TOKEN_DEL_DBL_MORE_EQUAL,
-    MP_TOKEN_DEL_DBL_LESS_EQUAL,
-    MP_TOKEN_DEL_DBL_STAR_EQUAL,
     MP_TOKEN_DEL_MINUS_MORE,
 } mp_token_kind_t;
 
@@ -177,8 +184,6 @@ void mp_lexer_to_next(mp_lexer_t *lex);
 // platform specific import function; must be implemented for a specific port
 // TODO tidy up, rename, or put elsewhere
 
-//mp_lexer_t *mp_import_open_file(qstr mod_name);
-
 typedef enum {
     MP_IMPORT_STAT_NO_EXIST,
     MP_IMPORT_STAT_DIR,

py/makemoduledefs.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+# This pre-processor parses provided objects' c files for
+# MP_REGISTER_MODULE(module_name, obj_module, enabled_define)
+# These are used to generate a header with the required entries for
+# "mp_rom_map_elem_t mp_builtin_module_table[]" in py/objmodule.c
+
+from __future__ import print_function
+
+import re
+import io
+import os
+import argparse
+
+
+pattern = re.compile(
+    r"[\n;]\s*MP_REGISTER_MODULE\((.*?),\s*(.*?),\s*(.*?)\);",
+    flags=re.DOTALL
+)
+
+
+def find_c_file(obj_file, vpath):
+    """ Search vpaths for the c file that matches the provided object_file.
+
+    :param str obj_file: object file to find the matching c file for
+    :param List[str] vpath: List of base paths, similar to gcc vpath
+    :return: str path to c file or None
+    """
+    c_file = None
+    relative_c_file = os.path.splitext(obj_file)[0] + ".c"
+    relative_c_file = relative_c_file.lstrip('/\\')
+    for p in vpath:
+        possible_c_file = os.path.join(p, relative_c_file)
+        if os.path.exists(possible_c_file):
+            c_file = possible_c_file
+            break
+
+    return c_file
+
+
+def find_module_registrations(c_file):
+    """ Find any MP_REGISTER_MODULE definitions in the provided c file.
+
+    :param str c_file: path to c file to check
+    :return: List[(module_name, obj_module, enabled_define)]
+    """
+    global pattern
+
+    if c_file is None:
+        # No c file to match the object file, skip
+        return set()
+
+    with io.open(c_file, encoding='utf-8') as c_file_obj:
+        return set(re.findall(pattern, c_file_obj.read()))
+
+
+def generate_module_table_header(modules):
+    """ Generate header with module table entries for builtin modules.
+
+    :param List[(module_name, obj_module, enabled_define)] modules: module defs
+    :return: None
+    """
+
+    # Print header file for all external modules.
+    mod_defs = []
+    print("// Automatically generated by makemoduledefs.py.\n")
+    for module_name, obj_module, enabled_define in modules:
+        mod_def = "MODULE_DEF_{}".format(module_name.upper())
+        mod_defs.append(mod_def)
+        print((
+            "#if ({enabled_define})\n"
+            "    extern const struct _mp_obj_module_t {obj_module};\n"
+            "    #define {mod_def} {{ MP_ROM_QSTR({module_name}), MP_ROM_PTR(&{obj_module}) }},\n"
+            "#else\n"
+            "    #define {mod_def}\n"
+            "#endif\n"
+            ).format(module_name=module_name, obj_module=obj_module,
+                     enabled_define=enabled_define, mod_def=mod_def)
+        )
+
+    print("\n#define MICROPY_REGISTERED_MODULES \\")
+
+    for mod_def in mod_defs:
+        print("    {mod_def} \\".format(mod_def=mod_def))
+
+    print("// MICROPY_REGISTERED_MODULES")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--vpath", default=".",
+                        help="comma separated list of folders to search for c files in")
+    parser.add_argument("files", nargs="*",
+                        help="list of c files to search")
+    args = parser.parse_args()
+
+    vpath = [p.strip() for p in args.vpath.split(',')]
+
+    modules = set()
+    for obj_file in args.files:
+        c_file = find_c_file(obj_file, vpath)
+        modules |= find_module_registrations(c_file)
+
+    generate_module_table_header(sorted(modules))
+
+
+if __name__ == '__main__':
+    main()

py/makeqstrdata.py

@@ -51,6 +51,176 @@ codepoint2name[ord('^')] = 'caret'
 codepoint2name[ord('|')] = 'pipe'
 codepoint2name[ord('~')] = 'tilde'
 
+# static qstrs, should be sorted
+
+static_qstr_list = [
+    "",
+    "__dir__", # Put __dir__ after empty qstr for builtin dir() to work
+    "\n",
+    " ",
+    "*",
+    "/",
+    "<module>",
+    "_",
+    "__call__",
+    "__class__",
+    "__delitem__",
+    "__enter__",
+    "__exit__",
+    "__getattr__",
+    "__getitem__",
+    "__hash__",
+    "__init__",
+    "__int__",
+    "__iter__",
+    "__len__",
+    "__main__",
+    "__module__",
+    "__name__",
+    "__new__",
+    "__next__",
+    "__qualname__",
+    "__repr__",
+    "__setitem__",
+    "__str__",
+    "ArithmeticError",
+    "AssertionError",
+    "AttributeError",
+    "BaseException",
+    "EOFError",
+    "Ellipsis",
+    "Exception",
+    "GeneratorExit",
+    "ImportError",
+    "IndentationError",
+    "IndexError",
+    "KeyError",
+    "KeyboardInterrupt",
+    "LookupError",
+    "MemoryError",
+    "NameError",
+    "NoneType",
+    "NotImplementedError",
+    "OSError",
+    "OverflowError",
+    "RuntimeError",
+    "StopIteration",
+    "SyntaxError",
+    "SystemExit",
+    "TypeError",
+    "ValueError",
+    "ZeroDivisionError",
+    "abs",
+    "all",
+    "any",
+    "append",
+    "args",
+    "bool",
+    "builtins",
+    "bytearray",
+    "bytecode",
+    "bytes",
+    "callable",
+    "chr",
+    "classmethod",
+    "clear",
+    "close",
+    "const",
+    "copy",
+    "count",
+    "dict",
+    "dir",
+    "divmod",
+    "end",
+    "endswith",
+    "eval",
+    "exec",
+    "extend",
+    "find",
+    "format",
+    "from_bytes",
+    "get",
+    "getattr",
+    "globals",
+    "hasattr",
+    "hash",
+    "id",
+    "index",
+    "insert",
+    "int",
+    "isalpha",
+    "isdigit",
+    "isinstance",
+    "islower",
+    "isspace",
+    "issubclass",
+    "isupper",
+    "items",
+    "iter",
+    "join",
+    "key",
+    "keys",
+    "len",
+    "list",
+    "little",
+    "locals",
+    "lower",
+    "lstrip",
+    "main",
+    "map",
+    "micropython",
+    "next",
+    "object",
+    "open",
+    "ord",
+    "pop",
+    "popitem",
+    "pow",
+    "print",
+    "range",
+    "read",
+    "readinto",
+    "readline",
+    "remove",
+    "replace",
+    "repr",
+    "reverse",
+    "rfind",
+    "rindex",
+    "round",
+    "rsplit",
+    "rstrip",
+    "self",
+    "send",
+    "sep",
+    "set",
+    "setattr",
+    "setdefault",
+    "sort",
+    "sorted",
+    "split",
+    "start",
+    "startswith",
+    "staticmethod",
+    "step",
+    "stop",
+    "str",
+    "strip",
+    "sum",
+    "super",
+    "throw",
+    "to_bytes",
+    "tuple",
+    "type",
+    "update",
+    "upper",
+    "utf-8",
+    "value",
+    "values",
+    "write",
+    "zip",
+]
+
 # this must match the equivalent function in qstr.c
 def compute_hash(qstr, bytes_hash):
     hash = 5381
@@ -70,9 +240,22 @@ def qstr_escape(qst):
     return re.sub(r'[^A-Za-z0-9_]', esc_char, qst)
 
 def parse_input_headers(infiles):
-    # read the qstrs in from the input files
     qcfgs = {}
     qstrs = {}
+
+    # add static qstrs
+    for qstr in static_qstr_list:
+        # work out the corresponding qstr name
+        ident = qstr_escape(qstr)
+
+        # don't add duplicates
+        assert ident not in qstrs
+
+        # add the qstr to the list, with order number to retain original order in file
+        order = len(qstrs) - 300000
+        qstrs[ident] = (order, ident, qstr)
+
+    # read the qstrs in from the input files
     for infile in infiles:
         with open(infile, 'rt') as f:
             for line in f:
@@ -96,9 +279,11 @@ def parse_input_headers(infiles):
                 # get the qstr value
                 qstr = match.group(1)
 
-                # special case to specify control characters
+                # special cases to specify control characters
                 if qstr == '\\n':
                     qstr = '\n'
+                elif qstr == '\\r\\n':
+                    qstr = '\r\n'
 
                 # work out the corresponding qstr name
                 ident = qstr_escape(qstr)
@@ -154,7 +339,7 @@ def print_qstr_data(qcfgs, qstrs):
     print('')
 
     # add NULL qstr with no hash or data
-    print('QDEF(MP_QSTR_NULL, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
+    print('QDEF(MP_QSTRnull, (const byte*)"%s%s" "")' % ('\\x00' * cfg_bytes_hash, '\\x00' * cfg_bytes_len))
 
     # go through each qstr and print it out
     for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):

py/makeqstrdefs.py

@@ -9,12 +9,9 @@ from __future__ import print_function
 
 import re
 import sys
+import io
 import os
 
-# Blacklist of qstrings that are specially handled in further
-# processing and should be ignored
-QSTRING_BLACK_LIST = set(['NULL', 'number_of'])
-
 
 def write_out(fname, output):
     if output:
@@ -45,8 +42,7 @@ def process_file(f):
             continue
         for match in re_qstr.findall(line):
             name = match.replace('MP_QSTR_', '')
-            if name not in QSTRING_BLACK_LIST:
-                output.append('Q(' + name + ')')
+            output.append('Q(' + name + ')')
 
     write_out(last_fname, output)
     return ""
@@ -108,7 +104,7 @@ if __name__ == "__main__":
         pass
 
     if args.command == "split":
-        with open(args.input_filename) as infile:
+        with io.open(args.input_filename, encoding='utf-8') as infile:
             process_file(infile)
 
     if args.command == "cat":

py/misc.h

@@ -47,8 +47,8 @@ typedef unsigned int uint;
 #endif
 
 // Classical double-indirection stringification of preprocessor macro's value
-#define _MP_STRINGIFY(x) #x
-#define MP_STRINGIFY(x) _MP_STRINGIFY(x)
+#define MP_STRINGIFY_HELPER(x) #x
+#define MP_STRINGIFY(x) MP_STRINGIFY_HELPER(x)
 
 // Static assertion macro
 #define MP_STATIC_ASSERT(cond) ((void)sizeof(char[1 - 2 * !(cond)]))

py/mkenv.mk

@@ -42,11 +42,14 @@ ECHO = @echo
 CP = cp
 MKDIR = mkdir
 SED = sed
+CAT = cat
+TOUCH = touch
 PYTHON = python3
 
 AS = $(CROSS_COMPILE)as
 CC = $(CROSS_COMPILE)gcc
 CXX = $(CROSS_COMPILE)g++
+GDB = $(CROSS_COMPILE)gdb
 LD = $(CROSS_COMPILE)ld
 OBJCOPY = $(CROSS_COMPILE)objcopy
 SIZE = $(CROSS_COMPILE)size
@@ -58,10 +61,13 @@ CXX += -m32
 LD += -m32
 endif
 
+MAKE_MANIFEST = $(PYTHON) $(TOP)/tools/makemanifest.py
 MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py
 MPY_CROSS = $(TOP)/mpy-cross/mpy-cross
 MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py
 
+MPY_LIB_DIR = $(TOP)/../micropython-lib
+
 all:
 .PHONY: all
 

py/mkrules.mk

@@ -20,12 +20,12 @@ endif
 # can be located. By following this scheme, it allows a single build rule
 # to be used to compile all .c files.
 
-vpath %.S . $(TOP)
+vpath %.S . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.S
 	$(ECHO) "CC $<"
 	$(Q)$(CC) $(CFLAGS) -c -o $@ $<
 
-vpath %.s . $(TOP)
+vpath %.s . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.s
 	$(ECHO) "AS $<"
 	$(Q)$(AS) -o $@ $<
@@ -42,18 +42,18 @@ $(Q)$(CC) $(CFLAGS) -c -MD -o $@ $<
   $(RM) -f $(@:.o=.d)
 endef
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 $(BUILD)/%.o: %.c
 	$(call compile_c)
 
 QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR
 QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
 
-vpath %.c . $(TOP)
+vpath %.c . $(TOP) $(USER_C_MODULES)
 
 $(BUILD)/%.pp: %.c
 	$(ECHO) "PreProcess $<"
-	$(Q)$(CC) $(CFLAGS) -E -Wp,-C,-dD,-dI -o $@ $<
+	$(Q)$(CPP) $(CFLAGS) -Wp,-C,-dD,-dI -o $@ $<
 
 # The following rule uses | to create an order only prerequisite. Order only
 # prerequisites only get built if they don't exist. They don't cause timestamp
@@ -70,14 +70,15 @@ $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h
 # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^)
 # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?)
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
-$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
+# See more information about this process in docs/develop/qstr.rst.
+$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS)
 	$(ECHO) "GEN $@"
-	$(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last;
+	$(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last
 
 $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last
 	$(ECHO) "GEN $@"
 	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
-	$(Q)touch $@
+	$(Q)$(TOUCH) $@
 
 $(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.split
 	$(ECHO) "GEN $@"
@@ -96,23 +97,35 @@ $(OBJ_DIRS):
 $(HEADER_BUILD):
 	$(MKDIR) -p $@
 
+ifneq ($(FROZEN_MANIFEST),)
+# to build frozen_content.c from a manifest
+$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h
+	$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) $(FROZEN_MANIFEST)
+
 ifneq ($(FROZEN_DIR),)
+$(error FROZEN_DIR cannot be used in conjunction with FROZEN_MANIFEST)
+endif
+
+ifneq ($(FROZEN_MPY_DIR),)
+$(error FROZEN_MPY_DIR cannot be used in conjunction with FROZEN_MANIFEST)
+endif
+endif
+
+ifneq ($(FROZEN_DIR),)
+$(info Warning: FROZEN_DIR is deprecated in favour of FROZEN_MANIFEST)
 $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DEPS)
 	$(ECHO) "GEN $@"
 	$(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@
 endif
 
 ifneq ($(FROZEN_MPY_DIR),)
-# to build the MicroPython cross compiler
-$(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/ports/windows/fmode.c
-	$(Q)$(MAKE) -C $(TOP)/mpy-cross
-
+$(info Warning: FROZEN_MPY_DIR is deprecated in favour of FROZEN_MANIFEST)
 # make a list of all the .py files that need compiling and freezing
 FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==')
 FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy))
 
 # to build .mpy files from .py files
-$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py $(TOP)/mpy-cross/mpy-cross
+$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py
 	@$(ECHO) "MPY $<"
 	$(Q)$(MKDIR) -p $(dir $@)
 	$(Q)$(MPY_CROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $<
@@ -146,6 +159,13 @@ clean-prog:
 .PHONY: clean-prog
 endif
 
+submodules:
+	$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
+ifneq ($(GIT_SUBMODULES),)
+	$(Q)git submodule update --init $(addprefix $(TOP)/,$(GIT_SUBMODULES))
+endif
+.PHONY: submodules
+
 LIBMICROPYTHON = libmicropython.a
 
 # We can execute extra commands after library creation using
@@ -190,7 +210,7 @@ print-cfg:
 
 print-def:
 	@$(ECHO) "The following defines are built into the $(CC) compiler"
-	touch __empty__.c
+	$(TOUCH) __empty__.c
 	@$(CC) -E -Wp,-dM __empty__.c
 	@$(RM) -f __empty__.c
 

py/modarray.c

@@ -29,15 +29,17 @@
 #if MICROPY_PY_ARRAY
 
 STATIC const mp_rom_map_elem_t mp_module_array_globals_table[] = {
-    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_array) },
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uarray) },
     { MP_ROM_QSTR(MP_QSTR_array), MP_ROM_PTR(&mp_type_array) },
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_array_globals, mp_module_array_globals_table);
 
-const mp_obj_module_t mp_module_array = {
+const mp_obj_module_t mp_module_uarray = {
     .base = { &mp_type_module },
     .globals = (mp_obj_dict_t*)&mp_module_array_globals,
 };
 
+MP_REGISTER_MODULE(MP_QSTR_uarray, mp_module_uarray, MICROPY_PY_ARRAY);
+
 #endif

py/modio.c

@@ -208,15 +208,8 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) {
     // package parameter being None, the path_in is interpreted as a
     // raw path.
     if (package_in != mp_const_none) {
-        mp_obj_t args[5];
-        args[0] = package_in;
-        args[1] = mp_const_none; // TODO should be globals
-        args[2] = mp_const_none; // TODO should be locals
-        args[3] = mp_const_true; // Pass sentinel "non empty" value to force returning of leaf module
-        args[4] = MP_OBJ_NEW_SMALL_INT(0);
-
-        // TODO lookup __import__ and call that instead of going straight to builtin implementation
-        mp_obj_t pkg = mp_builtin___import__(5, args);
+        // Pass "True" as sentinel value in fromlist to force returning of leaf module
+        mp_obj_t pkg = mp_import_name(mp_obj_str_get_qstr(package_in), mp_const_true, MP_OBJ_NEW_SMALL_INT(0));
 
         mp_obj_t dest[2];
         mp_load_method_maybe(pkg, MP_QSTR___path__, dest);

py/modmath.c

@@ -171,6 +171,42 @@ MATH_FUN_1(lgamma, lgamma)
 #endif
 //TODO: fsum
 
+#if MICROPY_PY_MATH_ISCLOSE
+STATIC mp_obj_t mp_math_isclose(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_a, ARG_b, ARG_rel_tol, ARG_abs_tol };
+    static const mp_arg_t allowed_args[] = {
+        {MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ},
+        {MP_QSTR_, MP_ARG_REQUIRED | MP_ARG_OBJ},
+        {MP_QSTR_rel_tol, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL}},
+        {MP_QSTR_abs_tol, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NEW_SMALL_INT(0)}},
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+    const mp_float_t a = mp_obj_get_float(args[ARG_a].u_obj);
+    const mp_float_t b = mp_obj_get_float(args[ARG_b].u_obj);
+    const mp_float_t rel_tol = args[ARG_rel_tol].u_obj == MP_OBJ_NULL
+        ? (mp_float_t)1e-9 : mp_obj_get_float(args[ARG_rel_tol].u_obj);
+    const mp_float_t abs_tol = mp_obj_get_float(args[ARG_abs_tol].u_obj);
+    if (rel_tol < (mp_float_t)0.0 || abs_tol < (mp_float_t)0.0) {
+        math_error();
+    }
+    if (a == b) {
+        return mp_const_true;
+    }
+    const mp_float_t difference = MICROPY_FLOAT_C_FUN(fabs)(a - b);
+    if (isinf(difference)) { // Either a or b is inf
+        return mp_const_false;
+    }
+    if ((difference <= abs_tol) ||
+        (difference <= MICROPY_FLOAT_C_FUN(fabs)(rel_tol * a)) ||
+        (difference <= MICROPY_FLOAT_C_FUN(fabs)(rel_tol * b))) {
+        return mp_const_true;
+    }
+    return mp_const_false;
+}
+MP_DEFINE_CONST_FUN_OBJ_KW(mp_math_isclose_obj, 2, mp_math_isclose);
+#endif
+
 // Function that takes a variable number of arguments
 
 // log(x[, base])
@@ -335,6 +371,9 @@ STATIC const mp_rom_map_elem_t mp_module_math_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_isfinite), MP_ROM_PTR(&mp_math_isfinite_obj) },
     { MP_ROM_QSTR(MP_QSTR_isinf), MP_ROM_PTR(&mp_math_isinf_obj) },
     { MP_ROM_QSTR(MP_QSTR_isnan), MP_ROM_PTR(&mp_math_isnan_obj) },
+    #if MICROPY_PY_MATH_ISCLOSE
+    { MP_ROM_QSTR(MP_QSTR_isclose), MP_ROM_PTR(&mp_math_isclose_obj) },
+    #endif
     { MP_ROM_QSTR(MP_QSTR_trunc), MP_ROM_PTR(&mp_math_trunc_obj) },
     { MP_ROM_QSTR(MP_QSTR_radians), MP_ROM_PTR(&mp_math_radians_obj) },
     { MP_ROM_QSTR(MP_QSTR_degrees), MP_ROM_PTR(&mp_math_degrees_obj) },

py/modmicropython.c

@@ -150,7 +150,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_micropython_kbd_intr_obj, mp_micropython_kbd
 #if MICROPY_ENABLE_SCHEDULER
 STATIC mp_obj_t mp_micropython_schedule(mp_obj_t function, mp_obj_t arg) {
     if (!mp_sched_schedule(function, arg)) {
-        mp_raise_msg(&mp_type_RuntimeError, "schedule stack full");
+        mp_raise_msg(&mp_type_RuntimeError, "schedule queue full");
     }
     return mp_const_none;
 }

py/modstruct.c

@@ -97,7 +97,7 @@ STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) {
             size += cnt;
         } else {
             total_cnt += cnt;
-            mp_uint_t align;
+            size_t align;
             size_t sz = mp_binary_get_size(fmt_type, *fmt, &align);
             while (cnt--) {
                 // Apply alignment
@@ -146,6 +146,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
         }
         p += offset;
     }
+    byte *p_base = p;
 
     // Check that the input buffer is big enough to unpack all the values
     if (p + total_sz > end_p) {
@@ -164,7 +165,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
             res->items[i++] = item;
         } else {
             while (cnt--) {
-                item = mp_binary_get_val(fmt_type, *fmt, &p);
+                item = mp_binary_get_val(fmt_type, *fmt, p_base, &p);
                 res->items[i++] = item;
             }
         }
@@ -179,6 +180,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c
     const char *fmt = mp_obj_str_get_str(fmt_in);
     char fmt_type = get_fmt_type(&fmt);
 
+    byte *p_base = p;
     size_t i;
     for (i = 0; i < n_args;) {
         mp_uint_t cnt = 1;
@@ -203,7 +205,7 @@ STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, c
         } else {
             // If we run out of args then we just finish; CPython would raise struct.error
             while (cnt-- && i < n_args) {
-                mp_binary_set_val(fmt_type, *fmt, args[i++], &p);
+                mp_binary_set_val(fmt_type, *fmt, args[i++], p_base, &p);
             }
         }
         fmt++;

py/modsys.c

@@ -34,6 +34,12 @@
 #include "py/stream.h"
 #include "py/smallint.h"
 #include "py/runtime.h"
+#include "py/persistentcode.h"
+
+#if MICROPY_PY_SYS_SETTRACE
+#include "py/objmodule.h"
+#include "py/profile.h"
+#endif
 
 #if MICROPY_PY_SYS
 
@@ -61,22 +67,36 @@ STATIC const mp_obj_tuple_t mp_sys_implementation_version_info_obj = {
     3,
     { I(MICROPY_VERSION_MAJOR), I(MICROPY_VERSION_MINOR), I(MICROPY_VERSION_MICRO) }
 };
+#if MICROPY_PERSISTENT_CODE_LOAD
+#define SYS_IMPLEMENTATION_ELEMS \
+    MP_ROM_QSTR(MP_QSTR_micropython), \
+    MP_ROM_PTR(&mp_sys_implementation_version_info_obj), \
+    MP_ROM_INT(MPY_FILE_HEADER_INT)
+#else
+#define SYS_IMPLEMENTATION_ELEMS \
+    MP_ROM_QSTR(MP_QSTR_micropython), \
+    MP_ROM_PTR(&mp_sys_implementation_version_info_obj)
+#endif
 #if MICROPY_PY_ATTRTUPLE
-STATIC const qstr impl_fields[] = { MP_QSTR_name, MP_QSTR_version };
+STATIC const qstr impl_fields[] = {
+    MP_QSTR_name,
+    MP_QSTR_version,
+    #if MICROPY_PERSISTENT_CODE_LOAD
+    MP_QSTR_mpy,
+    #endif
+};
 STATIC MP_DEFINE_ATTRTUPLE(
     mp_sys_implementation_obj,
     impl_fields,
-    2,
-        MP_ROM_QSTR(MP_QSTR_micropython),
-        MP_ROM_PTR(&mp_sys_implementation_version_info_obj)
+    2 + MICROPY_PERSISTENT_CODE_LOAD,
+        SYS_IMPLEMENTATION_ELEMS
 );
 #else
 STATIC const mp_rom_obj_tuple_t mp_sys_implementation_obj = {
     {&mp_type_tuple},
-    2,
+    2 + MICROPY_PERSISTENT_CODE_LOAD,
     {
-        MP_ROM_QSTR(MP_QSTR_micropython),
-        MP_ROM_PTR(&mp_sys_implementation_version_info_obj),
+        SYS_IMPLEMENTATION_ELEMS
     }
 };
 #endif
@@ -146,6 +166,24 @@ STATIC mp_obj_t mp_sys_getsizeof(mp_obj_t obj) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof);
 #endif
 
+#if MICROPY_PY_SYS_ATEXIT
+// atexit(callback): Callback is called when sys.exit is called.
+STATIC mp_obj_t mp_sys_atexit(mp_obj_t obj) {
+    mp_obj_t old = MP_STATE_VM(sys_exitfunc);
+    MP_STATE_VM(sys_exitfunc) = obj;
+    return old;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_atexit_obj, mp_sys_atexit);
+#endif
+
+#if MICROPY_PY_SYS_SETTRACE
+// settrace(tracefunc): Set the system’s trace function.
+STATIC mp_obj_t mp_sys_settrace(mp_obj_t obj) {
+    return mp_prof_settrace(obj);
+}
+MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_settrace_obj, mp_sys_settrace);
+#endif // MICROPY_PY_SYS_SETTRACE
+
 STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) },
 
@@ -180,6 +218,10 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR_exit), MP_ROM_PTR(&mp_sys_exit_obj) },
     #endif
 
+    #if MICROPY_PY_SYS_SETTRACE
+    { MP_ROM_QSTR(MP_QSTR_settrace), MP_ROM_PTR(&mp_sys_settrace_obj) },
+    #endif
+
     #if MICROPY_PY_SYS_STDFILES
     { MP_ROM_QSTR(MP_QSTR_stdin), MP_ROM_PTR(&mp_sys_stdin_obj) },
     { MP_ROM_QSTR(MP_QSTR_stdout), MP_ROM_PTR(&mp_sys_stdout_obj) },
@@ -201,6 +243,9 @@ STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = {
      */
 
     { MP_ROM_QSTR(MP_QSTR_print_exception), MP_ROM_PTR(&mp_sys_print_exception_obj) },
+    #if MICROPY_PY_SYS_ATEXIT
+    { MP_ROM_QSTR(MP_QSTR_atexit), MP_ROM_PTR(&mp_sys_atexit_obj) },
+    #endif
 };
 
 STATIC MP_DEFINE_CONST_DICT(mp_module_sys_globals, mp_module_sys_globals_table);