From f146bed94d6d76633bbed35df533e781ac186af8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 5 Feb 2026 00:49:06 +1100 Subject: [PATCH 01/11] py/modweakref: Implement weakref module with ref class. The behaviour follows CPython. It adds a new WTB table to the GC to efficiently track which heap pointers have a weak reference attached. Enabled at the "everything" level. Signed-off-by: Damien George --- py/gc.c | 118 ++++++++-- py/gc.h | 5 + py/modweakref.c | 314 +++++++++++++++++++++++++ py/mpconfig.h | 5 + py/mpstate.h | 3 + py/py.cmake | 1 + py/py.mk | 1 + py/runtime.c | 4 + tests/ports/unix/extra_coverage.py.exp | 2 +- 9 files changed, 433 insertions(+), 20 deletions(-) create mode 100644 py/modweakref.c diff --git a/py/gc.c b/py/gc.c index 0d4d19ce9e74c..5fe26ef8905c5 100644 --- a/py/gc.c +++ b/py/gc.c @@ -104,14 +104,21 @@ #if MICROPY_ENABLE_FINALISER // FTB = finaliser table byte // if set, then the corresponding block may have a finaliser - #define BLOCKS_PER_FTB (8) - #define FTB_GET(area, block) ((area->gc_finaliser_table_start[(block) / BLOCKS_PER_FTB] >> ((block) & 7)) & 1) #define FTB_SET(area, block) do { area->gc_finaliser_table_start[(block) / BLOCKS_PER_FTB] |= (1 << ((block) & 7)); } while (0) #define FTB_CLEAR(area, block) do { area->gc_finaliser_table_start[(block) / BLOCKS_PER_FTB] &= (~(1 << ((block) & 7))); } while (0) #endif +#if MICROPY_PY_WEAKREF +// WTB = weakref table byte +// if set, then the corresponding block may have a weakref in MP_STATE_VM(mp_weakref_map). +#define BLOCKS_PER_WTB (8) +#define WTB_GET(area, block) ((area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] >> ((block) & 7)) & 1) +#define WTB_SET(area, block) do { area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] |= (1 << ((block) & 7)); } while (0) +#define WTB_CLEAR(area, block) do { area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] &= (~(1 << ((block) & 7))); } while (0) +#endif + #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL #define GC_MUTEX_INIT() mp_thread_recursive_mutex_init(&MP_STATE_MEM(gc_mutex)) #define GC_ENTER() mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) @@ -138,17 +145,23 @@ static void gc_sweep_free_blocks(void); // TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, P=pool; all in bytes): - // T = A + F + P + // T = A + F + W + P // F = A * BLOCKS_PER_ATB / BLOCKS_PER_FTB + // W = A * BLOCKS_PER_ATB / BLOCKS_PER_WTB // P = A * BLOCKS_PER_ATB * BYTES_PER_BLOCK - // => T = A * (1 + BLOCKS_PER_ATB / BLOCKS_PER_FTB + BLOCKS_PER_ATB * BYTES_PER_BLOCK) + // => T = A * (1 + BLOCKS_PER_ATB / BLOCKS_PER_FTB + BLOCKS_PER_ATB / BLOCKS_PER_WTB + BLOCKS_PER_ATB * BYTES_PER_BLOCK) size_t total_byte_len = (byte *)end - (byte *)start; - #if MICROPY_ENABLE_FINALISER + #if MICROPY_ENABLE_FINALISER || MICROPY_PY_WEAKREF area->gc_alloc_table_byte_len = (total_byte_len - ALLOC_TABLE_GAP_BYTE) * MP_BITS_PER_BYTE / ( MP_BITS_PER_BYTE + #if MICROPY_ENABLE_FINALISER + MP_BITS_PER_BYTE * BLOCKS_PER_ATB / BLOCKS_PER_FTB + #endif + #if MICROPY_PY_WEAKREF + + MP_BITS_PER_BYTE * BLOCKS_PER_ATB / BLOCKS_PER_WTB + #endif + MP_BITS_PER_BYTE * BLOCKS_PER_ATB * BYTES_PER_BLOCK ); #else @@ -157,26 +170,36 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { area->gc_alloc_table_start = (byte *)start; + // Allocate FTB and WTB blocks if they are enabled. + byte *next_table = area->gc_alloc_table_start + area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE; + (void)next_table; #if MICROPY_ENABLE_FINALISER size_t gc_finaliser_table_byte_len = (area->gc_alloc_table_byte_len * BLOCKS_PER_ATB + BLOCKS_PER_FTB - 1) / BLOCKS_PER_FTB; - area->gc_finaliser_table_start = area->gc_alloc_table_start + area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE; + area->gc_finaliser_table_start = next_table; + next_table += gc_finaliser_table_byte_len; + #endif + #if MICROPY_PY_WEAKREF + size_t gc_weakref_table_byte_len = (area->gc_alloc_table_byte_len * BLOCKS_PER_ATB + BLOCKS_PER_WTB - 1) / BLOCKS_PER_WTB; + area->gc_weakref_table_start = next_table; + next_table += gc_weakref_table_byte_len; #endif + // Allocate the GC pool of heap blocks. size_t gc_pool_block_len = area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; area->gc_pool_start = (byte *)end - gc_pool_block_len * BYTES_PER_BLOCK; area->gc_pool_end = end; + assert(area->gc_pool_start >= next_table); - #if MICROPY_ENABLE_FINALISER - assert(area->gc_pool_start >= area->gc_finaliser_table_start + gc_finaliser_table_byte_len); - #endif - - #if MICROPY_ENABLE_FINALISER - // clear ATB's and FTB's - memset(area->gc_alloc_table_start, 0, gc_finaliser_table_byte_len + area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE); - #else - // clear ATB's - memset(area->gc_alloc_table_start, 0, area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE); - #endif + // Clear ATB's, and FTB's and WTB's if they are enabled. + memset(area->gc_alloc_table_start, 0, + area->gc_alloc_table_byte_len + ALLOC_TABLE_GAP_BYTE + #if MICROPY_ENABLE_FINALISER + + gc_finaliser_table_byte_len + #endif + #if MICROPY_PY_WEAKREF + + gc_weakref_table_byte_len + #endif + ); area->gc_last_free_atb_index = 0; area->gc_last_used_block = 0; @@ -196,6 +219,12 @@ static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { gc_finaliser_table_byte_len, gc_finaliser_table_byte_len * BLOCKS_PER_FTB); #endif + #if MICROPY_PY_WEAKREF + DEBUG_printf(" weakref table at %p, length " UINT_FMT " bytes, " + UINT_FMT " blocks\n", area->gc_weakref_table_start, + gc_weakref_table_byte_len, + gc_weakref_table_byte_len * BLOCKS_PER_WTB); + #endif DEBUG_printf(" pool at %p, length " UINT_FMT " bytes, " UINT_FMT " blocks\n", area->gc_pool_start, gc_pool_block_len * BYTES_PER_BLOCK, gc_pool_block_len); @@ -310,6 +339,9 @@ static bool gc_try_add_heap(size_t failed_alloc) { #if MICROPY_ENABLE_FINALISER + total_blocks / BLOCKS_PER_FTB #endif + #if MICROPY_PY_WEAKREF + + total_blocks / BLOCKS_PER_WTB + #endif + total_blocks * BYTES_PER_BLOCK + ALLOC_TABLE_GAP_BYTE + sizeof(mp_state_mem_area_t); @@ -556,6 +588,9 @@ void gc_collect_end(void) { } MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; GC_EXIT(); + #if MICROPY_PY_WEAKREF + gc_weakref_sweep(); + #endif } static void gc_deal_with_stack_overflow(void) { @@ -581,12 +616,16 @@ static void gc_deal_with_stack_overflow(void) { // Run finalisers for all to-be-freed blocks static void gc_sweep_run_finalisers(void) { - #if MICROPY_ENABLE_FINALISER + #if MICROPY_ENABLE_FINALISER || MICROPY_PY_WEAKREF + #if MICROPY_ENABLE_FINALISER && MICROPY_PY_WEAKREF + MP_STATIC_ASSERT(BLOCKS_PER_FTB == BLOCKS_PER_WTB); + #endif for (const mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); // Small speed optimisation: skip over empty FTB blocks size_t ftb_end = area->gc_last_used_block / BLOCKS_PER_FTB; // index is inclusive for (size_t ftb_idx = 0; ftb_idx <= ftb_end; ftb_idx++) { + #if MICROPY_ENABLE_FINALISER byte ftb = area->gc_finaliser_table_start[ftb_idx]; size_t block = ftb_idx * BLOCKS_PER_FTB; while (ftb) { @@ -616,9 +655,26 @@ static void gc_sweep_run_finalisers(void) { ftb >>= 1; block++; } + #endif + #if MICROPY_PY_WEAKREF + byte wtb = area->gc_weakref_table_start[ftb_idx]; + block = ftb_idx * BLOCKS_PER_WTB; + while (wtb) { + MICROPY_GC_HOOK_LOOP(block); + if (wtb & 1) { // WTB_GET(area, block) shortcut + if (ATB_GET_KIND(area, block) == AT_HEAD) { + mp_obj_base_t *obj = (mp_obj_base_t *)PTR_FROM_BLOCK(area, block); + gc_weakref_about_to_be_freed(obj); + WTB_CLEAR(area, block); + } + } + wtb >>= 1; + block++; + } + #endif } } - #endif // MICROPY_ENABLE_FINALISER + #endif // MICROPY_ENABLE_FINALISER || MICROPY_PY_WEAKREF } // Free unmarked heads and their tails @@ -769,6 +825,25 @@ void gc_info(gc_info_t *info) { GC_EXIT(); } +#if MICROPY_PY_WEAKREF +// Mark the GC heap pointer as having a weakref. +void gc_weakref_mark(void *ptr) { + mp_state_mem_area_t *area; + #if MICROPY_GC_SPLIT_HEAP + area = gc_get_ptr_area(ptr); + assert(area); + #else + assert(VERIFY_PTR(ptr)); + area = &MP_STATE_MEM(area); + #endif + + size_t block = BLOCK_FROM_PTR(area, ptr); + assert(ATB_GET_KIND(area, block) == AT_HEAD); + + WTB_SET(area, block); +} +#endif + void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { bool has_finaliser = alloc_flags & GC_ALLOC_FLAG_HAS_FINALISER; size_t n_blocks = ((n_bytes + BYTES_PER_BLOCK - 1) & (~(BYTES_PER_BLOCK - 1))) / BYTES_PER_BLOCK; @@ -967,6 +1042,11 @@ void gc_free(void *ptr) { FTB_CLEAR(area, block); #endif + #if MICROPY_PY_WEAKREF + // Objects that have a weak reference should not be explicitly freed. + assert(!WTB_GET(area, block)); + #endif + #if MICROPY_GC_SPLIT_HEAP if (MP_STATE_MEM(gc_last_free_area) != area) { // We freed something but it isn't the current area. Reset the diff --git a/py/gc.h b/py/gc.h index 36177633062b2..4679d6dc8632f 100644 --- a/py/gc.h +++ b/py/gc.h @@ -58,6 +58,11 @@ void gc_collect_end(void); // Use this function to sweep the whole heap and run all finalisers void gc_sweep_all(void); +// These functions are used to manage weakrefs. +void gc_weakref_mark(void *ptr); +void gc_weakref_about_to_be_freed(void *ptr); +void gc_weakref_sweep(void); + enum { GC_ALLOC_FLAG_HAS_FINALISER = 1, }; diff --git a/py/modweakref.c b/py/modweakref.c new file mode 100644 index 0000000000000..57a11690377be --- /dev/null +++ b/py/modweakref.c @@ -0,0 +1,314 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2026 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/gc.h" +#include "py/runtime.h" + +#if MICROPY_PY_WEAKREF + +// Macros to obfuscate a heap pointer as a small integer object. +#define PTR_TO_INT_OBJ(ptr) (MP_OBJ_NEW_SMALL_INT(((uintptr_t)ptr) >> 1)) +#define PTR_FROM_INT_OBJ(obj) ((void *)(MP_OBJ_SMALL_INT_VALUE((obj)) << 1)) + +// Macros to convert between a weak reference and a heap pointer. +#define WEAK_REFERENCE_FROM_HEAP_PTR(ptr) PTR_TO_INT_OBJ(ptr) +#define WEAK_REFERENCE_TO_HEAP_PTR(weak_ref) PTR_FROM_INT_OBJ(weak_ref) + +// Macros to manage ref-finalizer linked-list pointers. +// - mp_obj_ref_t is obfuscated as a small integer object so it's not traced by the GC. +// - mp_obj_finalize_t is stored as-is so it is traced by the GC. +#define REF_FIN_LIST_OBJ_IS_FIN(r) (!mp_obj_is_small_int((r))) +#define REF_FIN_LIST_OBJ_TO_PTR(r) ((mp_obj_ref_t *)(mp_obj_is_small_int((r)) ? PTR_FROM_INT_OBJ((r)) : MP_OBJ_TO_PTR((r)))) +#define REF_FIN_LIST_OBJ_FROM_REF(r) (PTR_TO_INT_OBJ((r))) +#define REF_FIN_LIST_OBJ_FROM_FIN(r) (MP_OBJ_FROM_PTR((r))) +#define REF_FIN_LIST_OBJ_TAIL (PTR_TO_INT_OBJ(NULL)) + +// weakref.ref() instance. +typedef struct _mp_obj_ref_t { + mp_obj_base_t base; + mp_obj_t ref_fin_next; + mp_obj_t obj_weak_ref; + mp_obj_t callback; +} mp_obj_ref_t; + +// weakref.finalize() instance. +// This is an extension of weakref.ref() and shares a lot of code with it. +typedef struct _mp_obj_finalize_t { + mp_obj_ref_t base; + size_t n_args; + size_t n_kw; + mp_obj_t *args; +} mp_obj_finalize_t; + +static const mp_obj_type_t mp_type_ref; +static const mp_obj_type_t mp_type_finalize; + +static mp_obj_t ref___del__(mp_obj_t self_in); + +void gc_weakref_about_to_be_freed(void *ptr) { + mp_obj_t idx = PTR_TO_INT_OBJ(ptr); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), idx, MP_MAP_LOOKUP); + if (elem != NULL) { + // Mark element as being freed. + elem->key = mp_const_none; + } +} + +void gc_weakref_sweep(void) { + mp_map_t *map = &MP_STATE_VM(mp_weakref_map); + for (size_t i = 0; i < map->alloc; i++) { + if (map->table[i].key == mp_const_none) { + // Element was just freed, so call all the registered callbacks. + --map->used; + map->table[i].key = MP_OBJ_SENTINEL; + mp_obj_ref_t *ref = REF_FIN_LIST_OBJ_TO_PTR(map->table[i].value); + map->table[i].value = MP_OBJ_NULL; + while (ref != NULL) { + // Invalidate the weak reference. + assert(ref->obj_weak_ref != mp_const_none); + ref->obj_weak_ref = mp_const_none; + + // Call any registered callbacks. + if (ref->callback != mp_const_none) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + if (ref->base.type == &mp_type_ref) { + // weakref.ref() type. + mp_call_function_1(ref->callback, MP_OBJ_FROM_PTR(ref)); + } else { + // weakref.finalize() type. + mp_obj_finalize_t *fin = (mp_obj_finalize_t *)ref; + mp_call_function_n_kw(fin->base.callback, fin->n_args, fin->n_kw, fin->args); + } + nlr_pop(); + } else { + mp_printf(MICROPY_ERROR_PRINTER, "Unhandled exception in weakref callback:\n"); + mp_obj_print_exception(MICROPY_ERROR_PRINTER, MP_OBJ_FROM_PTR(nlr.ret_val)); + } + } + + // Unlink the node. + mp_obj_ref_t *ref_fin_next = REF_FIN_LIST_OBJ_TO_PTR(ref->ref_fin_next); + ref->ref_fin_next = REF_FIN_LIST_OBJ_TAIL; + ref = ref_fin_next; + } + } + } +} + +static mp_obj_t mp_obj_ref_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + if (type == &mp_type_ref) { + // weakref.ref() type. + mp_arg_check_num(n_args, n_kw, 1, 2, false); + } else { + // weakref.finalize() type. + mp_arg_check_num(n_args, n_kw, 2, MP_OBJ_FUN_ARGS_MAX, true); + } + + // Validate the input object can have a weakref. + void *ptr = NULL; + if (mp_obj_is_obj(args[0])) { + ptr = MP_OBJ_TO_PTR(args[0]); + if (gc_nbytes(ptr) == 0) { + ptr = NULL; + } + } + if (ptr == NULL) { + mp_raise_TypeError(MP_ERROR_TEXT("not a heap object")); + } + + // Create or get the entry in mp_weakref_map corresponding to this object. + mp_obj_t obj_weak_reference = WEAK_REFERENCE_FROM_HEAP_PTR(ptr); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), obj_weak_reference, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + // This heap object does not have any existing weakref's, so initialise it. + elem->value = REF_FIN_LIST_OBJ_TAIL; + gc_weakref_mark(ptr); + } + + mp_obj_ref_t *self; + if (type == &mp_type_ref) { + // Create a new weakref.ref() object. + self = mp_obj_malloc_with_finaliser(mp_obj_ref_t, type); + // Link this new ref into the list of all refs/finalizers pointing to this object. + // To ensure it will *NOT* be traced by the GC (the user must manually hold onto it), + // store an integer version of the object after any weakref.finalize() objects (so + // the weakref.finalize() objects continue to be traced by the GC). + mp_obj_t *link = &elem->value; + while (REF_FIN_LIST_OBJ_IS_FIN(*link)) { + link = &REF_FIN_LIST_OBJ_TO_PTR(*link)->ref_fin_next; + } + self->ref_fin_next = *link; + *link = REF_FIN_LIST_OBJ_FROM_REF(self); + } else { + // Create a new weakref.finalize() object. + mp_obj_finalize_t *self_fin = mp_obj_malloc(mp_obj_finalize_t, type); + self_fin->n_args = n_args - 2; + self_fin->n_kw = n_kw; + size_t n_args_kw = self_fin->n_args + self_fin->n_kw * 2; + if (n_args_kw == 0) { + self_fin->args = NULL; + } else { + self_fin->args = m_new(mp_obj_t, n_args_kw); + memcpy(self_fin->args, args + 2, n_args_kw * sizeof(mp_obj_t)); + } + self = &self_fin->base; + // Link this new finalizer into the list of all refs/finalizers pointing to this object. + // To ensure it will be traced by the GC, store its pointer at the start of the list. + self->ref_fin_next = elem->value; + elem->value = REF_FIN_LIST_OBJ_FROM_FIN(self_fin); + } + + // Populate the object weak reference, and the callback. + self->obj_weak_ref = obj_weak_reference; + if (n_args > 1) { + self->callback = args[1]; + } else { + self->callback = mp_const_none; + } + + return MP_OBJ_FROM_PTR(self); +} + +static mp_obj_t mp_obj_ref_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_obj_ref_t *self = MP_OBJ_TO_PTR(self_in); + if (self->obj_weak_ref == mp_const_none) { + return mp_const_none; + } + if (self->base.type == &mp_type_ref) { + // weakref.ref() type. + return MP_OBJ_FROM_PTR(WEAK_REFERENCE_TO_HEAP_PTR(self->obj_weak_ref)); + } else { + // weakref.finalize() type. + mp_obj_finalize_t *self_fin = MP_OBJ_TO_PTR(self_in); + ref___del__(self_in); + return mp_call_function_n_kw(self_fin->base.callback, self_fin->n_args, self_fin->n_kw, self_fin->args); + } +} + +static mp_obj_t ref___del__(mp_obj_t self_in) { + mp_obj_ref_t *self = MP_OBJ_TO_PTR(self_in); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), self->obj_weak_ref, MP_MAP_LOOKUP); + if (elem != NULL) { + for (mp_obj_t *link = &elem->value; REF_FIN_LIST_OBJ_TO_PTR(*link) != NULL; link = &REF_FIN_LIST_OBJ_TO_PTR(*link)->ref_fin_next) { + if (self == REF_FIN_LIST_OBJ_TO_PTR(*link)) { + // Unlink and clear this node. + *link = self->ref_fin_next; + self->ref_fin_next = REF_FIN_LIST_OBJ_TAIL; + self->obj_weak_ref = mp_const_none; + break; + } + } + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(ref___del___obj, ref___del__); + +static mp_obj_t finalize_peek_detach_helper(mp_obj_t self_in, bool detach) { + mp_obj_finalize_t *self = MP_OBJ_TO_PTR(self_in); + if (self->base.obj_weak_ref == mp_const_none) { + return mp_const_none; + } + mp_obj_t tuple[4] = { + MP_OBJ_FROM_PTR(WEAK_REFERENCE_TO_HEAP_PTR(self->base.obj_weak_ref)), + self->base.callback, + mp_obj_new_tuple(self->n_args, self->args), + mp_obj_dict_make_new(&mp_type_dict, 0, self->n_kw, self->args + self->n_args), + }; + if (detach) { + ref___del__(self_in); + } + return mp_obj_new_tuple(MP_ARRAY_SIZE(tuple), tuple); +} + +static mp_obj_t finalize_peek(mp_obj_t self_in) { + return finalize_peek_detach_helper(self_in, false); +} +static MP_DEFINE_CONST_FUN_OBJ_1(finalize_peek_obj, finalize_peek); + +static mp_obj_t finalize_detach(mp_obj_t self_in) { + return finalize_peek_detach_helper(self_in, true); +} +static MP_DEFINE_CONST_FUN_OBJ_1(finalize_detach_obj, finalize_detach); + +static void mp_obj_finalize_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + if (dest[0] != MP_OBJ_NULL) { + // Store/delete attribute, unsupported. + return; + } + + if (attr == MP_QSTR_alive) { + mp_obj_finalize_t *self = MP_OBJ_TO_PTR(self_in); + dest[0] = mp_obj_new_bool(self->base.obj_weak_ref != mp_const_none); + return; + } else if (attr == MP_QSTR_peek) { + dest[0] = MP_OBJ_FROM_PTR(&finalize_peek_obj); + dest[1] = self_in; + } else if (attr == MP_QSTR_detach) { + dest[0] = MP_OBJ_FROM_PTR(&finalize_detach_obj); + dest[1] = self_in; + } +} + +static const mp_rom_map_elem_t ref_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&ref___del___obj) }, +}; +static MP_DEFINE_CONST_DICT(ref_locals_dict, ref_locals_dict_table); + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_ref, + MP_QSTR_ref, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_ref_make_new, + call, mp_obj_ref_call, + locals_dict, &ref_locals_dict + ); + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_finalize, + MP_QSTR_finalize, + MP_TYPE_FLAG_NONE, + make_new, mp_obj_ref_make_new, + call, mp_obj_ref_call, + attr, mp_obj_finalize_attr + ); + +static const mp_rom_map_elem_t mp_module_weakref_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_weakref) }, + { MP_ROM_QSTR(MP_QSTR_ref), MP_ROM_PTR(&mp_type_ref) }, + { MP_ROM_QSTR(MP_QSTR_finalize), MP_ROM_PTR(&mp_type_finalize) }, +}; +static MP_DEFINE_CONST_DICT(mp_module_weakref_globals, mp_module_weakref_globals_table); + +const mp_obj_module_t mp_module_weakref = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_weakref_globals, +}; + +MP_REGISTER_ROOT_POINTER(mp_map_t mp_weakref_map); +MP_REGISTER_MODULE(MP_QSTR_weakref, mp_module_weakref); + +#endif // MICROPY_PY_WEAKREF diff --git a/py/mpconfig.h b/py/mpconfig.h index e5b7b523a7c23..a9beec0445f77 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1908,6 +1908,11 @@ typedef time_t mp_timestamp_t; #define MICROPY_PY_THREAD_RECURSIVE_MUTEX (MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL) #endif +// Whether to provide the "weakref" module. +#ifndef MICROPY_PY_WEAKREF +#define MICROPY_PY_WEAKREF (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Extended modules #ifndef MICROPY_PY_ASYNCIO diff --git a/py/mpstate.h b/py/mpstate.h index 325c12217521f..32d1adb13ed74 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -107,6 +107,9 @@ typedef struct _mp_state_mem_area_t { #if MICROPY_ENABLE_FINALISER byte *gc_finaliser_table_start; #endif + #if MICROPY_PY_WEAKREF + byte *gc_weakref_table_start; + #endif byte *gc_pool_start; byte *gc_pool_end; diff --git a/py/py.cmake b/py/py.cmake index ec2a5d832d25f..a7b5feecd8d14 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -53,6 +53,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/modsys.c ${MICROPY_PY_DIR}/modthread.c ${MICROPY_PY_DIR}/moderrno.c + ${MICROPY_PY_DIR}/modweakref.c ${MICROPY_PY_DIR}/mpprint.c ${MICROPY_PY_DIR}/mpstate.c ${MICROPY_PY_DIR}/mpz.c diff --git a/py/py.mk b/py/py.mk index e7716a1adc9a6..930e583f33cb1 100644 --- a/py/py.mk +++ b/py/py.mk @@ -202,6 +202,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ modsys.o \ moderrno.o \ modthread.o \ + modweakref.o \ vm.o \ bc.o \ showbc.o \ diff --git a/py/runtime.c b/py/runtime.c index 3fc35c8c2daf6..e245f60e9f06d 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -179,6 +179,10 @@ void mp_init(void) { MP_STATE_VM(usbd) = MP_OBJ_NULL; #endif + #if MICROPY_PY_WEAKREF + mp_map_init(&MP_STATE_VM(mp_weakref_map), 0); + #endif + #if MICROPY_PY_THREAD_GIL mp_thread_mutex_init(&MP_STATE_VM(gil_mutex)); #endif diff --git a/tests/ports/unix/extra_coverage.py.exp b/tests/ports/unix/extra_coverage.py.exp index d11e5ee6f4215..8569f17e69312 100644 --- a/tests/ports/unix/extra_coverage.py.exp +++ b/tests/ports/unix/extra_coverage.py.exp @@ -75,7 +75,7 @@ json machine marshal math os platform random re select socket struct sys termios time tls uctypes -vfs websocket +vfs weakref websocket me micropython machine marshal math From fa801822540bc21365bf9c27cf1b44aaafe535e3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 13 Feb 2026 01:11:00 +1100 Subject: [PATCH 02/11] tests/basics: Add tests for weakref.ref and weakref.finalize. Signed-off-by: Damien George --- tests/basics/weakref_finalize_basic.py | 58 ++++++++++++++++++ tests/basics/weakref_finalize_collect.py | 75 +++++++++++++++++++++++ tests/basics/weakref_multiple_refs.py | 34 ++++++++++ tests/basics/weakref_multiple_refs.py.exp | 6 ++ tests/basics/weakref_ref_basic.py | 14 +++++ tests/basics/weakref_ref_collect.py | 68 ++++++++++++++++++++ tests/run-tests.py | 2 + 7 files changed, 257 insertions(+) create mode 100644 tests/basics/weakref_finalize_basic.py create mode 100644 tests/basics/weakref_finalize_collect.py create mode 100644 tests/basics/weakref_multiple_refs.py create mode 100644 tests/basics/weakref_multiple_refs.py.exp create mode 100644 tests/basics/weakref_ref_basic.py create mode 100644 tests/basics/weakref_ref_collect.py diff --git a/tests/basics/weakref_finalize_basic.py b/tests/basics/weakref_finalize_basic.py new file mode 100644 index 0000000000000..792cffacb1385 --- /dev/null +++ b/tests/basics/weakref_finalize_basic.py @@ -0,0 +1,58 @@ +# Test weakref.finalize() functionality that doesn't require gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.finalize(value, lambda: None) + except TypeError: + print(value, "TypeError") + + +# Convert (obj, func, args, kwargs) so CPython and MicroPython have a chance to match. +def convert_4_tuple(values): + if values is None: + return None + return (type(values[0]).__name__, type(values[1]), values[2], values[3]) + + +class A: + def __str__(self): + return "" + + +print("test alive, peek, detach") +a = A() +f = weakref.finalize(a, lambda: None, 1, 2, kwarg=3) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("detach", convert_4_tuple(f.detach())) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("detach", convert_4_tuple(f.detach())) +print("call", f()) +a = None + +print("test alive, peek, call") +a = A() +f = weakref.finalize(a, lambda *args, **kwargs: (args, kwargs), 1, 2, kwarg=3) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("call", f()) +print("alive", f.alive) +print("peek", convert_4_tuple(f.peek())) +print("call", f()) +print("detach", convert_4_tuple(f.detach())) + +print("test call which raises exception") +a = A() +f = weakref.finalize(a, lambda: 1 / 0) +try: + f() +except ZeroDivisionError as er: + print("call ZeroDivisionError") diff --git a/tests/basics/weakref_finalize_collect.py b/tests/basics/weakref_finalize_collect.py new file mode 100644 index 0000000000000..d364be9f62d67 --- /dev/null +++ b/tests/basics/weakref_finalize_collect.py @@ -0,0 +1,75 @@ +# Test weakref.finalize() functionality requiring gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + + +class A: + def __str__(self): + return "" + + +def callback(*args, **kwargs): + print("callback({}, {})".format(args, kwargs)) + return 42 + + +def test(): + print("test basic use of finalize() with a simple callback") + a = A() + f = weakref.finalize(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) + + print("test that a callback is passed the correct values") + a = A() + f = weakref.finalize(a, callback, 1, 2, kwarg=3) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) + + print("test that calling the finalizer cancels the finalizer") + a = A() + f = weakref.finalize(a, callback) + print(f()) + print(a) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + print("test that calling detach cancels the finalizer") + a = A() + f = weakref.finalize(a, callback) + print(len(f.detach())) + print(a) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + print("test that finalize does not get collected before its ref does") + a = A() + weakref.finalize(a, callback) + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("free a") + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + +test() diff --git a/tests/basics/weakref_multiple_refs.py b/tests/basics/weakref_multiple_refs.py new file mode 100644 index 0000000000000..373cd6bb691f1 --- /dev/null +++ b/tests/basics/weakref_multiple_refs.py @@ -0,0 +1,34 @@ +# Test weakref when multiple weak references are active. +# +# This test has different output to CPython due to the order that MicroPython +# executes weak reference callbacks. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + + +class A: + def __str__(self): + return "" + + +def test(): + print("test having multiple ref and finalize objects referencing the same thing") + a = A() + r1 = weakref.ref(a, lambda r: print("ref1", r())) + f1 = weakref.finalize(a, lambda: print("finalize1")) + r2 = weakref.ref(a, lambda r: print("ref2", r())) + f2 = weakref.finalize(a, lambda: print("finalize2")) + print(r1(), f1.alive, r2(), f2.alive) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + + +test() diff --git a/tests/basics/weakref_multiple_refs.py.exp b/tests/basics/weakref_multiple_refs.py.exp new file mode 100644 index 0000000000000..1f2d366f776fa --- /dev/null +++ b/tests/basics/weakref_multiple_refs.py.exp @@ -0,0 +1,6 @@ +test having multiple ref and finalize objects referencing the same thing + True True +finalize2 +finalize1 +ref2 None +ref1 None diff --git a/tests/basics/weakref_ref_basic.py b/tests/basics/weakref_ref_basic.py new file mode 100644 index 0000000000000..058045f6c33b3 --- /dev/null +++ b/tests/basics/weakref_ref_basic.py @@ -0,0 +1,14 @@ +# Test weakref.ref() functionality that doesn't require gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.ref(value) + except TypeError: + print(value, "TypeError") diff --git a/tests/basics/weakref_ref_collect.py b/tests/basics/weakref_ref_collect.py new file mode 100644 index 0000000000000..8b2d86fb3c79c --- /dev/null +++ b/tests/basics/weakref_ref_collect.py @@ -0,0 +1,68 @@ +# Test weakref.ref() functionality requiring gc.collect(). + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +# gc module must be available if weakref is. +import gc + +# Cannot reference non-heap objects. +for value in (None, False, True, Ellipsis, 0, "", ()): + try: + weakref.ref(value) + except TypeError: + print(value, "TypeError") + + +class A: + def __str__(self): + return "" + + +def callback(r): + print("callback", r()) + + +def test(): + print("test basic use of ref() with only one argument") + a = A() + r = weakref.ref(a) + print(r()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r()) + + print("test use of ref() with a callback") + a = A() + r = weakref.ref(a, callback) + print(r()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r()) + + print("test when weakref gets collected before the object it refs") + a = A() + r = weakref.ref(a, callback) + print(r()) + r = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + a = None + + print("test a double reference") + a = A() + r1 = weakref.ref(a, callback) + r2 = weakref.ref(a, callback) + print(r1(), r2()) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print(r1(), r2()) + + +test() diff --git a/tests/run-tests.py b/tests/run-tests.py index 22783dd7c621c..023d2e869c376 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -157,6 +157,8 @@ "webassembly": ( "basics/string_format_modulo.py", # can't print nulls to stdout "basics/string_strip.py", # can't print nulls to stdout + "basics/weakref_ref_collect.py", # requires custom test due to GC behaviour + "basics/weakref_finalize_collect.py", # requires custom test due to GC behaviour "extmod/asyncio_basic2.py", "extmod/asyncio_cancel_self.py", "extmod/asyncio_current_task.py", From 6fc21efb35198c106bdbe8446be7d3bffe2051c1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Feb 2026 15:16:57 +1100 Subject: [PATCH 03/11] tests/basics: Add test for weakref having exception in callback. Needs a native exp file because native code doesn't print line numbers in the traceback. Signed-off-by: Damien George --- tests/basics/weakref_callback_exception.py | 42 +++++++++++++++++++ .../basics/weakref_callback_exception.py.exp | 12 ++++++ .../weakref_callback_exception.py.native.exp | 8 ++++ tests/run-tests.py | 2 + 4 files changed, 64 insertions(+) create mode 100644 tests/basics/weakref_callback_exception.py create mode 100644 tests/basics/weakref_callback_exception.py.exp create mode 100644 tests/basics/weakref_callback_exception.py.native.exp diff --git a/tests/basics/weakref_callback_exception.py b/tests/basics/weakref_callback_exception.py new file mode 100644 index 0000000000000..df8e5129803f6 --- /dev/null +++ b/tests/basics/weakref_callback_exception.py @@ -0,0 +1,42 @@ +# Test weakref ref/finalize raising an exception within the callback. +# +# This test has different output to CPython due to the way that MicroPython +# prints the exception. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +import gc + + +class A: + def __str__(self): + return "" + + +def callback(*args): + raise ValueError("weakref callback", args) + + +def test(): + print("test ref with exception in the callback") + a = A() + r = weakref.ref(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("collect done") + + print("test finalize with exception in the callback") + a = A() + weakref.finalize(a, callback) + a = None + clean_the_stack = [0, 0, 0, 0] + gc.collect() + print("collect done") + + +test() diff --git a/tests/basics/weakref_callback_exception.py.exp b/tests/basics/weakref_callback_exception.py.exp new file mode 100644 index 0000000000000..c2f7796310c2b --- /dev/null +++ b/tests/basics/weakref_callback_exception.py.exp @@ -0,0 +1,12 @@ +test ref with exception in the callback +Unhandled exception in weakref callback: +Traceback (most recent call last): + File "\.\+weakref_callback_exception.py", line 21, in callback +ValueError: ('weakref callback', (,)) +collect done +test finalize with exception in the callback +Unhandled exception in weakref callback: +Traceback (most recent call last): + File "\.\+weakref_callback_exception.py", line 21, in callback +ValueError: ('weakref callback', ()) +collect done diff --git a/tests/basics/weakref_callback_exception.py.native.exp b/tests/basics/weakref_callback_exception.py.native.exp new file mode 100644 index 0000000000000..a06a35c3b7e94 --- /dev/null +++ b/tests/basics/weakref_callback_exception.py.native.exp @@ -0,0 +1,8 @@ +test ref with exception in the callback +Unhandled exception in weakref callback: +ValueError: ('weakref callback', (,)) +collect done +test finalize with exception in the callback +Unhandled exception in weakref callback: +ValueError: ('weakref callback', ()) +collect done diff --git a/tests/run-tests.py b/tests/run-tests.py index 023d2e869c376..0306324ce8570 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -157,6 +157,7 @@ "webassembly": ( "basics/string_format_modulo.py", # can't print nulls to stdout "basics/string_strip.py", # can't print nulls to stdout + "basics/weakref_callback_exception.py", # has different exception printing output "basics/weakref_ref_collect.py", # requires custom test due to GC behaviour "basics/weakref_finalize_collect.py", # requires custom test due to GC behaviour "extmod/asyncio_basic2.py", @@ -415,6 +416,7 @@ def detect_target_wiring_script(pyb, args): "micropython/meminfo.py", "basics/bytes_compare3.py", "basics/builtin_help.py", + "basics/weakref_callback_exception.py", "misc/sys_settrace_cov.py", "net_inet/tls_text_errors.py", "thread/thread_exc2.py", From 6973f7347acc3d436cbbedcf2181d5b722b430d2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 15 Feb 2026 23:06:47 +1100 Subject: [PATCH 04/11] webassembly/Makefile: Add test//% target. Following a69425b533932bbcac0ef463f9e27f79ff2150e3. Signed-off-by: Damien George --- ports/webassembly/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile index 9a673b757b29c..3dbe24188786b 100644 --- a/ports/webassembly/Makefile +++ b/ports/webassembly/Makefile @@ -130,7 +130,7 @@ OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o)) ################################################################################ # Main targets. -.PHONY: all repl min test test_min +.PHONY: all repl min test test//% test_min all: $(BUILD)/micropython.mjs @@ -150,6 +150,9 @@ min: $(BUILD)/micropython.min.mjs test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly +test//%: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py + cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly -i "$*" + test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly From 5f4bdfd0868e40db1a18640fce2652c5bb78e581 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 15 Feb 2026 23:07:39 +1100 Subject: [PATCH 05/11] webassembly/variants/pyscript: Enable weakref module and add tests. Signed-off-by: Damien George --- .../variants/pyscript/mpconfigvariant.h | 1 + tests/basics/weakref_finalize_collect.py | 1 + tests/basics/weakref_ref_collect.py | 1 + .../webassembly/weakref_finalize_collect.mjs | 86 +++++++++++++++++++ .../weakref_finalize_collect.mjs.exp | 28 ++++++ .../ports/webassembly/weakref_ref_collect.mjs | 69 +++++++++++++++ .../webassembly/weakref_ref_collect.mjs.exp | 16 ++++ 7 files changed, 202 insertions(+) create mode 100644 tests/ports/webassembly/weakref_finalize_collect.mjs create mode 100644 tests/ports/webassembly/weakref_finalize_collect.mjs.exp create mode 100644 tests/ports/webassembly/weakref_ref_collect.mjs create mode 100644 tests/ports/webassembly/weakref_ref_collect.mjs.exp diff --git a/ports/webassembly/variants/pyscript/mpconfigvariant.h b/ports/webassembly/variants/pyscript/mpconfigvariant.h index ed8e812803533..0b77efc4b32bf 100644 --- a/ports/webassembly/variants/pyscript/mpconfigvariant.h +++ b/ports/webassembly/variants/pyscript/mpconfigvariant.h @@ -1,3 +1,4 @@ #define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES) #define MICROPY_GC_SPLIT_HEAP (1) #define MICROPY_GC_SPLIT_HEAP_AUTO (1) +#define MICROPY_PY_WEAKREF (1) diff --git a/tests/basics/weakref_finalize_collect.py b/tests/basics/weakref_finalize_collect.py index d364be9f62d67..f6e7c14843e01 100644 --- a/tests/basics/weakref_finalize_collect.py +++ b/tests/basics/weakref_finalize_collect.py @@ -1,4 +1,5 @@ # Test weakref.finalize() functionality requiring gc.collect(). +# Should be kept in sync with tests/ports/webassembly/weakref_finalize_collect.py. try: import weakref diff --git a/tests/basics/weakref_ref_collect.py b/tests/basics/weakref_ref_collect.py index 8b2d86fb3c79c..0e8db977d7764 100644 --- a/tests/basics/weakref_ref_collect.py +++ b/tests/basics/weakref_ref_collect.py @@ -1,4 +1,5 @@ # Test weakref.ref() functionality requiring gc.collect(). +# Should be kept in sync with tests/ports/webassembly/weakref_ref_collect.py. try: import weakref diff --git a/tests/ports/webassembly/weakref_finalize_collect.mjs b/tests/ports/webassembly/weakref_finalize_collect.mjs new file mode 100644 index 0000000000000..1e0bc951350db --- /dev/null +++ b/tests/ports/webassembly/weakref_finalize_collect.mjs @@ -0,0 +1,86 @@ +// Test weakref.finalize() functionality requiring gc.collect(). +// Should be kept in sync with tests/basics/weakref_finalize_collect.py. +// +// This needs custom testing on the webassembly port since the GC can only +// run when Python code returns to JavaScript. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +// Set up. +mp.runPython(` +import gc, weakref + +class A: + def __str__(self): + return "" + +def callback(*args, **kwargs): + print("callback({}, {})".format(args, kwargs)) + return 42 +`); + +console.log("test basic use of finalize() with a simple callback"); +mp.runPython(` + a = A() + f = weakref.finalize(a, callback) + a = None + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) +`); + +console.log("test that a callback is passed the correct values"); +mp.runPython(` + a = A() + f = weakref.finalize(a, callback, 1, 2, kwarg=3) + a = None + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print("alive", f.alive) + print("peek", f.peek()) + print("detach", f.detach()) + print("call", f()) +`); + +console.log("test that calling the finalizer cancels the finalizer"); +mp.runPython(` + a = A() + f = weakref.finalize(a, callback) + print(f()) + print(a) + a = None + gc.collect() +`); +console.log("(outside Python)"); + +console.log("test that calling detach cancels the finalizer"); +mp.runPython(` + a = A() + f = weakref.finalize(a, callback) + print(len(f.detach())) + print(a) + a = None + gc.collect() +`); +console.log("(outside Python)"); + +console.log("test that finalize does not get collected before its ref does"); +mp.runPython(` + a = A() + weakref.finalize(a, callback) + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print("free a") + a = None + gc.collect() +`); +console.log("(outside Python)"); diff --git a/tests/ports/webassembly/weakref_finalize_collect.mjs.exp b/tests/ports/webassembly/weakref_finalize_collect.mjs.exp new file mode 100644 index 0000000000000..e8087a4ae9bd9 --- /dev/null +++ b/tests/ports/webassembly/weakref_finalize_collect.mjs.exp @@ -0,0 +1,28 @@ +test basic use of finalize() with a simple callback +callback((), {}) +(outside Python) +alive False +peek None +detach None +call None +test that a callback is passed the correct values +callback((1, 2), {'kwarg': 3}) +(outside Python) +alive False +peek None +detach None +call None +test that calling the finalizer cancels the finalizer +callback((), {}) +42 + +(outside Python) +test that calling detach cancels the finalizer +4 + +(outside Python) +test that finalize does not get collected before its ref does +(outside Python) +free a +callback((), {}) +(outside Python) diff --git a/tests/ports/webassembly/weakref_ref_collect.mjs b/tests/ports/webassembly/weakref_ref_collect.mjs new file mode 100644 index 0000000000000..546a851f0ac09 --- /dev/null +++ b/tests/ports/webassembly/weakref_ref_collect.mjs @@ -0,0 +1,69 @@ +// Test weakref.ref() functionality requiring gc.collect(). +// Should be kept in sync with tests/basics/weakref_ref_collect.py. +// +// This needs custom testing on the webassembly port since the GC can only +// run when Python code returns to JavaScript. + +const mp = await (await import(process.argv[2])).loadMicroPython(); + +// Set up. +mp.runPython(` +import gc, weakref + +class A: + def __str__(self): + return "" + +def callback(r): + print("callback", r()) +`); + +console.log("test basic use of ref() with only one argument"); +mp.runPython(` + a = A() + r = weakref.ref(a) + print(r()) + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print(r()) + a = None + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print(r()) +`); + +console.log("test use of ref() with a callback"); +mp.runPython(` + a = A() + r = weakref.ref(a, callback) + print(r()) + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print(r()) + a = None + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + print(r()) +`); + +console.log("test when weakref gets collected before the object it refs"); +mp.runPython(` + a = A() + r = weakref.ref(a, callback) + print(r()) + r = None + gc.collect() +`); +console.log("(outside Python)"); +mp.runPython(` + a = None + gc.collect() +`); diff --git a/tests/ports/webassembly/weakref_ref_collect.mjs.exp b/tests/ports/webassembly/weakref_ref_collect.mjs.exp new file mode 100644 index 0000000000000..f903d41702815 --- /dev/null +++ b/tests/ports/webassembly/weakref_ref_collect.mjs.exp @@ -0,0 +1,16 @@ +test basic use of ref() with only one argument + +(outside Python) + +(outside Python) +None +test use of ref() with a callback + +(outside Python) + +callback None +(outside Python) +None +test when weakref gets collected before the object it refs + +(outside Python) From b2fad5642455b1d3573c1582a21ae7bc70d6a1bf Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Feb 2026 15:19:39 +1100 Subject: [PATCH 06/11] tests/ports/webassembly: Update heap_expand expected output. The amount of GC heap got smaller because weakref WTB takes some of the available RAM. Signed-off-by: Damien George --- tests/ports/webassembly/heap_expand.mjs.exp | 46 ++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/ports/webassembly/heap_expand.mjs.exp b/tests/ports/webassembly/heap_expand.mjs.exp index 67ebe98e7fe02..4161fc7eaed83 100644 --- a/tests/ports/webassembly/heap_expand.mjs.exp +++ b/tests/ports/webassembly/heap_expand.mjs.exp @@ -1,27 +1,27 @@ -135241312 -135241280 -135241248 -135241216 -135241168 -135241120 -135241040 -135240896 -135240592 -135240064 -135239024 -135236960 +135233568 +135233536 +135233504 +135233472 +135233424 +135233376 +135233296 +135233152 135232848 -135224640 -135208240 -135175456 -135109840 -134978752 -134716592 -135216800 -136217168 -138217984 -142219568 -150222816 +135232320 +135231280 +135229216 +135225104 +135216896 +135200496 +135167712 +135102096 +134971008 +134708848 +135201312 +136186256 +138156160 +142095984 +149975648 1 2 4 From bcd4413e6302c757936fcee1b84cfc65b2b82671 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Feb 2026 13:22:09 +1100 Subject: [PATCH 07/11] docs/library/weakref: Add documentation for weakref module. Signed-off-by: Damien George --- docs/library/index.rst | 1 + docs/library/weakref.rst | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 docs/library/weakref.rst diff --git a/docs/library/index.rst b/docs/library/index.rst index e64809bfa8487..e9a3d2c0c8daf 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -81,6 +81,7 @@ library. struct.rst sys.rst time.rst + weakref.rst zlib.rst _thread.rst diff --git a/docs/library/weakref.rst b/docs/library/weakref.rst new file mode 100644 index 0000000000000..bae2204c6c878 --- /dev/null +++ b/docs/library/weakref.rst @@ -0,0 +1,74 @@ +:mod:`weakref` -- Python object lifetime management +=================================================== + +.. module:: weakref + :synopsis: Create weak references to Python objects + +|see_cpython_module| :mod:`python:weakref`. + +This module allows creation of weak references to Python objects. A weak reference +is a non-traceable reference to a heap-allocated Python object, so the garbage +collector can still reclaim the object even though the weak reference refers to it. + +Python callbacks can be registered to be called when an object is reclaimed by the +garbage collector. This provides a safe way to clean up when objects are no longer +needed. + +ref objects +----------- + +A ref object is the simplest way to make a weak reference. + +.. class:: ref(object [, callback], /) + + Return a weak reference to the given *object*. + + If *callback* is given and is not ``None`` then, when *object* is reclaimed + by the garbage collector and if the weak reference object is still alive, the + *callback* will be called. The *callback* will be passed the weak reference + object as its single argument. + +.. method:: ref.__call__() + + Calling the weak reference object will return its referenced object if that + object is still alive. Otherwise ``None`` will be returned. + +finalize objects +---------------- + +A finalize object is an extended version of a ref object that is more convenient to +use, and allows more control over the callback. + +.. class:: finalize(object, callback, /, *args, **kwargs) + + Return a weak reference to the given *object*. In contrast to *weakref.ref* + objects, finalize objects are held onto internally and will not be collected until + *object* is collected. + + A finalize object starts off alive. It transitions to the dead state when the + finalize object is called, either explicitly or when *object* is collected. It also + transitions to dead if the `finalize.detach()` method is called. + + When *object* is reclaimed by the garbage collector (or the finalize object is + explicitly called by user code) and the finalize object is still in the alive state, + the *callback* will be called. The *callback* will be passed arguments as: + ``callback(*args, **kwargs)``. + +.. method:: finalize.__call__() + + If the finalize object is alive then it transitions to the dead state and returns + the value of ``callback(*args, **kwargs)``. Otherwise ``None`` will be returned. + +.. method:: finalize.alive + + Read-only boolean attribute that indicates if the finalizer is in the alive state. + +.. method:: finalize.peek() + + If the finalize object is alive then return ``(object, callback, args, kwargs)``. + Otherwise return ``None``. + +.. method:: finalize.detach() + + If the finalize object is alive then it transitions to the dead state and returns + ``(object, callback, args, kwargs)``. Otherwise ``None`` will be returned. From 180db59eecfd6a0cc75837a608b776da727a3f09 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 16 Feb 2026 18:46:31 +1100 Subject: [PATCH 08/11] tools/ci.sh: Increase qemu_arm test run timeout. It takes longer now that weakref is enabled in the coverage build. Signed-off-by: Damien George --- tools/ci.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci.sh b/tools/ci.sh index 588bb31638c56..2743e1560a5ff 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -949,11 +949,11 @@ function ci_unix_qemu_arm_build { function ci_unix_qemu_arm_run_tests { # Issues with ARM tests: - # - thread/stress_aes.py takes around 70 seconds + # - thread/stress_aes.py takes around 90 seconds # - thread/stress_recurse.py is flaky # - thread/thread_gc1.py is flaky file ./ports/unix/build-coverage/micropython - (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython MICROPY_TEST_TIMEOUT=90 ./run-tests.py --exclude 'thread/stress_recurse.py|thread/thread_gc1.py') + (cd tests && MICROPY_MICROPYTHON=../ports/unix/build-coverage/micropython MICROPY_TEST_TIMEOUT=120 ./run-tests.py --exclude 'thread/stress_recurse.py|thread/thread_gc1.py') } function ci_unix_qemu_riscv64_setup { From 5955972f1aa6e5de8a40008c264701dd28810d28 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 24 Feb 2026 15:23:57 +1100 Subject: [PATCH 09/11] tests/basics/weakref_multiple_refs.py: Get working on webassembly. Signed-off-by: Damien George --- tests/basics/weakref_multiple_refs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/basics/weakref_multiple_refs.py b/tests/basics/weakref_multiple_refs.py index 373cd6bb691f1..400e03a17c7f8 100644 --- a/tests/basics/weakref_multiple_refs.py +++ b/tests/basics/weakref_multiple_refs.py @@ -19,6 +19,8 @@ def __str__(self): def test(): + global r1, r2 # needed for webassembly port to retain references to them + print("test having multiple ref and finalize objects referencing the same thing") a = A() r1 = weakref.ref(a, lambda r: print("ref1", r())) From 5b07252c8812379480d1f42afd85d6790aef0737 Mon Sep 17 00:00:00 2001 From: Anson Mansfield Date: Sun, 1 Mar 2026 13:21:47 -0500 Subject: [PATCH 10/11] tests/basics/weakref: Test that cyclic closures are not leaked. Signed-off-by: Anson Mansfield --- tests/basics/weakref_cyclic_closure.py | 48 ++++++++++++++++++++++ tests/basics/weakref_cyclic_closure.py.exp | 2 + 2 files changed, 50 insertions(+) create mode 100644 tests/basics/weakref_cyclic_closure.py create mode 100644 tests/basics/weakref_cyclic_closure.py.exp diff --git a/tests/basics/weakref_cyclic_closure.py b/tests/basics/weakref_cyclic_closure.py new file mode 100644 index 0000000000000..a957eb4b60bee --- /dev/null +++ b/tests/basics/weakref_cyclic_closure.py @@ -0,0 +1,48 @@ +# CPython's `weakref.finalize` is documented to leak memory in certain cases, +# per https://docs.python.org/3/library/weakref.html#weakref.finalize. +# This is not fundamental, though --- and MicroPython actually does NOT leak here. + +try: + import weakref +except ImportError: + print("SKIP") + raise SystemExit + +import gc + +item_count = 1024 # number of leaky finalizers to create +item_size = 1024 # size of each leaky finalizer's object + +item_fin_lb = item_count // 2 +mem_leak_ub = item_count // 2 * item_size + +item_fin = 0 +def fin(buf): + global item_fin + item_fin += 1 + +def leak(sz): + buf = bytearray(sz) + weakref.finalize(buf, fin, buf) + +gc.collect() +mem_before = gc.mem_alloc() + +for i in range(item_count): + leak(item_size) + gc.collect() + +gc.collect() +mem_after = gc.mem_alloc() + +mem_leak = mem_after - mem_before + +if item_fin > item_fin_lb: + print("collected >", item_fin_lb) +else: + print("collected =", item_fin) + +if mem_leak < mem_leak_ub: + print("leaked <", mem_leak_ub) +else: + print("leaked =", mem_leak) diff --git a/tests/basics/weakref_cyclic_closure.py.exp b/tests/basics/weakref_cyclic_closure.py.exp new file mode 100644 index 0000000000000..f01fa7f9a86d6 --- /dev/null +++ b/tests/basics/weakref_cyclic_closure.py.exp @@ -0,0 +1,2 @@ +collected > 512 +leaked < 524288 From 266ef6c1766e40ead4c65a1a9db22ea69f8e7097 Mon Sep 17 00:00:00 2001 From: Anson Mansfield Date: Mon, 2 Mar 2026 09:24:43 -0500 Subject: [PATCH 11/11] py/gc: Move `mp_weakref_map` state from VM to MEM. Signed-off-by: Anson Mansfield --- py/gc.c | 10 +++++++++- py/modweakref.c | 9 ++++----- py/mpstate.h | 4 ++++ py/runtime.c | 4 ---- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/py/gc.c b/py/gc.c index 5fe26ef8905c5..045c281a82c0d 100644 --- a/py/gc.c +++ b/py/gc.c @@ -112,7 +112,7 @@ #if MICROPY_PY_WEAKREF // WTB = weakref table byte -// if set, then the corresponding block may have a weakref in MP_STATE_VM(mp_weakref_map). +// if set, then the corresponding block may have a weakref in MP_STATE_MEM(mp_weakref_map). #define BLOCKS_PER_WTB (8) #define WTB_GET(area, block) ((area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] >> ((block) & 7)) & 1) #define WTB_SET(area, block) do { area->gc_weakref_table_start[(block) / BLOCKS_PER_WTB] |= (1 << ((block) & 7)); } while (0) @@ -255,6 +255,10 @@ void gc_init(void *start, void *end) { #endif GC_MUTEX_INIT(); + + #if MICROPY_PY_WEAKREF + mp_map_init(&MP_STATE_MEM(mp_weakref_map), 0); + #endif } #if MICROPY_GC_SPLIT_HEAP @@ -579,6 +583,10 @@ void gc_sweep_all(void) { void gc_collect_end(void) { gc_deal_with_stack_overflow(); gc_sweep_run_finalisers(); + #if MICROPY_PY_WEAKREF + gc_collect_root((void **)&MP_STATE_MEM(mp_weakref_map).table, 1); + gc_deal_with_stack_overflow(); + #endif gc_sweep_free_blocks(); #if MICROPY_GC_SPLIT_HEAP MP_STATE_MEM(gc_last_free_area) = &MP_STATE_MEM(area); diff --git a/py/modweakref.c b/py/modweakref.c index 57a11690377be..637a7496290c1 100644 --- a/py/modweakref.c +++ b/py/modweakref.c @@ -70,7 +70,7 @@ static mp_obj_t ref___del__(mp_obj_t self_in); void gc_weakref_about_to_be_freed(void *ptr) { mp_obj_t idx = PTR_TO_INT_OBJ(ptr); - mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), idx, MP_MAP_LOOKUP); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_MEM(mp_weakref_map), idx, MP_MAP_LOOKUP); if (elem != NULL) { // Mark element as being freed. elem->key = mp_const_none; @@ -78,7 +78,7 @@ void gc_weakref_about_to_be_freed(void *ptr) { } void gc_weakref_sweep(void) { - mp_map_t *map = &MP_STATE_VM(mp_weakref_map); + mp_map_t *map = &MP_STATE_MEM(mp_weakref_map); for (size_t i = 0; i < map->alloc; i++) { if (map->table[i].key == mp_const_none) { // Element was just freed, so call all the registered callbacks. @@ -142,7 +142,7 @@ static mp_obj_t mp_obj_ref_make_new(const mp_obj_type_t *type, size_t n_args, si // Create or get the entry in mp_weakref_map corresponding to this object. mp_obj_t obj_weak_reference = WEAK_REFERENCE_FROM_HEAP_PTR(ptr); - mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), obj_weak_reference, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_MEM(mp_weakref_map), obj_weak_reference, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); if (elem->value == MP_OBJ_NULL) { // This heap object does not have any existing weakref's, so initialise it. elem->value = REF_FIN_LIST_OBJ_TAIL; @@ -211,7 +211,7 @@ static mp_obj_t mp_obj_ref_call(mp_obj_t self_in, size_t n_args, size_t n_kw, co static mp_obj_t ref___del__(mp_obj_t self_in) { mp_obj_ref_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_VM(mp_weakref_map), self->obj_weak_ref, MP_MAP_LOOKUP); + mp_map_elem_t *elem = mp_map_lookup(&MP_STATE_MEM(mp_weakref_map), self->obj_weak_ref, MP_MAP_LOOKUP); if (elem != NULL) { for (mp_obj_t *link = &elem->value; REF_FIN_LIST_OBJ_TO_PTR(*link) != NULL; link = &REF_FIN_LIST_OBJ_TO_PTR(*link)->ref_fin_next) { if (self == REF_FIN_LIST_OBJ_TO_PTR(*link)) { @@ -308,7 +308,6 @@ const mp_obj_module_t mp_module_weakref = { .globals = (mp_obj_dict_t *)&mp_module_weakref_globals, }; -MP_REGISTER_ROOT_POINTER(mp_map_t mp_weakref_map); MP_REGISTER_MODULE(MP_QSTR_weakref, mp_module_weakref); #endif // MICROPY_PY_WEAKREF diff --git a/py/mpstate.h b/py/mpstate.h index 32d1adb13ed74..592732fd15559 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -156,6 +156,10 @@ typedef struct _mp_state_mem_t { // This is a global mutex used to make the GC thread-safe. mp_thread_recursive_mutex_t gc_mutex; #endif + + #if MICROPY_PY_WEAKREF + mp_map_t mp_weakref_map; + #endif } mp_state_mem_t; // This structure hold runtime and VM information. It includes a section diff --git a/py/runtime.c b/py/runtime.c index e245f60e9f06d..3fc35c8c2daf6 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -179,10 +179,6 @@ void mp_init(void) { MP_STATE_VM(usbd) = MP_OBJ_NULL; #endif - #if MICROPY_PY_WEAKREF - mp_map_init(&MP_STATE_VM(mp_weakref_map), 0); - #endif - #if MICROPY_PY_THREAD_GIL mp_thread_mutex_init(&MP_STATE_VM(gil_mutex)); #endif