diff --git a/mlx/backend/metal/allocator.cpp b/mlx/backend/metal/allocator.cpp
index 06e3d9ae49..c15bf3bdf7 100644
--- a/mlx/backend/metal/allocator.cpp
+++ b/mlx/backend/metal/allocator.cpp
@@ -226,6 +226,7 @@ void MetalAllocator::release(Buffer buffer) {
   std::unique_lock lk(mutex_);
   active_memory_ -= buf->length();
   num_resources_--;
+  residency_set_.erase(buf);
   lk.unlock();
   auto pool = metal::new_scoped_memory_pool();
   buf->release();