[5.x] Performance Optimisation for Queries - Optimise IteratorBuilder limit queries to avoid loading all items

src/Query/ItemQueryBuilder.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Query;
 
+use Generator;
 use Illuminate\Support\Collection;
 
 class ItemQueryBuilder extends IteratorBuilder
@@ -20,6 +21,13 @@ protected function getBaseItems()
         return $this->items;
     }
 
+    protected function getBaseItemsLazy(): Generator
+    {
+        foreach ($this->items as $item) {
+            yield $item;
+        }
+    }
+
     public function whereStatus($status)
     {
         return $this->where('status', $status);

src/Query/IteratorBuilder.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Query;
 
+use Generator;
 use Statamic\Support\Arr;
 
 abstract class IteratorBuilder extends Builder
@@ -39,11 +40,66 @@ public function pluck($column, $key = null)
 
     protected function getFilteredItems()
     {
-        $items = $this->getBaseItems();
+        // Can't optimize: no limit, has orderBy, or randomize
+        // These require all items to be loaded first
+        if (! $this->limit || $this->orderBys || $this->randomize) {
+            $items = $this->getBaseItems();
 
-        $items = $this->filterWheres($items);
+            return $this->filterWheres($items);
+        }
+
+        // No wheres - just get limited items directly
+        if (empty($this->wheres)) {
+            return $this->getBaseItemsLimited();
+        }
+
+        // Has limit AND wheres - batch hydrate until we have enough
+        return $this->getFilteredItemsInBatches();
+    }
+
+    protected function getBaseItemsLimited()
+    {
+        $needed = ($this->offset ?? 0) + $this->limit;
+        $collected = collect();
+
+        foreach ($this->getBaseItemsLazy() as $item) {
+            $collected->push($item);
+            if ($collected->count() >= $needed) {
+                break;
+            }
+        }
 
-        return $items;
+        return $collected;
+    }
+
+    protected function getFilteredItemsInBatches()
+    {
+        $needed = ($this->offset ?? 0) + $this->limit;
+        $batchSize = max(50, $this->limit * 2);
+        $collected = collect();
+        $batch = collect();
+
+        foreach ($this->getBaseItemsLazy() as $item) {
+            $batch->push($item);
+
+            if ($batch->count() >= $batchSize) {
+                $filtered = $this->filterWheres($batch);
+                $collected = $collected->concat($filtered);
+                $batch = collect();
+
+                if ($collected->count() >= $needed) {
+                    break;
+                }
+            }
+        }
+
+        // Process remaining items in final partial batch
+        if ($batch->isNotEmpty() && $collected->count() < $needed) {
+            $filtered = $this->filterWheres($batch);
+            $collected = $collected->concat($filtered);
+        }
+
+        return $collected;
     }
 
     protected function getFilteredAndLimitedItems()
@@ -362,6 +418,8 @@ protected function operatorToCarbonMethod($operator)
 
     abstract protected function getBaseItems();
 
+    abstract protected function getBaseItemsLazy(): Generator;
+
     public function inRandomOrder()
     {
         $this->randomize = true;

src/Search/QueryBuilder.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Search;
 
+use Generator;
 use Statamic\Contracts\Search\Result;
 use Statamic\Data\DataCollection;
 use Statamic\Query\Concerns\FakesQueries;
@@ -55,6 +56,35 @@ public function getBaseItems()
         return $this->transformResults($results);
     }
 
+    protected function getBaseItemsLazy(): Generator
+    {
+        $results = $this->getSearchResults($this->query);
+
+        // If withoutData mode, yield PlainResults directly (cheap, no hydration)
+        if (! $this->withData) {
+            foreach ($results as $i => $result) {
+                $plainResult = new PlainResult($result);
+                $plainResult->setIndex($this->index)->setScore($result['search_score'] ?? null);
+                yield $plainResult;
+            }
+
+            return;
+        }
+
+        // With data mode - batch hydrate to reduce database queries
+        // Use smaller batches when we know the limit and don't need filtering
+        $batchSize = $this->limit && empty($this->wheres) && empty($this->orderBys) && ! $this->randomize
+            ? ($this->offset ?? 0) + $this->limit
+            : 50;
+
+        foreach ($this->collect($results)->chunk($batchSize) as $batch) {
+            $hydrated = $this->transformResults($batch);
+            foreach ($hydrated as $item) {
+                yield $item;
+            }
+        }
+    }
+
     public function transformResults($results)
     {
         if (! $this->withData) {

tests/Fakes/Query/HydrationTrackingQueryBuilder.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace Tests\Fakes\Query;
+
+use Generator;
+use Mockery;
+use Statamic\Search\Index;
+use Statamic\Search\PlainResult;
+use Statamic\Search\QueryBuilder;
+
+class HydrationTrackingQueryBuilder extends QueryBuilder
+{
+    protected $results;
+    protected $hydrationCounter;
+
+    public function __construct($results, &$counter)
+    {
+        $this->results = $results;
+        $this->hydrationCounter = &$counter;
+        parent::__construct(Mockery::mock(Index::class));
+    }
+
+    public function getSearchResults($query)
+    {
+        return $this->results;
+    }
+
+    public function getBaseItems()
+    {
+        return $this->collect($this->results)->map(function ($item) {
+            $this->hydrationCounter++;
+            $result = new PlainResult($item);
+            $result->setScore($item['search_score'] ?? null);
+
+            return $result;
+        });
+    }
+
+    protected function getBaseItemsLazy(): Generator
+    {
+        foreach ($this->results as $item) {
+            $this->hydrationCounter++;
+            $result = new PlainResult($item);
+            $result->setScore($item['search_score'] ?? null);
+            yield $result;
+        }
+    }
+}

tests/Fakes/Query/TestIteratorBuilder.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Tests\Fakes\Query;
+
+use Generator;
+use Statamic\Data\DataCollection;
+use Statamic\Query\IteratorBuilder;
+
+class TestIteratorBuilder extends IteratorBuilder
+{
+    protected $items;
+    protected $loadCounter;
+
+    public function __construct($items, &$counter)
+    {
+        $this->items = $items;
+        $this->loadCounter = &$counter;
+    }
+
+    protected function getBaseItems()
+    {
+        $this->items->each(function () {
+            $this->loadCounter++;
+        });
+
+        return new DataCollection($this->items->all());
+    }
+
+    protected function getBaseItemsLazy(): Generator
+    {
+        foreach ($this->items as $item) {
+            $this->loadCounter++;
+            yield $item;
+        }
+    }
+}