From 64ec8d65e3cddf34d5603bb5dea4840f68444907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9?= Date: Wed, 15 May 2024 14:25:39 +0200 Subject: [PATCH] feat: empty core --- src/Models/CmsSearch.php | 7 ++- src/Models/Indexes/SolrIndex.php | 44 ++++++++---------- src/Services/Indexer/AbstractIndexService.php | 7 +-- src/Services/Indexer/IndexBuilderService.php | 46 +++++++++---------- src/Services/Indexer/SolrIndexService.php | 13 +++--- 5 files changed, 57 insertions(+), 60 deletions(-) diff --git a/src/Models/CmsSearch.php b/src/Models/CmsSearch.php index 7959891e..4d313c0d 100644 --- a/src/Models/CmsSearch.php +++ b/src/Models/CmsSearch.php @@ -40,7 +40,12 @@ class CmsSearch extends BaseModel { protected $table = 'cms_search'; - private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND']; + protected $casts = [ + 'updated_at' => 'datetime', + 'created_at' => 'datetime', + 'deleted_at' => 'datetime', + ]; + // private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND']; private static array $skipStatus = ['NOT_INDEXABLE', 'NOT_FOUND']; diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 79bac3c7..29ba3d42 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -71,27 +71,21 @@ public function __construct($debug = false) public function emptyCore() { - $searchItems = CmsSearch::all(); - if (count($searchItems) == 0) { - $curl = $this->solrHandler(); - $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt); - curl_setopt($curl, CURLOPT_URL, $url); - $payload = ['delete' => ['query' => '*:*']]; - - curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); - $result = curl_exec($curl); + $curl = $this->solrHandler(); + $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt); + curl_setopt($curl, CURLOPT_URL, $url); + $payload = ['delete' => ['query' => '*:*']]; - $json = json_decode($result); + curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); + $result = curl_exec($curl); - if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { - $this->mailQueryError($url, $result); + $json = json_decode($result); - return false; - } + if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) { + $this->mailQueryError($url, $result); - return true; + return false; } - return true; } @@ -100,7 +94,7 @@ private function solrHandler() { $handler = curl_init(); curl_setopt($handler, CURLOPT_RETURNTRANSFER, true); - curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass); + curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass); curl_setopt($handler, CURLOPT_POST, true); @@ -159,7 +153,7 @@ public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $doma $result = curl_exec($curl); if (curl_errno($curl) === 6) { - exit('[ERROR] Could not resolve solr host: '.$this->getSolrBaseUrl()); + exit('[ERROR] Could not resolve solr host: ' . $this->getSolrBaseUrl()); } $json = json_decode($result); if ($json && isset($json->responseHeader) && $json->responseHeader->status == 0) { @@ -171,7 +165,7 @@ public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $doma public function removeItem($url) { - if (! is_null($url)) { + if (!is_null($url)) { $curl = $this->solrHandler(); $payload = ['delete' => $url]; @@ -303,27 +297,27 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null, $lang ); if ($filter) { - $url .= '&fq='.$filter; + $url .= '&fq=' . $filter; } if ($start && is_int($start)) { - $url .= '&start='.$start; + $url .= '&start=' . $start; } if ($rows && is_int($rows)) { - $url .= '&rows='.$rows; + $url .= '&rows=' . $rows; } if (count($extraColumns) > 0) { } if ($sortField) { - $url .= '&sort='.urlencode($sortField.' '.$sortDirection); + $url .= '&sort=' . urlencode($sortField . ' ' . $sortDirection); } curl_setopt($curl, CURLOPT_URL, $url); $result = curl_exec($curl); $json = json_decode($result); $searchResults = new SolrItem($json, $query, false, $highlightLength); - if (! $searchResults->isValid()) { + if (!$searchResults->isValid()) { $this->mailQueryError($url, $result); } @@ -343,7 +337,7 @@ public function suggestItems($query, $filter = null) $result = curl_exec($curl); $json = json_decode($result); $suggestions = new SolrItem($json, $query); - if (! $suggestions->isValid()) { + if (!$suggestions->isValid()) { $this->buildSuggester(); $result = curl_exec($curl); $json = json_decode($result); diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php index bf2e4b1d..90b846bc 100644 --- a/src/Services/Indexer/AbstractIndexService.php +++ b/src/Services/Indexer/AbstractIndexService.php @@ -2,6 +2,7 @@ namespace NotFound\Framework\Services\Indexer; +use DateTime; use NotFound\Framework\Models\CmsSearch; abstract class AbstractIndexService @@ -12,7 +13,7 @@ abstract class AbstractIndexService public ?string $domain; - abstract public function __construct($debug = false); + abstract public function __construct(private bool $debug = false, private bool $fresh = false); abstract public function startUpdate(): bool; @@ -31,10 +32,10 @@ public function clean(): bool return true; } - public function urlNeedsUpdate(string $url, $updated): bool + public function urlNeedsUpdate(string $url, DateTime $updated): bool { $searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first(); - if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) { + if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at >= $updated)) { CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']); return false; diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index 5f8a002b..6511c066 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -10,10 +10,6 @@ class IndexBuilderService { - private bool $debug; - - private bool $fresh; - private $locales; private $domain; @@ -24,7 +20,7 @@ class IndexBuilderService private AbstractIndexService $searchServer; - public function __construct($debug = false, $fresh = false) + public function __construct(private bool $debug = false, private bool $fresh = false) { $serverType = config('indexer.engine'); $this->debug = $debug; @@ -34,16 +30,16 @@ public function __construct($debug = false, $fresh = false) $this->domain = rtrim(env('APP_URL', ''), '/'); switch ($serverType) { case 'solr': - $this->searchServer = new SolrIndexService($this->debug); + $this->searchServer = new SolrIndexService($this->debug, $this->fresh); break; default: exit('Unknown search index type'); } } - public function run() + public function run(): void { - if (! $this->searchServer->checkConnection()) { + if (!$this->searchServer->checkConnection()) { $this->writeDebug("\n\n Error connecting to search server! \n\n"); return; @@ -55,7 +51,7 @@ public function run() if (count($sites) > 0) { $startResult = $this->searchServer->startUpdate(); - if (! $startResult) { + if (!$startResult) { $this->writeDebug("\n\n Error when emptying core! \n\n"); } @@ -82,13 +78,14 @@ public function run() if ($this->sitemapFile) { fclose($this->sitemapFile); } - $finish = $this->searchServer->finishUpdate(); - - $this->writeDebug($finish->message); } } else { $this->writeDebug("No sites to index\n"); + return; } + $finish = $this->searchServer->finishUpdate(); + + $this->writeDebug($finish->message); } private function indexChildPages($parentId) @@ -98,7 +95,7 @@ private function indexChildPages($parentId) $this->writeDebug("┃\n"); $this->writeDebug(sprintf('%s (id: %d)', $page->url, $page->id), true, '┣━┓ 📂 Page '); - if (! isset($page->template->id)) { + if (!isset($page->template->id)) { $this->writeDebug(": ❌ Fail, skipping, no template found\n"); continue; @@ -106,12 +103,10 @@ private function indexChildPages($parentId) $menu = Menu::whereId($page->id)->firstOrFail(); - if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { + if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { $this->writeDebug(": ⏭️ Skipping, template excluded from search\n"); - } elseif (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) { $this->writeDebug(": ⏭️ Skipping, page excluded from search\n"); - } else { foreach ($this->locales as $lang) { @@ -148,7 +143,7 @@ private function updatePage($menu, $lang) // continue with customValues $customValues = []; - $className = 'App\Http\Controllers\Page\\'.$this->controllerName($menu).'Controller'; + $className = 'App\Http\Controllers\Page\\' . $this->controllerName($menu) . 'Controller'; $c = null; $priority = 1; @@ -163,10 +158,13 @@ private function updatePage($menu, $lang) } } $searchText = rtrim($searchText, ', '); - if (! empty($title) && ! empty($searchText)) { + if (!empty($title) && !empty($searchText)) { $searchItem = new SearchItem($url, $title); - $searchItem->setContent($searchText)->setLanguage($lang->url)->setPriority($priority)->setPublicationDate(new DateTime($menu->updated_at)); + $searchItem->setContent($searchText) + ->setLanguage($lang->url) + ->setPriority($priority) + ->setPublicationDate(new DateTime($menu->updated_at)); foreach ($customValues as $key => $value) { $searchItem->setCustomValue($key, $value); } @@ -198,7 +196,7 @@ private function updatePage($menu, $lang) private function updateSubPages($menu, $lang) { - $className = 'App\Http\Controllers\Page\\'.$this->controllerName($menu).'Controller'; + $className = 'App\Http\Controllers\Page\\' . $this->controllerName($menu) . 'Controller'; $c = null; // update subPage if necessary @@ -227,7 +225,7 @@ private function updateSubitems($class, $lang) // We need to check if the subPages is an array of arrays // If not we wrap it in an extra array - if (count($subPages) > 0 && ! is_array($subPages[0])) { + if (count($subPages) > 0 && !is_array($subPages[0])) { $subPages = [$subPages]; } foreach ($subPages as $subPage) { @@ -270,8 +268,8 @@ private function updateSubitems($class, $lang) private function createFolderIfNotExists($fullFilePath) { $path_parts = pathinfo($fullFilePath); - if (! file_exists($path_parts['dirname'])) { - if (! mkdir($path_parts['dirname'])) { + if (!file_exists($path_parts['dirname'])) { + if (!mkdir($path_parts['dirname'])) { printf("\n\n### Error creating sitemap folder"); } } @@ -285,7 +283,7 @@ private function writeDebug($text, $padding = false, $prefix = '') $text = substr($text, 0, $this->padding - strlen($prefix)); $text = str_pad($text, $this->padding - strlen($prefix), ' '); } - printf("\e[1m".$prefix."\e[0m".$text); + printf("\e[1m" . $prefix . "\e[0m" . $text); } } diff --git a/src/Services/Indexer/SolrIndexService.php b/src/Services/Indexer/SolrIndexService.php index 8e6e1b47..cb2fd290 100644 --- a/src/Services/Indexer/SolrIndexService.php +++ b/src/Services/Indexer/SolrIndexService.php @@ -8,23 +8,20 @@ class SolrIndexService extends AbstractIndexService { - private bool $debug = false; - public int $siteId; public ?string $domain; public $solrIndex; - public function __construct($debug = false) + public function __construct(private bool $debug = false, private bool $fresh = false) { - $this->debug = $debug; $this->solrIndex = new SolrIndex(); } public function retainItem(string $url): void { - $cmsSearchItem = CmsSearch::whereUrl($this->domain.$url)->first(); + $cmsSearchItem = CmsSearch::whereUrl($this->domain . $url)->first(); if ($cmsSearchItem) { $cmsSearchItem->search_status = 'UPDATED'; $cmsSearchItem->save(); @@ -72,7 +69,7 @@ public function upsertItem(SearchItem $searchItem): object $cmsSearchItem->setValues($searchItem, $cmsSearchItemStatus); $cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->domain); - if (in_array($cmsSearchItemStatus, ['NOT_FOUND', 'FAILED'])) { + if (in_array($cmsSearchItemStatus, ['NOT_FOUND', 'FAILED'])) { $cmsSearchItem->updated_at = null; } $cmsSearchItem->save(); @@ -85,7 +82,9 @@ public function startUpdate(): bool if ($this->debug) { printf("\n ** Starting SOLR update"); } - $emptyResult = $this->solrIndex->emptyCore(); + if ($this->fresh) { + $emptyResult = $this->solrIndex->emptyCore(); + } CmsSearch::setAllPending(); return $emptyResult;