Skip to content

Commit

Permalink
feat: empty core
Browse files Browse the repository at this point in the history
  • Loading branch information
@64knl
64knl committed May 15, 2024
1 parent ecb7ef9 commit 64ec8d6
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 60 deletions.
7 changes: 6 additions & 1 deletion src/Models/CmsSearch.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ class CmsSearch extends BaseModel
{
protected $table = 'cms_search';

private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];
protected $casts = [
'updated_at' => 'datetime',
'created_at' => 'datetime',
'deleted_at' => 'datetime',
];
// private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];

private static array $skipStatus = ['NOT_INDEXABLE', 'NOT_FOUND'];

Expand Down
44 changes: 19 additions & 25 deletions src/Models/Indexes/SolrIndex.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,27 +71,21 @@ public function __construct($debug = false)

public function emptyCore()
{
$searchItems = CmsSearch::all();
if (count($searchItems) == 0) {
$curl = $this->solrHandler();
$url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
curl_setopt($curl, CURLOPT_URL, $url);
$payload = ['delete' => ['query' => '*:*']];

curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
$result = curl_exec($curl);
$curl = $this->solrHandler();
$url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
curl_setopt($curl, CURLOPT_URL, $url);
$payload = ['delete' => ['query' => '*:*']];

$json = json_decode($result);
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
$result = curl_exec($curl);

if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) {
$this->mailQueryError($url, $result);
$json = json_decode($result);

return false;
}
if (!$json || !isset($json->responseHeader) || $json->responseHeader->status !== 0) {
$this->mailQueryError($url, $result);

return true;
return false;
}

return true;
}

Expand All @@ -100,7 +94,7 @@ private function solrHandler()
{
$handler = curl_init();
curl_setopt($handler, CURLOPT_RETURNTRANSFER, true);
curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser.':'.$this->solrPass);
curl_setopt($handler, CURLOPT_USERPWD, $this->solrUser . ':' . $this->solrPass);

curl_setopt($handler, CURLOPT_POST, true);

Expand Down Expand Up @@ -159,7 +153,7 @@ public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $doma
$result = curl_exec($curl);

if (curl_errno($curl) === 6) {
exit('[ERROR] Could not resolve solr host: '.$this->getSolrBaseUrl());
exit('[ERROR] Could not resolve solr host: ' . $this->getSolrBaseUrl());
}
$json = json_decode($result);
if ($json && isset($json->responseHeader) && $json->responseHeader->status == 0) {
Expand All @@ -171,7 +165,7 @@ public function upsertItem(SearchItem $indexItem, int $siteId = 1, ?string $doma

public function removeItem($url)
{
if (! is_null($url)) {
if (!is_null($url)) {
$curl = $this->solrHandler();

$payload = ['delete' => $url];
Expand Down Expand Up @@ -303,27 +297,27 @@ public function selectItems($query, $lang = 'nl', $filter = null, $start = null,
$lang
);
if ($filter) {
$url .= '&fq='.$filter;
$url .= '&fq=' . $filter;
}
if ($start && is_int($start)) {
$url .= '&start='.$start;
$url .= '&start=' . $start;
}

if ($rows && is_int($rows)) {
$url .= '&rows='.$rows;
$url .= '&rows=' . $rows;
}
if (count($extraColumns) > 0) {
}

if ($sortField) {
$url .= '&sort='.urlencode($sortField.' '.$sortDirection);
$url .= '&sort=' . urlencode($sortField . ' ' . $sortDirection);
}

curl_setopt($curl, CURLOPT_URL, $url);
$result = curl_exec($curl);
$json = json_decode($result);
$searchResults = new SolrItem($json, $query, false, $highlightLength);
if (! $searchResults->isValid()) {
if (!$searchResults->isValid()) {
$this->mailQueryError($url, $result);
}

Expand All @@ -343,7 +337,7 @@ public function suggestItems($query, $filter = null)
$result = curl_exec($curl);
$json = json_decode($result);
$suggestions = new SolrItem($json, $query);
if (! $suggestions->isValid()) {
if (!$suggestions->isValid()) {
$this->buildSuggester();
$result = curl_exec($curl);
$json = json_decode($result);
Expand Down
7 changes: 4 additions & 3 deletions src/Services/Indexer/AbstractIndexService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace NotFound\Framework\Services\Indexer;

use DateTime;
use NotFound\Framework\Models\CmsSearch;

abstract class AbstractIndexService
Expand All @@ -12,7 +13,7 @@ abstract class AbstractIndexService

public ?string $domain;

abstract public function __construct($debug = false);
abstract public function __construct(private bool $debug = false, private bool $fresh = false);

abstract public function startUpdate(): bool;

Expand All @@ -31,10 +32,10 @@ public function clean(): bool
return true;
}

public function urlNeedsUpdate(string $url, $updated): bool
public function urlNeedsUpdate(string $url, DateTime $updated): bool
{
$searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first();
if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) {
if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at >= $updated)) {
CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']);

return false;
Expand Down
46 changes: 22 additions & 24 deletions src/Services/Indexer/IndexBuilderService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@

class IndexBuilderService
{
private bool $debug;

private bool $fresh;

private $locales;

private $domain;
Expand All @@ -24,7 +20,7 @@ class IndexBuilderService

private AbstractIndexService $searchServer;

public function __construct($debug = false, $fresh = false)
public function __construct(private bool $debug = false, private bool $fresh = false)
{
$serverType = config('indexer.engine');
$this->debug = $debug;
Expand All @@ -34,16 +30,16 @@ public function __construct($debug = false, $fresh = false)
$this->domain = rtrim(env('APP_URL', ''), '/');
switch ($serverType) {
case 'solr':
$this->searchServer = new SolrIndexService($this->debug);
$this->searchServer = new SolrIndexService($this->debug, $this->fresh);
break;
default:
exit('Unknown search index type');
}
}

public function run()
public function run(): void
{
if (! $this->searchServer->checkConnection()) {
if (!$this->searchServer->checkConnection()) {
$this->writeDebug("\n\n Error connecting to search server! \n\n");

return;
Expand All @@ -55,7 +51,7 @@ public function run()

if (count($sites) > 0) {
$startResult = $this->searchServer->startUpdate();
if (! $startResult) {
if (!$startResult) {
$this->writeDebug("\n\n Error when emptying core! \n\n");
}

Expand All @@ -82,13 +78,14 @@ public function run()
if ($this->sitemapFile) {
fclose($this->sitemapFile);
}
$finish = $this->searchServer->finishUpdate();

$this->writeDebug($finish->message);
}
} else {
$this->writeDebug("No sites to index\n");
return;
}
$finish = $this->searchServer->finishUpdate();

$this->writeDebug($finish->message);
}

private function indexChildPages($parentId)
Expand All @@ -98,20 +95,18 @@ private function indexChildPages($parentId)
$this->writeDebug("┃\n");
$this->writeDebug(sprintf('%s (id: %d)', $page->url, $page->id), true, '┣━┓ 📂 Page ');

if (! isset($page->template->id)) {
if (!isset($page->template->id)) {
$this->writeDebug(": ❌ Fail, skipping, no template found\n");

continue;
}

$menu = Menu::whereId($page->id)->firstOrFail();

if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) {
if (!isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) {
$this->writeDebug(": ⏭️ Skipping, template excluded from search\n");

} elseif (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) {
$this->writeDebug(": ⏭️ Skipping, page excluded from search\n");

} else {

foreach ($this->locales as $lang) {
Expand Down Expand Up @@ -148,7 +143,7 @@ private function updatePage($menu, $lang)
// continue with customValues
$customValues = [];

$className = 'App\Http\Controllers\Page\\'.$this->controllerName($menu).'Controller';
$className = 'App\Http\Controllers\Page\\' . $this->controllerName($menu) . 'Controller';

$c = null;
$priority = 1;
Expand All @@ -163,10 +158,13 @@ private function updatePage($menu, $lang)
}
}
$searchText = rtrim($searchText, ', ');
if (! empty($title) && ! empty($searchText)) {
if (!empty($title) && !empty($searchText)) {

$searchItem = new SearchItem($url, $title);
$searchItem->setContent($searchText)->setLanguage($lang->url)->setPriority($priority)->setPublicationDate(new DateTime($menu->updated_at));
$searchItem->setContent($searchText)
->setLanguage($lang->url)
->setPriority($priority)
->setPublicationDate(new DateTime($menu->updated_at));
foreach ($customValues as $key => $value) {
$searchItem->setCustomValue($key, $value);
}
Expand Down Expand Up @@ -198,7 +196,7 @@ private function updatePage($menu, $lang)

private function updateSubPages($menu, $lang)
{
$className = 'App\Http\Controllers\Page\\'.$this->controllerName($menu).'Controller';
$className = 'App\Http\Controllers\Page\\' . $this->controllerName($menu) . 'Controller';
$c = null;
// update subPage if necessary

Expand Down Expand Up @@ -227,7 +225,7 @@ private function updateSubitems($class, $lang)

// We need to check if the subPages is an array of arrays
// If not we wrap it in an extra array
if (count($subPages) > 0 && ! is_array($subPages[0])) {
if (count($subPages) > 0 && !is_array($subPages[0])) {
$subPages = [$subPages];
}
foreach ($subPages as $subPage) {
Expand Down Expand Up @@ -270,8 +268,8 @@ private function updateSubitems($class, $lang)
private function createFolderIfNotExists($fullFilePath)
{
$path_parts = pathinfo($fullFilePath);
if (! file_exists($path_parts['dirname'])) {
if (! mkdir($path_parts['dirname'])) {
if (!file_exists($path_parts['dirname'])) {
if (!mkdir($path_parts['dirname'])) {
printf("\n\n### Error creating sitemap folder");
}
}
Expand All @@ -285,7 +283,7 @@ private function writeDebug($text, $padding = false, $prefix = '')
$text = substr($text, 0, $this->padding - strlen($prefix));
$text = str_pad($text, $this->padding - strlen($prefix), ' ');
}
printf("\e[1m".$prefix."\e[0m".$text);
printf("\e[1m" . $prefix . "\e[0m" . $text);
}
}

Expand Down
13 changes: 6 additions & 7 deletions src/Services/Indexer/SolrIndexService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,20 @@

class SolrIndexService extends AbstractIndexService
{
private bool $debug = false;

public int $siteId;

public ?string $domain;

public $solrIndex;

public function __construct($debug = false)
public function __construct(private bool $debug = false, private bool $fresh = false)
{
$this->debug = $debug;
$this->solrIndex = new SolrIndex();
}

public function retainItem(string $url): void
{
$cmsSearchItem = CmsSearch::whereUrl($this->domain.$url)->first();
$cmsSearchItem = CmsSearch::whereUrl($this->domain . $url)->first();
if ($cmsSearchItem) {
$cmsSearchItem->search_status = 'UPDATED';
$cmsSearchItem->save();
Expand Down Expand Up @@ -72,7 +69,7 @@ public function upsertItem(SearchItem $searchItem): object
$cmsSearchItem->setValues($searchItem, $cmsSearchItemStatus);
$cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->domain);

if (in_array($cmsSearchItemStatus, ['NOT_FOUND', 'FAILED'])) {
if (in_array($cmsSearchItemStatus, ['NOT_FOUND', 'FAILED'])) {
$cmsSearchItem->updated_at = null;
}
$cmsSearchItem->save();
Expand All @@ -85,7 +82,9 @@ public function startUpdate(): bool
if ($this->debug) {
printf("\n ** Starting SOLR update");
}
$emptyResult = $this->solrIndex->emptyCore();
if ($this->fresh) {
$emptyResult = $this->solrIndex->emptyCore();
}
CmsSearch::setAllPending();

return $emptyResult;
Expand Down

0 comments on commit 64ec8d6

Please sign in to comment.