diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ffef176 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +tests/ +phpunit.xml \ No newline at end of file diff --git a/Plugin.php b/Plugin.php index c2561ad..a05fb40 100644 --- a/Plugin.php +++ b/Plugin.php @@ -1,10 +1,14 @@ - 'bombozama.linkcheck::lang.details.description', 'author' => 'Gonzalo HenrĂ­quez', 'icon' => 'icon-chain-broken', - 'homepage' => 'https://github.com/bombozama/linkcheck' + 'homepage' => 'https://github.com/bombozama/linkcheck', ]; } + public function register() + { + Settings::extend(function ($model) { + $model->bindEvent('model.beforeSave', function () use ($model) { + if (!post('Settings[plugins]')) { + // If no plugin was selected, remove the selected model fields + // by setting modelators to an empty string. + $model->modelators = ''; + Flash::info(Lang::get('bombozama.linkcheck::lang.strings.plugins_empty')); + } + }); + }); + } + public function registerPermissions() { return [ @@ -38,6 +56,10 @@ public function registerPermissions() 'tab' => 'bombozama.linkcheck::lang.plugin.tab', 'label' => 'bombozama.linkcheck::lang.plugin.view', ], + 'bombozama.linkcheck.useragent' => [ + 'tab' => 'bombozama.linkcheck::lang.plugin.tab', + 'label' => 'bombozama.linkcheck::lang.plugin.useragent', + ], ]; } @@ -51,16 +73,25 @@ public function registerSettings() 'icon' => 'icon-chain-broken', 'class' => 'Bombozama\LinkCheck\Models\Settings', 'order' => 410, - 'permissions' => ['bombozama.linkcheck.manage'] + 'permissions' => ['bombozama.linkcheck.manage'], ], 'brokenlinks' => [ 'label' => 'bombozama.linkcheck::lang.menu.brokenlinks.label', 'description' => 'bombozama.linkcheck::lang.menu.brokenlinks.description', 'category' => 'bombozama.linkcheck::lang.plugin.category', 'icon' => 'icon-list', - 'url' => Backend::url('bombozama/linkcheck/brokenlinks'), + 'url' => Backend::url('bombozama/linkcheck/context'), 'order' => 411, - 'permissions' => ['bombozama.linkcheck.view'] + 'permissions' => ['bombozama.linkcheck.view'], + ], + 'userAgent' => [ + 'label' => 'bombozama.linkcheck::lang.menu.useragent.label', + 'description' => 'bombozama.linkcheck::lang.menu.useragent.description', + 'category' => 'bombozama.linkcheck::lang.plugin.category', + 'icon' => 'icon-user-secret', + 'url' => Backend::url('bombozama/linkcheck/useragent'), + 'order' => 412, + 'permissions' => ['bombozama.linkcheck.useragent'], ], ]; } @@ -74,15 +105,30 @@ public function registerListColumnTypes() public function httpStatus($value, $column, $record) { - return '' . $value . ''; + return '' . $value . ''; + } + + public function registerReportWidgets() + { + return [ + BrokenLinks::class => [ + 'label' => 'bombozama.linkcheck::lang.details.name', + 'context' => 'dashboard', + 'permissions' => [ + 'bombozama.linkcheck.view', + ], + ], + ]; } // Please do https://octobercms.com/docs/setup/installation#crontab-setup public function registerSchedule($schedule) { $settings = Settings::instance(); - $schedule->call(function(){ - BrokenLink::processLinks(); - })->cron($settings->time); + if ($settings->time) { + $schedule->call(function() { + Context::processLinks(); + })->cron($settings->time); + } } } diff --git a/Readme.md b/Readme.md index 9ec1a6a..91fa928 100644 --- a/Readme.md +++ b/Readme.md @@ -1,11 +1,20 @@ # LinkCheck Plugin for OctoberCMS Schedules a task to check for broken links in database fields and/or CMS pages. +## Refactoring +* The plugin has been revised and optimised for October v3.x. +* DB: Splitting of URL/status and contexts (CMS pages, model fields) in which the URLs are located +* URL-based approach: URLs are stored in a set so that URLs are only checked once +* cURL only calls headers + ## Features * Checks for broken links within database fields and CMS text files * User controls which fields to check * User controls which response codes should be logged * User controls task scheduling +* Management and use of user agents (new in v2.x) +* Dashboard report widget with overview (new in v2.x) +* Selection of plugins and directories for the check in the settings (new in v2.x) ## Usage A link status report can be found in the backend Settings tab. diff --git a/classes/Helper.php b/classes/Helper.php index 24444ab..22c26b8 100644 --- a/classes/Helper.php +++ b/classes/Helper.php @@ -1,12 +1,40 @@ -]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))#'; + preg_match_all($expression, $string, $matches); - return ($matches[0]); + + foreach($matches[0] as $match) { + try { + $url = parse_url($match); + + // Use URL without fragment + $parsed_url = $url['scheme'] . '://' . $url['host']; + + if (isset($url['path'])) { + $parsed_url .= $url['path']; + } + + if (isset($url['query'])) { + $parsed_url .= '?' . $url['query']; + } + + $urls[] = $parsed_url; + + } catch (\Exception $ex) { + // If URL is invalid / malformed, log an error + \Log::info('LinkCheck Error: Could not parse URL: ' . $match . ' due to the following error: ' . $ex); + } + } + + return $urls; } public static function getFullClassNameFromFile($pathToFile) @@ -15,47 +43,78 @@ public static function getFullClassNameFromFile($pathToFile) $class = $namespace = $buffer = ''; $i = 0; while (!$class) { - if (feof($fp)) + if (feof($fp)) { break; + } $buffer.= fread($fp, 512); $tokens = token_get_all($buffer); - if (strpos($buffer, '{') === false) + if (strpos($buffer, '{') === false) { continue; + } for (; $i < count($tokens); $i++) { - if($tokens[$i][0] === T_NAMESPACE) - for($j = $i+1; $j < count($tokens); $j++) - if($tokens[$j][0] === T_STRING) + if ($tokens[$i][0] === T_NAMESPACE) { + for ($j = $i+1; $j < count($tokens); $j++) { + if ($tokens[$j][0] === 265) { // PHP token id 265 == string $namespace .= '\\' . $tokens[$j][1]; - else if($tokens[$j] === '{' || $tokens[$j] === ';') + } elseif ($tokens[$j] === '{' || $tokens[$j] === ';') { break; - - if($tokens[$i][0] === T_CLASS) - for($j = $i+1; $j < count($tokens); $j++) - if($tokens[$j] === '{') - $class = $tokens[$i+2][1]; + } + } + } + if ($tokens[$i][0] === T_CLASS) { + for ($j = $i+1; $j < count($tokens); $j++) { + if ($tokens[$j] === '{') { + if (is_array($tokens[$i+2]) && $tokens[$i+2][0] !== 392) { // PHP token id 392 == whitespace + $class = $tokens[$i+2][1]; + } + } + } + } } } return $namespace . '\\' . $class; } - public static function getResponseCode($url) + public static function getResponseCode(string $url, mixed $userAgent): int { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); - curl_setopt($ch , CURLOPT_RETURNTRANSFER, 1); + curl_setopt($ch , CURLOPT_NOBODY, 1); + curl_setopt($ch, CURLOPT_TIMEOUT_MS, 30000); # Setting timeout to 30 seconds + if ($userAgent) { + curl_setopt($ch, CURLOPT_USERAGENT, $userAgent); + } $data = curl_exec($ch); $headers = curl_getinfo($ch); curl_close($ch); - # In case of timeouts, let's throw out a 408 error "Request timeout" - if($headers['http_code'] == 0) - $headers['http_code'] = '408'; - + if ($headers['http_code'] == 0) { + $headers['http_code'] = 408; + } return $headers['http_code']; } + /** + * Flatten an arbitrarily nested array + * + * see https://www.lambda-out-loud.com/posts/flatten-arrays-php/ + * + * @param array + * @return array + */ + public function flattenArray(array $array): array + { + $recursiveArrayIterator = new \RecursiveArrayIterator( + $array, + \RecursiveArrayIterator::CHILD_ARRAYS_ONLY + ); + $iterator = new \RecursiveIteratorIterator($recursiveArrayIterator); + + return iterator_to_array($iterator, false); + } + } \ No newline at end of file diff --git a/controllers/Context.php b/controllers/Context.php new file mode 100644 index 0000000..de46242 --- /dev/null +++ b/controllers/Context.php @@ -0,0 +1,37 @@ + $links['reported'], 'total' => $links['checked']])); + + return $this->listRefresh(); + } +} \ No newline at end of file diff --git a/controllers/UserAgent.php b/controllers/UserAgent.php new file mode 100644 index 0000000..be3d31b --- /dev/null +++ b/controllers/UserAgent.php @@ -0,0 +1,46 @@ + +
+ +
+ \ No newline at end of file diff --git a/controllers/context/config_list.yaml b/controllers/context/config_list.yaml new file mode 100644 index 0000000..c15dfac --- /dev/null +++ b/controllers/context/config_list.yaml @@ -0,0 +1,44 @@ +# =================================== +# List Behavior Config +# =================================== + +# Model List Column configuration +list: $/bombozama/linkcheck/models/context/columns.yaml + +# Model Class name +modelClass: Bombozama\LinkCheck\Models\Context + +# List Title +title: bombozama.linkcheck::lang.strings.list_title + +# Link URL for each record +recordUrl: null + +# Message to display if the list is empty +noRecordsMessage: bombozama.linkcheck::lang.strings.no_records + +# Records to display per page +recordsPerPage: 20 + +# Displays the list column set up button +showSetup: false + +# Displays the sorting link on each column +showSorting: true + +# Default sorting column +defaultSort: + column: status + direction: desc + +# Display checkboxes next to each record +showCheckboxes: false + +# Toolbar widget configuration +toolbar: + # Partial for toolbar buttons + buttons: list_toolbar + + # Search widget configuration + search: + prompt: backend::lang.list.search_prompt diff --git a/controllers/context/index.htm b/controllers/context/index.htm new file mode 100644 index 0000000..498d5dc --- /dev/null +++ b/controllers/context/index.htm @@ -0,0 +1 @@ +listRender() ?> \ No newline at end of file diff --git a/controllers/useragent/_list_toolbar.php b/controllers/useragent/_list_toolbar.php new file mode 100644 index 0000000..914ef68 --- /dev/null +++ b/controllers/useragent/_list_toolbar.php @@ -0,0 +1,22 @@ +
+ + + 'User Agent']) ?> + + +
+ + +
diff --git a/controllers/useragent/config_form.yaml b/controllers/useragent/config_form.yaml new file mode 100644 index 0000000..74b26a8 --- /dev/null +++ b/controllers/useragent/config_form.yaml @@ -0,0 +1,31 @@ +# =================================== +# Form Behavior Config +# =================================== + +# Record name +name: User Agent + +# Model Form Field configuration +form: $/bombozama/linkcheck/models/useragent/fields.yaml + +# Model Class name +modelClass: Bombozama\Linkcheck\Models\UserAgent + +# Default redirect location +defaultRedirect: bombozama/linkcheck/useragent + +# Create page +create: + title: backend::lang.form.create_title + redirect: bombozama/linkcheck/useragent/update/:id + redirectClose: bombozama/linkcheck/useragent + +# Update page +update: + title: backend::lang.form.update_title + redirect: bombozama/linkcheck/useragent + redirectClose: bombozama/linkcheck/useragent + +# Preview page +preview: + title: backend::lang.form.preview_title diff --git a/controllers/useragent/config_list.yaml b/controllers/useragent/config_list.yaml new file mode 100644 index 0000000..8275ce9 --- /dev/null +++ b/controllers/useragent/config_list.yaml @@ -0,0 +1,47 @@ +# =================================== +# List Behavior Config +# =================================== + +# Model List Column configuration +list: $/bombozama/linkcheck/models/useragent/columns.yaml + +# Model Class name +modelClass: Bombozama\Linkcheck\Models\UserAgent + +# List Title +title: Manage User Agents + +# Link URL for each record +recordUrl: bombozama/linkcheck/useragent/update/:id + +# Message to display if the list is empty +noRecordsMessage: backend::lang.list.no_records + +# Records to display per page +recordsPerPage: 20 + +# Display page numbers with pagination, disable to improve performance +showPageNumbers: true + +# Displays the list column set up button +showSetup: true + +# Displays the sorting link on each column +showSorting: true + +# Default sorting column +defaultSort: + column: id + direction: asc + +# Display checkboxes next to each record +showCheckboxes: true + +# Toolbar widget configuration +toolbar: + # Partial for toolbar buttons + buttons: list_toolbar + + # Search widget configuration + search: + prompt: backend::lang.list.search_prompt diff --git a/controllers/useragent/create.php b/controllers/useragent/create.php new file mode 100644 index 0000000..cd074a5 --- /dev/null +++ b/controllers/useragent/create.php @@ -0,0 +1,61 @@ + + + + +fatalError): ?> + + 'd-flex flex-column h-100']) ?> + +
+ formRender() ?> +
+ +
+
+ + + + + + + + +
+
+ + + + + +

+ fatalError) ?> +

+

+ + + +

+ + diff --git a/controllers/useragent/index.php b/controllers/useragent/index.php new file mode 100644 index 0000000..766877d --- /dev/null +++ b/controllers/useragent/index.php @@ -0,0 +1,2 @@ + +listRender() ?> diff --git a/controllers/useragent/preview.php b/controllers/useragent/preview.php new file mode 100644 index 0000000..a37963e --- /dev/null +++ b/controllers/useragent/preview.php @@ -0,0 +1,19 @@ + + + + +fatalError): ?> + +
+ formRenderPreview() ?> +
+ + + +

fatalError) ?>

+

+ + diff --git a/controllers/useragent/update.php b/controllers/useragent/update.php new file mode 100644 index 0000000..2bb9706 --- /dev/null +++ b/controllers/useragent/update.php @@ -0,0 +1,70 @@ + + + + +fatalError): ?> + + 'd-flex flex-column h-100']) ?> + +
+ formRender() ?> +
+ +
+
+ + + + + + + + + +
+
+ + + + + +

+ fatalError) ?> +

+

+ + + +

+ + diff --git a/lang/en/lang.php b/lang/en/lang.php index 9cea167..def9f08 100644 --- a/lang/en/lang.php +++ b/lang/en/lang.php @@ -1,20 +1,22 @@ [ 'tab' => 'LinkCheck Plugin', 'manage' => 'Manage Link check configuration', 'view' => 'Grant access to the link report', - 'category' => 'Link Check' + 'useragent' => 'Manage user agent configuration', + 'category' => 'Link Check', ], 'details' => [ 'name' => 'Link Check', - 'description' => 'Checks database daily for broken links...' + 'description' => 'Checks database daily for broken links...', ], 'strings' => [ 'refresh_list' => 'Refresh Link List', 'list_title' => 'Broken Link List', 'no_records' => 'No broken links have been detected at this time or the system is currently checking for broken links.', - 'total_links' => 'A total of :number broken links were found!', + 'total_links' => ':total links were checked. A total of :number broken links were found!', 'working' => 'Working... please hold', 'status' => 'Status', 'url' => 'URL', @@ -24,17 +26,53 @@ 'field' => 'Field', 'created_at' => 'Last check', 'time' => 'Time mask', - 'time_comment' => 'The plugin will check for broken links as scheduled here. Check http://crontab.org/ for more info.', + 'time_comment' => 'The plugin will check for broken links as scheduled here. To suspend the cron job, leave this field empty. Check http://crontab.org/ for more info.', 'codes' => 'Which HTTP response codes should be reported?', 'codes_opt_200' => 'Successful responses (200 - 206) [not recomended]', 'codes_opt_300' => 'Redirection responses (300 - 308)', 'codes_opt_400' => 'Client error responses (400 - 431)', 'codes_opt_500' => 'Server error responses (500 - 511)', 'check_cms' => 'Check links within CMS files?', + 'dirs' => 'CMS theme directories', + 'dirs_comment' => 'Select the directories whose files you wish to scan.', 'modelators_prompt' => 'Select a database field', - 'modelator' => 'Plugin/Model', - 'modelator_comment' => 'Select the Plugin/Model/Fields that you wish to scan.', + 'modelator' => 'Plugin Model Fields', + 'modelator_comment' => 'Select the model fields that you wish to scan.', 'modelator_empty' => 'Select at least one field to check for broken links [optional]', + 'plugins' => 'Plugins', + 'plugins_comment' => 'Select the Plugins that you wish to scan.', + 'plugins_empty' => 'Select at least one plugin to check for broken links [optional]', + 'CMS_dir_config' => 'No directories were specified for the link check in config.yaml.', + 'useragent' => [ + 'select_useragent' => 'Select a user agent for the link check.', + 'label' => 'User agent', + 'default_option' => 'none', + ], + 'log' => [ + 'check_started' => '[LinkCheck] Link check started.', + 'summary' => '[LinkCheck] Checking the links took :seconds seconds. :urls URLs were checked. :reported were reported.', + ], + ], + 'reportwidget' => [ + 'button' => [ + 'label' => 'Go to link report', + ], + 'categories' => [ + 'status' => 'Status', + 'broken_links' => 'Number of broken links', + 'total' => 'Total', + ], + 'last_check' => 'Last check on', + 'status_info' => 'For more information on HTTP status codes see: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status', + 'title' => 'Link Check Overview', + 'validation' => [ + 'required' => [ + 'message' => 'The Name field is required', + ], + 'regex' => [ + 'message' => 'The Name field can contain only Latin letters.', + ], + ], ], 'menu' => [ 'settings' => [ @@ -44,7 +82,11 @@ 'brokenlinks' => [ 'label' => 'Link report', 'description' => 'Shows list of broken links', - ] + ], + 'useragent' => [ + 'label' => 'User agent', + 'description' => 'Configure user agents', + ], ], 'codes' => [ '100' => 'Continue', @@ -123,5 +165,5 @@ '525' => 'SSL Handshake Failed', '526' => 'Invalid SSL Certificate', '527' => 'Railgun Error', - ] + ], ]; \ No newline at end of file diff --git a/models/BrokenLink.php b/models/BrokenLink.php index 8ee190b..276f088 100644 --- a/models/BrokenLink.php +++ b/models/BrokenLink.php @@ -1,112 +1,66 @@ - 'required', + 'status' => 'required', + ]; + public $hasMany = [ + 'context' => Context::class + ]; /** * Checks to see if links return selected response codes (in settings) * @param $url: the link url that should be checked - * @return bool|string + * @return array */ - public static function isBrokenLink($url) + public static function isBrokenLink(Settings $settings, String $url): array { - $response = Helper::getResponseCode($url); - - $settings = Settings::instance(); + $response = Helper::getResponseCode($url, $settings->user_agent); $report = []; - if(in_array(200, $settings->codes)) - for($i=200;$i<=206;$i++) $report[] = $i; - if(in_array(300, $settings->codes)) - for($i=300;$i<=308;$i++) $report[] = $i; - if(in_array(400, $settings->codes)) - for($i=400;$i<=431;$i++) $report[] = $i; - if(in_array(500, $settings->codes)) - for($i=500;$i<=511;$i++) $report[] = $i; - - if (in_array($response, $report)) - return $response; - - return false; - } - - /** - * Checks for broken links in selected database fields and/or all CMS files - * @return void - */ - public static function processLinks() - { - # Let's start by truncating the BrokenLinks table - BrokenLink::truncate(); - $brokenLinks = []; - $settings = Settings::instance(); - foreach($settings->modelators as $el) { - list($modelName, $field) = explode('::', $el['modelator']); - $models = $modelName::whereNotNull($field)->get(); - foreach($models as $model){ - $urls = Helper::scanForUrls($model->$field); - $modelParts = explode('\\', $modelName); - foreach($urls as $url){ - $status = BrokenLink::isBrokenLink($url); - if($status) - $brokenLinks[] = [ - 'status' => $status, - 'plugin' => $modelParts[1] . '.' . $modelParts[2], - 'model' => array_pop($modelParts), - 'model_id' => $model->id, - 'field' => $field, - 'context' => $model->$field, - 'url' => $url - ]; - } + if (in_array(200, $settings->codes)) { + for ($i = 200; $i <= 206; $i++) { + $report[] = $i; } } - - /** - * Go process the current theme - */ - $theme = Theme::getActiveTheme(); - $theme->getPath(); - - /** - * Should we process theme pages? - */ - if($settings['checkCMS'] == '1') - foreach(File::directories($theme->getPath()) as $themeSubDir){ - # Skip the assets folder - if(basename($themeSubDir) == 'assets') - continue; - - foreach(File::allFiles($themeSubDir) as $filePath){ - $urls = Helper::scanForUrls(file_get_contents($filePath)); - foreach($urls as $url){ - $status = BrokenLink::isBrokenLink($url); - if($status) - $brokenLinks[] = [ - 'status' => $status, - 'plugin' => 'CMS', - 'model' => str_replace($theme->getPath() . DIRECTORY_SEPARATOR, '', $filePath), - 'url' => $url - ]; - } - } + if (in_array(300, $settings->codes)) { + for ($i = 300; $i <= 308; $i++) { + $report[] = $i; + } + } + if (in_array(400, $settings->codes)) { + for ($i = 400; $i <= 431; $i++) { + $report[] = $i; } + } + if (in_array(500, $settings->codes)) { + for ($i = 500; $i <= 511; $i++) { + $report[] = $i; + } + } - /** - * Lets seed the BrokenLink table with any and all found links. - */ - foreach($brokenLinks as $brokenLink) - BrokenLink::create($brokenLink); + if (in_array($response, $report)) { + $new_broken_link = BrokenLink::create([ + 'url' => $url, + 'status' => $response, + ]); + return $new_broken_link->toArray(); + } - return count($brokenLinks); + return array( + 'id' => null, + 'url' => $url, + 'status' => false, + ); } } \ No newline at end of file diff --git a/models/Context.php b/models/Context.php new file mode 100644 index 0000000..33fe824 --- /dev/null +++ b/models/Context.php @@ -0,0 +1,172 @@ + \Bombozama\LinkCheck\Models\BrokenLink::class + ]; + + /** + * Checks for broken links in selected database fields and/or all CMS files + * @return array + */ + public static function processLinks(): array + { + \Log::info(__('bombozama.linkcheck::lang.strings.log.check_started')); + $start = microtime(true); + $helper = new Helper(); + $url_set = array(); + $check_count = 0; + $url_count = 0; + $settings = Settings::instance(); + + # Remove all data from tables + Context::truncate(); + BrokenLink::truncate(); + + if ($modulators = $settings->modelators) { + foreach ($modulators as $el) { + list($modelName, $field) = explode('::', $el); + $models = $modelName::whereNotNull($field)->get(); + + foreach ($models as $model) { + $model_field; + + if (is_array($model->$field)) { + $model_field = implode(', ', $helper->flattenArray($model->$field)); + } else { + $model_field = $model->$field; + } + + $urls = $helper->scanForUrls($model_field); + + if (!$urls) { + continue; + } + $modelParts = explode('\\', $modelName); + + foreach ($urls as $url) { + $broken_link_id = null; + $status = false; + + if (isset($url_set[$url])) { + + if ($url_set[$url] === false) { + continue; + } + + $existing_url = BrokenLink::where('url', $url)->first(); + $broken_link_id = $existing_url->id; + $status = $existing_url->status; + } else { + $status = BrokenLink::isBrokenLink($settings, $url); + $url_set[$url] = $status['status']; + + if ($status['status'] === false) { + continue; + } + + $broken_link_id = $status['id']; + } + + Context::insert([ + 'broken_link_id' => $broken_link_id, + 'plugin' => (sizeof($modelParts) >= 3) ? $modelParts[1] . '.' . $modelParts[2] : $modelName, + 'model' => (sizeof($modelParts) > 3) ? $modelParts[4] : $modelName, + 'model_id' => $model->id, + 'field' => $field, + 'last_checked' => Carbon::now(new \DateTimeZone('UTC')), + ]); + } + } + } + } + + /** + * Go process the current theme + */ + $theme = Theme::getActiveTheme(); + $theme->getPath(); + + /** + * Should we process theme pages? + */ + if ($settings['checkCMS'] == '1') { + $include_dirs = ($settings->dirs !== "") ? $settings->dirs : []; + + foreach (File::directories($theme->getPath()) as $themeSubDir) { + + # Skip all folders except whitlisted folders + if (!in_array(basename($themeSubDir), $include_dirs)) { + continue; + } + + foreach (File::allFiles($themeSubDir) as $filePath) { + set_time_limit(120); // Increasing PHP script execution time + + if (str_contains($filePath, 'static-pages-en') === false) { + $urls = $helper->scanForUrls(file_get_contents($filePath)); + + if (!$urls) { + continue; + } + + foreach ($urls as $url) { + $broken_link_id = null; + $status = false; + + if (isset($url_set[$url])) { + + if ($url_set[$url] === false) { + continue; + } + + $existing_url = BrokenLink::where('url', $url)->first(); + $broken_link_id = $existing_url->id; + $status = $existing_url->status; + } else { + $status = BrokenLink::isBrokenLink($settings, $url); + $url_set[$url] = $status['status']; + + if ($status['status'] === false) { + continue; + } + + $broken_link_id = $status['id']; + } + + Context::insert([ + 'broken_link_id' => $broken_link_id, + 'plugin' => 'CMS', + 'model' => str_replace($theme->getPath() . DIRECTORY_SEPARATOR, '', $filePath), + 'last_checked' => Carbon::now(new \DateTimeZone('UTC')), + ]); + } + } + } + } + } + + $checked = count($url_set); + $reported = array_filter($url_set, fn($url) => $url !== false); + \Log::info(__('bombozama.linkcheck::lang.strings.log.summary', ['seconds' => round(microtime(true) - $start, 2), 'urls' => $checked, 'reported' => count($reported)])); + + return array( + 'checked' => $checked, + 'reported' => count($reported), + ); + } +} \ No newline at end of file diff --git a/models/Settings.php b/models/Settings.php index 729fbdb..db8a56d 100644 --- a/models/Settings.php +++ b/models/Settings.php @@ -1,58 +1,143 @@ - 'required' ]; + + public function beforeValidate(): void + { + $plugins = $this->plugins; + $modelators = $this->modelators; + $checkCMS = $this->checkCMS; + $dirs = $this->dirs; + + if ($checkCMS && !$dirs) { + throw new ValidationException(['dirs' => 'You must select at least one option if the feature is enabled.']); + } + if ($plugins && !$modelators) { + throw new ValidationException(['modelators' => 'You must select at least one option if the feature is enabled.']); + } + } + + # List all CMS Theme dirs for custom selection + public function getDirOptions(): array + { + $dirs = []; + $theme = Theme::getActiveTheme(); + $theme->getPath(); + foreach (File::directories($theme->getPath()) as $themeSubDir) { + $dirs[basename($themeSubDir)] = basename($themeSubDir); + } + return $dirs; + } + + # List all CMS Theme dirs for custom selection + public function getPluginOptions(): array + { + $plugins = []; + foreach (File::directories(plugins_path()) as $author) { + foreach (File::directories($author) as $plugin) { + $dir = basename($author) . '/' . basename($plugin); + if ($dir === 'bombozama/linkcheck') { + continue; + } + $plugins[$dir] = $dir; + } + } + return $plugins; + } # Render options for dropdowns on settings/fields.yaml - public function getModelatorOptions( $keyValue = null ) { + public function getModelatorOptions(): array + { + $include_plugins = $this->plugins ?? []; $models = $out = []; - $authors = File::directories( plugins_path() ); + $authors = File::directories(plugins_path()); + $plugin_pattern = '/\/(\w+\/\w+)$/'; foreach ( $authors as $author ) { - foreach ( File::directories( $author ) as $plugin ) { + foreach (File::directories($author) as $plugin) { + preg_match($plugin_pattern, $plugin, $matches); + + if (!in_array($matches[1], $include_plugins)) { + continue; + } + $modelPath = $plugin . DIRECTORY_SEPARATOR . 'models'; - if ( ! File::exists( $modelPath ) ) { + + if (!File::exists($modelPath)) { continue; } - foreach ( File::files( $modelPath ) as $modelFile ) { + foreach (File::files($modelPath) as $modelFile) { # All links in the LinkCheck plugin table are broken. Skip. $linkCheckPluginPath = plugins_path() . DIRECTORY_SEPARATOR . 'bombozama' . DIRECTORY_SEPARATOR . 'linkcheck'; - if ( $plugin == $linkCheckPluginPath ) { + + if ($plugin == $linkCheckPluginPath) { continue; } - $models[] = Helper::getFullClassNameFromFile( (string) $modelFile ); + $models[] = Helper::getFullClassNameFromFile((string) $modelFile); } } } - foreach ( $models as $model ) { - if ( substr( $model, - 5 ) == 'Pivot' ) { + foreach ($models as $model) { + if (substr($model, - 5) == 'Pivot') { continue; } + // Check if class is abstact, because abstract classes cannot be instanciated + $class = new \ReflectionClass($model); + if (!$class->isAbstract()) { + $object = new $model(); + + if (!isset($object->table)) { + continue; + } - $object = new $model(); - foreach ( Schema::getColumnListing( $object->table ) as $column ) { - $type = DB::connection()->getDoctrineColumn( $object->table, $column )->getType()->getName(); - if ( in_array( $type, [ 'string', 'text' ] ) ) { - $out[ $model . '::' . $column ] = $model . '::' . $column; + foreach (Schema::getColumnListing($object->table) as $column) { + $type = DB::connection()->getDoctrineColumn( $object->table, $column )->getType()->getName(); + + if (in_array($type, ['string', 'text'])) { + $out[$model . '::' . $column] = $model . '::' . $column; + } } } } return $out; } + + public function getUserAgentOptions() + { + return UserAgent::pluck('user_agent', 'id'); + } + + public function filterFields($fields, $context = null) + { + if ($this->checkCMS) { + $fields->dirs->hidden = false; + } + + if ($this->plugins) { + $fields->modelators->hidden = false; + } + } } diff --git a/models/UserAgent.php b/models/UserAgent.php new file mode 100644 index 0000000..11026a7 --- /dev/null +++ b/models/UserAgent.php @@ -0,0 +1,27 @@ + 'required', + ]; +} diff --git a/models/brokenlink/columns.yaml b/models/brokenlink/columns.yaml index 6965df5..99618e4 100644 --- a/models/brokenlink/columns.yaml +++ b/models/brokenlink/columns.yaml @@ -11,27 +11,6 @@ columns: url: label: bombozama.linkcheck::lang.strings.url searchable: true - select: IF( CHAR_LENGTH(url) > 99, CONCAT( LEFT(url, 99), '...' ), url ) - plugin: - label: bombozama.linkcheck::lang.strings.plugin - searchable: true - model: - label: bombozama.linkcheck::lang.strings.model - searchable: true - model_id: - label: bombozama.linkcheck::lang.strings.model_id - searchable: true - select: IF(model_id IS NULL, '--', model_id) - width: 110px - context: - invisible: true - searchable: false - field: - label: bombozama.linkcheck::lang.strings.field - searchable: true - width: 110px - created_at: - label: bombozama.linkcheck::lang.strings.created_at - type: timetense - searchable: false - width: 145px \ No newline at end of file + type: linkage + attributes: + target: _blank diff --git a/models/brokenlink/fields.yaml b/models/brokenlink/fields.yaml new file mode 100644 index 0000000..1a53e32 --- /dev/null +++ b/models/brokenlink/fields.yaml @@ -0,0 +1,16 @@ +# =================================== +# Form Field Definitions +# =================================== + +fields: + status: + disabled: true + label: bombozama.linkcheck::lang.strings.status + width: 100px + type: number + url: + disabled: true + label: bombozama.linkcheck::lang.strings.url + type: text + attributes: + target: _blank \ No newline at end of file diff --git a/models/context/columns.yaml b/models/context/columns.yaml new file mode 100644 index 0000000..b2a5720 --- /dev/null +++ b/models/context/columns.yaml @@ -0,0 +1,39 @@ +# =================================== +# List Column Definitions +# =================================== + +columns: + status: + label: bombozama.linkcheck::lang.strings.status + searchable: true + width: 100px + relation: brokenLink + select: status + type: httpstatus + url: + label: bombozama.linkcheck::lang.strings.url + searchable: true + relation: brokenLink + select: url + type: linkage + attributes: + target: _blank + plugin: + label: bombozama.linkcheck::lang.strings.plugin + searchable: true + model: + label: bombozama.linkcheck::lang.strings.model + searchable: true + model_id: + label: 'Model ID' + searchable: true + field: + label: bombozama.linkcheck::lang.strings.field + searchable: true + width: 110px + last_checked: + label: bombozama.linkcheck::lang.strings.created_at + type: datetime + useTimezone: true + searchable: false + width: 145px \ No newline at end of file diff --git a/models/context/fields.yaml b/models/context/fields.yaml new file mode 100644 index 0000000..e16d7be --- /dev/null +++ b/models/context/fields.yaml @@ -0,0 +1,27 @@ +# =================================== +# Form Field Definitions +# =================================== + +fields: + url_id: + disabled: true + label: bombozama.linkcheck::lang.strings.url_id + plugin: + disabled: true + label: bombozama.linkcheck::lang.strings.plugin + searchable: true + model: + disabled: true + label: bombozama.linkcheck::lang.strings.model + searchable: true + model_id: + disabled: true + label: bombozama.linkcheck::lang.strings.model_id + searchable: true + select: IF(model_id IS NULL, '--', model_id) + width: 110px + field: + disabled: true + label: bombozama.linkcheck::lang.strings.field + searchable: true + width: 110px diff --git a/models/settings/fields.yaml b/models/settings/fields.yaml index 2b5e93e..00851a3 100644 --- a/models/settings/fields.yaml +++ b/models/settings/fields.yaml @@ -2,35 +2,60 @@ # Form Field Definitions # =================================== -fields: - time: - label: bombozama.linkcheck::lang.strings.time - comment: bombozama.linkcheck::lang.strings.time_comment - placeholder: '* * * * *' - span: left - codes: - label: bombozama.linkcheck::lang.strings.codes - type: checkboxlist - span: right - default: 400,500 - options: - 200: bombozama.linkcheck::lang.strings.codes_opt_200 - 300: bombozama.linkcheck::lang.strings.codes_opt_300 - 400: bombozama.linkcheck::lang.strings.codes_opt_400 - 500: bombozama.linkcheck::lang.strings.codes_opt_500 - checkCMS: - label: bombozama.linkcheck::lang.strings.check_cms - type: checkbox - default: true - span: left - modelators: - type: repeater - prompt: bombozama.linkcheck::lang.strings.modelators_prompt - label: bombozama.linkcheck::lang.strings.modelator - commentAbove: bombozama.linkcheck::lang.strings.modelator_comment - form: - fields: - modelator: - emptyOption: bombozama.linkcheck::lang.strings.modelator_empty - span: full - type: dropdown \ No newline at end of file +tabs: + fields: + time: + label: bombozama.linkcheck::lang.strings.time + comment: bombozama.linkcheck::lang.strings.time_comment + placeholder: '* * * * *' + span: left + tab: General + codes: + label: bombozama.linkcheck::lang.strings.codes + type: checkboxlist + span: right + default: 400,500 + options: + 200: bombozama.linkcheck::lang.strings.codes_opt_200 + 300: bombozama.linkcheck::lang.strings.codes_opt_300 + 400: bombozama.linkcheck::lang.strings.codes_opt_400 + 500: bombozama.linkcheck::lang.strings.codes_opt_500 + tab: General + user_agent: + commentAbove: 'bombozama.linkcheck::lang.strings.useragent.select_useragent' + label: 'bombozama.linkcheck::lang.strings.useragent.label' + type: dropdown + emptyOption: 'bombozama.linkcheck::lang.strings.useragent.default_option' + tab: General + checkCMS: + label: bombozama.linkcheck::lang.strings.check_cms + type: switch + default: false + span: left + tab: CMS + dirs: + type: taglist + mode: array + label: bombozama.linkcheck::lang.strings.dirs + commentAbove: bombozama.linkcheck::lang.strings.dirs_comment + options: getDirOptions + dependsOn: checkCMS + hidden: true + tab: CMS + plugins: + type: taglist + mode: array + label: bombozama.linkcheck::lang.strings.plugins + commentAbove: bombozama.linkcheck::lang.strings.plugins_comment + options: getPluginOptions + tab: Plugins + modelators: + type: checkboxlist + quickselect: true + label: bombozama.linkcheck::lang.strings.modelator + commentAbove: bombozama.linkcheck::lang.strings.modelator_comment + options: getModelatorOptions + dependsOn: plugins + hidden: true + tab: Plugins + diff --git a/models/useragent/columns.yaml b/models/useragent/columns.yaml new file mode 100644 index 0000000..134a67c --- /dev/null +++ b/models/useragent/columns.yaml @@ -0,0 +1,8 @@ +# =================================== +# List Column Definitions +# =================================== + +columns: + user_agent: + label: 'bombozama.linkcheck::lang.menu.useragent.label' + searchable: true diff --git a/models/useragent/fields.yaml b/models/useragent/fields.yaml new file mode 100644 index 0000000..2190315 --- /dev/null +++ b/models/useragent/fields.yaml @@ -0,0 +1,8 @@ +# =================================== +# Form Field Definitions +# =================================== + +fields: + user_agent: + label: 'bombozama.linkcheck::lang.menu.useragent.label' + type: text diff --git a/reportwidgets/BrokenLinks.php b/reportwidgets/BrokenLinks.php new file mode 100644 index 0000000..12f37a3 --- /dev/null +++ b/reportwidgets/BrokenLinks.php @@ -0,0 +1,59 @@ + [ + 'title' => 'Name', + 'default' => __('bombozama.linkcheck::lang.reportwidget.title'), + 'type' => 'string', + 'validation' => [ + 'required' => [ + 'message' => __('bombozama.linkcheck::lang.reportwidget.validation.required.message'), + ], + 'regex' => [ + 'message' => __('bombozama.linkcheck::lang.reportwidget.validation.regex.message'), + 'pattern' => '^[a-zA-Z\s]+$', + ] + ] + ], + ]; + } + + public function render() + { + try { + $this->getBrokenLinks(); + } + catch (Exception $ex) { + $this->vars['error'] = $ex->getMessage(); + } + + return $this->makePartial('brokenlinks'); + } + + public function getBrokenLinks() + { + $last_check = new Carbon(Context::orderBy('id', 'desc')->first()->last_checked); + $this->vars['total'] = BrokenLink::all()->count(); + $this->vars['grouped'] = BrokenLink::all()->groupBy('status')->toArray(); + $this->vars['last_check'] = $last_check->toDayDateTimeString(); + } +} diff --git a/reportwidgets/brokenlinks/partials/_brokenlinks.php b/reportwidgets/brokenlinks/partials/_brokenlinks.php new file mode 100644 index 0000000..c851acd --- /dev/null +++ b/reportwidgets/brokenlinks/partials/_brokenlinks.php @@ -0,0 +1,48 @@ +
+

+ +
+ +
+ +

+ +
diff --git a/updates/create_context_table.php b/updates/create_context_table.php new file mode 100644 index 0000000..29cafba --- /dev/null +++ b/updates/create_context_table.php @@ -0,0 +1,37 @@ +id(); + $table->integer('broken_link_id'); + $table->string('plugin'); + $table->string('model'); + $table->integer('model_id')->nullable(); + $table->string('field')->nullable(); + $table->dateTime('last_checked'); + }); + } + + /** + * down reverses the migration + */ + public function down() + { + Schema::dropIfExists('bombozama_linkcheck_context'); + } +}; diff --git a/updates/create_user_agents_table.php b/updates/create_user_agents_table.php new file mode 100644 index 0000000..e0a9c89 --- /dev/null +++ b/updates/create_user_agents_table.php @@ -0,0 +1,33 @@ +id(); + $table->string('user_agent'); + $table->timestamps(); + }); + } + + /** + * down reverses the migration + */ + public function down() + { + Schema::dropIfExists('bombozama_linkcheck_user_agents'); + } +}; diff --git a/updates/update_broken_links_table.php b/updates/update_broken_links_table.php index 35e9de5..7bea8a9 100755 --- a/updates/update_broken_links_table.php +++ b/updates/update_broken_links_table.php @@ -1,27 +1,27 @@ -text('context')->nullable(); - $table->text('url')->nullable()->change(); - }); - } - - public function down() - { - Schema::table('bombozama_linkcheck_broken_links', function($table) - { - $table->dropColumn('context'); - $table->string('url', 255)->nullable()->change(); - }); - } - +text('context')->nullable(); + $table->text('url')->nullable()->change(); + }); + } + + public function down() + { + Schema::table('bombozama_linkcheck_broken_links', function($table) + { + $table->dropColumn('context'); + $table->string('url', 255)->nullable()->change(); + }); + } + } \ No newline at end of file diff --git a/updates/update_broken_links_table2.php b/updates/update_broken_links_table2.php new file mode 100644 index 0000000..38c1e8c --- /dev/null +++ b/updates/update_broken_links_table2.php @@ -0,0 +1,33 @@ +dropColumn('plugin'); + $table->dropColumn('model'); + $table->dropColumn('model_id'); + $table->dropColumn('context'); + $table->dropColumn('field'); + }); + } + + public function down() + { + Schema::table('bombozama_linkcheck_broken_links', function($table) + { + $table->string('plugin'); + $table->string('model'); + $table->integer('model_id')->nullable(); + $table->string('field')->nullable(); + $table->text('context')->nullable(); + }); + } + +} \ No newline at end of file diff --git a/updates/update_context_table.php b/updates/update_context_table.php new file mode 100644 index 0000000..153a418 --- /dev/null +++ b/updates/update_context_table.php @@ -0,0 +1,25 @@ +string('last_checked')->change(); + }); + } + + public function down() + { + Schema::table('bombozama_linkcheck_context', function($table) + { + $table->dateTime('last_checked'); + }); + } + +} \ No newline at end of file diff --git a/updates/update_context_table2.php b/updates/update_context_table2.php new file mode 100644 index 0000000..4f772ec --- /dev/null +++ b/updates/update_context_table2.php @@ -0,0 +1,25 @@ +datetime('last_checked')->change(); + }); + } + + public function down() + { + Schema::table('bombozama_linkcheck_context', function($table) + { + $table->string('last_checked'); + }); + } + +} \ No newline at end of file diff --git a/updates/version.yaml b/updates/version.yaml index 20b2be3..6d92d36 100755 --- a/updates/version.yaml +++ b/updates/version.yaml @@ -8,3 +8,23 @@ 1.0.5: - 'Updated table bombozama_linkcheck_broken_links' - update_broken_links_table.php +2.0.0: + - 'Refactored Plugin; compatibility with October 3.x' + - update_broken_links_table2.php + - create_context_table.php +2.1.0: + - 'Added user agent feature' + - create_user_agents_table.php +2.1.1: + - 'Changed field last_checked from datetime to string' + - update_context_table.php +2.2.0: + - 'New Feature: Report widget for dashboard' +2.2.1: + - 'Optimized data display on widget' +2.3.0: + - 'Added config.yaml to include plugins and/or directories for link check' +2.3.1: + - 'Changed field last_checked from string to datetime' +2.4.0: + - Config for CMS directories and plugins was integrated in backend form. \ No newline at end of file