From 12f5d34cb4dbbc12adcdc9a3876abcc4e51dbb2e Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 28 Nov 2023 13:52:53 +0100 Subject: [PATCH] Hide main method with option to skip local network check + local test. --- safe_url_check/lib/safe_url_check.dart | 122 ++------------- safe_url_check/lib/src/safe_url_check.dart | 144 ++++++++++++++++++ .../test/relative_redirect_test.dart | 61 ++++++++ 3 files changed, 215 insertions(+), 112 deletions(-) create mode 100644 safe_url_check/lib/src/safe_url_check.dart create mode 100644 safe_url_check/test/relative_redirect_test.dart diff --git a/safe_url_check/lib/safe_url_check.dart b/safe_url_check/lib/safe_url_check.dart index c7c6ba27..ff8986d5 100644 --- a/safe_url_check/lib/safe_url_check.dart +++ b/safe_url_check/lib/safe_url_check.dart @@ -39,12 +39,7 @@ import 'dart:async'; import 'package:retry/retry.dart'; -import 'src/private_ip.dart'; -import 'src/unique_local_ip.dart'; -import 'src/version.dart'; - -const _defaultUserAgent = 'package:safe_url_check/$packageVersion ' - '(+https://github.com/google/dart-neats/tree/master/safe_url_check)'; +import 'src/safe_url_check.dart'; /// Check if [url] is available, without allowing access to private networks. /// @@ -63,114 +58,17 @@ const _defaultUserAgent = 'package:safe_url_check/$packageVersion ' Future safeUrlCheck( Uri url, { int maxRedirects = 8, - String userAgent = _defaultUserAgent, + String userAgent = defaultUserAgent, HttpClient? client, RetryOptions retryOptions = const RetryOptions(maxAttempts: 3), Duration timeout = const Duration(seconds: 90), }) async { - ArgumentError.checkNotNull(url, 'url'); - ArgumentError.checkNotNull(maxRedirects, 'maxRedirects'); - ArgumentError.checkNotNull(userAgent, 'userAgent'); - ArgumentError.checkNotNull(retryOptions, 'retryOptions'); - if (maxRedirects < 0) { - throw ArgumentError.value( - maxRedirects, - 'maxRedirects', - 'must be a positive integer', - ); - } - - try { - // Create client if one wasn't given. - var c = client; - c ??= HttpClient(); - try { - return await _safeUrlCheck( - url, - maxRedirects, - c, - userAgent, - retryOptions, - timeout, - ); - } finally { - // Close client, if it was created here. - if (client == null) { - c.close(force: true); - } - } - } on Exception { - return false; - } -} - -Future _safeUrlCheck( - Uri url, - int maxRedirects, - HttpClient client, - String userAgent, - RetryOptions retryOptions, - Duration timeout, -) async { - assert(maxRedirects >= 0); - - // If no scheme or not http or https, we fail. - if (!url.hasScheme || (!url.isScheme('http') && !url.isScheme('https'))) { - return false; - } - - final ips = await retryOptions.retry(() async { - final ips = await InternetAddress.lookup(url.host).timeout(timeout); - if (ips.isEmpty) { - throw Exception('DNS resolution failed'); - } - return ips; - }); - for (final ip in ips) { - // If given a loopback, linklocal or multicast IP, return false - if (ip.isLoopback || - ip.isLinkLocal || - ip.isMulticast || - isPrivateIpV4(ip) || - isUniqueLocalIpV6(ip)) { - return false; - } - } - - final response = await retryOptions.retry(() async { - // We can't use the HttpClient from dart:io with a custom socket, so instead - // of making a connection to one of the IPs resolved above, and specifying - // the host header, we rely on the OS caching DNS queries and not returning - // different IPs for a second lookup. - final request = await client.headUrl(url).timeout(timeout); - request.followRedirects = false; - request.headers.set(HttpHeaders.userAgentHeader, userAgent); - final response = await request.close().timeout(timeout); - await response.drain().catchError((e) => null).timeout(timeout); - if (500 <= response.statusCode && response.statusCode < 600) { - // retry again, when we hit a 5xx response - throw Exception('internal server error'); - } - return response; - }); - if (200 <= response.statusCode && response.statusCode < 300) { - return true; - } - if (response.isRedirect && - response.headers[HttpHeaders.locationHeader]!.isNotEmpty && - maxRedirects > 0) { - final loc = Uri.parse(response.headers[HttpHeaders.locationHeader]![0]); - final nextUri = url.resolveUri(loc); - return _safeUrlCheck( - nextUri, - maxRedirects - 1, - client, - userAgent, - retryOptions, - timeout, - ); - } - - // Response is 4xx, or some other unsupported code. - return false; + return doSafeUrlCheck( + url, + maxRedirects: maxRedirects, + userAgent: userAgent, + client: client, + retryOptions: retryOptions, + timeout: timeout, + ); } diff --git a/safe_url_check/lib/src/safe_url_check.dart b/safe_url_check/lib/src/safe_url_check.dart new file mode 100644 index 00000000..13a18e87 --- /dev/null +++ b/safe_url_check/lib/src/safe_url_check.dart @@ -0,0 +1,144 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:retry/retry.dart'; + +import 'private_ip.dart'; +import 'unique_local_ip.dart'; +import 'version.dart'; + +const defaultUserAgent = 'package:safe_url_check/$packageVersion ' + '(+https://github.com/google/dart-neats/tree/master/safe_url_check)'; + +Future doSafeUrlCheck( + Uri url, { + int maxRedirects = 8, + String userAgent = defaultUserAgent, + HttpClient? client, + RetryOptions retryOptions = const RetryOptions(maxAttempts: 3), + Duration timeout = const Duration(seconds: 90), +}) async { + ArgumentError.checkNotNull(url, 'url'); + ArgumentError.checkNotNull(maxRedirects, 'maxRedirects'); + ArgumentError.checkNotNull(userAgent, 'userAgent'); + ArgumentError.checkNotNull(retryOptions, 'retryOptions'); + if (maxRedirects < 0) { + throw ArgumentError.value( + maxRedirects, + 'maxRedirects', + 'must be a positive integer', + ); + } + + try { + // Create client if one wasn't given. + var c = client; + c ??= HttpClient(); + try { + return await SafeUrlChecker( + client: c, + userAgent: userAgent, + retryOptions: retryOptions, + timeout: timeout, + ).checkUrl(url, maxRedirects); + } finally { + // Close client, if it was created here. + if (client == null) { + c.close(force: true); + } + } + } on Exception { + return false; + } +} + +class SafeUrlChecker { + final HttpClient client; + final String userAgent; + final RetryOptions retryOptions; + final Duration timeout; + final bool skipLocalNetworkCheck; + + SafeUrlChecker({ + required this.client, + required this.userAgent, + required this.retryOptions, + required this.timeout, + this.skipLocalNetworkCheck = false, + }); + + Future checkUrl( + Uri url, + int maxRedirects, + ) async { + assert(maxRedirects >= 0); + + // If no scheme or not http or https, we fail. + if (!url.hasScheme || (!url.isScheme('http') && !url.isScheme('https'))) { + return false; + } + + final ips = await retryOptions.retry(() async { + final ips = await InternetAddress.lookup(url.host).timeout(timeout); + if (ips.isEmpty) { + throw Exception('DNS resolution failed'); + } + return ips; + }); + if (!skipLocalNetworkCheck) { + for (final ip in ips) { + // If given a loopback, linklocal or multicast IP, return false + if (ip.isLoopback || + ip.isLinkLocal || + ip.isMulticast || + isPrivateIpV4(ip) || + isUniqueLocalIpV6(ip)) { + return false; + } + } + } + + final response = await retryOptions.retry(() async { + // We can't use the HttpClient from dart:io with a custom socket, so instead + // of making a connection to one of the IPs resolved above, and specifying + // the host header, we rely on the OS caching DNS queries and not returning + // different IPs for a second lookup. + final request = await client.headUrl(url).timeout(timeout); + request.followRedirects = false; + request.headers.set(HttpHeaders.userAgentHeader, userAgent); + final response = await request.close().timeout(timeout); + await response.drain().catchError((e) => null).timeout(timeout); + if (500 <= response.statusCode && response.statusCode < 600) { + // retry again, when we hit a 5xx response + throw Exception('internal server error'); + } + return response; + }); + if (200 <= response.statusCode && response.statusCode < 300) { + return true; + } + if (response.isRedirect && + response.headers[HttpHeaders.locationHeader]!.isNotEmpty && + maxRedirects > 0) { + final loc = Uri.parse(response.headers[HttpHeaders.locationHeader]![0]); + final nextUri = url.resolveUri(loc); + return checkUrl(nextUri, maxRedirects - 1); + } + + // Response is 4xx, or some other unsupported code. + return false; + } +} diff --git a/safe_url_check/test/relative_redirect_test.dart b/safe_url_check/test/relative_redirect_test.dart new file mode 100644 index 00000000..4ea7ea08 --- /dev/null +++ b/safe_url_check/test/relative_redirect_test.dart @@ -0,0 +1,61 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:retry/retry.dart'; +import 'package:safe_url_check/src/safe_url_check.dart'; +import 'package:test/test.dart'; + +void main() { + late HttpServer server; + setUpAll(() async { + server = await HttpServer.bind(InternetAddress.anyIPv4, 0); + server.listen((e) async { + switch (e.requestedUri.path) { + case '/redirect/local': + e.response.statusCode = 303; + e.response.headers.set('location', 'target'); + await e.response.close(); + return; + case '/redirect/target': + e.response.write('OK'); + await e.response.close(); + return; + default: + e.response.statusCode = 404; + await e.response.close(); + } + }); + }); + + tearDownAll(() async { + await server.close(); + }); + + test('relative redirect', () async { + final client = HttpClient(); + final checker = SafeUrlChecker( + client: client, + userAgent: defaultUserAgent, + retryOptions: RetryOptions(), + timeout: Duration(seconds: 2), + skipLocalNetworkCheck: true, + ); + expect( + await checker.checkUrl( + Uri.parse('http://localhost:${server.port}/redirect/local'), 2), + isTrue); + }); +}