From e246c209f850e23e0f812a28f8348f332956865d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20So=C3=B3s?= Date: Mon, 4 Dec 2023 17:21:14 +0100 Subject: [PATCH] Allow relative URLs in the redirect location header. (#229) * Allow relative URLs in the redirect location header. * updated code + minimal test * Hide main method with option to skip local network check + local test. * @visibleForTesting * keep _safeUrlCheck without context class --- safe_url_check/CHANGELOG.md | 3 + safe_url_check/lib/safe_url_check.dart | 120 ++------------ safe_url_check/lib/src/safe_url_check.dart | 146 ++++++++++++++++++ safe_url_check/lib/src/version.dart | 2 +- safe_url_check/pubspec.yaml | 5 +- .../test/relative_redirect_test.dart | 61 ++++++++ safe_url_check/test/safe_url_check_test.dart | 1 + 7 files changed, 225 insertions(+), 113 deletions(-) create mode 100644 safe_url_check/lib/src/safe_url_check.dart create mode 100644 safe_url_check/test/relative_redirect_test.dart diff --git a/safe_url_check/CHANGELOG.md b/safe_url_check/CHANGELOG.md index 91fc6051..d2292af1 100644 --- a/safe_url_check/CHANGELOG.md +++ b/safe_url_check/CHANGELOG.md @@ -1,3 +1,6 @@ +## v.1.1.2 + * Allow relative URLs in the redirect `location` header. + ## v1.1.1 * Added `topics` to `pubspec.yaml`. diff --git a/safe_url_check/lib/safe_url_check.dart b/safe_url_check/lib/safe_url_check.dart index 0baaa076..ff8986d5 100644 --- a/safe_url_check/lib/safe_url_check.dart +++ b/safe_url_check/lib/safe_url_check.dart @@ -39,12 +39,7 @@ import 'dart:async'; import 'package:retry/retry.dart'; -import 'src/private_ip.dart'; -import 'src/unique_local_ip.dart'; -import 'src/version.dart'; - -const _defaultUserAgent = 'package:safe_url_check/$packageVersion ' - '(+https://github.com/google/dart-neats/tree/master/safe_url_check)'; +import 'src/safe_url_check.dart'; /// Check if [url] is available, without allowing access to private networks. /// @@ -63,112 +58,17 @@ const _defaultUserAgent = 'package:safe_url_check/$packageVersion ' Future safeUrlCheck( Uri url, { int maxRedirects = 8, - String userAgent = _defaultUserAgent, + String userAgent = defaultUserAgent, HttpClient? client, RetryOptions retryOptions = const RetryOptions(maxAttempts: 3), Duration timeout = const Duration(seconds: 90), }) async { - ArgumentError.checkNotNull(url, 'url'); - ArgumentError.checkNotNull(maxRedirects, 'maxRedirects'); - ArgumentError.checkNotNull(userAgent, 'userAgent'); - ArgumentError.checkNotNull(retryOptions, 'retryOptions'); - if (maxRedirects < 0) { - throw ArgumentError.value( - maxRedirects, - 'maxRedirects', - 'must be a positive integer', - ); - } - - try { - // Create client if one wasn't given. - var c = client; - c ??= HttpClient(); - try { - return await _safeUrlCheck( - url, - maxRedirects, - c, - userAgent, - retryOptions, - timeout, - ); - } finally { - // Close client, if it was created here. - if (client == null) { - c.close(force: true); - } - } - } on Exception { - return false; - } -} - -Future _safeUrlCheck( - Uri url, - int maxRedirects, - HttpClient client, - String userAgent, - RetryOptions retryOptions, - Duration timeout, -) async { - assert(maxRedirects >= 0); - - // If no scheme or not http or https, we fail. - if (!url.hasScheme || (!url.isScheme('http') && !url.isScheme('https'))) { - return false; - } - - final ips = await retryOptions.retry(() async { - final ips = await InternetAddress.lookup(url.host).timeout(timeout); - if (ips.isEmpty) { - throw Exception('DNS resolution failed'); - } - return ips; - }); - for (final ip in ips) { - // If given a loopback, linklocal or multicast IP, return false - if (ip.isLoopback || - ip.isLinkLocal || - ip.isMulticast || - isPrivateIpV4(ip) || - isUniqueLocalIpV6(ip)) { - return false; - } - } - - final response = await retryOptions.retry(() async { - // We can't use the HttpClient from dart:io with a custom socket, so instead - // of making a connection to one of the IPs resolved above, and specifying - // the host header, we rely on the OS caching DNS queries and not returning - // different IPs for a second lookup. - final request = await client.headUrl(url).timeout(timeout); - request.followRedirects = false; - request.headers.set(HttpHeaders.userAgentHeader, userAgent); - final response = await request.close().timeout(timeout); - await response.drain().catchError((e) => null).timeout(timeout); - if (500 <= response.statusCode && response.statusCode < 600) { - // retry again, when we hit a 5xx response - throw Exception('internal server error'); - } - return response; - }); - if (200 <= response.statusCode && response.statusCode < 300) { - return true; - } - if (response.isRedirect && - response.headers[HttpHeaders.locationHeader]!.isNotEmpty && - maxRedirects > 0) { - return _safeUrlCheck( - Uri.parse(response.headers[HttpHeaders.locationHeader]![0]), - maxRedirects - 1, - client, - userAgent, - retryOptions, - timeout, - ); - } - - // Response is 4xx, or some other unsupported code. - return false; + return doSafeUrlCheck( + url, + maxRedirects: maxRedirects, + userAgent: userAgent, + client: client, + retryOptions: retryOptions, + timeout: timeout, + ); } diff --git a/safe_url_check/lib/src/safe_url_check.dart b/safe_url_check/lib/src/safe_url_check.dart new file mode 100644 index 00000000..317feeb0 --- /dev/null +++ b/safe_url_check/lib/src/safe_url_check.dart @@ -0,0 +1,146 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:retry/retry.dart'; + +import 'private_ip.dart'; +import 'unique_local_ip.dart'; +import 'version.dart'; + +const defaultUserAgent = 'package:safe_url_check/$packageVersion ' + '(+https://github.com/google/dart-neats/tree/master/safe_url_check)'; + +Future doSafeUrlCheck( + Uri url, { + int maxRedirects = 8, + String userAgent = defaultUserAgent, + HttpClient? client, + RetryOptions retryOptions = const RetryOptions(maxAttempts: 3), + Duration timeout = const Duration(seconds: 90), + @visibleForTesting bool skipLocalNetworkCheck = false, +}) async { + ArgumentError.checkNotNull(url, 'url'); + ArgumentError.checkNotNull(maxRedirects, 'maxRedirects'); + ArgumentError.checkNotNull(userAgent, 'userAgent'); + ArgumentError.checkNotNull(retryOptions, 'retryOptions'); + if (maxRedirects < 0) { + throw ArgumentError.value( + maxRedirects, + 'maxRedirects', + 'must be a positive integer', + ); + } + + try { + // Create client if one wasn't given. + var c = client; + c ??= HttpClient(); + try { + return await _safeUrlCheck( + url, + maxRedirects, + client: c, + userAgent: userAgent, + retryOptions: retryOptions, + timeout: timeout, + skipLocalNetworkCheck: skipLocalNetworkCheck, + ); + } finally { + // Close client, if it was created here. + if (client == null) { + c.close(force: true); + } + } + } on Exception { + return false; + } +} + +Future _safeUrlCheck( + Uri url, + int maxRedirects, { + required HttpClient client, + required String userAgent, + required RetryOptions retryOptions, + required Duration timeout, + required bool skipLocalNetworkCheck, +}) async { + assert(maxRedirects >= 0); + + // If no scheme or not http or https, we fail. + if (!url.hasScheme || (!url.isScheme('http') && !url.isScheme('https'))) { + return false; + } + + final ips = await retryOptions.retry(() async { + final ips = await InternetAddress.lookup(url.host).timeout(timeout); + if (ips.isEmpty) { + throw Exception('DNS resolution failed'); + } + return ips; + }); + if (!skipLocalNetworkCheck) { + for (final ip in ips) { + // If given a loopback, linklocal or multicast IP, return false + if (ip.isLoopback || + ip.isLinkLocal || + ip.isMulticast || + isPrivateIpV4(ip) || + isUniqueLocalIpV6(ip)) { + return false; + } + } + } + + final response = await retryOptions.retry(() async { + // We can't use the HttpClient from dart:io with a custom socket, so instead + // of making a connection to one of the IPs resolved above, and specifying + // the host header, we rely on the OS caching DNS queries and not returning + // different IPs for a second lookup. + final request = await client.headUrl(url).timeout(timeout); + request.followRedirects = false; + request.headers.set(HttpHeaders.userAgentHeader, userAgent); + final response = await request.close().timeout(timeout); + await response.drain().catchError((e) => null).timeout(timeout); + if (500 <= response.statusCode && response.statusCode < 600) { + // retry again, when we hit a 5xx response + throw Exception('internal server error'); + } + return response; + }); + if (200 <= response.statusCode && response.statusCode < 300) { + return true; + } + if (response.isRedirect && + response.headers[HttpHeaders.locationHeader]!.isNotEmpty && + maxRedirects > 0) { + final loc = Uri.parse(response.headers[HttpHeaders.locationHeader]![0]); + final nextUri = url.resolveUri(loc); + return _safeUrlCheck( + nextUri, + maxRedirects - 1, + client: client, + retryOptions: retryOptions, + userAgent: userAgent, + timeout: timeout, + skipLocalNetworkCheck: skipLocalNetworkCheck, + ); + } + + // Response is 4xx, or some other unsupported code. + return false; +} diff --git a/safe_url_check/lib/src/version.dart b/safe_url_check/lib/src/version.dart index cf2dd1f8..28abf73c 100644 --- a/safe_url_check/lib/src/version.dart +++ b/safe_url_check/lib/src/version.dart @@ -1,2 +1,2 @@ // Generated code. Do not modify. -const packageVersion = '1.1.0'; +const packageVersion = '1.1.2'; diff --git a/safe_url_check/pubspec.yaml b/safe_url_check/pubspec.yaml index 5596e889..cc525eaa 100644 --- a/safe_url_check/pubspec.yaml +++ b/safe_url_check/pubspec.yaml @@ -1,5 +1,5 @@ name: safe_url_check -version: 1.1.1 +version: 1.1.2 description: >- Check if an untrusted URL is broken, without allowing connections to a private IP address. @@ -14,10 +14,11 @@ environment: sdk: '>=2.12.0 <3.0.0' dependencies: + meta: ^1.11.0 retry: ^3.0.0+1 dev_dependencies: test: ^1.5.1 - lints: ^2.0.1 + lints: ^3.0.0 build_runner: ^2.3.3 build_version: ^2.0.1 diff --git a/safe_url_check/test/relative_redirect_test.dart b/safe_url_check/test/relative_redirect_test.dart new file mode 100644 index 00000000..dc006690 --- /dev/null +++ b/safe_url_check/test/relative_redirect_test.dart @@ -0,0 +1,61 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'dart:io'; + +import 'package:retry/retry.dart'; +import 'package:safe_url_check/src/safe_url_check.dart'; +import 'package:test/test.dart'; + +void main() { + late HttpServer server; + setUpAll(() async { + server = await HttpServer.bind(InternetAddress.anyIPv4, 0); + server.listen((e) async { + switch (e.requestedUri.path) { + case '/redirect/local': + e.response.statusCode = 303; + e.response.headers.set('location', 'target'); + await e.response.close(); + return; + case '/redirect/target': + e.response.write('OK'); + await e.response.close(); + return; + default: + e.response.statusCode = 404; + await e.response.close(); + } + }); + }); + + tearDownAll(() async { + await server.close(); + }); + + test('relative redirect', () async { + final client = HttpClient(); + expect( + await doSafeUrlCheck( + Uri.parse('http://localhost:${server.port}/redirect/local'), + maxRedirects: 2, + client: client, + userAgent: defaultUserAgent, + retryOptions: RetryOptions(), + timeout: Duration(seconds: 2), + skipLocalNetworkCheck: true, + ), + isTrue); + }); +} diff --git a/safe_url_check/test/safe_url_check_test.dart b/safe_url_check/test/safe_url_check_test.dart index b2d7f0ab..b06a3f88 100644 --- a/safe_url_check/test/safe_url_check_test.dart +++ b/safe_url_check/test/safe_url_check_test.dart @@ -27,5 +27,6 @@ void main() { testValidUrl('https://google.com'); testValidUrl('https://github.com'); testValidUrl('https://github.com/google/dart-neats.git'); + testValidUrl('https://httpbin.org/redirect-to?url=status%2F200'); testInvalidUrl('https://github.com/google/dart-neats.git/bad-url'); }