diff --git a/gslib/tests/test_rm.py b/gslib/tests/test_rm.py index 4a902c0d1..70df2cfd5 100644 --- a/gslib/tests/test_rm.py +++ b/gslib/tests/test_rm.py @@ -24,6 +24,7 @@ import sys from unittest import mock +from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_PREFIX from gslib.exception import NO_URLS_MATCHED_TARGET import gslib.tests.testcase as testcase @@ -751,6 +752,25 @@ def test_rm_multiple_nonexistent_objects(self): self.assertIn('2 files/objects could not be removed.', stderr) +class TestRmUnitTests(testcase.GsUtilUnitTestCase): + """Unit tests for gsutil rm command.""" + + def test_rm_no_arguments_raises(self): + with self.assertRaisesRegex( + CommandException, r'expects at least one URL\.'): + self.RunCommand('rm', []) + + def test_rm_stdin_with_arguments_raises(self): + with self.assertRaisesRegex( + CommandException, r'No arguments allowed with the -I flag\.'): + self.RunCommand('rm', ['-I', 'gs://bucket/obj']) + + def test_rm_local_file_fails(self): + with self.assertRaisesRegex( + CommandException, r'does not support "file://" URLs'): + self.RunCommand('rm', ['file:///tmp/foo.txt']) + + class TestRmUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for gsutil rm with shim.""" diff --git a/gslib/tests/test_rpo.py b/gslib/tests/test_rpo.py index ead6ae103..74e6ff1b8 100644 --- a/gslib/tests/test_rpo.py +++ b/gslib/tests/test_rpo.py @@ -20,6 +20,7 @@ import textwrap from gslib.commands.rpo import RpoCommand +from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.storage_url import StorageUrlFromString @@ -86,6 +87,40 @@ def test_invalid_subcommand_raises_error(self): CommandException, 'Invalid subcommand "blah", use get|set instead'): self.RunCommand('rpo', ['blah', 'DEFAULT', 'gs://boo*']) + def test_rpo_xml_api_disabled_raises(self): + # Mock api selector to return XML for gs. + with mock.patch( + 'gslib.cloud_api_delegator.CloudApiDelegator.GetApiSelector', + return_value=ApiSelector.XML): + with self.assertRaisesRegex( + CommandException, + r'command can only be with the Cloud Storage JSON API'): + self.RunCommand('rpo', ['get', 'gs://bucket']) + + def test_rpo_missing_args_raises(self): + # Min args is 2 (action subcommand + URL). + with self.assertRaisesRegex( + CommandException, r'requires at least 2 arguments'): + self.RunCommand('rpo', []) + + with self.assertRaisesRegex( + CommandException, r'requires at least 2 arguments'): + self.RunCommand('rpo', ['get']) + + with self.assertRaisesRegex( + CommandException, r'requires at least 2 arguments'): + self.RunCommand('rpo', ['set', 'ASYNC_TURBO']) + + def test_rpo_non_gcs_rejection(self): + s3_bucket = self.CreateBucket(provider='s3') + with self.assertRaisesRegex( + CommandException, r'command can only be used for GCS buckets'): + self.RunCommand('rpo', ['get', suri(s3_bucket)]) + + with self.assertRaisesRegex( + CommandException, r'command can only be used for GCS buckets'): + self.RunCommand('rpo', ['set', 'ASYNC_TURBO', suri(s3_bucket)]) + class TestRpoUnitWithShim(testcase.ShimUnitTestBase): diff --git a/gslib/tests/test_setmeta.py b/gslib/tests/test_setmeta.py index ca5a2ac51..338a2ce43 100644 --- a/gslib/tests/test_setmeta.py +++ b/gslib/tests/test_setmeta.py @@ -26,6 +26,7 @@ import six from gslib.commands import setmeta +from gslib.exception import CommandException from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 @@ -307,6 +308,66 @@ def test_setmeta_raises_error_if_not_provided_headers(self): ' -h flag. See "gsutil help setmeta" for more information.', stderr) +class TestSetMetaUnit(testcase.GsUtilUnitTestCase): + """Unit tests for gsutil setmeta command.""" + + def test_setmeta_no_headers_raises(self): + with self.assertRaisesRegex( + CommandException, r'requires one or more headers to be provided'): + self.RunCommand('setmeta', ['gs://bucket/obj']) + + def test_setmeta_bucket_url_raises(self): + with self.assertRaisesRegex( + CommandException, r'must name an object'): + self.RunCommand( + 'setmeta', ['-h', 'Content-Type:text/html', 'gs://bucket']) + + def test_setmeta_canned_acl_disallowed(self): + with self.assertRaisesRegex( + CommandException, r'no longer allows canned ACLs'): + self.RunCommand( + 'setmeta', ['-h', 'x-goog-acl:public-read', 'gs://bucket/obj']) + + with self.assertRaisesRegex( + CommandException, r'no longer allows canned ACLs'): + self.RunCommand( + 'setmeta', ['-h', 'x-amz-acl:public-read', 'gs://bucket/obj']) + + def test_setmeta_duplicate_headers_raises(self): + with self.assertRaisesRegex( + CommandException, r'Each header must appear at most once\.'): + self.RunCommand( + 'setmeta', + ['-h', 'Content-Type:text/html', '-h', 'content-type:text/plain', + 'gs://bucket/obj']) + + with self.assertRaisesRegex( + CommandException, r'Each header must appear at most once\.'): + self.RunCommand( + 'setmeta', + ['-h', 'Content-Type:text/html', '-h', 'Content-Type', + 'gs://bucket/obj']) + + def test_setmeta_invalid_header_raises(self): + with self.assertRaisesRegex( + CommandException, r'Invalid or disallowed header'): + self.RunCommand( + 'setmeta', ['-h', 'Invalid-Header:val', 'gs://bucket/obj']) + + def test_setmeta_non_ascii_header_raises(self): + with self.assertRaisesRegex( + CommandException, r'Invalid non-ASCII'): + self.RunCommand( + 'setmeta', ['-h', 'x-goog-meta-héader:val', 'gs://bucket/obj']) + + def test_setmeta_non_ascii_standard_value_raises(self): + with self.assertRaisesRegex( + CommandException, r'Invalid non-ASCII'): + self.RunCommand( + 'setmeta', + ['-h', 'Cache-Control:public, max-age=3600ã', 'gs://bucket/obj']) + + class TestSetMetaShim(testcase.ShimUnitTestBase): @mock.patch.object(setmeta.SetMetaCommand, 'RunCommand', new=mock.Mock()) diff --git a/gslib/tests/test_signurl.py b/gslib/tests/test_signurl.py index 8b9a16f26..957915a4f 100644 --- a/gslib/tests/test_signurl.py +++ b/gslib/tests/test_signurl.py @@ -277,6 +277,23 @@ def testSignUrlMissingUrlArgs(self): self.assertIn('The command requires a key file argument and one or more ' 'URL arguments', stderr) + def testSignUrlNonGsUrlRaises(self): + """Tests signurl with a non-gs:// URL.""" + stderr = self.RunGsUtil( + ['signurl', self._GetJSONKsFile(), 's3://bucket/obj'], + return_stderr=True, + expected_status=1) + self.assertIn('Can only create signed urls from gs:// urls', stderr) + + def testSignUrlResumableBucketRaises(self): + """Tests signurl with RESUMABLE method on a bucket URL.""" + stderr = self.RunGsUtil( + ['signurl', '-m', 'RESUMABLE', '-r', 'us-east1', + self._GetJSONKsFile(), 'gs://bucket'], + return_stderr=True, + expected_status=1) + self.assertIn('Resumable signed URLs require an object name.', stderr) + @unittest.skipUnless(HAVE_CRYPTO, 'signurl requires cryptography library.') class UnitTestSignUrl(testcase.GsUtilUnitTestCase): diff --git a/gslib/tests/test_storage_url.py b/gslib/tests/test_storage_url.py index eea7440ad..19458b5fd 100644 --- a/gslib/tests/test_storage_url.py +++ b/gslib/tests/test_storage_url.py @@ -20,6 +20,7 @@ from __future__ import unicode_literals import os +import stat import sys from gslib.exception import CommandException @@ -171,3 +172,65 @@ def test_not_raising_error_if_multiple_objects_with_recursion(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b/p'])) storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=True) + + def test_urls_are_for_single_provider(self): + self.assertTrue( + storage_url.UrlsAreForSingleProvider(['gs://b1', 'gs://b2/o'])) + self.assertFalse( + storage_url.UrlsAreForSingleProvider(['gs://b1', 's3://b2'])) + self.assertFalse( + storage_url.UrlsAreForSingleProvider([])) + + def test_storage_url_with_generation_or_version(self): + url = storage_url.StorageUrlFromString('gs://bucket/obj#12345') + self.assertTrue(url.IsCloudUrl()) + self.assertEqual(url.generation, '12345') + self.assertEqual(url.object_name, 'obj') + + url = storage_url.StorageUrlFromString('s3://bucket/obj#versionid') + self.assertTrue(url.IsCloudUrl()) + self.assertEqual(url.generation, 'versionid') + self.assertEqual(url.object_name, 'obj') + + def test_raises_error_for_invalid_root_level_object_name(self): + with self.assertRaises(InvalidUrlError): + storage_url.StorageUrlFromString('gs://bucket/.') + with self.assertRaises(InvalidUrlError): + storage_url.StorageUrlFromString('gs://bucket/..') + + def test_raises_error_for_unrecognized_scheme(self): + with self.assertRaisesRegex( + InvalidUrlError, r'Unrecognized scheme "http"'): + storage_url.StorageUrlFromString('http://example.com/file') + + def test_storage_url_equality_and_hashing(self): + url1 = storage_url.StorageUrlFromString('gs://bucket/obj') + url2 = storage_url.StorageUrlFromString('gs://bucket/obj') + url3 = storage_url.StorageUrlFromString('gs://bucket/other') + + self.assertEqual(url1, url2) + self.assertNotEqual(url1, url3) + self.assertEqual(hash(url1), hash(url2)) + self.assertNotEqual(hash(url1), hash(url3)) + + def test_file_url_stream(self): + url = storage_url.StorageUrlFromString('-') + self.assertTrue(url.IsStream()) + + url2 = storage_url.StorageUrlFromString('file://-') + self.assertTrue(url2.IsStream()) + + url3 = storage_url.StorageUrlFromString('gs://bucket/obj') + with self.assertRaises(NotImplementedError): + url3.IsStream() + + @mock.patch('os.stat') + def test_file_url_fifo(self, mock_os_stat): + mock_os_stat.return_value.st_mode = stat.S_IFIFO + url = storage_url.StorageUrlFromString('file:///tmp/fake-fifo') + self.assertTrue(url.IsFifo()) + + # CloudUrl does not support IsFifo + url2 = storage_url.StorageUrlFromString('gs://bucket/obj') + with self.assertRaises(NotImplementedError): + url2.IsFifo() diff --git a/gslib/tests/test_tabcomplete.py b/gslib/tests/test_tabcomplete.py index c5f22a3db..43043b665 100644 --- a/gslib/tests/test_tabcomplete.py +++ b/gslib/tests/test_tabcomplete.py @@ -20,11 +20,18 @@ from __future__ import unicode_literals import os -import time import sys +import time +from unittest import mock from gslib.command import CreateOrGetGsutilLogger from gslib.tab_complete import CloudObjectCompleter +from gslib.tab_complete import CloudOrLocalObjectCompleter +from gslib.tab_complete import CompleterType +from gslib.tab_complete import LocalObjectCompleter +from gslib.tab_complete import LocalObjectOrCannedACLCompleter +from gslib.tab_complete import MakeCompleter +from gslib.tab_complete import NoOpCompleter from gslib.tab_complete import TAB_COMPLETE_CACHE_TTL from gslib.tab_complete import TabCompletionCache import gslib.tests.testcase as testcase @@ -366,3 +373,63 @@ def test_prefix_caching_partial_results(self): results = completer(request) self.assertEqual([str(object_uri)], results) + + def test_make_completer(self): + api = self.MakeGsUtilApi() + self.assertIsInstance( + MakeCompleter(CompleterType.NO_OP, api), NoOpCompleter) + self.assertIsInstance( + MakeCompleter(CompleterType.LOCAL_OBJECT, api), LocalObjectCompleter) + self.assertIsInstance( + MakeCompleter(CompleterType.LOCAL_OBJECT_OR_CANNED_ACL, api), + LocalObjectOrCannedACLCompleter) + self.assertIsInstance( + MakeCompleter(CompleterType.CLOUD_OR_LOCAL_OBJECT, api), + CloudOrLocalObjectCompleter) + + with self.assertRaisesRegex(RuntimeError, 'Unknown completer'): + MakeCompleter('invalid_completer', api) + + def test_no_op_completer(self): + completer = NoOpCompleter() + self.assertEqual([], completer('any_prefix')) + + @mock.patch.object(LocalObjectCompleter, '__call__', autospec=True) + def test_local_object_or_canned_acl_completer(self, mock_local_completer): + mock_local_completer.return_value = ['/tmp/pubfile'] + completer = LocalObjectOrCannedACLCompleter() + results = completer('pub') + # Should contain local file and canned ACLs starting with 'pub': + self.assertIn('/tmp/pubfile', results) + self.assertIn('public-read', results) + self.assertIn('public-read-write', results) + + @mock.patch.object(LocalObjectCompleter, '__call__', autospec=True) + @mock.patch.object(CloudObjectCompleter, '__call__', autospec=True) + def test_cloud_or_local_object_completer(self, mock_cloud_completer, + mock_local_completer): + completer = CloudOrLocalObjectCompleter(self.MakeGsUtilApi()) + + # Cloud URL prefix -> routes to CloudObjectCompleter + completer('gs://') + mock_cloud_completer.assert_called_once() + mock_local_completer.assert_not_called() + + # Local file path prefix -> routes to LocalObjectCompleter + mock_cloud_completer.reset_mock() + completer('/tmp/') + mock_local_completer.assert_called_once() + mock_cloud_completer.assert_not_called() + + @mock.patch('gslib.tab_complete.CloudListingRequestThread', autospec=True) + @mock.patch('argcomplete.warn') + def test_cloud_object_completer_timeout(self, mock_warn, mock_thread_class): + mock_thread_class.return_value.is_alive.return_value = True + + completer = CloudObjectCompleter(self.MakeGsUtilApi()) + with SetBotoConfigForTest([('GSUtil', 'tab_completion_timeout', '1'), + ('GSUtil', 'state_dir', self.CreateTempDir())]): + results = completer('gs://prefix') + + self.assertEqual([], results) + mock_warn.assert_called_once() diff --git a/gslib/tests/test_ui.py b/gslib/tests/test_ui.py index 78396b8b6..628950849 100644 --- a/gslib/tests/test_ui.py +++ b/gslib/tests/test_ui.py @@ -28,6 +28,7 @@ import os import pickle +from unittest import mock import crcmod import six @@ -52,6 +53,7 @@ from gslib.thread_message import MetadataMessage from gslib.thread_message import ProducerThreadMessage from gslib.thread_message import ProgressMessage +from gslib.thread_message import RetryableErrorMessage from gslib.thread_message import SeekAheadMessage from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths @@ -59,6 +61,7 @@ from gslib.tracker_file import TrackerFileType from gslib.ui_controller import BytesToFixedWidthString from gslib.ui_controller import DataManager +from gslib.ui_controller import EstimationSource from gslib.ui_controller import MainThreadUIQueue from gslib.ui_controller import MetadataManager from gslib.ui_controller import UIController @@ -1570,3 +1573,70 @@ def test_ui_spinner(self): # old_spinner3. self.assertNotEqual(old_spinner3, current_spinner) self.assertNotEqual(old_spinner1, current_spinner) + + def test_ui_estimation_source_priority(self): + stream = six.StringIO() + start_time = self.start_time + ui_controller = UIController(custom_time=start_time) + status_queue = MainThreadUIQueue(stream, ui_controller) + + # 1. Initialize data manager using a FileMessage + PutToQueueWithTimeout( + status_queue, + FileMessage(StorageUrlFromString('foo'), + None, + start_time, + message_type=FileMessage.FILE_UPLOAD, + size=100)) + self.assertEqual(ui_controller.manager.num_objects, 1) + self.assertEqual(ui_controller.manager.total_size, 100) + self.assertEqual(ui_controller.manager.num_objects_source, + EstimationSource.INDIVIDUAL_MESSAGES) + + # 2. Put a SeekAheadMessage (higher priority than INDIVIDUAL_MESSAGES) + PutToQueueWithTimeout( + status_queue, SeekAheadMessage(5, 500, start_time + 1)) + self.assertEqual(ui_controller.manager.num_objects, 5) + self.assertEqual(ui_controller.manager.total_size, 500) + self.assertEqual(ui_controller.manager.num_objects_source, + EstimationSource.SEEK_AHEAD_THREAD) + + # 3. Put a lower-priority ProducerThreadMessage. + # finished=False -> PRODUCER_THREAD_ESTIMATE (3), which has lower + # priority than SEEK_AHEAD_THREAD (2), so it should be ignored. + PutToQueueWithTimeout( + status_queue, + ProducerThreadMessage(10, 1000, start_time + 2, finished=False)) + self.assertEqual(ui_controller.manager.num_objects, 5) + self.assertEqual(ui_controller.manager.total_size, 500) + self.assertEqual(ui_controller.manager.num_objects_source, + EstimationSource.SEEK_AHEAD_THREAD) + + # 4. Put a higher-priority ProducerThreadMessage. + # finished=True -> PRODUCER_THREAD_FINAL (1), which has higher + # priority than SEEK_AHEAD_THREAD (2), so it should overwrite. + PutToQueueWithTimeout( + status_queue, + ProducerThreadMessage(20, 2000, start_time + 3, finished=True)) + self.assertEqual(ui_controller.manager.num_objects, 20) + self.assertEqual(ui_controller.manager.total_size, 2000) + self.assertEqual(ui_controller.manager.num_objects_source, + EstimationSource.PRODUCER_THREAD_FINAL) + + @mock.patch('gslib.ui_controller.LogRetryableError') + def test_ui_retryable_error_message(self, mock_log_retryable): + stream = six.StringIO() + start_time = self.start_time + ui_controller = UIController(custom_time=start_time) + status_queue = MainThreadUIQueue(stream, ui_controller) + + # Put a metadata message first to initialize the manager + PutToQueueWithTimeout(status_queue, MetadataMessage(start_time)) + + # Send a RetryableErrorMessage + error_msg = RetryableErrorMessage( + Exception('fake error'), start_time + 1, num_retries=1) + PutToQueueWithTimeout(status_queue, error_msg) + + # Verify that the metrics log handler was invoked + mock_log_retryable.assert_called_once_with(error_msg) diff --git a/gslib/tests/test_user_agent_helper.py b/gslib/tests/test_user_agent_helper.py index 31af55e9a..7031afd75 100644 --- a/gslib/tests/test_user_agent_helper.py +++ b/gslib/tests/test_user_agent_helper.py @@ -140,3 +140,12 @@ def testCloudSdk(self, mock_invoked, mock_version): mock_invoked.return_value = False mock_version.return_value = '500.1' self.assertRegex(GetUserAgent(['help']), r"command/help$") + + @mock.patch.object(system_util, 'CloudSdkVersion') + @mock.patch.object(system_util, 'InvokedViaCloudSdk') + def testCloudSdkNoVersion(self, mock_invoked, mock_version): + mock_invoked.return_value = True + mock_version.return_value = None + ua = GetUserAgent(['help']) + self.assertRegex(ua, r"google-cloud-sdk$") + self.assertNotRegex(ua, r"google-cloud-sdk/") diff --git a/gslib/tests/test_util.py b/gslib/tests/test_util.py index bb21f9531..e0a2df263 100644 --- a/gslib/tests/test_util.py +++ b/gslib/tests/test_util.py @@ -428,3 +428,20 @@ def testGetMaxConcurrentCompressedUploadsMinimum(self, mock_config): self.assertEqual(boto_util.GetMaxConcurrentCompressedUploads(), 1) mock_config.return_value = -1 self.assertEqual(boto_util.GetMaxConcurrentCompressedUploads(), 1) + + def testCalculateThroughput(self): + self.assertEqual(unit_util.CalculateThroughput(100, 10.0), 10.0) + # Elapsed time < 0.01 is treated as 0.01 + self.assertEqual(unit_util.CalculateThroughput(100, 0.005), 10000.0) + + def testDivideAndCeil(self): + self.assertEqual(unit_util.DivideAndCeil(10, 3), 4) + self.assertEqual(unit_util.DivideAndCeil(10, 5), 2) + self.assertEqual(unit_util.DivideAndCeil(0, 5), 0) + + def testPercentile(self): + self.assertEqual(unit_util.Percentile([10, 20, 30], 0.5), 20.0) + self.assertEqual(unit_util.Percentile([10, 20, 30], 0.25), 15.0) + self.assertEqual(unit_util.Percentile([10, 20, 30], 0.0), 10.0) + self.assertEqual(unit_util.Percentile([10, 20, 30], 1.0), 30.0) + self.assertIsNone(unit_util.Percentile([], 0.5)) diff --git a/gslib/tests/test_wildcard_iterator.py b/gslib/tests/test_wildcard_iterator.py index 64f9ca1dd..8aa981945 100644 --- a/gslib/tests/test_wildcard_iterator.py +++ b/gslib/tests/test_wildcard_iterator.py @@ -26,6 +26,7 @@ from __future__ import division from __future__ import unicode_literals +import os import re import six import tempfile @@ -33,10 +34,13 @@ from gslib import wildcard_iterator from gslib.exception import InvalidUrlError from gslib.storage_url import ContainsWildcard +from gslib.utils import system_util from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetDummyProjectForUnitTest +from gslib.tests.util import unittest +from unittest import mock class CloudWildcardIteratorTests(testcase.GsUtilUnitTestCase): @@ -366,6 +370,38 @@ def testDoubleWildcardBeforeAndAfterPrefix(self): self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) + def testWildcardWithGenerationMatch(self): + """Tests that wildcard matching works with a specific generation.""" + bucket_uri = self.CreateBucket() + key_uri = self.CreateObject(bucket_uri=bucket_uri, + object_name='abcd', + contents=b'test contents') + + boto_key = key_uri.get_key() + boto_key.generation = 123456 + + # Get the object and its generation + blrs = list(self._test_wildcard_iterator( + bucket_uri.clone_replace_name('abcd')).IterAll()) + self.assertEqual(1, len(blrs)) + blr = blrs[0] + self.assertTrue(blr.IsObject()) + generation = blr.root_object.generation + self.assertEqual(generation, 123456) + + # 1. Query with exact generation + url_with_gen = '%s/abc*#%s' % (suri(bucket_uri), generation) + blrs_match = list(self._test_wildcard_iterator(url_with_gen).IterAll()) + self.assertEqual(1, len(blrs_match)) + self.assertEqual('%s/abcd#%s' % (suri(bucket_uri), generation), + blrs_match[0].url_string) + + # 2. Query with wrong generation + url_with_wrong_gen = '%s/abc*#999999' % suri(bucket_uri) + blrs_mismatch = list(self._test_wildcard_iterator( + url_with_wrong_gen).IterAll()) + self.assertEqual(0, len(blrs_mismatch)) + class FileIteratorTests(testcase.GsUtilUnitTestCase): """Unit tests for FileWildcardIterator.""" @@ -525,3 +561,55 @@ def testExcludeTupleButExcludeDirFalse(self): for u in self._test_wildcard_iterator(uri, exclude_tuple=exclude_tuple). IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) + + @unittest.skipIf(system_util.IS_WINDOWS, 'Symlinks not supported on Windows') + def testSymlinkHandling(self): + # 1. Create a symlink to 'abcd' + target_path = os.path.join(self.test_dir, 'abcd') + link_path = os.path.join(self.test_dir, 'link_abcd') + os.symlink(target_path, link_path) + + # Create a symlink to 'dir1' + dir_target_path = os.path.join(self.test_dir, 'dir1') + dir_link_path = os.path.join(self.test_dir, 'link_dir1') + os.symlink(dir_target_path, dir_link_path) + + # 2. Iterate without ignore_symlinks=True (default is False) + wildcard_url_str = suri(self.test_dir, '*') + iterator = wildcard_iterator.CreateWildcardIterator( + wildcard_url_str, self.MakeGsUtilApi()) + results = set(blr.url_string for blr in iterator) + + # It should include the symlink files + self.assertIn(suri(link_path), results) + self.assertIn(suri(dir_link_path), results) + + # 3. Iterate with ignore_symlinks=True + mock_logger = mock.Mock() + iterator_ignored = wildcard_iterator.CreateWildcardIterator( + wildcard_url_str, self.MakeGsUtilApi(), ignore_symlinks=True, + logger=mock_logger) + results_ignored = set(blr.url_string for blr in iterator_ignored) + + # It should NOT include the symlink files + self.assertNotIn(suri(link_path), results_ignored) + self.assertNotIn(suri(dir_link_path), results_ignored) + + # Verify that info log was called for skipping + mock_logger.info.assert_any_call( + 'Skipping symbolic link %s...', link_path) + mock_logger.info.assert_any_call( + 'Skipping symbolic link %s...', dir_link_path) + + # 4. Check recursive walk log output when it skips dir symlink + mock_logger.reset_mock() + recursive_wildcard_str = suri(self.test_dir, '**', '*') + recursive_iterator = wildcard_iterator.CreateWildcardIterator( + recursive_wildcard_str, self.MakeGsUtilApi(), ignore_symlinks=True, + logger=mock_logger) + + list(recursive_iterator) + + # Ensure it logged skipped symlink directories during walk + mock_logger.info.assert_any_call( + 'Skipping symlink directory "%s"', dir_link_path) diff --git a/gslib/tests/test_wrapped_credentials.py b/gslib/tests/test_wrapped_credentials.py index ece95c173..c54972a01 100644 --- a/gslib/tests/test_wrapped_credentials.py +++ b/gslib/tests/test_wrapped_credentials.py @@ -16,6 +16,7 @@ import datetime import json +import os import httplib2 from google.auth import aws @@ -351,3 +352,46 @@ def testFromJsonExternalAccountAuthorizedUserCredentials(self): self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, external_account_authorized_user.Credentials) + + def testInvalidCredentialsRaiseTypeError(self): + with self.assertRaises(TypeError): + WrappedCredentials(None) + with self.assertRaises(TypeError): + WrappedCredentials(object()) + + def testForExternalAccountFromFile(self): + info = { + "type": "external_account", + "audience": "foo", + "subject_token_type": "bar", + "token_url": "https://sts.googleapis.com", + "credential_source": {"url": "google.com"} + } + temp_dir = self.CreateTempDir() + temp_file = os.path.join(temp_dir, 'creds.json') + with open(temp_file, 'w') as f: + json.dump(info, f) + + creds = WrappedCredentials.for_external_account(temp_file) + self.assertIsInstance(creds, WrappedCredentials) + self.assertIsInstance(creds._base, identity_pool.Credentials) + + def testForExternalAccountAuthorizedUserFromFile(self): + info = { + "type": "external_account_authorized_user", + "audience": "//iam.googleapis.com/locations/global/workforcePools/1", + "refresh_token": "refreshToken", + "token_url": "https://sts.googleapis.com/v1/oauth/token", + "token_info_url": "https://sts.googleapis.com/v1/instrospect", + "client_id": "clientId", + "client_secret": "clientSecret" + } + temp_dir = self.CreateTempDir() + temp_file = os.path.join(temp_dir, 'creds.json') + with open(temp_file, 'w') as f: + json.dump(info, f) + + creds = WrappedCredentials.for_external_account_authorized_user(temp_file) + self.assertIsInstance(creds, WrappedCredentials) + self.assertIsInstance(creds._base, + external_account_authorized_user.Credentials)