Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions gslib/commands/cp.py
Original file line number Diff line number Diff line change
Expand Up @@ -1043,8 +1043,15 @@ def _ConstructNameExpansionIteratorDstTupleIterator(self, src_url_strs_iter,
self.has_file_dst = self.has_file_dst or exp_dst_url.IsFileUrl()
self.has_cloud_dst = self.has_cloud_dst or exp_dst_url.IsCloudUrl()
self.provider_types.add(exp_dst_url.scheme)
self.combined_src_urls = list(itertools.chain(self.combined_src_urls,
src_url_str))
# combined_src_urls is only consumed by SeekAheadNameExpansionIterator,
# which is not used when reading sources from stdin (-I). Do not touch
# src_url_str in that case: it is a one-shot stdin iterator that is
# already being consumed by the NameExpansionIterator above, and
# materializing it here would silently drop every path that has not
# yet been read by the copy loop.
if not copy_helper_opts.read_args_from_stdin:
self.combined_src_urls = list(
itertools.chain(self.combined_src_urls, src_url_str))

yield name_expansion_iterator_dst_tuple

Expand Down
27 changes: 27 additions & 0 deletions gslib/tests/test_cp.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import binascii
import datetime
import gzip
import io
import logging
import os
import pickle
Expand Down Expand Up @@ -5117,6 +5118,32 @@ def test_storage_class_on_local_destination_fails(self):
suri(bucket_uri, 'object'), 'local_file'
])

def test_read_args_from_stdin_copies_all_paths(self):
"""Tests that cp -I copies every path from stdin, not just the first two.

Regression test for a bug where the stdin iterator was materialized into a
list while the name expansion iterator (which had already buffered two
entries for its plurality check) still held a reference to it, causing
all paths after the second one to be silently dropped.
"""
src_dir = self.CreateTempDir()
dst_dir = self.CreateTempDir()
num_files = 5
fpaths = [
self.CreateTempFile(tmpdir=src_dir,
file_name='f%d' % i,
contents=('data%d' % i).encode('ascii'))
for i in range(num_files)
]
stdin_lines = '\n'.join(fpaths) + '\n'
with mock.patch('sys.stdin', io.StringIO(stdin_lines)):
self.RunCommand('cp', ['-I', dst_dir])
copied = sorted(os.listdir(dst_dir))
self.assertEqual(copied, ['f%d' % i for i in range(num_files)])
for i in range(num_files):
with open(os.path.join(dst_dir, 'f%d' % i), 'rb') as f:
self.assertEqual(f.read(), ('data%d' % i).encode('ascii'))

def test_read_args_from_stdin_with_source_urls_fails(self):
bucket_uri = self.CreateBucket()
with self.assertRaisesRegex(
Expand Down