fixup! FEAT: implemented sas7bdat adapter using Pandas

Author: gdementen

larray_editor/arrayadapter.py

@@ -3693,7 +3693,10 @@ def __init__(self, data, attributes=None):
         super().__init__(data, attributes=attributes)
         reader = data
         index_cols = set(reader.index) if reader.index is not None else set()
-        self._chunk_size = reader.chunksize or 1_000
+        default_chunksize = max(1_000_000 // reader.row_length, 1)
+        chunksize = reader.chunksize or default_chunksize
+        logger.debug(f'{chunksize=}')
+        self._chunk_size = chunksize
         self._colnames = [col for col in reader.column_names
                           if col not in index_cols]
 
@@ -3725,12 +3728,23 @@ def get_values(self, h_start, v_start, h_stop, v_stop):
             reader.close()
             reader.__init__(fpath, **kwargs)
             current_row = reader._current_row_in_file_index
+        expected_num_rows = v_stop - v_start
+        expected_num_cols = h_stop - h_start
+
         # skip to v_start
         num_rows_to_skip = v_start - self._chunk_size - current_row
         if num_rows_to_skip > 0:
             logger.debug(f"must skip {num_rows_to_skip} rows")
-            if num_rows_to_skip > 10_000_000:
-                raise RuntimeError('File is too large to display non top rows')
+            if num_rows_to_skip * reader.row_lenth > 10_000_000:
+                # An exception would be eaten by the adapter so the user
+                # would never see it
+                msg = 'File is too large to display non top rows'
+                first_row = [[msg] + [''] * (expected_num_cols - 1)]
+                other_rows = [
+                    [''] * expected_num_cols
+                ] * (expected_num_rows - 1)
+                return first_row + other_rows
+
         while current_row < v_start - self._chunk_size:
             reader.read(self._chunk_size)
             current_row = reader._current_row_in_file_index
@@ -3740,8 +3754,8 @@ def get_values(self, h_start, v_start, h_stop, v_stop):
         logger.debug(f'{len(df)} rows read')
         assert v_start >= current_row, f"{v_start} < {current_row}"
         chunk = df.iloc[v_start - current_row:]
-        assert len(chunk) == (v_stop - v_start), \
-            f"{len(chunk)} != {v_stop - v_start}"
+        assert len(chunk) == expected_num_rows, \
+            f"{len(chunk)=} != {expected_num_rows=}"
 
         chunk_columns = [chunk.iloc[:, i].values
                          for i in range(h_start, h_stop)]