Fix for not retrieving all items when response has multiple pages of items

tabcmd/commands/server.py

@@ -50,31 +50,52 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional
             container_name: str = "[{0}] {1}".format(container.__class__.__name__, container.name)
             item_log_name = "{0}/{1}".format(container_name, item_log_name)
         logger.debug(_("export.status").format(item_log_name))
-        req_option = TSC.RequestOptions()
-        req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name))
-        all_items, pagination_item = item_endpoint.get(req_option)
-        if all_items is None or all_items == []:
-            raise TSC.ServerResponseError(
-                code="404",
-                summary=_("errors.xmlapi.not_found"),
-                detail=_("errors.xmlapi.not_found") + ": " + item_log_name,
+
+        result = []
+        total_available_items = None
+        page_number = 1
+        total_retrieved_items = 0
+
+        while True:
+            req_option = TSC.RequestOptions(pagenumber=page_number)
+            req_option.filter.add(
+                TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name)
             )
-        if len(all_items) == 1:
-            logger.debug("Exactly one result found")
-            result = all_items
-        if len(all_items) > 1:
+            all_items, pagination_item = item_endpoint.get(req_option)
+
+            if all_items is None or all_items == []:
+                raise TSC.ServerResponseError(
+                    code="404",
+                    summary=_("errors.xmlapi.not_found"),
+                    detail=_("errors.xmlapi.not_found") + ": " + item_log_name,
+                )
+
+            if total_available_items is None:
+                total_available_items = pagination_item.total_available
+
+            total_retrieved_items += len(all_items)
+
             logger.debug(
-                "{}+ items of this name were found: {}".format(
-                    len(all_items), all_items[0].name + ", " + all_items[1].name + ", ..."
+                "{} items of name: {} were found for query page number: {}, page size: {} & total available: {}".format(
+                    len(all_items),
+                    item_name,
+                    pagination_item.page_number,
+                    pagination_item.page_size,
+                    pagination_item.total_available,
                 )
             )
 
             if container:
                 container_id = container.id
                 logger.debug("Filtering to items in project {}".format(container.id))
-                result = list(filter(lambda item: item.project_id == container_id, all_items))
+                result.extend(list(filter(lambda item: item.project_id == container_id, all_items)))
             else:
-                result = all_items
+                result.extend(all_items)
+
+            if total_retrieved_items >= total_available_items:
+                break
+
+            page_number = pagination_item.page_number + 1
 
         return result
 

tabcmd/execution/parent_parser.py

@@ -134,6 +134,14 @@ def parent_parser_with_global_options():
         version=strings[6] + "v" + version + "\n \n",
         help=strings[7],
     )
+
+    parser.add_argument(
+        "--query-page-size",
+        type=int,
+        default=None,
+        metavar="<PAGE_SIZE>",
+        help="Specify the page size for query results.",
+    )
     return parser
 
 

tabcmd/execution/tabcmd_controller.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import sys
 
 from .localize import set_client_locale
@@ -37,6 +38,8 @@ def run(parser, user_input=None):
             logger.debug(namespace)
         if namespace.language:
             set_client_locale(namespace.language, logger)
+        if namespace.query_page_size:
+            os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size)
         try:
             func = namespace.func
             # if a subcommand was identified, call the function assigned to it

tests/commands/test_projects_utils.py

@@ -7,7 +7,11 @@
 fake_item = mock.MagicMock()
 fake_item.name = "fake-name"
 fake_item.id = "fake-id"
-getter = mock.MagicMock("get", return_value=([fake_item], 1))
+fake_item_pagination = mock.MagicMock()
+fake_item_pagination.page_number = 1
+fake_item_pagination.total_available = 1
+fake_item_pagination.page_size = 100
+getter = mock.MagicMock("get", return_value=([fake_item], fake_item_pagination))
 
 
 class ProjectsTest(unittest.TestCase):

tests/commands/test_publish_command.py

@@ -18,12 +18,17 @@
 fake_item.pdf = b"/pdf-representation-of-view"
 fake_item.extract_encryption_mode = "Disabled"
 
+fake_item_pagination = MagicMock()
+fake_item_pagination.page_number = 1
+fake_item_pagination.total_available = 1
+fake_item_pagination.page_size = 100
+
 fake_job = MagicMock()
 fake_job.id = "fake-job-id"
 
 creator = MagicMock()
 getter = MagicMock()
-getter.get = MagicMock("get", return_value=([fake_item], 1))
+getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination))
 getter.publish = MagicMock("publish", return_value=fake_item)
 
 

tests/commands/test_run_commands.py

@@ -37,15 +37,21 @@
 fake_item = MagicMock()
 fake_item.name = "fake-name"
 fake_item.id = "fake-id"
+fake_item.project_id = "fake-id"
 fake_item.pdf = b"/pdf-representation-of-view"
 fake_item.extract_encryption_mode = "Disabled"
 
+fake_item_pagination = MagicMock()
+fake_item_pagination.page_number = 1
+fake_item_pagination.total_available = 1
+fake_item_pagination.page_size = 100
+
 fake_job = MagicMock()
 fake_job.id = "fake-job-id"
 
 creator = MagicMock()
 getter = MagicMock()
-getter.get = MagicMock("get", return_value=([fake_item], 1))
+getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination))
 getter.publish = MagicMock("publish", return_value=fake_item)
 getter.create_extract = MagicMock("create_extract", return_value=fake_job)
 getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job)
@@ -214,7 +220,6 @@ def test_refresh_extract(self, mock_session, mock_server):
         mock_args.removecalculations = None
         mock_args.incremental = None
         mock_args.synchronous = None
-        print(mock_args)
 
         refresh_extracts_command.RefreshExtracts.run_command(mock_args)
         mock_session.assert_called()

tests/commands/test_server.py

@@ -0,0 +1,222 @@
+import unittest
+from unittest.mock import MagicMock, patch
+import tableauserverclient as TSC
+from tabcmd.commands.server import Server
+
+
+class TestServer(unittest.TestCase):
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_returns_items(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = None
+
+        pagination_item = MagicMock()
+        pagination_item.total_available = 1
+        pagination_item.page_number = 1
+        pagination_item.page_size = 1
+
+        item = MagicMock()
+        item_endpoint.get.return_value = ([item], pagination_item)
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [item])
+        logger.debug.assert_called()
+        item_endpoint.get.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_no_items_found(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = None
+
+        pagination_item = MagicMock()
+        pagination_item.total_available = 0
+        pagination_item.page_number = 1
+        pagination_item.page_size = 1
+
+        item_endpoint.get.return_value = ([], pagination_item)
+
+        with self.assertRaises(TSC.ServerResponseError):
+            Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        logger.debug.assert_called()
+        item_endpoint.get.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = MagicMock()
+        container.id = "container_id"
+
+        pagination_item = MagicMock()
+        pagination_item.total_available = 1
+        pagination_item.page_number = 1
+        pagination_item.page_size = 1
+
+        item = MagicMock()
+        item.project_id = "container_id"
+        item_endpoint.get.return_value = ([item], pagination_item)
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [item])
+        logger.debug.assert_called()
+        item_endpoint.get.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_with_container_no_match(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = MagicMock()
+        container.id = "container_id"
+
+        pagination_item = MagicMock()
+        pagination_item.total_available = 1
+        pagination_item.page_number = 1
+        pagination_item.page_size = 1
+
+        item = MagicMock()
+        item.project_id = "different_container_id"
+        item_endpoint.get.return_value = ([item], pagination_item)
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [])
+        logger.debug.assert_called()
+        item_endpoint.get.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = None
+
+        pagination_item_1 = MagicMock()
+        pagination_item_1.total_available = 3
+        pagination_item_1.page_number = 1
+        pagination_item_1.page_size = 1
+
+        pagination_item_2 = MagicMock()
+        pagination_item_2.total_available = 3
+        pagination_item_2.page_number = 2
+        pagination_item_2.page_size = 1
+
+        pagination_item_3 = MagicMock()
+        pagination_item_3.total_available = 3
+        pagination_item_3.page_number = 3
+        pagination_item_3.page_size = 1
+
+        item_1 = MagicMock()
+        item_2 = MagicMock()
+        item_3 = MagicMock()
+
+        item_endpoint.get.side_effect = [
+            ([item_1], pagination_item_1),
+            ([item_2], pagination_item_2),
+            ([item_3], pagination_item_3),
+        ]
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [item_1, item_2, item_3])
+        self.assertEqual(item_endpoint.get.call_count, 3)
+        logger.debug.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_multiple_pages_with_container(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = MagicMock()
+        container.id = "container_id"
+
+        pagination_item_1 = MagicMock()
+        pagination_item_1.total_available = 3
+        pagination_item_1.page_number = 1
+        pagination_item_1.page_size = 1
+
+        pagination_item_2 = MagicMock()
+        pagination_item_2.total_available = 3
+        pagination_item_2.page_number = 2
+        pagination_item_2.page_size = 1
+
+        pagination_item_3 = MagicMock()
+        pagination_item_3.total_available = 3
+        pagination_item_3.page_number = 3
+        pagination_item_3.page_size = 1
+
+        item_1 = MagicMock()
+        item_1.project_id = "container_id_1"
+        item_2 = MagicMock()
+        item_2.project_id = "container_id"
+        item_3 = MagicMock()
+        item_3.project_id = "container_id_2"
+
+        item_endpoint.get.side_effect = [
+            ([item_1], pagination_item_1),
+            ([item_2], pagination_item_2),
+            ([item_3], pagination_item_3),
+        ]
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [item_2])
+        self.assertEqual(item_endpoint.get.call_count, 3)
+        logger.debug.assert_called()
+
+    @patch("tabcmd.commands.server.TSC.RequestOptions")
+    @patch("tabcmd.commands.server.TSC.Filter")
+    def test_get_items_by_name_multiple_pages_no_container_match(self, MockFilter, MockRequestOptions):
+        logger = MagicMock()
+        item_endpoint = MagicMock()
+        item_name = "test_item"
+        container = MagicMock()
+        container.id = "container_id"
+
+        pagination_item_1 = MagicMock()
+        pagination_item_1.total_available = 3
+        pagination_item_1.page_number = 1
+        pagination_item_1.page_size = 1
+
+        pagination_item_2 = MagicMock()
+        pagination_item_2.total_available = 3
+        pagination_item_2.page_number = 2
+        pagination_item_2.page_size = 1
+
+        pagination_item_3 = MagicMock()
+        pagination_item_3.total_available = 3
+        pagination_item_3.page_number = 3
+        pagination_item_3.page_size = 1
+
+        item_1 = MagicMock()
+        item_1.project_id = "different_container_id_1"
+        item_2 = MagicMock()
+        item_2.project_id = "different_container_id_2"
+        item_3 = MagicMock()
+        item_3.project_id = "different_container_id_3"
+
+        item_endpoint.get.side_effect = [
+            ([item_1], pagination_item_1),
+            ([item_2], pagination_item_2),
+            ([item_3], pagination_item_3),
+        ]
+
+        result = Server.get_items_by_name(logger, item_endpoint, item_name, container)
+
+        self.assertEqual(result, [])
+        self.assertEqual(item_endpoint.get.call_count, 3)
+        logger.debug.assert_called()