diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index 4091c0ba..46e77d01 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -50,31 +50,52 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional container_name: str = "[{0}] {1}".format(container.__class__.__name__, container.name) item_log_name = "{0}/{1}".format(container_name, item_log_name) logger.debug(_("export.status").format(item_log_name)) - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name)) - all_items, pagination_item = item_endpoint.get(req_option) - if all_items is None or all_items == []: - raise TSC.ServerResponseError( - code="404", - summary=_("errors.xmlapi.not_found"), - detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + + result = [] + total_available_items = None + page_number = 1 + total_retrieved_items = 0 + + while True: + req_option = TSC.RequestOptions(pagenumber=page_number) + req_option.filter.add( + TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name) ) - if len(all_items) == 1: - logger.debug("Exactly one result found") - result = all_items - if len(all_items) > 1: + all_items, pagination_item = item_endpoint.get(req_option) + + if all_items is None or all_items == []: + raise TSC.ServerResponseError( + code="404", + summary=_("errors.xmlapi.not_found"), + detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + ) + + if total_available_items is None: + total_available_items = pagination_item.total_available + + total_retrieved_items += len(all_items) + logger.debug( - "{}+ items of this name were found: {}".format( - len(all_items), all_items[0].name + ", " + all_items[1].name + ", ..." + "{} items of name: {} were found for query page number: {}, page size: {} & total available: {}".format( + len(all_items), + item_name, + pagination_item.page_number, + pagination_item.page_size, + pagination_item.total_available, ) ) if container: container_id = container.id logger.debug("Filtering to items in project {}".format(container.id)) - result = list(filter(lambda item: item.project_id == container_id, all_items)) + result.extend(list(filter(lambda item: item.project_id == container_id, all_items))) else: - result = all_items + result.extend(all_items) + + if total_retrieved_items >= total_available_items: + break + + page_number = pagination_item.page_number + 1 return result diff --git a/tabcmd/execution/parent_parser.py b/tabcmd/execution/parent_parser.py index bc81f59c..41362bed 100644 --- a/tabcmd/execution/parent_parser.py +++ b/tabcmd/execution/parent_parser.py @@ -134,6 +134,14 @@ def parent_parser_with_global_options(): version=strings[6] + "v" + version + "\n \n", help=strings[7], ) + + parser.add_argument( + "--query-page-size", + type=int, + default=None, + metavar="", + help="Specify the page size for query results.", + ) return parser diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 6e06dfe1..2fbf270d 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -1,4 +1,5 @@ import logging +import os import sys from .localize import set_client_locale @@ -37,6 +38,8 @@ def run(parser, user_input=None): logger.debug(namespace) if namespace.language: set_client_locale(namespace.language, logger) + if namespace.query_page_size: + os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size) try: func = namespace.func # if a subcommand was identified, call the function assigned to it diff --git a/tests/commands/test_projects_utils.py b/tests/commands/test_projects_utils.py index 9ba3c09d..5e92a10d 100644 --- a/tests/commands/test_projects_utils.py +++ b/tests/commands/test_projects_utils.py @@ -7,7 +7,11 @@ fake_item = mock.MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" -getter = mock.MagicMock("get", return_value=([fake_item], 1)) +fake_item_pagination = mock.MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 +getter = mock.MagicMock("get", return_value=([fake_item], fake_item_pagination)) class ProjectsTest(unittest.TestCase): diff --git a/tests/commands/test_publish_command.py b/tests/commands/test_publish_command.py index 0c9049a7..800c81fc 100644 --- a/tests/commands/test_publish_command.py +++ b/tests/commands/test_publish_command.py @@ -18,12 +18,17 @@ fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) diff --git a/tests/commands/test_run_commands.py b/tests/commands/test_run_commands.py index c29341d0..c24e3e74 100644 --- a/tests/commands/test_run_commands.py +++ b/tests/commands/test_run_commands.py @@ -37,15 +37,21 @@ fake_item = MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" +fake_item.project_id = "fake-id" fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) getter.create_extract = MagicMock("create_extract", return_value=fake_job) getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job) @@ -214,7 +220,6 @@ def test_refresh_extract(self, mock_session, mock_server): mock_args.removecalculations = None mock_args.incremental = None mock_args.synchronous = None - print(mock_args) refresh_extracts_command.RefreshExtracts.run_command(mock_args) mock_session.assert_called() diff --git a/tests/commands/test_server.py b/tests/commands/test_server.py new file mode 100644 index 00000000..b8a7d293 --- /dev/null +++ b/tests/commands/test_server.py @@ -0,0 +1,222 @@ +import unittest +from unittest.mock import MagicMock, patch +import tableauserverclient as TSC +from tabcmd.commands.server import Server + + +class TestServer(unittest.TestCase): + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_returns_items(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_no_items_found(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 0 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item_endpoint.get.return_value = ([], pagination_item) + + with self.assertRaises(TSC.ServerResponseError): + Server.get_items_by_name(logger, item_endpoint, item_name, container) + + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item.project_id = "container_id" + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_with_container_no_match(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item.project_id = "different_container_id" + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, []) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_2 = MagicMock() + item_3 = MagicMock() + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item_1, item_2, item_3]) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages_with_container(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_1.project_id = "container_id_1" + item_2 = MagicMock() + item_2.project_id = "container_id" + item_3 = MagicMock() + item_3.project_id = "container_id_2" + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item_2]) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages_no_container_match(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_1.project_id = "different_container_id_1" + item_2 = MagicMock() + item_2.project_id = "different_container_id_2" + item_3 = MagicMock() + item_3.project_id = "different_container_id_3" + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, []) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called()