diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index d9be64c3..60975117 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,8 +1,9 @@ -name: Release-Executable +name: Package-and-release # Pyinstaller requires that executables for each OS are built on that OS # This action is intended to build on each of the supported OS's: mac, windows, linux. -# and then upload all three files to a new release +# (We run two macos versions to build for arm and x64) +# and then upload all four files to a new release # reference material: # https://data-dive.com/multi-os-deployment-in-cloud-using-pyinstaller-and-github-actions @@ -15,7 +16,6 @@ on: workflow_dispatch: jobs: - buildexe: name: Build executables and upload them to the existing release runs-on: ${{ matrix.os }} @@ -29,22 +29,34 @@ jobs: pyinstaller tabcmd-windows.spec --clean --noconfirm --distpath ./dist/windows OUT_FILE_NAME: tabcmd.exe ASSET_MIME: application/vnd.microsoft.portable-executable + CMD_INVOKE: > + ./dist/windows/tabcmd.exe - os: macos-13 - TARGET: macos - CMD_BUILD: > + TARGET: macos-x86 + CMD_BUILD: | pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/ + ls + pwd + ./mac_bundle_step.sh BUNDLE_NAME: tabcmd.app OUT_FILE_NAME: tabcmd.app APP_BINARY_FILE_NAME: tabcmd ASSET_MIME: application/zip + CMD_INVOKE: > + ./dist/macos/tabcmd.app/Contents/MacOS/tabcmd - os: macos-latest - TARGET: macos - CMD_BUILD: > + TARGET: macos-arm + CMD_BUILD: | pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos/ - BUNDLE_NAME: tabcmd_arm64.app + ls + pwd + ./mac_bundle_step.sh + BUNDLE_NAME: tabcmd.app OUT_FILE_NAME: tabcmd.app APP_BINARY_FILE_NAME: tabcmd ASSET_MIME: application/zip + CMD_INVOKE: > + ./dist/macos/tabcmd.app/Contents/MacOS/tabcmd - os: ubuntu-latest TARGET: ubuntu # https://stackoverflow.com/questions/31259856 @@ -53,6 +65,8 @@ jobs: pyinstaller --clean -y --distpath ./dist/ubuntu tabcmd-linux.spec && chown -R --reference=. ./dist/ubuntu OUT_FILE_NAME: tabcmd + CMD_INVOKE: > + ./dist/ubuntu/tabcmd steps: - uses: actions/checkout@v4 @@ -71,44 +85,24 @@ jobs: doit version python -m build - - name: Package with pyinstaller for ${{matrix.TARGET}} + - name: Build executable with pyinstaller for ${{matrix.TARGET}} run: ${{matrix.CMD_BUILD}} - - - name: Validate package for ${{matrix.TARGET}} - if: matrix.TARGET != 'macos' - run: ./dist/${{ matrix.TARGET }}/${{matrix.OUT_FILE_NAME}} - - - name: Validate package for Mac - if: matrix.TARGET == 'macos' - run: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }}/Contents/MacOS/${{ matrix.APP_BINARY_FILE_NAME }} - - - name: Tar app bundle for Mac - if: matrix.TARGET == 'macos' - run: | - rm -f dist/${{ matrix.TARGET }}/${{ matrix.APP_BINARY_FILE_NAME }} - cd dist/${{ matrix.TARGET }} - tar -cvf ${{ matrix.BUNDLE_NAME }}.tar ${{ matrix.OUT_FILE_NAME }} - - - name: Upload artifact - if: matrix.TARGET != 'macos' + + - name: Validate executable for ${{matrix.TARGET}} + run: ${{matrix.CMD_INVOKE}} + + - name: Upload binary uses: actions/upload-artifact@v4 with: name: ${{ matrix.OUT_FILE_NAME }} path: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }} - - - name: Upload artifact for Mac - if: matrix.TARGET == 'macos' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.BUNDLE_NAME }} - path: ./dist/${{ matrix.TARGET }}/${{ matrix.BUNDLE_NAME }}.tar - - name: Upload binaries to release + - name: Copy binary to release uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }}/ + file: ./dist/${{ matrix.TARGET }}/${{ matrix.OUT_FILE_NAME }} + asset_name: ${{ matrix.TARGET }}-${{ matrix.OUT_FILE_NAME }} tag: ${{ github.ref_name }} overwrite: true promote: true - diff --git a/mac_bundle_step.sh b/mac_bundle_step.sh new file mode 100644 index 00000000..ed17d7c9 --- /dev/null +++ b/mac_bundle_step.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Pyinstaller bundles cannot be run by double-clicking them +# https://github.com/pyinstaller/pyinstaller/issues/5154#issuecomment-2508011279 +APP_NAME="tabcmd" + +cd dist/$APP_NAME.app/Contents/MacOS +mv $APP_NAME ${APP_NAME}_cli + +cat << EOF > $APP_NAME +#!/bin/bash +# This is the launcher for OSX, this way the app will be opened +# when you double click it from the apps folder +open -n /Applications/${APP_NAME}.app/Contents/MacOS/${APP_NAME}_cli +EOF + +chmod +x $APP_NAME + diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index 4091c0ba..19136141 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -50,31 +50,59 @@ def get_items_by_name(logger, item_endpoint, item_name: str, container: Optional container_name: str = "[{0}] {1}".format(container.__class__.__name__, container.name) item_log_name = "{0}/{1}".format(container_name, item_log_name) logger.debug(_("export.status").format(item_log_name)) - req_option = TSC.RequestOptions() - req_option.filter.add(TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name)) - all_items, pagination_item = item_endpoint.get(req_option) - if all_items is None or all_items == []: - raise TSC.ServerResponseError( - code="404", - summary=_("errors.xmlapi.not_found"), - detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + + result = [] + total_available_items = None + page_number = 1 + total_retrieved_items = 0 + + while True: + req_option = TSC.RequestOptions(pagenumber=page_number) + req_option.filter.add( + TSC.Filter(TSC.RequestOptions.Field.Name, TSC.RequestOptions.Operator.Equals, item_name) ) - if len(all_items) == 1: - logger.debug("Exactly one result found") - result = all_items - if len(all_items) > 1: + + # todo - this doesn't filter if the project is in the top level. + # todo: there is no guarantee that these fields are the same for different content types. + # probably better if we move that type specific logic out to a wrapper + if container: + # the name of the filter field is different if you are finding a project or any other item + if type(item_endpoint).__name__.find("Projects") < 0: + parentField = TSC.RequestOptions.Field.ProjectName + parentValue = container.name + else: + parentField = TSC.RequestOptions.Field.ParentProjectId + parentValue = container.id + logger.debug("filtering for parent with {}".format(parentField)) + + req_option.filter.add(TSC.Filter(parentField, TSC.RequestOptions.Operator.Equals, parentValue)) + + all_items, pagination_item = item_endpoint.get(req_option) + + if pagination_item.total_available == 0: + raise TSC.ServerResponseError( + code="404", + summary=_("errors.xmlapi.not_found"), + detail=_("errors.xmlapi.not_found") + ": " + item_log_name, + ) + + total_retrieved_items += len(all_items) + logger.debug( - "{}+ items of this name were found: {}".format( - len(all_items), all_items[0].name + ", " + all_items[1].name + ", ..." + "{} items of name: {} were found for query page number: {}, page size: {} & total available: {}".format( + len(all_items), + item_name, + pagination_item.page_number, + pagination_item.page_size, + pagination_item.total_available, ) ) - if container: - container_id = container.id - logger.debug("Filtering to items in project {}".format(container.id)) - result = list(filter(lambda item: item.project_id == container_id, all_items)) - else: - result = all_items + result.extend(all_items) + if total_retrieved_items >= pagination_item.total_available: + break + + page_number = pagination_item.page_number + 1 return result @@ -146,12 +174,7 @@ def _parse_project_path_to_list(project_path: str): def _get_project_by_name_and_parent(logger, server, project_name: str, parent: Optional[TSC.ProjectItem]): # logger.debug("get by name and parent: {0}, {1}".format(project_name, parent)) # get by name to narrow down the list - projects = Server.get_items_by_name(logger, server.projects, project_name) - if parent is not None: - parent_id = parent.id - for project in projects: - if project.parent_id == parent_id: - return project + projects = Server.get_items_by_name(logger, server.projects, project_name, parent) return projects[0] @staticmethod diff --git a/tabcmd/execution/parent_parser.py b/tabcmd/execution/parent_parser.py index bc81f59c..41362bed 100644 --- a/tabcmd/execution/parent_parser.py +++ b/tabcmd/execution/parent_parser.py @@ -134,6 +134,14 @@ def parent_parser_with_global_options(): version=strings[6] + "v" + version + "\n \n", help=strings[7], ) + + parser.add_argument( + "--query-page-size", + type=int, + default=None, + metavar="", + help="Specify the page size for query results.", + ) return parser diff --git a/tabcmd/execution/tabcmd_controller.py b/tabcmd/execution/tabcmd_controller.py index 6e06dfe1..2fbf270d 100644 --- a/tabcmd/execution/tabcmd_controller.py +++ b/tabcmd/execution/tabcmd_controller.py @@ -1,4 +1,5 @@ import logging +import os import sys from .localize import set_client_locale @@ -37,6 +38,8 @@ def run(parser, user_input=None): logger.debug(namespace) if namespace.language: set_client_locale(namespace.language, logger) + if namespace.query_page_size: + os.environ["TSC_PAGE_SIZE"] = str(namespace.query_page_size) try: func = namespace.func # if a subcommand was identified, call the function assigned to it diff --git a/tests/commands/test_projects_utils.py b/tests/commands/test_projects_utils.py index 9ba3c09d..5e92a10d 100644 --- a/tests/commands/test_projects_utils.py +++ b/tests/commands/test_projects_utils.py @@ -7,7 +7,11 @@ fake_item = mock.MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" -getter = mock.MagicMock("get", return_value=([fake_item], 1)) +fake_item_pagination = mock.MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 +getter = mock.MagicMock("get", return_value=([fake_item], fake_item_pagination)) class ProjectsTest(unittest.TestCase): diff --git a/tests/commands/test_publish_command.py b/tests/commands/test_publish_command.py index 0c9049a7..800c81fc 100644 --- a/tests/commands/test_publish_command.py +++ b/tests/commands/test_publish_command.py @@ -18,12 +18,17 @@ fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) diff --git a/tests/commands/test_run_commands.py b/tests/commands/test_run_commands.py index c29341d0..c24e3e74 100644 --- a/tests/commands/test_run_commands.py +++ b/tests/commands/test_run_commands.py @@ -37,15 +37,21 @@ fake_item = MagicMock() fake_item.name = "fake-name" fake_item.id = "fake-id" +fake_item.project_id = "fake-id" fake_item.pdf = b"/pdf-representation-of-view" fake_item.extract_encryption_mode = "Disabled" +fake_item_pagination = MagicMock() +fake_item_pagination.page_number = 1 +fake_item_pagination.total_available = 1 +fake_item_pagination.page_size = 100 + fake_job = MagicMock() fake_job.id = "fake-job-id" creator = MagicMock() getter = MagicMock() -getter.get = MagicMock("get", return_value=([fake_item], 1)) +getter.get = MagicMock("get", return_value=([fake_item], fake_item_pagination)) getter.publish = MagicMock("publish", return_value=fake_item) getter.create_extract = MagicMock("create_extract", return_value=fake_job) getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job) @@ -214,7 +220,6 @@ def test_refresh_extract(self, mock_session, mock_server): mock_args.removecalculations = None mock_args.incremental = None mock_args.synchronous = None - print(mock_args) refresh_extracts_command.RefreshExtracts.run_command(mock_args) mock_session.assert_called() diff --git a/tests/commands/test_server.py b/tests/commands/test_server.py new file mode 100644 index 00000000..aaf287e5 --- /dev/null +++ b/tests/commands/test_server.py @@ -0,0 +1,113 @@ +import pytest +import unittest +from unittest.mock import MagicMock, patch +import tableauserverclient as TSC +from tabcmd.commands.server import Server + + +class TestServer(unittest.TestCase): + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_returns_items(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_no_items_found(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item = MagicMock() + pagination_item.total_available = 0 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item_endpoint.get.return_value = ([], pagination_item) + + with self.assertRaises(TSC.ServerResponseError): + Server.get_items_by_name(logger, item_endpoint, item_name, container) + + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_with_container(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = MagicMock() + container.id = "container_id" + + pagination_item = MagicMock() + pagination_item.total_available = 1 + pagination_item.page_number = 1 + pagination_item.page_size = 1 + + item = MagicMock() + item.project_id = "container_id" + item_endpoint.get.return_value = ([item], pagination_item) + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item]) + logger.debug.assert_called() + item_endpoint.get.assert_called() + + @patch("tabcmd.commands.server.TSC.RequestOptions") + @patch("tabcmd.commands.server.TSC.Filter") + def test_get_items_by_name_multiple_pages(self, MockFilter, MockRequestOptions): + logger = MagicMock() + item_endpoint = MagicMock() + item_name = "test_item" + container = None + + pagination_item_1 = MagicMock() + pagination_item_1.total_available = 3 + pagination_item_1.page_number = 1 + pagination_item_1.page_size = 1 + + pagination_item_2 = MagicMock() + pagination_item_2.total_available = 3 + pagination_item_2.page_number = 2 + pagination_item_2.page_size = 1 + + pagination_item_3 = MagicMock() + pagination_item_3.total_available = 3 + pagination_item_3.page_number = 3 + pagination_item_3.page_size = 1 + + item_1 = MagicMock() + item_2 = MagicMock() + item_3 = MagicMock() + + item_endpoint.get.side_effect = [ + ([item_1], pagination_item_1), + ([item_2], pagination_item_2), + ([item_3], pagination_item_3), + ] + + result = Server.get_items_by_name(logger, item_endpoint, item_name, container) + + self.assertEqual(result, [item_1, item_2, item_3]) + self.assertEqual(item_endpoint.get.call_count, 3) + logger.debug.assert_called()