remove product bundles json from repo, and fetch it dynamically, add helper function to get valid item types and bundles in cli (#1094)

* get product bundles from orders endpoint, don't test every bundle and item type for maintainability

* remove unused var

* use lazy loading for bundles spec

* tmp

* remove skip validation, clean up error for user

* clean up

* remove PSOrthoTile from tests, replace with PSScene, mock bundles spec in async tests

* update tests to use a mocked bundles spec, clean up linting/formatting

* mock spec in data api cli tests

* use real bundle name in mock

* print to debug test failing in ci

* print to debug test failing in ci

* move fake bundles spec into conftest to avoid collision

* add commands for show-bundles and show-item-types in cli with tests

* remove 'show' from cli commands, rework error message

* reformat cache dict

* mock call to spec url in the fixture

Author: charcey

design-docs/CLI-Data.md

@@ -737,7 +737,7 @@ with a utc offset using a file to specify the filter.
 
 ```
 $ planet data stats --utc-offset +1:00 \
-PSScene,PSOrthoTile day filter-desc.json
+PSScene,REOrthoTile day filter-desc.json
 
 <unedited API response json>
 ```

planet/cli/data.py

@@ -27,9 +27,10 @@
                                  SEARCH_SORT_DEFAULT,
                                  STATS_INTERVAL)
 
-from planet.specs import (get_data_item_types,
-                          validate_data_item_type,
-                          SpecificationException)
+from planet.specs import (FetchBundlesSpecError,
+                          get_item_types,
+                          SpecificationException,
+                          validate_data_item_type)
 
 from . import types
 from .cmds import coro, translate_exceptions
@@ -38,9 +39,6 @@
 from .session import CliSession
 from .validators import check_geom
 
-valid_item_string = "Valid entries for ITEM_TYPES: " + "|".join(
-    get_data_item_types())
-
 
 @asynccontextmanager
 async def data_client(ctx):
@@ -79,6 +77,8 @@ def check_item_types(ctx, param, item_types) -> Optional[List[dict]]:
         return item_types
     except SpecificationException as e:
         raise click.BadParameter(str(e))
+    except FetchBundlesSpecError as e:
+        raise click.ClickException(str(e))
 
 
 def check_item_type(ctx, param, item_type) -> Optional[List[dict]]:
@@ -88,12 +88,14 @@ def check_item_type(ctx, param, item_type) -> Optional[List[dict]]:
         validate_data_item_type(item_type)
     except SpecificationException as e:
         raise click.BadParameter(str(e))
+    except FetchBundlesSpecError as e:
+        raise click.ClickException(str(e))
 
     return item_type
 
 
 def check_search_id(ctx, param, search_id) -> str:
-    """Ensure search id is a valix hex string"""
+    """Ensure search id is a valid hex string"""
     try:
         _ = DataClient._check_search_id(search_id)
     except exceptions.ClientError as e:
@@ -274,7 +276,7 @@ def filter(ctx,
     echo_json(filt, pretty)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -320,7 +322,7 @@ async def search(ctx, item_types, geom, filter, limit, name, sort, pretty):
             echo_json(item, pretty)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -417,7 +419,7 @@ async def search_run(ctx, search_id, sort, limit, pretty):
             echo_json(item, pretty)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -477,7 +479,7 @@ async def search_delete(ctx, search_id):
         await cl.delete_search(search_id)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -526,7 +528,7 @@ async def search_update(ctx,
         echo_json(items, pretty)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -588,7 +590,7 @@ async def asset_download(ctx,
             cl.validate_checksum(asset, path)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -602,7 +604,7 @@ async def asset_activate(ctx, item_type, item_id, asset_type):
         await cl.activate_asset(asset)
 
 
-@data.command(epilog=valid_item_string)  # type: ignore
+@data.command()  # type: ignore
 @click.pass_context
 @translate_exceptions
 @coro
@@ -652,3 +654,13 @@ async def asset_wait(ctx, item_type, item_id, asset_type, delay, max_attempts):
 
 # TODO: search_run()".
 # TODO: item_get()".
+
+
+@data.command()  # type: ignore
+@click.pass_context
+@translate_exceptions
+def item_types(ctx):
+    """Show valid item types."""
+    click.echo("Valid item types:")
+    for it in get_item_types():
+        click.echo(f"- {it}")

planet/cli/orders.py

@@ -15,6 +15,7 @@
 from contextlib import asynccontextmanager
 import logging
 from pathlib import Path
+from typing import List, Optional
 
 import click
 
@@ -26,10 +27,47 @@
 from .io import echo_json
 from .options import limit, pretty
 from .session import CliSession
+from ..specs import (FetchBundlesSpecError,
+                     get_bundle_names,
+                     get_item_types,
+                     SpecificationException,
+                     validate_bundle,
+                     validate_data_item_type)
 
 LOGGER = logging.getLogger(__name__)
 
 
+def check_item_type(ctx, param, item_type):
+    """Validates the item type provided by comparing it to all supported
+    item types."""
+    try:
+        validate_data_item_type(item_type)
+        return item_type
+    except SpecificationException as e:
+        raise click.BadParameter(str(e))
+    except FetchBundlesSpecError as e:
+        raise click.ClickException(str(e))
+
+
+def check_bundle(ctx, param, bundle) -> Optional[List[dict]]:
+    """Validates the bundle provided by comparing it to all supported
+    bundles."""
+
+    # Get the value of item_type from the context
+    item_type = ctx.params.get('item_type')
+    if not item_type:
+        raise click.BadParameter("Item type is required to validate a bundle.")
+
+    try:
+        validate_bundle(item_type, bundle)
+    except SpecificationException as e:
+        raise click.BadParameter(str(e))
+    except FetchBundlesSpecError as e:
+        raise click.ClickException(str(e))
+
+    return bundle
+
+
 @asynccontextmanager
 async def orders_client(ctx):
     base_url = ctx.obj['BASE_URL']
@@ -314,13 +352,13 @@ async def create(ctx, request, pretty, **kwargs):
 @click.option('--item-type',
               required=True,
               help='Item type for requested item ids.',
-              type=click.Choice(planet.specs.get_item_types(),
-                                case_sensitive=False))
+              type=str,
+              callback=check_item_type)
 @click.option('--bundle',
               required=True,
-              help='Asset type for the item.',
-              type=click.Choice(planet.specs.get_product_bundles(),
-                                case_sensitive=False))
+              help='Bundle type for the item.',
+              type=str,
+              callback=check_bundle)
 @click.option('--name',
               required=True,
               help='Order name. Does not need to be unique.',
@@ -444,3 +482,23 @@ async def request(ctx,
         create_configuration=create_configuration)
 
     echo_json(request, pretty)
+
+
+@orders.command()  # type: ignore
+@click.pass_context
+@translate_exceptions
+def item_types(ctx):
+    """Show valid item types for ordering."""
+    click.echo("Valid item types:")
+    for it in get_item_types():
+        click.echo(f"- {it}")
+
+
+@orders.command()  # type: ignore
+@click.pass_context
+@translate_exceptions
+def bundles(ctx):
+    """Show valid bundle names for ordering."""
+    click.echo("Valid bundles:")
+    for it in get_bundle_names():
+        click.echo(f"- {it}")

planet/cli/subscriptions.py

@@ -12,12 +12,9 @@
 from planet.clients.subscriptions import SubscriptionsClient
 from .. import subscription_request
 from ..subscription_request import sentinel_hub
-from ..specs import get_item_types, validate_item_type, SpecificationException
+from ..specs import FetchBundlesSpecError, get_item_types, SpecificationException, validate_item_type
 from .validators import check_geom
 
-ALL_ITEM_TYPES = get_item_types()
-valid_item_string = "Valid entries for ITEM_TYPES: " + "|".join(ALL_ITEM_TYPES)
-
 
 def check_item_types(ctx, param, item_types) -> Optional[List[dict]]:
     """Validates each item types provided by comparing them to all supported
@@ -28,17 +25,8 @@ def check_item_types(ctx, param, item_types) -> Optional[List[dict]]:
         return item_types
     except SpecificationException as e:
         raise click.BadParameter(str(e))
-
-
-def check_item_type(ctx, param, item_type) -> Optional[List[dict]]:
-    """Validates the item type provided by comparing it to all supported
-    item types."""
-    try:
-        validate_item_type(item_type)
-    except SpecificationException as e:
-        raise click.BadParameter(str(e))
-
-    return item_type
+    except FetchBundlesSpecError as e:
+        raise click.ClickException(str(e))
 
 
 @asynccontextmanager
@@ -418,7 +406,7 @@ def request(name,
     echo_json(res, pretty)
 
 
-@subscriptions.command(epilog=valid_item_string)  # type: ignore
+@subscriptions.command()  # type: ignore
 @translate_exceptions
 @click.option('--item-types',
               required=True,
@@ -496,7 +484,7 @@ def request_catalog(item_types,
 @click.option(
     '--var-id',
     required=True,
-    help='A Planetary Variable ID. See documenation for all available IDs.')
+    help='A Planetary Variable ID. See documentation for all available IDs.')
 @click.option(
     '--geometry',
     required=True,
@@ -528,3 +516,13 @@ def request_pv(var_type, var_id, geometry, start_time, end_time, pretty):
         end_time=end_time,
     )
     echo_json(res, pretty)
+
+
+@subscriptions.command()  # type: ignore
+@click.pass_context
+@translate_exceptions
+def item_types(ctx):
+    """Show valid item types for catalog subscriptions."""
+    click.echo("Valid item types:")
+    for it in get_item_types():
+        click.echo(f"- {it}")

planet/clients/orders.py

@@ -32,7 +32,7 @@
 ORDERS_PATH = '/orders/v2'
 BULK_PATH = '/bulk/orders/v2'
 
-# Order states https://developers.planet.com/docs/orders/ordering/#order-states
+# Order states https://developers.planet.com/apis/orders/states/
 # this is in order of state progression except for final states
 ORDER_STATE_SEQUENCE = \
     ('queued', 'running', 'failed', 'success', 'partial', 'cancelled')

planet/data/README.md

@@ -1,18 +1,6 @@
 # data
 This directory contains static external resources used by the codebase.
 
-## Orders Spec
-
-### Files
-
-* `orders_product_bundle_2024-08-12.json`
-
-### Description
-
-This file was downloaded from the Planet
-[Product Bundles Reference](https://developers.planet.com/docs/orders/product-bundles-reference/)
-page and is stored without any adjustments.
-
 ## GeoJSON Schemas
 
 ### Files