Content-Type overly restrictive

[jsignell]

I was messing around on pystac-client and trying to understand what's going on in stac-utils/pystac#1097. I tried looking at earth-search and the _repr_html_ was raising an error:

from pystac_client import Client

# set pystac_client logger to DEBUG to see API calls
import logging
logging.basicConfig()
logger = logging.getLogger('pystac_client')
logger.setLevel(logging.DEBUG)

Client.open("https://earth-search.aws.element84.com/v1")

DEBUG:pystac_client.stac_api_io:GET https://earth-search.aws.element84.com/v1 Headers: {'User-Agent': 'python-requests/2.28.2', 'Accept-Encoding': 'gzip, deflate, br', 'Accept': '*/*', 'Connection': 'keep-alive'}
DEBUG:pystac_client.stac_api_io:GET https://earth-search.aws.element84.com/v1/collections/naip Headers: {'User-Agent': 'python-requests/2.28.2', 'Accept-Encoding': 'gzip, deflate, br', 'Accept': '*/*', 'Connection': 'keep-alive'}
DEBUG:pystac_client.stac_api_io:GET https://earth-search.aws.element84.com/v1/collections/naip/items?limit=100 Headers: {'User-Agent': 'python-requests/2.28.2', 'Accept-Encoding': 'gzip, deflate, br', 'Accept': '*/*', 'Connection': 'keep-alive'}
DEBUG:pystac_client.stac_api_io:GET https://earth-search.aws.element84.com/v1/collections/naip/items?limit=100 Headers: {'User-Agent': 'python-requests/2.28.2', 'Accept-Encoding': 'gzip, deflate, br', 'Accept': '*/*', 'Connection': 'keep-alive'}
DEBUG:pystac_client.stac_api_io:POST https://earth-search.aws.element84.com/v1/search Headers: {'User-Agent': 'python-requests/2.28.2', 'Accept-Encoding': 'gzip, deflate, br', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Length': '14', 'Content-Type': 'application/json'} Payload: {"limit": 100}

---------------------------------------------------------------------------
APIError                                  Traceback (most recent call last)
File ~/conda/envs/pystac-client/lib/python3.11/site-packages/IPython/core/formatters.py:344, in BaseFormatter.__call__(self, obj)
    342     method = get_real_method(obj, self.print_method)
    343     if method is not None:
--> 344         return method()
    345     return None
    346 else:

File ~/pystac/pystac/catalog.py:214, in Catalog._repr_html_(self)
    212 if jinja_env:
    213     template = jinja_env.get_template("Catalog.jinja2")
--> 214     return str(template.render(catalog=self))
    215 else:
    216     return escape(repr(self))

File ~/conda/envs/pystac-client/lib/python3.11/site-packages/jinja2/environment.py:1301, in Template.render(self, *args, **kwargs)
   1299     return self.environment.concat(self.root_render_func(ctx))  # type: ignore
   1300 except Exception:
-> 1301     self.environment.handle_exception()

File ~/conda/envs/pystac-client/lib/python3.11/site-packages/jinja2/environment.py:936, in Environment.handle_exception(self, source)
    931 """Exception handling helper.  This is used internally to either raise
    932 rewritten exceptions or return a rendered traceback for the template.
    933 """
    934 from .debug import rewrite_traceback_stack
--> 936 raise rewrite_traceback_stack(source=source)

File ~/pystac/pystac/html/Catalog.jinja2:25, in top-level template code()
     23 {{ macros.stac_extensions(catalog) }}
     24 {{ macros.children(catalog) }}
---> 25 {{ macros.items(catalog) }}
     26 {{ macros.links(catalog) }}
     27 {% block subclass_details %} {% endblock %}

File ~/conda/envs/pystac-client/lib/python3.11/site-packages/jinja2/runtime.py:777, in Macro._invoke(self, arguments, autoescape)
    774 if self._environment.is_async:
    775     return self._async_invoke(arguments, autoescape)  # type: ignore
--> 777 rv = self._func(*arguments)
    779 if autoescape:
    780     rv = Markup(rv)

File ~/pystac/pystac/html/Macros.jinja2:39, in template()
     37 
     38 {% macro items(parent) -%}
---> 39     {% if parent.get_items()|is_nonempty_generator %}
     40         <details>
     41             <summary style="margin-bottom: 10px; margin-top: 10px">

File ~/pystac/pystac/html/jinja_env.py:17, in get_jinja_env.<locals>.<lambda>(x)
     12 environment = Environment(
     13     loader=PackageLoader("pystac", "html"), autoescape=select_autoescape()
     14 )
     16 environment.filters["first"] = lambda x: islice(x, 1)
---> 17 environment.filters["is_nonempty_generator"] = lambda x: next(x, None) is not None
     19 return environment

File ~/pystac-client/pystac_client/client.py:301, in Client.get_items(self)
    299 if self._conforms_to(ConformanceClasses.ITEM_SEARCH):
    300     search = self.search()
--> 301     yield from search.items()
    302 else:
    303     for item in super().get_items():

File ~/pystac-client/pystac_client/item_search.py:682, in ItemSearch.items(self)
    675 def items(self) -> Iterator[Item]:
    676     """Iterator that yields :class:`pystac.Item` instances for each item matching
    677     the given search parameters.
    678 
    679     Yields:
    680         Item : each Item matching the search criteria
    681     """
--> 682     for item in self.items_as_dicts():
    683         # already signed in items_as_dicts
    684         yield Item.from_dict(item, root=self.client, preserve_dict=False)

File ~/pystac-client/pystac_client/item_search.py:694, in ItemSearch.items_as_dicts(self)
    687 """Iterator that yields :class:`dict` instances for each item matching
    688 the given search parameters.
    689 
    690 Yields:
    691     Item : each Item matching the search criteria
    692 """
    693 nitems = 0
--> 694 for page in self._stac_io.get_pages(
    695     self.url, self.method, self.get_parameters()
    696 ):
    697     for item in page.get("features", []):
    698         call_modifier(self.modifier, item)

File ~/pystac-client/pystac_client/stac_api_io.py:246, in StacApiIO.get_pages(self, url, method, parameters)
    234 def get_pages(
    235     self,
    236     url: str,
    237     method: Optional[str] = None,
    238     parameters: Optional[Dict[str, Any]] = None,
    239 ) -> Iterator[Dict[str, Any]]:
    240     """Iterator that yields dictionaries for each page at a STAC paging
    241     endpoint, e.g., /collections, /search
    242 
    243     Return:
    244         Dict[str, Any] : JSON content from a single page
    245     """
--> 246     page = self.read_json(url, method=method, parameters=parameters)
    247     if not (page.get("features") or page.get("collections")):
    248         return None

File ~/pystac/pystac/stac_io.py:206, in StacIO.read_json(self, source, *args, **kwargs)
    189 def read_json(self, source: HREF, *args: Any, **kwargs: Any) -> Dict[str, Any]:
    190     """Read a dict from the given source.
    191 
    192     See :func:`StacIO.read_text <pystac.StacIO.read_text>` for usage of
   (...)
    204         given source.
    205     """
--> 206     txt = self.read_text(source, *args, **kwargs)
    207     return self.json_loads(txt)

File ~/pystac-client/pystac_client/stac_api_io.py:123, in StacApiIO.read_text(self, source, *args, **kwargs)
    121 href = str(source)
    122 if _is_url(href):
--> 123     return self.request(href, *args, **kwargs)
    124 else:
    125     with open(href) as f:

File ~/pystac-client/pystac_client/stac_api_io.py:169, in StacApiIO.request(self, href, method, headers, parameters)
    167     raise APIError(str(err))
    168 if resp.status_code != 200:
--> 169     raise APIError.from_response(resp)
    170 try:
    171     return resp.content.decode("utf-8")

APIError: {"message":"Forbidden"}

that I think essentially boils down to the API needing the value for Content-Type to end in ;

import requests

response = requests.post(
    "https://earth-search.aws.element84.com/v1/search", 
    data={"limit": 100}, 
    headers={
        'User-Agent': 'python-requests/2.28.2', 
        'Accept-Encoding': 'gzip, deflate, br', 
        'Accept': '*/*', 
        'Connection': 'keep-alive', 
        'Content-Length': '14', 
        'Content-Type': 'application/json'
    }
)
assert not response.ok

import requests

response = requests.post(
    "https://earth-search.aws.element84.com/v1/search", 
    data={"limit": 100}, 
    headers={
        'User-Agent': 'python-requests/2.28.2', 
        'Accept-Encoding': 'gzip, deflate, br', 
        'Accept': '*/*', 
        'Connection': 'keep-alive', 
        'Content-Length': '14', 
        'Content-Type': 'application/json;'
    }
)
assert response.ok

[jsignell]

This was a misdiagnosis of a known bug with earth-search specifically. Thanks for taking a look Phil!