avoid misparsing references as other types using Pydantic discriminated unions (#1216)

I’ve found a corner case, I think due to changes to Pydantic’s [smart
mode](https://docs.pydantic.dev/latest/concepts/unions/#smart-mode) for
disambiguating unions, where a subdocument with a `$ref` key and
additional keys (title, description, etc) is validated as a `Schema`
instead of as a `Reference` model. I believe these should be ignored per
["any properties added SHALL be
ignored"](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#reference-object)
() This was a problem for me since I want to use a reference to an enum
as a header parameter in FastAPI. The OpenAPI document generated by
FastAPI included several fields (title, description, etc) along with
`$ref`. The`Schema` instance it was validate to lacked any actual type
constraint information, so openapi-python-client needed to treat this as
an `AnyParameter`, and disallowed it in headers.

I have attached an example OpenAPI document
[`oa.json`](https://github.com/user-attachments/files/19170207/oa.json)
you can use to replicate this with the current `openapi-python-client`,
or you can use the test cases from the first commit in this PR, which
resolves it by using a Pydantic discriminated union testing for the
presence of `$ref` for all the unions including `Reference`.

(I had created an [issue
upstream](kuimono/openapi-schema-pydantic#36),
but figured I'd share the workaround here.)

---------

Co-authored-by: Natarajan Krishnaswami <[email protected]>
Co-authored-by: Dylan Anthony <[email protected]>

Author: 3 people

.changeset/always_parse_ref_as_a_reference.md

@@ -0,0 +1,11 @@
+---
+default: patch
+---
+
+# Always parse `$ref` as a reference
+
+If additional attributes were included with a `$ref` (for example `title` or `description`), the property could be 
+interpreted as a new type instead of a reference, usually resulting in `Any` in the generated code.
+Now, any sibling properties to `$ref` will properly be ignored, as per the OpenAPI specification.
+
+Thanks @nkrishnaswami!

end_to_end_tests/baseline_openapi_3.0.json

@@ -394,7 +394,16 @@
           "content": {
             "multipart/form-data": {
               "schema": {
-                "$ref": "#/components/schemas/Body_upload_file_tests_upload_post"
+                "$ref": "#/components/schemas/Body_upload_file_tests_upload_post",
+                "title": "Body_upload_file_tests_upload_post",
+                "required": [
+                  "some_file",
+                  "some_object",
+                  "some_nullable_object",
+                  "some_required_number"
+                ],
+                "properties": {
+                }
               }
             }
           },

openapi_python_client/schema/openapi_schema_pydantic/components.py

@@ -1,4 +1,4 @@
-from typing import Optional, Union
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict
 
@@ -7,7 +7,7 @@
 from .header import Header
 from .link import Link
 from .parameter import Parameter
-from .reference import Reference
+from .reference import ReferenceOr
 from .request_body import RequestBody
 from .response import Response
 from .schema import Schema
@@ -25,15 +25,15 @@ class Components(BaseModel):
         - https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#componentsObject
     """
 
-    schemas: Optional[dict[str, Union[Schema, Reference]]] = None
-    responses: Optional[dict[str, Union[Response, Reference]]] = None
-    parameters: Optional[dict[str, Union[Parameter, Reference]]] = None
-    examples: Optional[dict[str, Union[Example, Reference]]] = None
-    requestBodies: Optional[dict[str, Union[RequestBody, Reference]]] = None
-    headers: Optional[dict[str, Union[Header, Reference]]] = None
-    securitySchemes: Optional[dict[str, Union[SecurityScheme, Reference]]] = None
-    links: Optional[dict[str, Union[Link, Reference]]] = None
-    callbacks: Optional[dict[str, Union[Callback, Reference]]] = None
+    schemas: Optional[dict[str, ReferenceOr[Schema]]] = None
+    responses: Optional[dict[str, ReferenceOr[Response]]] = None
+    parameters: Optional[dict[str, ReferenceOr[Parameter]]] = None
+    examples: Optional[dict[str, ReferenceOr[Example]]] = None
+    requestBodies: Optional[dict[str, ReferenceOr[RequestBody]]] = None
+    headers: Optional[dict[str, ReferenceOr[Header]]] = None
+    securitySchemes: Optional[dict[str, ReferenceOr[SecurityScheme]]] = None
+    links: Optional[dict[str, ReferenceOr[Link]]] = None
+    callbacks: Optional[dict[str, ReferenceOr[Callback]]] = None
     model_config = ConfigDict(
         # `Callback` contains an unresolvable forward reference, will rebuild in `__init__.py`:
         defer_build=True,

openapi_python_client/schema/openapi_schema_pydantic/encoding.py

@@ -1,8 +1,8 @@
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import BaseModel, ConfigDict
 
-from .reference import Reference
+from .reference import ReferenceOr
 
 if TYPE_CHECKING:  # pragma: no cover
     from .header import Header
@@ -17,7 +17,7 @@ class Encoding(BaseModel):
     """
 
     contentType: Optional[str] = None
-    headers: Optional[dict[str, Union["Header", Reference]]] = None
+    headers: Optional[dict[str, ReferenceOr["Header"]]] = None
     style: Optional[str] = None
     explode: bool = False
     allowReserved: bool = False

openapi_python_client/schema/openapi_schema_pydantic/media_type.py

@@ -1,10 +1,10 @@
-from typing import Any, Optional, Union
+from typing import Any, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from .encoding import Encoding
 from .example import Example
-from .reference import Reference
+from .reference import ReferenceOr
 from .schema import Schema
 
 
@@ -16,9 +16,9 @@ class MediaType(BaseModel):
         - https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#mediaTypeObject
     """
 
-    media_type_schema: Optional[Union[Reference, Schema]] = Field(default=None, alias="schema")
+    media_type_schema: Optional[ReferenceOr[Schema]] = Field(default=None, alias="schema")
     example: Optional[Any] = None
-    examples: Optional[dict[str, Union[Example, Reference]]] = None
+    examples: Optional[dict[str, ReferenceOr[Example]]] = None
     encoding: Optional[dict[str, Encoding]] = None
     model_config = ConfigDict(
         # `Encoding` is not build yet, will rebuild in `__init__.py`:

openapi_python_client/schema/openapi_schema_pydantic/operation.py

@@ -1,11 +1,11 @@
-from typing import Optional, Union
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from .callback import Callback
 from .external_documentation import ExternalDocumentation
 from .parameter import Parameter
-from .reference import Reference
+from .reference import ReferenceOr
 from .request_body import RequestBody
 from .responses import Responses
 from .security_requirement import SecurityRequirement
@@ -25,8 +25,8 @@ class Operation(BaseModel):
     description: Optional[str] = None
     externalDocs: Optional[ExternalDocumentation] = None
     operationId: Optional[str] = None
-    parameters: Optional[list[Union[Parameter, Reference]]] = None
-    request_body: Optional[Union[RequestBody, Reference]] = Field(None, alias="requestBody")
+    parameters: Optional[list[ReferenceOr[Parameter]]] = None
+    request_body: Optional[ReferenceOr[RequestBody]] = Field(None, alias="requestBody")
     responses: Responses
     callbacks: Optional[dict[str, Callback]] = None
 

openapi_python_client/schema/openapi_schema_pydantic/parameter.py

@@ -1,11 +1,11 @@
-from typing import Any, Optional, Union
+from typing import Any, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from ..parameter_location import ParameterLocation
 from .example import Example
 from .media_type import MediaType
-from .reference import Reference
+from .reference import ReferenceOr
 from .schema import Schema
 
 
@@ -30,9 +30,9 @@ class Parameter(BaseModel):
     style: Optional[str] = None
     explode: bool = False
     allowReserved: bool = False
-    param_schema: Optional[Union[Reference, Schema]] = Field(default=None, alias="schema")
+    param_schema: Optional[ReferenceOr[Schema]] = Field(default=None, alias="schema")
     example: Optional[Any] = None
-    examples: Optional[dict[str, Union[Example, Reference]]] = None
+    examples: Optional[dict[str, ReferenceOr[Example]]] = None
     content: Optional[dict[str, MediaType]] = None
     model_config = ConfigDict(
         # `MediaType` is not build yet, will rebuild in `__init__.py`:

openapi_python_client/schema/openapi_schema_pydantic/path_item.py

@@ -1,9 +1,9 @@
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from .parameter import Parameter
-from .reference import Reference
+from .reference import ReferenceOr
 from .server import Server
 
 if TYPE_CHECKING:
@@ -34,7 +34,7 @@ class PathItem(BaseModel):
     patch: Optional["Operation"] = None
     trace: Optional["Operation"] = None
     servers: Optional[list[Server]] = None
-    parameters: Optional[list[Union[Parameter, Reference]]] = None
+    parameters: Optional[list[ReferenceOr[Parameter]]] = None
     model_config = ConfigDict(
         # `Operation` is an unresolvable forward reference, will rebuild in `__init__.py`:
         defer_build=True,

openapi_python_client/schema/openapi_schema_pydantic/reference.py

@@ -1,4 +1,7 @@
-from pydantic import BaseModel, ConfigDict, Field
+from typing import Annotated, Any, Literal, TypeVar, Union
+
+from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag
+from typing_extensions import TypeAlias
 
 
 class Reference(BaseModel):
@@ -24,3 +27,17 @@ class Reference(BaseModel):
             "examples": [{"$ref": "#/components/schemas/Pet"}, {"$ref": "Pet.json"}, {"$ref": "definitions.json#/Pet"}]
         },
     )
+
+
+T = TypeVar("T")
+
+
+def _reference_discriminator(obj: Any) -> Literal["ref", "other"]:
+    if isinstance(obj, dict):
+        return "ref" if "$ref" in obj else "other"
+    return "ref" if isinstance(obj, Reference) else "other"
+
+
+ReferenceOr: TypeAlias = Annotated[
+    Union[Annotated[Reference, Tag("ref")], Annotated[T, Tag("other")]], Discriminator(_reference_discriminator)
+]

openapi_python_client/schema/openapi_schema_pydantic/response.py

@@ -1,11 +1,11 @@
-from typing import Optional, Union
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict
 
 from .header import Header
 from .link import Link
 from .media_type import MediaType
-from .reference import Reference
+from .reference import ReferenceOr
 
 
 class Response(BaseModel):
@@ -19,9 +19,9 @@ class Response(BaseModel):
     """
 
     description: str
-    headers: Optional[dict[str, Union[Header, Reference]]] = None
+    headers: Optional[dict[str, ReferenceOr[Header]]] = None
     content: Optional[dict[str, MediaType]] = None
-    links: Optional[dict[str, Union[Link, Reference]]] = None
+    links: Optional[dict[str, ReferenceOr[Link]]] = None
     model_config = ConfigDict(
         # `MediaType` is not build yet, will rebuild in `__init__.py`:
         defer_build=True,

openapi_python_client/schema/openapi_schema_pydantic/responses.py

@@ -1,9 +1,7 @@
-from typing import Union
-
-from .reference import Reference
+from .reference import ReferenceOr
 from .response import Response
 
-Responses = dict[str, Union[Response, Reference]]
+Responses = dict[str, ReferenceOr[Response]]
 """
 A container for the expected responses of an operation.
 The container maps a HTTP response code to the expected response.

openapi_python_client/schema/openapi_schema_pydantic/schema.py

@@ -5,7 +5,7 @@
 from ..data_type import DataType
 from .discriminator import Discriminator
 from .external_documentation import ExternalDocumentation
-from .reference import Reference
+from .reference import ReferenceOr
 from .xml import XML
 
 
@@ -38,14 +38,14 @@ class Schema(BaseModel):
     enum: Union[None, list[Any]] = Field(default=None, min_length=1)
     const: Union[None, StrictStr, StrictInt, StrictFloat, StrictBool] = None
     type: Union[DataType, list[DataType], None] = Field(default=None)
-    allOf: list[Union[Reference, "Schema"]] = Field(default_factory=list)
-    oneOf: list[Union[Reference, "Schema"]] = Field(default_factory=list)
-    anyOf: list[Union[Reference, "Schema"]] = Field(default_factory=list)
-    schema_not: Optional[Union[Reference, "Schema"]] = Field(default=None, alias="not")
-    items: Optional[Union[Reference, "Schema"]] = None
-    prefixItems: list[Union[Reference, "Schema"]] = Field(default_factory=list)
-    properties: Optional[dict[str, Union[Reference, "Schema"]]] = None
-    additionalProperties: Optional[Union[bool, Reference, "Schema"]] = None
+    allOf: list[ReferenceOr["Schema"]] = Field(default_factory=list)
+    oneOf: list[ReferenceOr["Schema"]] = Field(default_factory=list)
+    anyOf: list[ReferenceOr["Schema"]] = Field(default_factory=list)
+    schema_not: Optional[ReferenceOr["Schema"]] = Field(default=None, alias="not")
+    items: Optional[ReferenceOr["Schema"]] = None
+    prefixItems: list[ReferenceOr["Schema"]] = Field(default_factory=list)
+    properties: Optional[dict[str, ReferenceOr["Schema"]]] = None
+    additionalProperties: Optional[Union[bool, ReferenceOr["Schema"]]] = None
     description: Optional[str] = None
     schema_format: Optional[str] = Field(default=None, alias="format")
     default: Optional[Any] = None