feat(cdk): improve connector builder field descriptions and examples #681
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -1,5 +1,3 @@ | ||||||
| # generated by datamodel-codegen: | ||||||
| # filename: declarative_component_schema.yaml | ||||||
|
|
||||||
|
|
@@ -623,7 +621,9 @@ class OAuthAuthenticator(BaseModel): | |||||
| scopes: Optional[List[str]] = Field( | ||||||
| None, | ||||||
| description="List of scopes that should be granted to the access token.", | ||||||
| examples=[ | ||||||
| ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] | ||||||
| ], | ||||||
| title="Scopes", | ||||||
| ) | ||||||
| token_expiry_date: Optional[str] = Field( | ||||||
|
|
@@ -1134,24 +1134,28 @@ class OAuthConfigSpecification(BaseModel): | |||||
| class Config: | ||||||
| extra = Extra.allow | ||||||
|
|
||||||
| oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( | ||||||
| Field( | ||||||
| None, | ||||||
| description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", | ||||||
| examples=[ | ||||||
| {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, | ||||||
| { | ||||||
| "app_id": { | ||||||
| "type": "string", | ||||||
| "path_in_connector_config": ["info", "app_id"], | ||||||
| } | ||||||
| }, | ||||||
| ], | ||||||
| title="OAuth user input", | ||||||
| ) | ||||||
| ) | ||||||
| oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( | ||||||
| Field( | ||||||
| None, | ||||||
| description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', | ||||||
| title="DeclarativeOAuth Connector Specification", | ||||||
| ) | ||||||
| ) | ||||||
| complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( | ||||||
| None, | ||||||
|
|
@@ -1169,7 +1173,9 @@ class Config: | |||||
| complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( | ||||||
| None, | ||||||
| description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", | ||||||
| examples=[ | ||||||
| {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} | ||||||
| ], | ||||||
| title="OAuth input specification", | ||||||
| ) | ||||||
| complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( | ||||||
|
|
@@ -1665,7 +1671,9 @@ class CustomConfigTransformation(BaseModel): | |||||
| class_name: str = Field( | ||||||
| ..., | ||||||
| description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_<name>.<package>.<class_name>`.", | ||||||
| examples=[ | ||||||
| "source_declarative_manifest.components.MyCustomConfigTransformation" | ||||||
| ], | ||||||
| ) | ||||||
| parameters: Optional[Dict[str, Any]] = Field( | ||||||
| None, | ||||||
|
|
@@ -2050,7 +2058,9 @@ class RecordSelector(BaseModel): | |||||
| description="Responsible for filtering records to be emitted by the Source.", | ||||||
| title="Record Filter", | ||||||
| ) | ||||||
| schema_normalization: Optional[ | ||||||
| Union[SchemaNormalization, CustomSchemaNormalization] | ||||||
| ] = Field( | ||||||
| None, | ||||||
| description="Responsible for normalization according to the schema.", | ||||||
| title="Schema Normalization", | ||||||
|
|
@@ -2086,10 +2096,12 @@ class DpathValidator(BaseModel): | |||||
| ], | ||||||
| title="Field Path", | ||||||
| ) | ||||||
| validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( | ||||||
| Field( | ||||||
| ..., | ||||||
| description="The condition that the specified config value will be evaluated against", | ||||||
| title="Validation Strategy", | ||||||
| ) | ||||||
| ) | ||||||
|
|
||||||
|
|
||||||
|
|
@@ -2106,10 +2118,12 @@ class PredicateValidator(BaseModel): | |||||
| ], | ||||||
| title="Value", | ||||||
| ) | ||||||
| validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( | ||||||
| Field( | ||||||
| ..., | ||||||
| description="The validation strategy to apply to the value.", | ||||||
| title="Validation Strategy", | ||||||
| ) | ||||||
| ) | ||||||
|
|
||||||
|
|
||||||
|
|
@@ -2299,9 +2313,9 @@ class Config: | |||||
|
|
||||||
| type: Literal["DeclarativeSource"] | ||||||
| check: Union[CheckStream, CheckDynamicStream] | ||||||
| streams: Optional[ | ||||||
| List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] | ||||||
| ] = None | ||||||
| dynamic_streams: List[DynamicDeclarativeStream] | ||||||
| version: str = Field( | ||||||
| ..., | ||||||
|
|
@@ -2426,7 +2440,9 @@ class Config: | |||||
| extra = Extra.allow | ||||||
|
|
||||||
| type: Literal["DeclarativeStream"] | ||||||
| name: Optional[str] = Field( | ||||||
| "", description="The stream name.", example=["Users"], title="Name" | ||||||
| ) | ||||||
| retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( | ||||||
| ..., | ||||||
| description="Component used to coordinate how records are extracted across stream slices and request pages.", | ||||||
|
|
@@ -2611,18 +2627,20 @@ class HttpRequester(BaseModelWithDeprecations): | |||||
| description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", | ||||||
| title="Query Properties", | ||||||
| ) | ||||||
| request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( | ||||||
| Field( | ||||||
| None, | ||||||
| description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", | ||||||
| examples=[ | ||||||
| {"unit": "day"}, | ||||||
| { | ||||||
| "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' | ||||||
| }, | ||||||
| {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, | ||||||
| {"sort_by[asc]": "updated_at"}, | ||||||
| ], | ||||||
| title="Query Parameters", | ||||||
| ) | ||||||
| ) | ||||||
| request_headers: Optional[Union[Dict[str, str], str]] = Field( | ||||||
| None, | ||||||
|
|
@@ -2789,7 +2807,9 @@ class QueryProperties(BaseModel): | |||||
|
|
||||||
| class StateDelegatingStream(BaseModel): | ||||||
| type: Literal["StateDelegatingStream"] | ||||||
| name: str = Field( | ||||||
| ..., description="The stream name.", example=["Users"], title="Name" | ||||||
|
||||||
| ..., description="The stream name.", example=["Users"], title="Name" | |
| ..., description="The stream name.", example="Users", title="Name" |
Copilot uses AI. Check for mistakes.
There was a problem hiding this comment.
Ditto my comment above. I don't know if that's correct.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The example for stream name should be a string, not a list. Stream names are strings like "Users", not arrays. Change
example=["Users"]toexample="Users".Copilot uses AI. Check for mistakes.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@copilot - I don't think that is correct. Confirm if you can view the docs, but despite the weird making, I think this aligns with "examples" in the JSON schema spec, which can be an array.