From bc09b28a649d591a45ec28008d3b6038ecb5fcd8 Mon Sep 17 00:00:00 2001 From: cccs-cat001 <56204545+cccs-cat001@users.noreply.github.com> Date: Thu, 30 Jan 2025 00:56:35 -0500 Subject: [PATCH 1/5] refactor(operation_config): change logging on is_profiling_enabled (#12416) Co-authored-by: Harshal Sheth --- metadata-ingestion/setup.py | 12 +++++++----- .../src/datahub/ingestion/source/sql/sql_config.py | 10 ---------- .../ingestion/source_config/operation_config.py | 9 +++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index e603b5f6ac1d30..f7e6482fd26f87 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -251,6 +251,7 @@ # Iceberg Python SDK # Kept at 0.4.0 due to higher versions requiring pydantic>2, as soon as we are fine with it, bump this dependency "pyiceberg>=0.4.0", + *cachetools_lib, } mssql_common = { @@ -407,13 +408,14 @@ # UnsupportedProductError # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/release-notes.html#rn-7-14-0 # https://github.com/elastic/elasticsearch-py/issues/1639#issuecomment-883587433 - "elasticsearch": {"elasticsearch==7.13.4"}, + "elasticsearch": {"elasticsearch==7.13.4", *cachetools_lib}, "cassandra": { "cassandra-driver>=3.28.0", # We were seeing an error like this `numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject` # with numpy 2.0. This likely indicates a mismatch between scikit-learn and numpy versions. # https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility "numpy<2", + *cachetools_lib, }, "feast": { "feast>=0.34.0,<1", @@ -425,7 +427,7 @@ "numpy<2", }, "grafana": {"requests"}, - "glue": aws_common, + "glue": aws_common | cachetools_lib, # hdbcli is supported officially by SAP, sqlalchemy-hana is built on top but not officially supported "hana": sql_common | { @@ -482,11 +484,11 @@ | classification_lib | {"db-dtypes"} # Pandas extension data types | cachetools_lib, - "s3": {*s3_base, *data_lake_profiling}, + "s3": {*s3_base, *data_lake_profiling, *cachetools_lib}, "gcs": {*s3_base, *data_lake_profiling}, - "abs": {*abs_base, *data_lake_profiling}, + "abs": {*abs_base, *data_lake_profiling, *cachetools_lib}, "sagemaker": aws_common, - "salesforce": {"simple-salesforce"}, + "salesforce": {"simple-salesforce", *cachetools_lib}, "snowflake": snowflake_common | usage_common | sqlglot_lib, "snowflake-summary": snowflake_common | usage_common | sqlglot_lib, "snowflake-queries": snowflake_common | usage_common | sqlglot_lib, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py index 7d82d99412ffe8..3ead59eed2d39a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py @@ -2,8 +2,6 @@ from abc import abstractmethod from typing import Any, Dict, Optional -import cachetools -import cachetools.keys import pydantic from pydantic import Field from sqlalchemy.engine import URL @@ -29,7 +27,6 @@ StatefulIngestionConfigBase, ) from datahub.ingestion.source_config.operation_config import is_profiling_enabled -from datahub.utilities.cachetools_keys import self_methodkey logger: logging.Logger = logging.getLogger(__name__) @@ -118,13 +115,6 @@ class SQLCommonConfig( # Custom Stateful Ingestion settings stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None - # TRICKY: The operation_config is time-dependent. Because we don't want to change - # whether or not we're running profiling mid-ingestion, we cache the result of this method. - # TODO: This decorator should be moved to the is_profiling_enabled(operation_config) method. - @cachetools.cached( - cache=cachetools.LRUCache(maxsize=1), - key=self_methodkey, - ) def is_profiling_enabled(self) -> bool: return self.profiling.enabled and is_profiling_enabled( self.profiling.operation_config diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py index a670173aa47519..1846dcb4fdd3d0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/operation_config.py @@ -2,10 +2,12 @@ import logging from typing import Any, Dict, Optional +import cachetools import pydantic from pydantic.fields import Field from datahub.configuration.common import ConfigModel +from datahub.utilities.cachetools_keys import self_methodkey logger = logging.getLogger(__name__) @@ -62,6 +64,13 @@ def validate_profile_date_of_month(cls, v: Optional[int]) -> Optional[int]: return profile_date_of_month +# TRICKY: The operation_config is time-dependent. Because we don't want to change +# whether or not we're running profiling mid-ingestion, we cache the result of this method. +# An additional benefit is that we only print the log lines on the first call. +@cachetools.cached( + cache=cachetools.LRUCache(maxsize=1), + key=self_methodkey, +) def is_profiling_enabled(operation_config: OperationConfig) -> bool: if operation_config.lower_freq_profile_enabled is False: return True From 6acd94b436647c5c309317b8a1f58319c13ef82f Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Thu, 30 Jan 2025 03:13:25 -0500 Subject: [PATCH 2/5] feat(docs) assertion execution behavior (#12484) --- docs/managed-datahub/observe/assertions.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/managed-datahub/observe/assertions.md b/docs/managed-datahub/observe/assertions.md index e63d051a0096b2..b8789bb92e858f 100644 --- a/docs/managed-datahub/observe/assertions.md +++ b/docs/managed-datahub/observe/assertions.md @@ -46,3 +46,21 @@ With Acryl Observe, you can get the Assertion Change event by getting API events ## Cost We provide a plethora of ways to run your assertions, aiming to allow you to use the cheapest possible means to do so and/or the most accurate means to do so, depending on your use case. For example, for Freshness (SLA) assertions, it is relatively cheap to use either their Audit Log or Information Schema as a means to run freshness checks, and we support both of those as well as Last Modified Column, High Watermark Column, and DataHub Operation ([see the docs for more details](/docs/managed-datahub/observe/freshness-assertions.md#3-change-source)). + +## Execution details - Where and How + +There are a few ways DataHub Cloud assertions can be executed: +1. Directly query the source system: + a. `Information Schema` tables are used by default to power cheap, fast checks on a table's freshness or row count. + b. `Audit log` or `Operation log` tables can be used to granularly monitor table operations. + c. The table itself can also be queried directly. This is useful for freshness checks referencing `last_updated` columns, row count checks targetting a subset of the data, and column value checks. We offer several optimizations to reduce query costs for these checks. +2. Reference DataHub profiling information + a. `Operation`s that are reported via ingestion or our SDKs can power monitoring table freshness. + b. `DatasetProfile` and `SchemaFieldProfile` ingested or reported via SDKs can power monitoring table metrics and column metrics. + +### Privacy: Execute In-Network, avoid exposing data externally +As a part of DataHub Cloud, we offer a [Remote Executor](/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md) deployment model. If this model is used, assertions will execute within your network, and only the results will be sent back to DataHub Cloud. Neither your actual credentials, nor your source data will leave your network. + +### Source system selection +Assertions will execute queries using the same source system that was used to initially ingest the table. +There are some scenarios where customers may have multiple ingestion sources for, i.e. a BigQuery table. In this case, by default the executor will take the ingestion source that was used to ingest the table's `DatasetProperties`. This behavior can be modified by your customer success rep. From c8e88aef097c043ee4f9515e46db9f7505d1725d Mon Sep 17 00:00:00 2001 From: hao zhang Date: Thu, 30 Jan 2025 16:13:47 +0800 Subject: [PATCH 3/5] feat(ingest/dbt-core): support fetching using the s3a protocol (#12465) Co-authored-by: hzhang --- metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py index 04de763370c951..fbb54d211e1b21 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py @@ -488,7 +488,7 @@ def load_file_as_json( ) -> Dict: if re.match("^https?://", uri): return json.loads(requests.get(uri).text) - elif re.match("^s3://", uri): + elif is_s3_uri(uri): u = urlparse(uri) assert aws_connection response = aws_connection.get_s3_client().get_object( From aaaa655c21bf66e38a318043942e36fee8a01c30 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 30 Jan 2025 00:39:56 -0800 Subject: [PATCH 4/5] feat(dataProcessInstance): Support data process instance entity page and lineage (#12499) --- .../datahub/graphql/GmsGraphQLEngine.java | 20 +- .../DataPlatformInstanceAspectMapper.java | 1 + .../DataProcessInstanceType.java | 3 +- .../mappers/DataProcessInstanceMapper.java | 41 ++++- .../src/main/resources/entity.graphql | 29 ++- .../DataProcessInstanceTypeTest.java | 7 +- .../DataProcessInstanceMapperTest.java | 23 ++- .../DataProcessInstanceEntity.tsx | 42 ++--- .../DataProcessInstanceEntity.tsx | 173 +++++------------- .../dataProcessInstance/preview/Preview.tsx | 28 +-- .../search/EmbeddedListSearchSection.tsx | 27 ++- .../profile/header/DefaultEntityHeader.tsx | 15 +- .../profile/sidebar/SidebarEntityHeader.tsx | 16 +- .../app/ingest/source/builder/constants.ts | 4 +- .../src/app/lineageV2/LineageExplorer.tsx | 5 +- .../LineageFilterNode/computeOrFilters.ts | 19 +- .../useFetchFilterNodeContents.ts | 8 +- .../LineageTransformationNode.tsx | 9 +- .../src/app/lineageV2/NodeBuilder.ts | 1 - datahub-web-react/src/app/lineageV2/common.ts | 23 ++- .../lineageV2/controls/LineageControls.tsx | 8 +- .../controls/LineageSearchFilters.tsx | 13 ++ .../app/lineageV2/pruneAllDuplicateEdges.ts | 135 ++++++++++++++ .../src/app/lineageV2/useBulkEntityLineage.ts | 7 +- .../app/lineageV2/useColumnHighlighting.ts | 15 +- .../lineageV2/useComputeGraph/filterNodes.ts | 85 ++++++++- .../useComputeGraph/getDisplayedNodes.ts | 2 +- .../useComputeGraph/useComputeGraph.tsx | 3 + .../app/lineageV2/useSearchAcrossLineage.ts | 97 +--------- .../src/app/previewV2/EntityHeader.tsx | 4 +- .../src/graphql/dataProcessInstance.graphql | 103 ++++++----- datahub-web-react/src/graphql/lineage.graphql | 3 + datahub-web-react/src/images/mlflowlogo2.png | Bin 0 -> 20876 bytes .../openapi/openapi_mces_golden.json | 6 +- .../integration/openapi/openapi_to_file.yml | 5 +- .../bootstrap_mcps/data-platforms.yaml | 2 +- .../e2e/lineageV2/v2_lineage_column_level.js | 2 +- 37 files changed, 611 insertions(+), 373 deletions(-) create mode 100644 datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts create mode 100644 datahub-web-react/src/images/mlflowlogo2.png diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 68f1d851420258..c142dbf7eeb229 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -3141,13 +3141,26 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b "DataProcessInstance", typeWiring -> typeWiring + .dataFetcher("exists", new EntityExistsResolver(entityService)) + .dataFetcher( + "platform", + new LoadableTypeResolver<>( + dataPlatformType, + (env) -> { + final DataProcessInstance dataProcessInstance = env.getSource(); + return dataProcessInstance != null + && dataProcessInstance.getPlatform() != null + ? dataProcessInstance.getPlatform().getUrn() + : null; + })) .dataFetcher( "dataPlatformInstance", new LoadableTypeResolver<>( dataPlatformInstanceType, (env) -> { final DataProcessInstance dataProcessInstance = env.getSource(); - return dataProcessInstance.getDataPlatformInstance() != null + return dataProcessInstance != null + && dataProcessInstance.getDataPlatformInstance() != null ? dataProcessInstance.getDataPlatformInstance().getUrn() : null; })) @@ -3160,6 +3173,11 @@ private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder b final DataProcessInstance dpi = env.getSource(); return dpi.getContainer() != null ? dpi.getContainer().getUrn() : null; })) + .dataFetcher( + "parentTemplate", + new EntityTypeResolver( + entityTypes, + (env) -> ((DataProcessInstance) env.getSource()).getParentTemplate())) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher( "lineage", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java index ab3127a3ae232b..7355b19762924c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/DataPlatformInstanceAspectMapper.java @@ -29,6 +29,7 @@ public DataPlatformInstance apply( result.setType(EntityType.DATA_PLATFORM_INSTANCE); result.setUrn(input.getInstance().toString()); } + // Warning: This often cannot be read properly: overwritten by LoadableTypeResolver result.setPlatform( DataPlatform.builder() .setUrn(input.getPlatform().toString()) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java index eeaaaa96f51704..0b79d6ee220ebf 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceType.java @@ -41,7 +41,8 @@ public class DataProcessInstanceType DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, ML_TRAINING_RUN_PROPERTIES_ASPECT_NAME, SUB_TYPES_ASPECT_NAME, - CONTAINER_ASPECT_NAME); + CONTAINER_ASPECT_NAME, + STATUS_ASPECT_NAME); private final EntityClient _entityClient; private final FeatureFlags _featureFlags; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java index d721f5a5fb522d..66aae8d555c431 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapper.java @@ -3,6 +3,7 @@ import static com.linkedin.metadata.Constants.*; import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.Status; import com.linkedin.common.SubTypes; import com.linkedin.common.urn.Urn; import com.linkedin.data.DataMap; @@ -13,12 +14,15 @@ import com.linkedin.datahub.graphql.types.common.mappers.AuditStampMapper; import com.linkedin.datahub.graphql.types.common.mappers.CustomPropertiesMapper; import com.linkedin.datahub.graphql.types.common.mappers.DataPlatformInstanceAspectMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; import com.linkedin.datahub.graphql.types.common.mappers.SubTypesMapper; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLHyperParamMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLMetricMapper; import com.linkedin.dataprocess.DataProcessInstanceProperties; +import com.linkedin.dataprocess.DataProcessInstanceRelationships; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.ml.metadata.MLTrainingRunProperties; @@ -68,7 +72,7 @@ public DataProcessInstance apply( mappingHelper.mapToResult( DATA_PROCESS_INSTANCE_PROPERTIES_ASPECT_NAME, (dataProcessInstance, dataMap) -> - mapDataProcessProperties(context, dataProcessInstance, dataMap, entityUrn)); + mapDataProcessInstanceProperties(context, dataProcessInstance, dataMap, entityUrn)); mappingHelper.mapToResult( ML_TRAINING_RUN_PROPERTIES_ASPECT_NAME, (dataProcessInstance, dataMap) -> @@ -77,8 +81,10 @@ public DataProcessInstance apply( DATA_PLATFORM_INSTANCE_ASPECT_NAME, (dataProcessInstance, dataMap) -> { DataPlatformInstance dataPlatformInstance = new DataPlatformInstance(dataMap); - dataProcessInstance.setDataPlatformInstance( - DataPlatformInstanceAspectMapper.map(context, dataPlatformInstance)); + com.linkedin.datahub.graphql.generated.DataPlatformInstance value = + DataPlatformInstanceAspectMapper.map(context, dataPlatformInstance); + dataProcessInstance.setPlatform(value.getPlatform()); + dataProcessInstance.setDataPlatformInstance(value); }); mappingHelper.mapToResult( SUB_TYPES_ASPECT_NAME, @@ -87,6 +93,14 @@ public DataProcessInstance apply( mappingHelper.mapToResult( CONTAINER_ASPECT_NAME, (dataProcessInstance, dataMap) -> mapContainers(context, dataProcessInstance, dataMap)); + mappingHelper.mapToResult( + STATUS_ASPECT_NAME, + (dataProcessInstance, dataMap) -> + dataProcessInstance.setStatus(StatusMapper.map(context, new Status(dataMap)))); + mappingHelper.mapToResult( + DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, + (dataProcessInstance, dataMap) -> + mapDataProcessInstanceRelationships(context, dataProcessInstance, dataMap)); return mappingHelper.getResult(); } @@ -124,8 +138,8 @@ private void mapTrainingRunProperties( dpi.setMlTrainingRunProperties(properties); } - private void mapDataProcessProperties( - @Nonnull QueryContext context, + private void mapDataProcessInstanceProperties( + @Nullable QueryContext context, @Nonnull DataProcessInstance dpi, @Nonnull DataMap dataMap, @Nonnull Urn entityUrn) { @@ -146,9 +160,20 @@ private void mapDataProcessProperties( CustomPropertiesMapper.map( dataProcessInstanceProperties.getCustomProperties(), entityUrn)); } - if (dataProcessInstanceProperties.hasCreated()) { - dpi.setCreated(AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); - } + dpi.setCreated(AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); + properties.setCreated( + AuditStampMapper.map(context, dataProcessInstanceProperties.getCreated())); dpi.setProperties(properties); } + + private void mapDataProcessInstanceRelationships( + @Nullable QueryContext context, @Nonnull DataProcessInstance dpi, @Nonnull DataMap dataMap) { + DataProcessInstanceRelationships dataProcessInstanceRelationships = + new DataProcessInstanceRelationships(dataMap); + + if (dataProcessInstanceRelationships.getParentTemplate() != null) { + dpi.setParentTemplate( + UrnToEntityMapper.map(context, dataProcessInstanceRelationships.getParentTemplate())); + } + } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index f6adf884b2badc..25615d9b6aa4fe 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -6733,6 +6733,16 @@ type DataProcessInstance implements EntityWithRelationships & Entity { """ type: EntityType! + """ + Whether or not this entity exists on DataHub + """ + exists: Boolean + + """ + Status metadata of the data process instance + """ + status: Status + """ The history of state changes for the run """ @@ -6741,12 +6751,12 @@ type DataProcessInstance implements EntityWithRelationships & Entity { """ When the run was kicked off """ - created: AuditStamp + created: AuditStamp @deprecated(reason: "Use `properties.created`") """ The name of the data process """ - name: String + name: String @deprecated(reason: "Use `properties.name`") """ Edges extending from this entity. @@ -13154,7 +13164,7 @@ type DataProcessInstanceProperties { """ When this process instance was created """ - created: AuditStamp + created: AuditStamp! """ Additional custom properties specific to this process instance @@ -13188,9 +13198,8 @@ type MLTrainingRunProperties { } extend type DataProcessInstance { - """ - Additional read only properties associated with the Data Job + Additional read only properties associated with the Data Process Instance """ properties: DataProcessInstanceProperties @@ -13209,6 +13218,11 @@ extend type DataProcessInstance { """ container: Container + """ + Standardized platform urn where the data process instance is defined + """ + platform: DataPlatform + """ Recursively get the lineage of containers for this entity """ @@ -13218,4 +13232,9 @@ extend type DataProcessInstance { Additional properties when subtype is Training Run """ mlTrainingRunProperties: MLTrainingRunProperties + + """ + The parent entity whose run instance it is + """ + parentTemplate: Entity } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java index 437c74ab669146..9535830407117f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/DataProcessInstanceTypeTest.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.FabricType; import com.linkedin.common.Status; @@ -61,7 +62,11 @@ public class DataProcessInstanceTypeTest { private static final DataProcessInstanceKey TEST_DPI_1_KEY = new DataProcessInstanceKey().setId("id-1"); private static final DataProcessInstanceProperties TEST_DPI_1_PROPERTIES = - new DataProcessInstanceProperties().setName("Test DPI").setType(DataProcessType.STREAMING); + new DataProcessInstanceProperties() + .setName("Test DPI") + .setType(DataProcessType.STREAMING) + .setCreated( + new AuditStamp().setTime(1234L).setActor(UrnUtils.getUrn("urn:li:corpuser:1"))); private static final DataProcessInstanceInput TEST_DPI_1_DPI_INPUT = new DataProcessInstanceInput().setInputs(new UrnArray(ImmutableList.of(DATASET_URN))); private static final DataProcessInstanceOutput TEST_DPI_1_DPI_OUTPUT = diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java index cd9d58b54e6b3a..cac3c6f594932c 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceMapperTest.java @@ -3,6 +3,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; +import com.linkedin.common.AuditStamp; import com.linkedin.common.DataPlatformInstance; import com.linkedin.common.url.Url; import com.linkedin.common.urn.Urn; @@ -12,6 +13,7 @@ import com.linkedin.datahub.graphql.generated.DataProcessInstance; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.dataprocess.DataProcessInstanceProperties; +import com.linkedin.dataprocess.DataProcessInstanceRelationships; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; @@ -28,6 +30,7 @@ public class DataProcessInstanceMapperTest { private static final String TEST_INSTANCE_URN = "urn:li:dataProcessInstance:(test-workflow,test-instance)"; private static final String TEST_CONTAINER_URN = "urn:li:container:testContainer"; + private static final String TEST_USER_URN = "urn:li:corpuser:test"; private static final String TEST_EXTERNAL_URL = "https://example.com/process"; private static final String TEST_NAME = "Test Process Instance"; @@ -52,11 +55,15 @@ public void testMapBasicFields() throws Exception { } @Test - public void testMapDataProcessProperties() throws Exception { + public void testMapDataProcessInstanceProperties() throws Exception { // Create DataProcessInstanceProperties DataProcessInstanceProperties properties = new DataProcessInstanceProperties(); properties.setName(TEST_NAME); properties.setExternalUrl(new Url(TEST_EXTERNAL_URL)); + AuditStamp created = new AuditStamp(); + created.setTime(123456789L); + created.setActor(Urn.createFromString(TEST_USER_URN)); + properties.setCreated(created); // Add properties aspect addAspect(Constants.DATA_PROCESS_INSTANCE_PROPERTIES_ASPECT_NAME, properties); @@ -66,6 +73,20 @@ public void testMapDataProcessProperties() throws Exception { assertNotNull(instance.getProperties()); assertEquals(instance.getName(), TEST_NAME); assertEquals(instance.getExternalUrl(), TEST_EXTERNAL_URL); + assertEquals(instance.getCreated().getTime(), 123456789L); + assertEquals(instance.getCreated().getActor(), TEST_USER_URN); + } + + @Test + public void testMapDataProcessInstanceRelationships() throws Exception { + DataProcessInstanceRelationships relationships = new DataProcessInstanceRelationships(); + relationships.setParentTemplate(Urn.createFromString(TEST_INSTANCE_URN)); + + addAspect(Constants.DATA_PROCESS_INSTANCE_RELATIONSHIPS_ASPECT_NAME, relationships); + + DataProcessInstance instance = DataProcessInstanceMapper.map(null, entityResponse); + assertNotNull(instance.getParentTemplate()); + assertEquals(instance.getParentTemplate().getUrn(), TEST_INSTANCE_URN); } @Test diff --git a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx index bdf77959e97c7f..239681609323eb 100644 --- a/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProcessInstance/DataProcessInstanceEntity.tsx @@ -5,7 +5,10 @@ import { DataProcessInstance, EntityType, OwnershipType, SearchResult } from '.. import { Preview } from './preview/Preview'; import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; import { EntityProfile } from '../shared/containers/profile/EntityProfile'; -import { useGetDataProcessInstanceQuery } from '../../../graphql/dataProcessInstance.generated'; +import { + GetDataProcessInstanceQuery, + useGetDataProcessInstanceQuery, +} from '../../../graphql/dataProcessInstance.generated'; import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; @@ -15,7 +18,6 @@ import { GenericEntityProperties } from '../shared/types'; import { getDataForEntityType } from '../shared/containers/profile/utils'; import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; import { EntityMenuItems } from '../shared/EntityDropdown/EntityDropdown'; -import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; import { getDataProduct } from '../shared/utils'; import SummaryTab from './profile/DataProcessInstanceSummary'; @@ -58,9 +60,9 @@ export class DataProcessInstanceEntity implements Entity { ); }; - isSearchEnabled = () => true; + isSearchEnabled = () => false; - isBrowseEnabled = () => true; + isBrowseEnabled = () => false; isLineageEnabled = () => true; @@ -127,12 +129,9 @@ export class DataProcessInstanceEntity implements Entity { ]; getOverridePropertiesFromEntity = (processInstance?: DataProcessInstance | null): GenericEntityProperties => { - const name = processInstance?.name; - const externalUrl = processInstance?.externalUrl; return { - name, - externalUrl, - platform: processInstance?.dataPlatformInstance?.platform, + name: processInstance && this.displayName(processInstance), + platform: (processInstance as GetDataProcessInstanceQuery['dataProcessInstance'])?.optionalPlatform, }; }; @@ -142,14 +141,11 @@ export class DataProcessInstanceEntity implements Entity { return ( { return ( { }; getLineageVizConfig = (entity: DataProcessInstance) => { + const properties = this.getGenericEntityProperties(entity); return { urn: entity?.urn, name: this.displayName(entity), type: EntityType.DataProcessInstance, subtype: entity?.subTypes?.typeNames?.[0], - icon: entity?.dataPlatformInstance?.platform?.properties?.logoUrl || undefined, - platform: entity?.dataPlatformInstance?.platform, + icon: properties?.platform?.properties?.logoUrl ?? undefined, + platform: properties?.platform ?? undefined, container: entity?.container, }; }; diff --git a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx index 3ab8c2b268aac3..09301b101b259c 100644 --- a/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx +++ b/datahub-web-react/src/app/entityV2/dataProcessInstance/DataProcessInstanceEntity.tsx @@ -1,36 +1,18 @@ +import { GenericEntityProperties } from '@app/entity/shared/types'; +import { globalEntityRegistryV2 } from '@app/EntityRegistryProvider'; +import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '@app/entityV2/Entity'; +import { EntityProfile } from '@app/entityV2/shared/containers/profile/EntityProfile'; +import SidebarEntityHeader from '@app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader'; +import { getDataForEntityType } from '@app/entityV2/shared/containers/profile/utils'; +import { EntityMenuItems } from '@app/entityV2/shared/EntityDropdown/EntityMenuActions'; +import { LineageTab } from '@app/entityV2/shared/tabs/Lineage/LineageTab'; +import { PropertiesTab } from '@app/entityV2/shared/tabs/Properties/PropertiesTab'; +import { getDataProduct } from '@app/entityV2/shared/utils'; +import { GetDataProcessInstanceQuery, useGetDataProcessInstanceQuery } from '@graphql/dataProcessInstance.generated'; +import { ArrowsClockwise } from 'phosphor-react'; import React from 'react'; -import { ApiOutlined } from '@ant-design/icons'; -import { GenericEntityProperties } from '@src/app/entity/shared/types'; -import { - DataProcessInstance, - Entity as GeneratedEntity, - EntityType, - OwnershipType, - SearchResult, -} from '../../../types.generated'; -import { Preview } from './preview/Preview'; -import { Entity, EntityCapabilityType, IconStyleType, PreviewType } from '../Entity'; -import { EntityProfile } from '../shared/containers/profile/EntityProfile'; -import { useGetDataProcessInstanceQuery } from '../../../graphql/dataProcessInstance.generated'; -import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab'; -import { LineageTab } from '../shared/tabs/Lineage/LineageTab'; -import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection'; -import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection'; -import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection'; -import { getDataForEntityType } from '../shared/containers/profile/utils'; -import { SidebarDomainSection } from '../shared/containers/profile/sidebar/Domain/SidebarDomainSection'; -import { capitalizeFirstLetterOnly } from '../../shared/textUtil'; -import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection'; -import { getDataProduct } from '../shared/utils'; -// import SummaryTab from './profile/DataProcessInstaceSummary'; - -// const getProcessPlatformName = (data?: DataProcessInstance): string => { -// return ( -// data?.dataPlatformInstance?.platform?.properties?.displayName || -// capitalizeFirstLetterOnly(data?.dataPlatformInstance?.platform?.name) || -// '' -// ); -// }; +import { DataProcessInstance, Entity as GeneratedEntity, EntityType, SearchResult } from '../../../types.generated'; +import Preview from './preview/Preview'; const getParentEntities = (data: DataProcessInstance): GeneratedEntity[] => { const parentEntity = data?.relationships?.relationships?.find( @@ -48,6 +30,7 @@ const getParentEntities = (data: DataProcessInstance): GeneratedEntity[] => { }, ]; }; + /** * Definition of the DataHub DataProcessInstance entity. */ @@ -56,15 +39,15 @@ export class DataProcessInstanceEntity implements Entity { icon = (fontSize?: number, styleType?: IconStyleType, color?: string) => { if (styleType === IconStyleType.TAB_VIEW) { - return ; + return ; } if (styleType === IconStyleType.HIGHLIGHT) { - return ; + return ; } return ( - { ); }; - isSearchEnabled = () => true; + isSearchEnabled = () => false; - isBrowseEnabled = () => true; + isBrowseEnabled = () => false; isLineageEnabled = () => true; @@ -98,15 +81,10 @@ export class DataProcessInstanceEntity implements Entity { useEntityQuery={this.useEntityQuery} // useUpdateQuery={useUpdateDataProcessInstanceMutation} getOverrideProperties={this.getOverridePropertiesFromEntity} + headerDropdownItems={ + new Set([EntityMenuItems.UPDATE_DEPRECATION, EntityMenuItems.RAISE_INCIDENT, EntityMenuItems.SHARE]) + } tabs={[ - // { - // name: 'Documentation', - // component: DocumentationTab, - // }, - // { - // name: 'Summary', - // component: SummaryTab, - // }, { name: 'Lineage', component: LineageTab, @@ -115,51 +93,26 @@ export class DataProcessInstanceEntity implements Entity { name: 'Properties', component: PropertiesTab, }, - // { - // name: 'Incidents', - // component: IncidentTab, - // getDynamicName: (_, processInstance) => { - // const activeIncidentCount = processInstance?.dataProcessInstance?.activeIncidents.total; - // return `Incidents${(activeIncidentCount && ` (${activeIncidentCount})`) || ''}`; - // }, - // }, ]} sidebarSections={this.getSidebarSections()} /> ); - getSidebarSections = () => [ - { - component: SidebarAboutSection, - }, - { - component: SidebarOwnerSection, - properties: { - defaultOwnerType: OwnershipType.TechnicalOwner, - }, - }, - { - component: SidebarTagsSection, - properties: { - hasTags: true, - hasTerms: true, - }, - }, - { - component: SidebarDomainSection, - }, - { - component: DataProductSection, - }, - ]; + getSidebarSections = () => [{ component: SidebarEntityHeader }]; getOverridePropertiesFromEntity = (processInstance?: DataProcessInstance | null): GenericEntityProperties => { - const name = processInstance?.name; - const externalUrl = processInstance?.externalUrl; + const parent = + processInstance?.parentTemplate && + globalEntityRegistryV2.getGenericEntityProperties( + processInstance.parentTemplate.type, + processInstance.parentTemplate, + ); return { - name, - externalUrl, - platform: processInstance?.dataPlatformInstance?.platform, + name: processInstance && this.displayName(processInstance), + platform: + (processInstance as GetDataProcessInstanceQuery['dataProcessInstance'])?.optionalPlatform || + parent?.platform, + parent, }; }; @@ -169,73 +122,35 @@ export class DataProcessInstanceEntity implements Entity { return ( ); }; - renderSearch = (result: SearchResult) => { - const data = result.entity as DataProcessInstance; - const genericProperties = this.getGenericEntityProperties(data); - const parentEntities = getParentEntities(data); - return ( - - ); - }; + renderSearch = (result: SearchResult) => + this.renderPreview(PreviewType.SEARCH, result.entity as DataProcessInstance); getLineageVizConfig = (entity: DataProcessInstance) => { + const properties = this.getGenericEntityProperties(entity); return { urn: entity?.urn, name: this.displayName(entity), type: EntityType.DataProcessInstance, subtype: entity?.subTypes?.typeNames?.[0], - icon: entity?.dataPlatformInstance?.platform?.properties?.logoUrl || undefined, - platform: entity?.dataPlatformInstance?.platform, + icon: properties?.platform?.properties?.logoUrl ?? undefined, + platform: properties?.platform ?? undefined, container: entity?.container, - // health: entity?.health || undefined, }; }; diff --git a/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx b/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx index 3a3b0340695d96..4f95d13600374a 100644 --- a/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx +++ b/datahub-web-react/src/app/entityV2/dataProcessInstance/preview/Preview.tsx @@ -1,3 +1,5 @@ +import { GenericEntityProperties } from '@app/entity/shared/types'; +import DefaultPreviewCard from '@app/previewV2/DefaultPreviewCard'; import React from 'react'; import { DataProduct, @@ -11,15 +13,14 @@ import { Owner, SearchInsight, Container, - ParentContainersResult, } from '../../../../types.generated'; -import DefaultPreviewCard from '../../../preview/DefaultPreviewCard'; import { useEntityRegistry } from '../../../useEntityRegistry'; import { IconStyleType } from '../../Entity'; -export const Preview = ({ +export default function Preview({ urn, name, + data, subType, description, platformName, @@ -38,13 +39,10 @@ export const Preview = ({ paths, health, parentEntities, - parentContainers, -}: // duration, -// status, -// startTime, -{ +}: { urn: string; name: string; + data: GenericEntityProperties | null; subType?: string | null; description?: string | null; platformName?: string; @@ -63,17 +61,15 @@ export const Preview = ({ paths?: EntityPath[]; health?: Health[] | null; parentEntities?: Array | null; - parentContainers?: ParentContainersResult | null; - // duration?: number | null; - // status?: string | null; - // startTime?: number | null; -}): JSX.Element => { +}): JSX.Element { const entityRegistry = useEntityRegistry(); return ( ); -}; +} diff --git a/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx b/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx index a9601cc12df30b..2dd17f21e8e0eb 100644 --- a/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx +++ b/datahub-web-react/src/app/entityV2/shared/components/styled/search/EmbeddedListSearchSection.tsx @@ -4,7 +4,7 @@ import { useHistory, useLocation } from 'react-router'; import { ApolloError } from '@apollo/client'; import useSortInput from '@src/app/searchV2/sorting/useSortInput'; import { useSelectedSortOption } from '@src/app/search/context/SearchContext'; -import { FacetFilterInput } from '../../../../../../types.generated'; +import { EntityType, FacetFilterInput } from '../../../../../../types.generated'; import useFilters from '../../../../../search/utils/useFilters'; import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl'; import { FilterSet, GetSearchResultsParams, SearchResultsInterface } from './types'; @@ -19,6 +19,30 @@ import { import { decodeComma } from '../../../utils'; const FILTER = 'filter'; +const SEARCH_ENTITY_TYPES = [ + EntityType.Dataset, + EntityType.Dashboard, + EntityType.Chart, + EntityType.Mlmodel, + EntityType.MlmodelGroup, + EntityType.MlfeatureTable, + EntityType.Mlfeature, + EntityType.MlprimaryKey, + EntityType.DataFlow, + EntityType.DataJob, + EntityType.GlossaryTerm, + EntityType.GlossaryNode, + EntityType.Tag, + EntityType.Role, + EntityType.CorpUser, + EntityType.CorpGroup, + EntityType.Container, + EntityType.Domain, + EntityType.DataProduct, + EntityType.Notebook, + EntityType.BusinessAttribute, + EntityType.DataProcessInstance, +]; function getParamsWithoutFilters(params: QueryString.ParsedQuery) { const paramsCopy = { ...params }; @@ -146,6 +170,7 @@ export const EmbeddedListSearchSection = ({ return ( @@ -222,11 +231,7 @@ export const DefaultEntityHeader = ({ type={displayedEntityType} entityType={entityType} browsePaths={entityData?.browsePathV2} - parentEntities={ - entityData?.parentContainers?.containers || - entityData?.parentDomains?.domains || - entityData?.parentNodes?.nodes - } + parentEntities={parentEntities} contentRef={contentRef} isContentTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx b/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx index 4d8eeba24408c7..eaef45ad00dcbd 100644 --- a/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx +++ b/datahub-web-react/src/app/entityV2/shared/containers/profile/sidebar/SidebarEntityHeader.tsx @@ -1,7 +1,7 @@ import React from 'react'; import styled from 'styled-components'; -import { Container, DataPlatform, EntityType, Post } from '../../../../../../types.generated'; import { useEntityData, useRefetch } from '../../../../../entity/shared/EntityContext'; +import { Container, DataPlatform, EntityType, Post, Entity } from '../../../../../../types.generated'; import ContextPath from '../../../../../previewV2/ContextPath'; import HealthIcon from '../../../../../previewV2/HealthIcon'; import NotesIcon from '../../../../../previewV2/NotesIcon'; @@ -56,6 +56,14 @@ const SidebarEntityHeader = () => { const platforms = entityType === EntityType.SchemaField ? entityData?.parent?.siblingPlatforms : entityData?.siblingPlatforms; + const containerPath = + entityData?.parentContainers?.containers || + entityData?.parentDomains?.domains || + entityData?.parentNodes?.nodes || + []; + const parentPath: Entity[] = entityData?.parent ? [entityData.parent as Entity] : []; + const parentEntities = containerPath.length ? containerPath : parentPath; + if (loading) { return ; } @@ -90,11 +98,7 @@ const SidebarEntityHeader = () => { type={displayedEntityType} entityType={entityType} browsePaths={entityData?.browsePathV2} - parentEntities={ - entityData?.parentContainers?.containers || - entityData?.parentDomains?.domains || - entityData?.parentNodes?.nodes - } + parentEntities={parentEntities} contentRef={contentRef} isContentTruncated={isContentTruncated} /> diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 58525b3e88f975..9711dfe2f03d73 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -29,7 +29,7 @@ import powerbiLogo from '../../../../images/powerbilogo.png'; import modeLogo from '../../../../images/modelogo.png'; import databricksLogo from '../../../../images/databrickslogo.png'; import verticaLogo from '../../../../images/verticalogo.png'; -import mlflowLogo from '../../../../images/mlflowlogo.png'; +import mlflowLogo2 from '../../../../images/mlflowlogo2.png'; import dynamodbLogo from '../../../../images/dynamodblogo.png'; import fivetranLogo from '../../../../images/fivetranlogo.png'; import csvLogo from '../../../../images/csv-logo.png'; @@ -159,7 +159,7 @@ export const PLATFORM_URN_TO_LOGO = { [LOOKER_URN]: lookerLogo, [MARIA_DB_URN]: mariadbLogo, [METABASE_URN]: metabaseLogo, - [MLFLOW_URN]: mlflowLogo, + [MLFLOW_URN]: mlflowLogo2, [MODE_URN]: modeLogo, [MONGO_DB_URN]: mongodbLogo, [MSSQL_URN]: mssqlLogo, diff --git a/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx b/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx index d9f64696625ca6..ff30a54dc36db9 100644 --- a/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx +++ b/datahub-web-react/src/app/lineageV2/LineageExplorer.tsx @@ -36,10 +36,11 @@ export default function LineageExplorer(props: Props) { const [columnEdgeVersion, setColumnEdgeVersion] = useState(0); const [displayVersion, setDisplayVersion] = useState<[number, string[]]>([0, []]); const [hideTransformations, setHideTransformations] = useShouldHideTransformations(); + const [showDataProcessInstances, setShowDataProcessInstances] = useState(false); const [showGhostEntities, setShowGhostEntities] = useState(false); - const context = { + const context: NodeContext = { rootUrn: urn, rootType: type, nodes, @@ -55,6 +56,8 @@ export default function LineageExplorer(props: Props) { setColumnEdgeVersion, hideTransformations, setHideTransformations, + showDataProcessInstances, + setShowDataProcessInstances, showGhostEntities, setShowGhostEntities, }; diff --git a/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts b/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts index 252f1d2054ea38..44b523efe49acb 100644 --- a/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts +++ b/datahub-web-react/src/app/lineageV2/LineageFilterNode/computeOrFilters.ts @@ -13,10 +13,27 @@ import { AndFilterInput, EntityType, FacetFilterInput } from '@types'; export default function computeOrFilters( defaultFilters: FacetFilterInput[], hideTransformations = true, + hideDataProcessInstances = true, ): AndFilterInput[] { - if (!hideTransformations) { + if (!hideTransformations && !hideDataProcessInstances) { return [{ and: defaultFilters }]; } + + if (!hideTransformations) { + return [ + { + and: [ + ...defaultFilters, + { + field: ENTITY_FILTER_NAME, + values: [EntityType.DataProcessInstance], + negated: true, + }, + ], + }, + ]; + } + return [ { and: [ diff --git a/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts b/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts index bab6e1dc4705bf..cdfd7f66f3a1ff 100644 --- a/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts +++ b/datahub-web-react/src/app/lineageV2/LineageFilterNode/useFetchFilterNodeContents.ts @@ -20,9 +20,13 @@ interface Return { export default function useFetchFilterNodeContents(parent: string, direction: LineageDirection, skip: boolean): Return { const { startTimeMillis, endTimeMillis } = useGetLineageTimeParams(); - const { hideTransformations } = useContext(LineageNodesContext); + const { hideTransformations, showDataProcessInstances } = useContext(LineageNodesContext); - const orFilters = computeOrFilters([{ field: DEGREE_FILTER_NAME, values: ['1'] }], hideTransformations); + const orFilters = computeOrFilters( + [{ field: DEGREE_FILTER_NAME, values: ['1'] }], + hideTransformations, + showDataProcessInstances, + ); const { data } = useAggregateAcrossLineageQuery({ skip, fetchPolicy: 'cache-first', diff --git a/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx b/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx index 84e4b8fb39a883..aef9ecf9833f28 100644 --- a/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx +++ b/datahub-web-react/src/app/lineageV2/LineageTransformationNode/LineageTransformationNode.tsx @@ -2,6 +2,7 @@ import { ConsoleSqlOutlined, HomeOutlined, LoadingOutlined } from '@ant-design/i import LineageVisualizationContext from '@app/lineageV2/LineageVisualizationContext'; import { Skeleton, Spin } from 'antd'; import { Tooltip } from '@components'; +import { useEntityRegistryV2 } from '@app/useEntityRegistry'; import React, { useContext } from 'react'; import { Handle, NodeProps, Position } from 'reactflow'; import styled from 'styled-components'; @@ -68,6 +69,7 @@ const NodeWrapper = styled.div<{ `; const IconWrapper = styled.div<{ isGhost: boolean }>` + display: flex; opacity: ${({ isGhost }) => (isGhost ? 0.5 : 1)}; `; @@ -92,7 +94,9 @@ const CustomIcon = styled.img` export default function LineageTransformationNode(props: NodeProps) { const { data, selected, dragging } = props; const { urn, type, entity, fetchStatus } = data; + const entityRegistry = useEntityRegistryV2(); const isQuery = type === EntityType.Query; + const isDataProcessInstance = type === EntityType.DataProcessInstance; const { rootUrn } = useContext(LineageNodesContext); const { cllHighlightedNodes, setHoveredNode } = useContext(LineageDisplayContext); @@ -131,8 +135,11 @@ export default function LineageTransformationNode(props: NodeProps {icon && } + {!icon && isDataProcessInstance && entityRegistry.getIcon(EntityType.DataProcessInstance, 18)} {!icon && isQuery && } - {!icon && !isQuery && } + {!icon && !isQuery && !isDataProcessInstance && ( + + )} {fetchStatus[LineageDirection.Upstream] === FetchStatus.LOADING && ( diff --git a/datahub-web-react/src/app/lineageV2/NodeBuilder.ts b/datahub-web-react/src/app/lineageV2/NodeBuilder.ts index f369f95ceefc4f..07eacabc974f1e 100644 --- a/datahub-web-react/src/app/lineageV2/NodeBuilder.ts +++ b/datahub-web-react/src/app/lineageV2/NodeBuilder.ts @@ -180,7 +180,6 @@ export default class NodeBuilder { createEdges(edges: NodeContext['edges']): Edge[] { const baseEdges = new Map>(); edges.forEach((edge, edgeId) => { - if (!edge.isDisplayed) return; const [upstream, downstream] = parseEdgeId(edgeId); if (upstream in this.nodeInformation && downstream in this.nodeInformation) { const upstreamDirection = this.nodeInformation[upstream].direction; diff --git a/datahub-web-react/src/app/lineageV2/common.ts b/datahub-web-react/src/app/lineageV2/common.ts index 1110460a6a0537..65ae9736f2cbb2 100644 --- a/datahub-web-react/src/app/lineageV2/common.ts +++ b/datahub-web-react/src/app/lineageV2/common.ts @@ -75,7 +75,7 @@ export interface LineageFilter extends NodeBase { export type LineageNode = LineageEntity | LineageFilter; -const TRANSFORMATION_TYPES: string[] = [EntityType.Query, EntityType.DataJob]; +const TRANSFORMATION_TYPES: string[] = [EntityType.Query, EntityType.DataJob, EntityType.DataProcessInstance]; export function useIgnoreSchemaFieldStatus(): boolean { return useAppConfig().config.featureFlags.schemaFieldLineageIgnoreStatus; @@ -128,6 +128,11 @@ export function isUrnQuery(urn: string, entityRegistry: EntityRegistry): boolean return type === EntityType.Query; } +export function isUrnDataProcessInstance(urn: string, entityRegistry: EntityRegistry): boolean { + const type = getEntityTypeFromEntityUrn(urn, entityRegistry); + return type === EntityType.DataProcessInstance; +} + export function isUrnTransformational(urn: string, entityRegistry: EntityRegistry): boolean { const type = getEntityTypeFromEntityUrn(urn, entityRegistry); return (!!type && TRANSFORMATION_TYPES.includes(type)) || isUrnDbt(urn, entityRegistry); @@ -211,6 +216,8 @@ export interface NodeContext { setColumnEdgeVersion: Dispatch>; hideTransformations: boolean; setHideTransformations: (hide: boolean) => void; + showDataProcessInstances: boolean; + setShowDataProcessInstances: (hide: boolean) => void; showGhostEntities: boolean; setShowGhostEntities: (hide: boolean) => void; } @@ -234,6 +241,8 @@ export const LineageNodesContext = React.createContext({ setColumnEdgeVersion: () => {}, hideTransformations: false, setHideTransformations: () => {}, + showDataProcessInstances: false, + setShowDataProcessInstances: () => {}, showGhostEntities: false, setShowGhostEntities: () => {}, }); @@ -342,6 +351,7 @@ export function onClickPreventSelect(event: React.MouseEvent): true { const DATA_STORE_COLOR = '#ffd279'; const BI_TOOL_COLOR = '#8682a2'; +const ML_COLOR = '#206de8'; const DEFAULT_COLOR = '#ff7979'; export function getNodeColor(type?: EntityType): [string, string] { @@ -351,5 +361,14 @@ export function getNodeColor(type?: EntityType): [string, string] { if (type === EntityType.Dataset) { return [DATA_STORE_COLOR, 'Column']; } - return [DEFAULT_COLOR, 'Column']; + if ( + type === EntityType.Mlmodel || + type === EntityType.MlmodelGroup || + type === EntityType.Mlfeature || + type === EntityType.MlfeatureTable || + type === EntityType.MlprimaryKey + ) { + return [ML_COLOR, '']; + } + return [DEFAULT_COLOR, '']; } diff --git a/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx b/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx index 4dfde23ea3427c..df306d5759869b 100644 --- a/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx +++ b/datahub-web-react/src/app/lineageV2/controls/LineageControls.tsx @@ -56,7 +56,8 @@ const ControlsColumn = styled.div``; type PanelType = 'filters' | 'timeRange'; export default function LineageControls() { - const { rootUrn, hideTransformations, showGhostEntities } = useContext(LineageNodesContext); + const { rootUrn, hideTransformations, showDataProcessInstances, showGhostEntities } = + useContext(LineageNodesContext); const { isTabFullsize, setTabFullsize } = useContext(TabFullsizedContext); const { isDefault: isLineageTimeUnchanged } = useGetLineageTimeParams(); const { fitView } = useReactFlow(); @@ -104,7 +105,10 @@ export default function LineageControls() { > {showExpandedText ? 'Filter' : null} diff --git a/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx b/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx index 59d4a801a5147c..25a34748bf9c43 100644 --- a/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx +++ b/datahub-web-react/src/app/lineageV2/controls/LineageSearchFilters.tsx @@ -41,6 +41,8 @@ export default function LineageSearchFilters() { nodeVersion, hideTransformations, setHideTransformations, + showDataProcessInstances, + setShowDataProcessInstances, showGhostEntities, setShowGhostEntities, } = useContext(LineageNodesContext); @@ -75,6 +77,17 @@ export default function LineageSearchFilters() { /> + + + + Show Process Instances + Show task runs. Will not hide home node.} + /> + + + + diff --git a/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts b/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts new file mode 100644 index 00000000000000..9873b5906ac44a --- /dev/null +++ b/datahub-web-react/src/app/lineageV2/pruneAllDuplicateEdges.ts @@ -0,0 +1,135 @@ +import EntityRegistry from '@app/entityV2/EntityRegistry'; +import { + createEdgeId, + getEdgeId, + isUrnDataProcessInstance, + isUrnTransformational, + NodeContext, +} from '@app/lineageV2/common'; +import { LineageDirection } from '@types'; + +enum HideOption { + TRANSFORMATIONS = 'transformations', + DATA_PROCESS_INSTANCES = 'dataProcessInstances', +} + +const hideOptionIncludeUrnFunctions: Record boolean> = { + [HideOption.TRANSFORMATIONS]: isUrnTransformational, + [HideOption.DATA_PROCESS_INSTANCES]: isUrnDataProcessInstance, +}; + +/** + * Remove direct edges between non-transformational nodes, if there is a path between them through a transformational node. + * Remove direct edges between non-data process instances, if there is a path between them through data process instances. + * This prevents the graph from being cluttered with effectively duplicate edges. + * @param urn Urn for which to remove parent edges. + * @param direction Direction to look for parents. + * @param context Lineage node context. + * @param entityRegistry EntityRegistry, used to get EntityType from an urn. + */ +export default function pruneAllDuplicateEdges( + urn: string, + direction: LineageDirection | null, + context: Pick, + entityRegistry: EntityRegistry, +) { + let changed = false; + Object.values(HideOption).forEach((hideOption) => { + changed ||= pruneDuplicateEdges(urn, direction, hideOption, context, entityRegistry); + }); + if (changed) { + context.setDisplayVersion(([version, nodes]) => [version + 1, nodes]); + } +} + +/** + * Remove direct edges between a certain set of "excluded" nodes, if there is a path between them through only "included" nodes. + */ +export function pruneDuplicateEdges( + urn: string, + direction: LineageDirection | null, + hideOption: HideOption, + context: Pick, + entityRegistry: EntityRegistry, +): boolean { + const { edges } = context; + const neighbors: Record> = { + [LineageDirection.Downstream]: new Set(), + [LineageDirection.Upstream]: new Set(), + }; + + const includeUrn = hideOptionIncludeUrnFunctions[hideOption]; + const isUrnIncluded = includeUrn(urn, entityRegistry); + + function getNeighbors(d: LineageDirection) { + return getNeighborsByFunction(urn, d, includeUrn, context, entityRegistry); + } + + if (direction) { + neighbors[direction] = getNeighbors(direction); + } else { + neighbors[LineageDirection.Upstream] = getNeighbors(LineageDirection.Upstream); + neighbors[LineageDirection.Downstream] = getNeighbors(LineageDirection.Downstream); + } + + let changed = false; + if (isUrnIncluded) { + neighbors[LineageDirection.Upstream].forEach((source) => { + neighbors[LineageDirection.Downstream].forEach((destination) => { + const edge = edges.get(createEdgeId(source, destination)); + if (edge?.isDisplayed) { + edge.isDisplayed = false; + changed = true; + } + }); + }); + } else { + Object.values(LineageDirection).forEach((d) => { + neighbors[d].forEach((source) => { + const edge = edges.get(getEdgeId(urn, source, d)); + if (edge?.isDisplayed) { + edge.isDisplayed = false; + changed = true; + } + }); + }); + } + + return changed; +} + +/** + * Get the non-transformational nodes that are reachable from `urn` in `direction` via a transformational path. + * @param urn Urn for which to get neighbors. + * @param direction Direction to look for neighbors. + * @param includeUrn Function to determine if a node should be included, based on its urn. + * @param adjacencyList Adjacency list of the lineage graph. + * @param entityRegistry EntityRegistry, used to get EntityType from an urn. + */ +function getNeighborsByFunction( + urn: string, + direction: LineageDirection, + includeUrn: (urn: string, entityRegistry: EntityRegistry) => boolean, + { adjacencyList }: Pick, + entityRegistry: EntityRegistry, +) { + const neighbors = new Set(); + // If urn is included, then direct neighbors can be included + const stack = includeUrn(urn, entityRegistry) + ? [urn] + : Array.from(adjacencyList[direction].get(urn) || []).filter((p) => includeUrn(p, entityRegistry)); + const seen = new Set(stack); + for (let u = stack.pop(); u; u = stack.pop()) { + Array.from(adjacencyList[direction].get(u) || []).forEach((parent) => { + if (includeUrn(parent, entityRegistry)) { + if (!seen.has(parent)) { + stack.push(parent); + seen.add(parent); + } + } else { + neighbors.add(parent); + } + }); + } + return neighbors; +} diff --git a/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts b/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts index 1cfc6c892fc3e4..54769a0a1243fe 100644 --- a/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts +++ b/datahub-web-react/src/app/lineageV2/useBulkEntityLineage.ts @@ -1,3 +1,4 @@ +import pruneAllDuplicateEdges from '@app/lineageV2/pruneAllDuplicateEdges'; import { useAppConfig } from '@app/useAppConfig'; import { useCallback, useContext, useEffect, useState } from 'react'; import { useGetBulkEntityLineageV2Query } from '../../graphql/lineage.generated'; @@ -18,7 +19,7 @@ import { useIgnoreSchemaFieldStatus, } from './common'; import { FetchedEntityV2Relationship } from './types'; -import { addQueryNodes, pruneDuplicateEdges, setEntityNodeDefault } from './useSearchAcrossLineage'; +import { addQueryNodes, setEntityNodeDefault } from './useSearchAcrossLineage'; const BATCH_SIZE = 10; @@ -100,7 +101,7 @@ export default function useBulkEntityLineage(shownUrns: string[]): (urn: string) const entityRegistry = useEntityRegistryV2(); useEffect(() => { - const smallContext = { nodes, edges, adjacencyList }; + const smallContext = { nodes, edges, adjacencyList, setDisplayVersion }; let changed = false; data?.entities?.forEach((rawEntity) => { if (!rawEntity) return; @@ -122,7 +123,7 @@ export default function useBulkEntityLineage(shownUrns: string[]): (urn: string) entity.upstreamRelationships?.forEach((relationship) => { processEdge(node, relationship, LineageDirection.Upstream, smallContext); }); - pruneDuplicateEdges(node.urn, null, smallContext, entityRegistry); + pruneAllDuplicateEdges(node.urn, null, smallContext, entityRegistry); } } }); diff --git a/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts b/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts index cd44609e0630df..82de5330346429 100644 --- a/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts +++ b/datahub-web-react/src/app/lineageV2/useColumnHighlighting.ts @@ -33,8 +33,17 @@ export default function useColumnHighlighting( } { const entityRegistry = useEntityRegistryV2(); const { setEdges } = useReactFlow(); - const { nodes, adjacencyList, edges, rootUrn, rootType, nodeVersion, columnEdgeVersion, hideTransformations } = - useContext(LineageNodesContext); + const { + nodes, + adjacencyList, + edges, + rootUrn, + rootType, + nodeVersion, + columnEdgeVersion, + hideTransformations, + showDataProcessInstances, + } = useContext(LineageNodesContext); const { cllHighlightedNodes, highlightedColumns, columnEdges } = useMemo(() => { const displayedNodeIds = new Set(shownUrns); @@ -73,7 +82,7 @@ export default function useColumnHighlighting( }), 0, ); - }, [nodeVersion, hideTransformations, columnEdges, setEdges]); + }, [nodeVersion, hideTransformations, showDataProcessInstances, columnEdges, setEdges]); return { cllHighlightedNodes, highlightedColumns }; } diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts b/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts index 224986962b89aa..edfd36dd53e0ea 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/filterNodes.ts @@ -1,19 +1,22 @@ +import { globalEntityRegistryV2 } from '@app/EntityRegistryProvider'; import { addToAdjacencyList, EdgeId, getEdgeId, isGhostEntity, isTransformational, + isUrnQuery, LineageAuditStamp, LineageEdge, NodeContext, parseEdgeId, setDefault, } from '@app/lineageV2/common'; -import { LineageDirection } from '@types'; +import { EntityType, LineageDirection } from '@types'; export interface HideNodesConfig { hideTransformations: boolean; + hideDataProcessInstances: boolean; hideGhostEntities: boolean; ignoreSchemaFieldStatus: boolean; } @@ -25,7 +28,7 @@ type ContextSubset = Pick; */ export default function hideNodes( rootUrn: string, - { hideTransformations, hideGhostEntities, ignoreSchemaFieldStatus }: HideNodesConfig, + { hideTransformations, hideDataProcessInstances, hideGhostEntities, ignoreSchemaFieldStatus }: HideNodesConfig, { nodes, edges, adjacencyList }: ContextSubset, ): ContextSubset { let newNodes = nodes; @@ -51,6 +54,25 @@ export default function hideNodes( adjacencyList: newAdjacencyList, })); } + if (hideDataProcessInstances) { + // Note: Will only pick one query node if there is lineage t1 -> q1 -> dpi1 -> q2 -> t2 + // Currently data process instances can't have lineage to queries so this is fine + newNodes = new Map( + Array.from(newNodes).filter( + ([urn, node]) => urn === rootUrn || node?.entity?.type !== EntityType.DataProcessInstance, + ), + ); + ({ newEdges, newAdjacencyList } = connectEdges(rootUrn, { + nodes: newNodes, + edges: newEdges, + adjacencyList: newAdjacencyList, + })); + } + ({ newEdges, newAdjacencyList } = removeHiddenEdges({ + nodes: newNodes, + adjacencyList: newAdjacencyList, + edges: newEdges, + })); return { nodes: newNodes, edges: newEdges, adjacencyList: newAdjacencyList }; } @@ -99,24 +121,30 @@ function connectEdges(rootUrn: string, { nodes, edges, adjacencyList }: ContextS seen.add(id); adjacencyList[direction].get(id)?.forEach((neighbor) => { + if (isUrnQuery(neighbor, globalEntityRegistryV2)) { + return; + } if (nodes.has(neighbor)) { addToAdjacencyList(newAdjacencyList, direction, id, neighbor); const edgeId = getEdgeId(id, neighbor, direction); - // isDisplayed always true -- only set to false right now to deduplicate edges through dbt - newEdges.set(edgeId, { ...edges.get(edgeId), via: undefined, isDisplayed: true }); + const existingEdge = newEdges.get(edgeId); + newEdges.set(edgeId, mergeEdges(edges.get(edgeId), existingEdge)); buildNewAdjacencyList(neighbor, direction); } else { buildNewAdjacencyList(neighbor, direction)?.forEach((child) => { addToAdjacencyList(newAdjacencyList, direction, id, child); const edgeId = getEdgeId(id, child, direction); const firstEdge = edges.get(getEdgeId(id, neighbor, direction)); - const secondEdge = edges.get(getEdgeId(neighbor, child, direction)); - newEdges.set(edgeId, { + const secondEdge = newEdges.get(getEdgeId(neighbor, child, direction)); + const existingEdge = newEdges.get(edgeId); + const newEdge = { isManual: (firstEdge?.isManual || secondEdge?.isManual) ?? false, created: getLatestTimestamp(firstEdge?.created, secondEdge?.created), updated: getLatestTimestamp(firstEdge?.updated, secondEdge?.updated), - isDisplayed: true, - }); + isDisplayed: (firstEdge?.isDisplayed && secondEdge?.isDisplayed) ?? false, + via: firstEdge?.via || secondEdge?.via, + }; + newEdges.set(edgeId, mergeEdges(newEdge, existingEdge)); }); } }); @@ -126,9 +154,50 @@ function connectEdges(rootUrn: string, { nodes, edges, adjacencyList }: ContextS buildNewAdjacencyList(rootUrn, LineageDirection.Upstream); seen.clear(); buildNewAdjacencyList(rootUrn, LineageDirection.Downstream); + + newEdges.forEach((edge, edgeId) => { + const [upstream, downstream] = parseEdgeId(edgeId); + if (edge.via && nodes.has(edge.via) && nodes.get(edge.via)?.type === EntityType.Query) { + setDefault(newAdjacencyList[LineageDirection.Upstream], edge.via, new Set()).add(upstream); + setDefault(newAdjacencyList[LineageDirection.Downstream], edge.via, new Set()).add(downstream); + } + }); + return { newAdjacencyList, newEdges }; } +/** Merge two edges, each representing a different path between two nodes. */ +function mergeEdges(edgeA?: LineageEdge, edgeB?: LineageEdge): LineageEdge { + return { + isManual: edgeA?.isManual && edgeB?.isManual, + created: getLatestTimestamp(edgeA?.created, edgeB?.created), + updated: getLatestTimestamp(edgeA?.updated, edgeB?.updated), + isDisplayed: (edgeA?.isDisplayed || edgeB?.isDisplayed) ?? false, + via: edgeA?.via || edgeB?.via, + }; +} + +function removeHiddenEdges({ edges }: ContextSubset) { + const newEdges = new Map(); + const newAdjacencyList: NodeContext['adjacencyList'] = { + [LineageDirection.Upstream]: new Map(), + [LineageDirection.Downstream]: new Map(), + }; + + edges.forEach((edge, edgeId) => { + const [upstream, downstream] = parseEdgeId(edgeId); + if (edge.isDisplayed) { + addToAdjacencyList(newAdjacencyList, LineageDirection.Upstream, downstream, upstream); + newEdges.set(edgeId, edge); + if (edge.via) { + setDefault(newAdjacencyList[LineageDirection.Upstream], edge.via, new Set()).add(upstream); + setDefault(newAdjacencyList[LineageDirection.Downstream], edge.via, new Set()).add(downstream); + } + } + }); + return { newEdges, newAdjacencyList }; +} + function getLatestTimestamp( a: LineageAuditStamp | undefined, b: LineageAuditStamp | undefined, diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts b/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts index 81d213a158cd53..cfe4e442884b0b 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/getDisplayedNodes.ts @@ -254,7 +254,7 @@ function getTransformationalNodes( } if (nodesInBetween.has(child) || leafUrns.has(child)) { const edge = edges.get(getEdgeId(urn, child, direction)); - if (edge?.isDisplayed && edge?.via) { + if (edge?.via) { const queryNode = nodes.get(edge.via); if (queryNode) { result.push(queryNode); diff --git a/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx b/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx index eebcc84b3d9bda..4d1040ded3896d 100644 --- a/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx +++ b/datahub-web-react/src/app/lineageV2/useComputeGraph/useComputeGraph.tsx @@ -27,6 +27,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe dataVersion, displayVersion, hideTransformations, + showDataProcessInstances, showGhostEntities, } = useContext(LineageNodesContext); const entityRegistry = useEntityRegistryV2(); @@ -56,6 +57,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe const config: HideNodesConfig = { hideTransformations, + hideDataProcessInstances: !showDataProcessInstances, hideGhostEntities: !showGhostEntities, ignoreSchemaFieldStatus, }; @@ -88,6 +90,7 @@ export default function useComputeGraph(urn: string, type: EntityType): Processe displayVersionNumber, hideTransformations, prevHideTransformations, + showDataProcessInstances, showGhostEntities, ignoreSchemaFieldStatus, dataVersion, diff --git a/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts b/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts index f114eac6288ee3..a1eba9ba01e5f3 100644 --- a/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts +++ b/datahub-web-react/src/app/lineageV2/useSearchAcrossLineage.ts @@ -1,20 +1,18 @@ +import pruneAllDuplicateEdges from '@app/lineageV2/pruneAllDuplicateEdges'; import { useEffect, useState } from 'react'; import { useSearchAcrossLineageStructureLazyQuery } from '../../graphql/search.generated'; import { Entity, EntityType, LineageDirection, Maybe, SearchAcrossLineageInput } from '../../types.generated'; -import EntityRegistry from '../entityV2/EntityRegistry'; import { DBT_URN } from '../ingest/source/builder/constants'; import { useGetLineageTimeParams } from '../lineage/utils/useGetLineageTimeParams'; import { DEGREE_FILTER_NAME } from '../search/utils/constants'; import { useEntityRegistryV2 } from '../useEntityRegistry'; import { addToAdjacencyList, - createEdgeId, FetchStatus, Filters, getEdgeId, isQuery, isTransformational, - isUrnTransformational, LINEAGE_FILTER_PAGINATION, LineageEntity, NodeContext, @@ -80,6 +78,7 @@ export default function useSearchAcrossLineage( platforms: [DBT_URN], }, { entityType: EntityType.DataJob }, + { entityType: EntityType.DataProcessInstance }, ], }, searchFlags: { @@ -101,7 +100,7 @@ export default function useSearchAcrossLineage( }, [fetchLineage, lazy]); useEffect(() => { - const smallContext = { nodes, edges, adjacencyList }; + const smallContext = { nodes, edges, adjacencyList, setDisplayVersion }; let addedNode = false; data?.searchAcrossLineage?.searchResults.forEach((result) => { @@ -137,7 +136,7 @@ export default function useSearchAcrossLineage( } if (data) { - pruneDuplicateEdges(urn, direction, smallContext, entityRegistry); + pruneAllDuplicateEdges(urn, direction, smallContext, entityRegistry); processed.add(urn); if (addedNode) setNodeVersion((version) => version + 1); @@ -162,94 +161,6 @@ export default function useSearchAcrossLineage( return { fetchLineage, processed: processed.has(urn) }; } -/** - * Remove direct edges between non-transformational nodes, if there is a path between them through only transformational nodes. - * This prevents the graph from being cluttered with effectively duplicate edges. - * @param urn Urn for which to remove parent edges. - * @param direction Direction to look for parents. - * @param context Lineage node context. - * @param entityRegistry EntityRegistry, used to get EntityType from an urn. - */ -export function pruneDuplicateEdges( - urn: string, - direction: LineageDirection | null, - context: Pick, - entityRegistry: EntityRegistry, -) { - const { edges } = context; - const neighbors: Record> = { - [LineageDirection.Downstream]: new Set(), - [LineageDirection.Upstream]: new Set(), - }; - - const urnIsTransformational = isUrnTransformational(urn, entityRegistry); - function getNeighbors(d: LineageDirection) { - return getNonTransformationalNeighbors(urn, d, urnIsTransformational, context, entityRegistry); - } - - if (direction) { - neighbors[direction] = getNeighbors(direction); - } else { - neighbors[LineageDirection.Upstream] = getNeighbors(LineageDirection.Upstream); - neighbors[LineageDirection.Downstream] = getNeighbors(LineageDirection.Downstream); - } - - if (urnIsTransformational) { - neighbors[LineageDirection.Upstream].forEach((source) => { - neighbors[LineageDirection.Downstream].forEach((destination) => { - const edge = edges.get(createEdgeId(source, destination)); - if (edge?.isDisplayed) { - edge.isDisplayed = false; - } - }); - }); - } else { - Object.values(LineageDirection).forEach((d) => { - neighbors[d].forEach((source) => { - const edge = edges.get(getEdgeId(urn, source, d)); - if (edge?.isDisplayed) { - edge.isDisplayed = false; - } - }); - }); - } -} - -/** - * Get the non-transformational nodes that are reachable from `urn` in `direction` via a transformational path. - * @param urn Urn for which to get neighbors. - * @param direction Direction to look for neighbors. - * @param includeDirect If false, only include non-transformational neighbors through a transformational node. - * @param adjacencyList Adjacency list of the lineage graph. - * @param entityRegistry EntityRegistry, used to get EntityType from an urn. - */ -function getNonTransformationalNeighbors( - urn: string, - direction: LineageDirection, - includeDirect: boolean, - { adjacencyList }: Pick, - entityRegistry: EntityRegistry, -) { - const neighbors = new Set(); - const stack = includeDirect - ? [urn] - : Array.from(adjacencyList[direction].get(urn) || []).filter((p) => isUrnTransformational(p, entityRegistry)); - const seen = new Set(stack); - for (let u = stack.pop(); u; u = stack.pop()) { - Array.from(adjacencyList[direction].get(u) || []).forEach((parent) => { - if (isUrnTransformational(parent, entityRegistry)) { - if (!seen.has(parent)) { - stack.push(parent); - seen.add(parent); - } - } else { - neighbors.add(parent); - } - }); - } - return neighbors; -} - export function setEntityNodeDefault( urn: string, type: EntityType, diff --git a/datahub-web-react/src/app/previewV2/EntityHeader.tsx b/datahub-web-react/src/app/previewV2/EntityHeader.tsx index be986d6293eac2..9ebf30732b9d7b 100644 --- a/datahub-web-react/src/app/previewV2/EntityHeader.tsx +++ b/datahub-web-react/src/app/previewV2/EntityHeader.tsx @@ -96,12 +96,12 @@ const EntityHeader: React.FC = ({ {previewType === PreviewType.HOVER_CARD ? ( - {name || ''} + {name || urn} ) : ( - + )} diff --git a/datahub-web-react/src/graphql/dataProcessInstance.graphql b/datahub-web-react/src/graphql/dataProcessInstance.graphql index 442f8db0a933b2..548aa746b3a85b 100644 --- a/datahub-web-react/src/graphql/dataProcessInstance.graphql +++ b/datahub-web-react/src/graphql/dataProcessInstance.graphql @@ -67,6 +67,10 @@ fragment processInstanceRelationshipResults on EntityRelationshipsResult { fragment dataProcessInstanceFields on DataProcessInstance { urn type + exists + status { + removed + } parentContainers { ...parentContainersFields } @@ -78,7 +82,7 @@ fragment dataProcessInstanceFields on DataProcessInstance { } properties { name - createdTS: created { + created { time actor } @@ -88,6 +92,7 @@ fragment dataProcessInstanceFields on DataProcessInstance { } } mlTrainingRunProperties { + id outputUrls trainingMetrics { name @@ -100,67 +105,79 @@ fragment dataProcessInstanceFields on DataProcessInstance { value } } + optionalPlatform: platform { + ...platformFields + } dataPlatformInstance { ...dataPlatformInstanceFields } - state(startTimeMillis: null, endTimeMillis: null, limit: 1) { - status - attempt - result { - resultType - nativeResultType - } - timestampMillis - durationMillis - } - relationships(input: { types: ["InstanceOf", "Consumes", "Produces"], direction: OUTGOING, start: 0, count: 50 }) { - ...processInstanceRelationshipResults - } -} - -query getDataProcessInstance($urn: String!) { - dataProcessInstance(urn: $urn) { + parentTemplate { urn type - parentContainers { - ...parentContainersFields - } - subTypes { - typeNames - } - container { - ...entityContainer - } - name - properties { + # TODO: Clean up fields below; support DataFlow + ... on Dataset { name - created { - time - actor + properties { + name + description + qualifiedName + } + editableProperties { + description + } + platform { + ...platformFields + } + subTypes { + typeNames + } + status { + removed } } - mlTrainingRunProperties { - id - outputUrls - trainingMetrics { + ... on DataJob { + urn + type + dataFlow { + ...nonRecursiveDataFlowFields + } + jobId + properties { name description - value + externalUrl + customProperties { + key + value + } } - hyperParams { - name + deprecation { + ...deprecationFields + } + dataPlatformInstance { + ...dataPlatformInstanceFields + } + subTypes { + typeNames + } + editableProperties { description - value + } + status { + removed } } + } +} + +query getDataProcessInstance($urn: String!) { + dataProcessInstance(urn: $urn) { + ...dataProcessInstanceFields relationships( input: { types: ["InstanceOf", "Consumes", "Produces"], direction: OUTGOING, start: 0, count: 50 } ) { ...processInstanceRelationshipResults } - dataPlatformInstance { - ...dataPlatformInstanceFields - } state(startTimeMillis: null, endTimeMillis: null, limit: 1) { status attempt diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index 2a89789f1c50d1..5daf200fef5fc7 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -716,6 +716,9 @@ fragment entityLineageV2 on Entity { ...platformFields } } + ... on DataProcessInstance { + ...dataProcessInstanceFields + } ... on Domain { id properties { diff --git a/datahub-web-react/src/images/mlflowlogo2.png b/datahub-web-react/src/images/mlflowlogo2.png new file mode 100644 index 0000000000000000000000000000000000000000..d2b7dbea783415f85d60e3386fcca732d32107c8 GIT binary patch literal 20876 zcmZ^L1z6k9vS<OC=Ny1BE_9j zC{pBw|2g;G^S%3C^6_Q!+u7OO*_qj~5TmXtkB38n0{{T<6cuDN0RUj~{T~Di{RG)5 z@CE$=wv|+o1OV!i9$cGaqQAo}6*N@0C;l&0J~-YfJi0)K<1YF zQ9~5np>F*`(MClD@C4n406;(@00z1RM85%1_5kodZ2$m`3H`z-0RB&J0qDPafyo6J z|I@w)gzigD0RSK#d+irqFI1F8EM1*B%&lB4tU3Ig-R=PZqJARirn9w|Io!|L$;DH| zPmKN_2oZGq{xK&#{2vf6M=|;rD(Y})R}X8r00$Qb7ri(R91a)tu(A=+l#%=QaP&7Z zdOI&KHxW)wUteDiUtSJZ4_i)dVPRoTE*?%E9(FVYyQja4m$@Iiizmasko*@O8Ea2V z4|_K+dsi3uJzjGQS8p#ddiwi`{`>W>bz1w`|IbV=p8wt!dV`$zJ)GPeT%7-}VP5t& z|DR#^J^vc^&%XXOo#;I>5p{b%YbSjfduMAGPxR8n`M9`5|C#20_59C5|6}Bf|L@5E z==r}#O1nC_dT6^RTx@lk}c&_u&8Q*}vgMIq#|ZA5{L= z5&rWOeTL#VqMZND58^m4RCy!-00cl$MpD}kc#sFpeK^~6cho-1ixJ&*iW@z+oBAlm zJqJ(5n%JUfGb`G$NTWDHM)F}zQSC<_FBKfd1!C!}zg%fE5e~O{ zYEl8!c${T2D8#w0iJulvP_qBWG8M1DlcM}Qb%;u2F5~uO+mrU{dlYwm3xQJZ*ungH z$8?;P)3dyJ#gJzT%c^4wy8R2P#vW(EEE!I03rmcz=6i|vUq?i{QkaUmX$fxyu*hBj zK;le@WjreQ3N2`wCS%c-!7y@Lgq9AIlC@MkOi^6QO&~0ILS9{U#*~v_>#<~qvxTxt z0lX=HDdf52fi&Gllu>k&#U~5euPTp9!iex;$D%+mY@AF(h+5P#;mfz_`UATw^BWwo zSz0BDR#Q46)6&kA({cw~N%6<6wFH&A(vq^lyFTgS zR@%pxtdAV#mJa|$|Bg2=!~o7pOd49U>`1X$Yo@E$J}w-Qeipq_EewL$dL~_NQz7DZIid+VRZ= zSc*4#N;qhNpk5j1poO1w6h9u@c)(*cB<3 zoPn>V_b2h$@d@XpOs~l*bT3}YyjIHRq`0V*_9+lp0mBd=lP*;&wKL1p*yzgK-wiwR z+mu$tUC?I}J2*go)Sk1$Nq`C2&#dX#6I1DfUXsI5t}p;mw1t;Au~xeCqY`;~({j@s zd#i&}eBy&#LHq|ULugtvOJ$tckH58R8y*A*iJHRQjfuUVmW!WH45U2`Qf*%=xv>bO>}M zee=GpD97$y+6kcwu`fGoV#gla)@qeC=;Bvx{Fo_pfYX(RVs#&TCQ>Y1ly;r-Xtnw+ zWMb;c$adYjYLM?Kt5@6K%JnaQV*wWVSs|JeyAiMTnw1L++}VXyq+RPGqC;>vVfi(s z2+cQxgy&!Lk~`U9!cxp4gb2V+1j8d*RReaIGl5qGzR`_Qf!FT>n<9K*_LRg z`6JYT-11NvL#;C>de4FWHadpiaDP#`GYPxn-QbO&c)9RFKcMQ?nKya2!CG1d#|Gyk zLTiE`2t1grj^_HtBHeFH`O0UkaAcB3#tjE7DqB$fR_55yL8)jwV3=FTG8Qc|tVT;j z`5yy*7uH4m;T`j$i2W2o;OZ(2bw$82Lz@vrzm{i6<;lunJxobN2*5A9?h+5j>)nG_ zDKdNBjaLUose5_PN#_tK5IzjeCg4bCA57fbcFlIbSfy8}p zE%E9ySoWs{MG+!I4F+H9IcGif4#xfv#cnQUAm=v158}>F_M@F%>#SgXW>xH{YyOxW z2u^N{{!Ws4?J!cuL+@%?@4VUisoxwA+@@moEJ5H}rP1}4I!^AluIncX{394bRC5d! zF3Rz!6IC<)aOP*c8^TCdYobFs0xSzsF%!Ir zzTxO}Jq?+x+6We=N4AdV<~0Qby&Wh1z*iq@D!*B0?L;~ZV8)PN!Jw$;FK@0gQ(W=b zT*akBwi*c$>ow>0l~yq}^cV}^dBR~{AimuUvE)JGIg3j_-&hJ@FZGzy^{`4f1CIyW zkJV+*A;vFA#0#iXK~y8HZU%`swJyMAgA&+_IYpcyXxcBFBVRV5cnEUM1cVBJQvW!} zJNHiJ0D--dZMJQ7mS&Ds-gH?)1CqSzcp!LkZWmVndVKcO=T0WFZjyKC!>dYFdY$M% zt<_qv(2$C!$PhaM2Y~R23ur&erf#UicjoqME7dCpBKULS0*568snX_Ods0@m$A)2p z8b@Ci;)APT1^&8Q9;8~I;HWYyDgnYSf?O5&o9vi(&1r6V5;CrxR>cMtARufozKhiO zL@6U(cGl;kkP{=^X@^1vfGke5^)SU$V0{lCIAuwUTXB7uE9ieS;=P* zJnJYQ>=`0%oxO{INK`Tv7||sTa!^(1YgRb@I2Y&lMZ9TZCItXekf&f?Rx|ienyGjz z#6)S&!`dqe0OvB)#strEbX&xY0bSmS9H zqnzbmjK<o0HjqJ>3JJZI>!+h>~u$(q0(bSfaZh%B+un+x@9Tz+d?IVz}9#z z^cD_@qo=1l1T{?us^dK=z%e}!(10J)K6g-K(3VXHeKPti4+Df&!IEjA%$$COUMss* znNgD^;E9+e;OIJuAxB}}7wra;FnMeazXu?F#^af#uL|5@@}x@eSN1cAKi!4qAN8gc z*qUZ|GfP^{-JoD2`r4?k^;h;P`rUaP3*qDsz{i$~Jzgx;P90f4`_D)(=%5fJ@&+sp zGs?Y_QoJ?%^Da{E8Pv|_$?u#R`y{1<31jxgji+zd1eaGiA>gzwVxu)%9WwbnjO0j1bN-mod!`VWy7)rv+&rt>8U}d? zWL@YKUfY^WlU>Hw?w3l=n&3BAG5Im|7Bx7l-CQ17@G?afcFYIFp(3zvw_HYXP|UB} z%dZ+Gfu(dX7Fr(uCCYxKy~u!<3q~JeT$gG#OsKew5tW_devyXgEf3D|`A4p1K1PWt zK?y~br=>x#+BaRDeBI-wnp8n^y4C8NBBAQajN07{bu3#)GGwE=HOm%U5D+$`g87~J z;*te82}Ak%n&X&1xp}2d@Laf`F)Zrsk@m%*FX)*a`@0?f(nrC$=aPI=_p*w-M*JOtpEQKGxp$ z-D|g_AW=1WxZiJtoim$3{(4f{ued2|nmM#!Qqk3BKXCT&Ry-wAtbth@2ZM18zXLSJ)*qJ^&ZF7oqh|R%nusn zlR2N-o!Y*$9>sX%lZP+^4VG-MlW}p}Nr5r+-g(^AASGd3lE`2g-Cl2Z$sd{5-pr|z zd;l&=wemMikJH4DaRUFg^>&*`>!)0L*CSV@(<`R){hQ#G7cZ|8HQ~=M9 z7^Eu;kwegxj;89c$OaZI0O_bn%of{TLZEhi#7rbs0cm9u6;YZ zEklSZRE~T5yi3CcE~bP04juxRy^GXLCQno-u72SHe>wlo(Ru}K-+?F?B*nNgR;tag z{kT@q9IlRUa6a`?dGC<}bHdIrcv>^y`@}hW;<7W-mLYFM8`fK0IOr|4j$Ff@xawAqkNpUpxv=FnQVrXtj22wC(W0cD?}w9*6vb@ zT`cTXX!+JF<+11mDN4umn{a50xW70W-=t`yO&f~D$2}sv3!86lg>|f{kKLl{on4|w z#{{c_MCDct+pg;1@@5sgG8EUS$qN zT`9HByxU$Cntehs)6R0o$o93pZ!_L#uPji`*cC2&UKaRfqnzZ@?Zzz`x!(hM17HQ6Wl z<%H&0k3BWJ`jr-3smM$TNTa6V5plfZ1IIY*w)n)wrz8}%Nt?S)S3kL5eQQ{k6dKk2 znzZRLZmGRT=}d%(dAuJ;d=@nUc5tPpe&#H?wh*Di&ho(cYl?I}o?CwC27!EbMT|q? z46*NA8g&u#$>nB&KiqaI92d_KulJX5#&z~?{WdK|3Z;Z~QT0&lU8vEa^Q1GXv6go} zcXixQ`AC6h{05Ri*h<1m7RDnWyx|$MLh__-K}2yrn0++1!f53eF}WFl%HB9j{al z*3}|yZ`i|p>64740BA38Wyq=DBv#Ffgr#NI)f5!At4&RqEQa#`jk3K$lyzL`^U=g~ zN(){m?^}jqGeew;chp)~JY-~HTZ9+Q1Q89sep30SVr9oc#MaPBW$5=`zvztOL=~2y zXI%SV=oZR(`a6e%`KtQJG5VVV%CLyYeqlmr`X7v8q<*SwI~*DvZMXjwgYEhUDdA>L z3)~G1P*`sodlAOW=6e(~d^H_&6K>^ITO29;OnZIDS5qr}val=h!&(f0$6dBz*Gy!& z&Pb{*5BzhcEYLTR`3_q}$H$^cb`F2&;xv0|o1xh*oX8S9Y#l+u_g$fTZ7Z$o&704a zZ}vY`p>FL39D9lTbG_uA>;bKY>+!;g;?0?>csCs1=~8r>Fc4iYX4hdJ&7&R-JHg@0 z4>cw3Ue!xFI;mrk$U(LFX@;yBs?)pGc(QvrgbfbspK}Y*U%;SbRaJ2XW-3>i`mO6lthM-zu<-~ z!snzHJI3#N(i4$%~B~Lg%fAIu$7Bq|H{HId=winyFR3Pn*Km4_+v#_p1YWjy--_(R4CL#ls zI~HP(fP?wZYEnuCO8sI21nayMA~)gL^ zluCY2w^Wj_2{20u)S{O^DG%gvk5KcT4Wp{3cRck6JpJD4fJQz{sG>L$#B+5x%|yw8 z4lcI8CE{dQw8>;CQ=zMEqu?12Mv^l2kE-_!y(4C8-e9%fBv^IY(_G&IrO!*P8jleRc`j< z`w9B!q)Dx-I>|AvGvT)bsIQ~84RRg3+TzsJp}E(?o8gr^>lv!T1nsO41>HP`4HP~;=U#b&0%_&L846TtQFrxwc+REx`asJ2nA=G* zJlp7^ig>~2DY8N_{1kI&7-Xu7Jc(09H7{u!>K6FO^{k44&eo@1@$KF5AAKM-OF~h^ zvT9G4sqG&Fws`EnR**`uDGY7p`kA@cK;z_V%Bdz(4}^>ux)wNLqDIBe!?3OEECqTG z`f9<*74ujltj`A{DGGmZ?4j1j^YTYBK6*ETJn$^ON(Qv|v50Lcq3l3Hw$EI^{MJ}W zLe+P%`q32+Ga_*CH>G3jZ&&Z1P%g`(Fw{y70roGW6x0Y!b@uFgNc&1I%ElS zAT-#+XN=7S_Pe1;f7%+vkm$g%B0C0hF-mOXwd#XDeDS#K>O&>8jb20S=iusCa`9>^ z)S;IEUl7vBDgXI`Xz_GMT~PUI2y5#7nf*W zTCHPF-ScOQsjzjIZn!->oi+5?9KqJB1n-%-k}|yc+fV70f%yvK_LK?e>+ImPPj>?K zM-n4Z<8N(4MC$fWWfNm9BID$B9#z|@t`IPpXDew)Tvh&lz4da|nsVB= zoyKaQmFYooOCG+Io-fE@CvJD@bL^p>8*?Cd-IVAPu`Jr^dPmDcKkSA-yWG>`imjhC;_?*#dl?-er!cMdaCug&`iJpk8Mz)vjZd)0kDk2Rt}4-U z?{)^=O$oC}y?!$?KkLD0q!msK6Rj$gh-1H4ANiWWCMD4 zWf#cV=dmxK)l`v~&fu@GN5ZCWpn04xOQK+t-gV7?BO}|>!`E;Hd>)Xzu%5iEhI}CA zG7?_F4lD_2y99YSZwH?r_q%}hGUMN#IZY=2@L`J-p9cF(ZMPgvdN5`@6?Qr1n66Ce zq6m$Rq~|zUDzFA}HkJV@MTiRMax)sfQ5I8=>-)PZ0SXH}J*ytcQ%HlG9mC%0X!#7v zNl(E%QoA}Up7u{{w@5s*2Y-A9volSMXt4JB6OFHTEWiuGuA}nzw|V09h*RvcX>@HU z=BF~Xoq5@5YO}PHhR-FrDvPGZa~G?gax3Ex-!0xj0@hmzWK=Ym(LFNeS*CeSJ%c2 z!(X|8TgBg|n{`Hv(JmcOs9Gr?63sC2wvtn2bx9`EKzjovBsrZjZ!*sn!`W5LzluT4 zo{!xcXi?=w@~44bG!LcK&oSslRXJi+TaNvayhKE?J)oIW8hfGNW-l@rA zJ)z6neBB!!Fu`{S90A_MHC5r4+-G0^GU~>|?#i=>au|`gy${$o;_mn9;xQO&f&%S? zE`0OQJ^O3*xtnzy~Pwt8l-^C zOBeXhyDFw+zliSP0Hvx}H*iNLcf z;T^|%E!${sJR8Go-1iCALiY;Tp3A#Fo`Czsvj!7GV@>3&@c|=_@hX#BUl;WZ)k|YP z;Czl}3YUon7Zrk!S^J>JZMJ{uDduy-FqRAc&|yLz4A25Im+gfmAFoRFd>n%`h)>-i zdooB07aK+q>>V5#O!t8vuCH@8zx@me1i{TXfmsvpnG-&?8%S3=!`Uh%W);TpFq4Pg z8P4GLamziFVrB|WZ|ANSEd_Ig0U$3JxO~@hjOrx}mEyIjp%bSd>6;&spy-)WiXd5I zxDXG0*TP-VnkYk<0xpOZx-Io71^lPv;pe_auwCk=upBc91rB^>-^0fs4yGVqATB6j zkfjQ_4{Pzjx%O)1P_Mqf3B_V`X6VAMv(wTLd1Q|<*fA7dX2Vj(OVPjANIu3%O51lH z$$d1)3u#qEj3-=kE_^#I8$~Ssyb_$+U~>ixjZBuOrYeZYw&tgH1+G2EnGnb!BIF1| zs({hXhZKR6?|cb+qk~=8CixD}TLC0jjp3plW8SK|amIC-#~8N=qxP%BFE+1{`NW-h z`>rEp1i)J3-oq<5b=1oeT=?{2S)->8Mz?uhSGJw>E&bk53K2T8&KJ5P|5C0vc!JD= z!l)b^#^j-Sds~hRH;i)Ck@kJ?tg;u|6D)LhMsILBYDBMq4kueZN7()pn5yH|5;Irj zwA!_)N&`mdMyh>_A+l-R`OP5_TQd^E*iOZWj$t{ma0S}&&PpBD8M{V5oT@rytt8b` zCP&yU@?A;re9&)grEW@62GFxZtmF`CGMgz_PtRSbErH*ROtYW*PXUTZ)k-dDO;yri zThl~#lP3nW3T&7_7y!5hF%l~Yb)Zudv4w-POEWW{W+-Ar_cu;ICafDUj{(@eQA(t2 z8__giM5iKZ698wg%oJR)`0OnB;lBVOL~*$fgsnq{pM z8jQg-Ys#7Rl;XhWNKZLP^eed%I0gv5LKNCAHQMLkK4ftG#qI0b$)OllvL7S%4eeLg z&~wM6*hP-Iwwj*E1jMr)* z#z#-DMmSO0`b`V}a)?}!KiMOje12owr|95g+S;eV)naZ?b?%16&jN1UWDw97t$oUT zK*k3(lU*<3>OV)m_Zn4c0r1%O{Q(h)l)ZzCm$Sf`0&B_|O+5n?70{Z&Djd)RD*G-U`j~}1V+Gfz!hwdZqmNuth{J;^Hv=wL4&dvWctv4PsBM6rUiE2;*g@1syH0p z*Ral%UH64dhbE4I<_EWaIbf<8NpBhD{je)g8l1iM?7=Rsq7(#;^Ap%E%XfD(i1m}| z#mvVVUx)tPhX`<7!R4^Qhudlw&;W4)zmEbYsD`*8h$G+QhaCVHhvPLijG0Mfg8d8@7}alBH^a74VEa z{Jqg`D-^Fj>=7EOD?N|%wZtdUjn}lftD1eW&6XhmLaDuGb=^-elh;Ic0fkNz*=VZ{ zh?e|M;9>ceDTd#UOWqSX`Ou+6hjXV8Kq2N}U8wdmlJNJ9j%%9XMKBB?>=+JQ^j&2l z__$%8VaKR@P7%FM1nhOHQ`EN=fUcu?S5yXRDOGqmmHgu`!cKt_6s?#VAQL!gMHG$J z7k)8_qYE4l+xHnpC!w5Cb*N@Zy{c-cZBr>UO9rh9kgmEdZ z)N2Gj?lWZ8dS#r8W!Ti*7uO6(oetzp*xT?oFSp>h(-ix#g1+;Qg@F{Yol{%-Zp%T& zu6x+_58%);4M4YfYM00B6Bs&=hF*GJoequH$TtpmaU>(u~Q6L!s32DziHs$?Ykw3Zfi}4-XQQIY%GhQBe;G;OWX|j z`e;0^#061yRbL!Y(Y-LHTUp}tM1Vysbp`}phLn7SPIV);_QhNSxA~9OVY(c$fh;1; zb^IFQr;TAbqhQYHFyt0=yCO96X)%?6*#Uz4K7O8dZ1y^)hn?fYu2&D5{a_D4VV0V; z3-u2zq6deR8k?-MR48pO{TR1;xVC8hu|vBEqn)bCMf)%^I9YffnoG+>#7MC?gVoQQ zKNbZWf-*U+py1C+W&s-UJj*Z>0I5g*#K|iTVifd2lsm{VK2GZX&JYAMN{@tPeI{~3 zuB6J3mV`dl(lMq&OC^v2!REn-`BZ7lMt&F=W-|dtW_EvBV6o|?dw^Djl%Tz!wMuNN z(hcRS2V=t7waO2%X6IoD<>Ro&kc(G3;uKQW%=pHIErZf3qKKVCEOGE3JWNe4*7_KGmC-owSVLWT{H#c5D0_t z0MWB6YEPO6B-r{fu(v z$)zTF88{fc0xiI@Xp25a0g9N%SGZ)5Y$pH03@7?twUFPSJA?Sb-G{{RY=h&0f*B?T zT=8v3O@|g8;E(jJ+R(}1_=Ye>$OSt29zChK)>e%b2C#Ve$g(}*UyKoTKdX@)5kIV_ zssq+LYEAU0R}#%G!=(;W@@(Ogpx0H^SVV$P2!y}-UTemv3ZNYjm<;IE+jXsGE}Rj& zg?l>4&T=l~Fk0~`EVzqvraHiLkKC5j-JY)Zo>!DVyyrMchA!de za{;ZgT-RcD=M4c~`_CLX6JU_Syenv3A`{%&>-$Y+$Rp30n6Cw0YyswD?xXuRHxh%4 zXm*L!+E_iA!?miH<$^f`~ z9@N8Eh(V2?Xh6{)>K9S{V#Qb%SQHL$8@qq8)pPL~!LfjD4OM=aL#ia2i_LL)S@7-A z;%^FDAzF6!#B9=0phI6A*avQ&Ka;W0N=g)n3QkVCaLIY{xLSWO!ioD0)@O!tF%Et+ zw336sLM6Tulzl(NQ%}qtk6mF8`@?8-)K-enA6yW1qC|ZReWGc_dvc!WMQB~yc~c#fm`E!>b~RW z=P%7ff(D7VMWg}HOEKg=HnNK>C<_ALj6L!ED*6)vh*1aySm47Cz+Y5z|Kxw+&%*tQ zW-Al=+Jaj<5A0`z*-wZsX?0cq5>8F(V(1&`*5)m_a+P9yZg1INiZy^7V!)1hfvo+1 z7jB#`#+X7nf4CR|7S4f9$v#i8Ol74cW(ywoqGeZ90T~P!2%KP7`&8n2d*p*nh*(s7 z;lFXs$2OFVg;pFG;EFqGp2(xxg`(tua|%fH(J5_10DN3kP6BcWf*rGlqOh9Ee$a`Z ziO0Fb9rDAv*R!7ntm{n&wHz!8^uHhLafLznfkWdE@7ro?@$Res^MTkKl^6736wxI4 zP_bX$Y&-W8p^YJ-t&78K&M@zN^ewyi`hWm_Bf+7rVeR`DvyFlQ(c@oE@jBumn%K4w zZResN*c(FQ7i*OA7dh0%KWGt)>E0RpKN87)EkS_6*E6#@1pM?+ax`-%!6wd?nbl52 ze zuLD_Pb|c(5U3)wrG>;#&0zmdqMx63w+>ZD5lHSBGMr;V25B-|805LKTmJ8$R5rq(i31|+>zBR@GV6-^{EKIX{D1F4*WiIv-7?4Wp-!PM-^$%_&5p! zUFru<&PQAZeweXv^9hht@M0=Zx(%4~%mYP#ROr#1jGujzS~sr6!oD1>U)}#g5(iu_ zT~P3z;#f18dQH4#`0NxYvs<5&Jl=4S{+Dp5$=-!LoElBE-{6YUl~0>mbeMz_mF+0Q zjUq!l8Hh8vn?3{lk&CM1I=aZ!2+nmk9cqO+lJP={ec6Ur-svE_(GVc~!LREZV`KF{ z7CgfFT`94yK$0pa6pBVW2esqY`8(+T!v%or!C|t`O8En%Anf|h9;Yce-;P{Jeoxxe z9Dz%Z=5!9?f}>3WBLVV9o*y~(0)~}a07VguYnW zdYudXg62re8kZWP{`(-C7}Fj%l8b{75m02k{~alw65ipg{7@5(*acS5@q_lGIZJ`S zlFf9K(+;aE+VdHg@gjj*S})~ClMAzKJY}#*MjL!?p$OR?7%S7)cMZ|myi1mMQJQm> z0RQ&rX(z9a&qZR4mQG(H=9<$$V6;2&2RgAmD7-G~@<;4fIkt%HCMbHC+4h7f#Ac4t z#4*hD08ecN6B3cx_4zEwZtFLjv@(3UN{MtbwLgr;5k5@sT{36oApB*=Ug7A&T81D2 z9xui^HMm|Cp5L1Lvg?cI9_Fur`dflh9(#b|{#Zytjiq?B_pvaA<=hLk#5DBA$O-+7 z9E_g7#O6gRuxo^ko7va9zoA@+=fAsgXNi7x zZjh`m_q;ViT{GgrpQLas{Je)|4-GdkG}OVSx%cY?p^dZR^LL)pR>w|5Do(#^(;OMf z@Zt1@APE1~TCFm&;ltYUcabiS0LJ?pO3lo2`W4^9?U7Aldf8_{ijKcjY_5h`B{$kF zb7{bm8mkE`gXNo1mrA$KxS_X;CP6h>;AFJ>JcvZ1g53TrliMLQ=@EZJO7`#6lQpqm zOo)+B1#jicwB9WEOL0$%}x(rof@Qa9NA+)UOhSu zFDcHvz3Prp81vrggWtPAgrVOk1E&Yyy|pdDh#?6jTOh&kEs;(UKt8OBLy_-@)r6<2 z3fK73a|QlvQ1g&6kfP`_&GCN#9Cvnv#A!YEH5hCN(3 zGSk#l?!ON+QHEabsTN#f&2tT76{PMYk?UAKMpoE7)r_`zTCA-#l91P0zw!N_pe5Wa z6CjrCPIn0RHxPr!M}jRPe!Z9TeuM*mPVS8obqQ#U7zt;i?_J8LiNkhb_;Qxs;M9e{gq6THYi1UF_6lD`##$PUL>$>dE zeg@j_{AtyHgATynmw9R1_Xh9SD3&Mdspzx!Altzit^Oa*=4l+h`Rt65!1wEkX>=*77u3GTrS~kpd3hGA%D*SMed6CUtDkErU7-X3~bJ3_Fed<+%TNVD5zQoiwwF0X}@V zV(cO>>yi9o2{-`Ckx$hb5NNImHF8TwpTB=q{i?QY0wT}=go)P)&uW+&mh8d_wVkgr zr!`QV&u1nDGdb!;CZOgMjXKUqO3=OM0K&{ZCy7w@!`isnHrloZ%%+5$l_ zM?~*6h<_4LH-cv6AYa}~L^hWPMa%GBCGc20DdBmyP(}FOD*HVCf$g|S;n(7YHcPSN!1Xs;SWcz<>$Huj(2^ZxB+l7Td-Wo;i3#()Pe+(qA|X1B_9nr1YP!b2 zoZ6{Nr9dGmY6u2?NiMTb@2qt=uErXYfQ3zDGh7X2Q_BS8&9p5X;eDRYhq|%q2eCA ztHcY@e`49%{I<(y=t7pi6jMUxcz(S1Qcy=_!1MCb!6Ip8Rn&Qn^a;iLa+^1l@hr#^ zS&ECv3qU!*PQPpHohG6a2Z)SFYQ6UTLuWK!l z2-%z2#>z!@EJnIiI-o;+wS_0akKDp+)gfm*cvIhG-we63G@bh?0!dRpe{HwC-MUx= zq&;(^F3*LdOYtN$7LFka^#RD4bNA0+F9SyPSwE&xh8$aN{?sgnTBW%^Hlpse?-c)D zAr|>PRbMZ!HyqIZdd`>m{Yyk^BYPJpIuQJXBDP}mT*JXo;Tl2sXFxWmc#Sn@yU@VpQz93Itrz$_1f z6q}QXO!tV;@5RW}2)Wq1Ka|dXw(K|-R7A_7yKq5lFtRJV>rSznOJ~ zIYZ{1P#j;Y)ENi# z_qX?Zr-1bI=?`zTH|I-91Dxm)#De~--Q()fofFIn>Xx}(a_wqI%j9H&oI5W78a3)# zpOa?e{9CTygn>3Xy_tt=sbWSpjl5Rc*`6os1k%Wp-IUuaH~Wff{Jr zKhsWX0*s+>hdC*S&mT+Q zxl07;j-+g-zEz~&(Bq^SHN08>^ggKC?$KvKfv*$Uik&4d?K;Mk)brj~z3nBwqD*Ul z9iA=QxgM82dt}WU*}0eASp$Z>$?82E-<%#1Ia_kY#$$LlG`a4sG*T3Do^#akNAHHq z{f|g>x>xr5--Oy*Bi1F=HiH+xPmw~8D;7>QIt=W(d*6QAdG81;ars;!rg@y@Vl#$i z=pdYGN-`r~E^f+qtVMY$`HB#P9h^B~WgVd>V0CElER{^)YoSdh-i6q=lJ1x}UTjk3 z{e?v`+nw{tAC@P>>E-spgA0cSO;*(@Uz0yne0+)`UGG^X1@ikM;@+h=>$~C?+Ke>J z?-M$>t5Z>i3N`e36cGc+YgIK|jmL5((B0(#g9wYr#1RTYr5% zZDtW)yQW0=^!>K{i(R<3%&R>uqNx)?n(^zeLJhq7Dfl~!dXo=V#v<(?*MZ^TRYUpT z%d@1u)|o!1mhh<0YX-!+f%^AVXm*k7DhF@H{#^BWFfv^(%N@JbS2lSMoC_2=-*Md^ZS3-M~z!huP0s_8vy39PEOfl$At`4uiHHx#7O@4dtUje^m zXl-42HFe!0x~_F>Z0}efTdDH4A1u)z{>5BpEyXB9rx8B+kT!r8D}(ZM+|!P@bDyea z4xufu$9ya#<~&x^Qqw%gB&|$%UD$glGx>lCLNoM8*rTIpwX>o#93+6&bYM5E=dP~% zKLkEMANH+m!s%wfNK<8gV>D(XLEbi^NLlG+TZ&agrI|PVT9;2k!Ie{Gqo8NbNBz@k z{BCMRws2ezJLvb^IyA+XQcSEO3PNSVpgw!yym%U;>%I{cILd!9or1h(%i;gIkZAio z_-zxQh$lPe_21X+b?wUq6kUiGUe!Nnbwg$$_hDpx_INy^EW7HtaWk&x&ObK9gWzZE z#bW$v=j?hD!GSsS6KFoog4V>iS zR__;fRcP1J70Bw>CT#-uv#NU>pIsKJpjYRJzig+y#QLG?D?+;rHG=atztv)-NBez zv)Ain+vfc--b{O61L$RZyS`ZW3PRJorczjox+UW1d!C9e5eyw_Qt}NFicUA|%u4Sy zl{amCn_>(mm|OW~5v|L%s#otQ?{72Wq&9Zc58%c&-Hvqw~f=w0lK!c!$}_uA`oUB;FNjCHQ9!9zTj) zJva}Le$p!TUcmxiBNdZGyNKSL>1ixX(g$|WCyZspv1;;?#BBw=eQ=54026=GAn^9lJ z!i=?lskK&`=D>*QMit7?_13Z>^2d-(Ag^UUF8n>&ulck$qhySXTKq9y?-~vZE-@SI z-YrD6#Rzqo=wZ~Nmy)WdcVtLHG4qUq0;^HnbirKE@i+E4>BJf`r7nJgOKqN zm0xIVIG3rZVf~711|4eD$nlAR{J+vdA=2Bw9+*6)Y?LeQUL^|kcf>x9_Tgmr|K*(V zG;$8NnzCQ!P^kNY{2|{M9W1K2j4<|F;Tx=?+@sRQiDG#G0N?%oe*w^a64N+$0%(kB z%3pXA>osam9ol++6jB=vtD+ao4!QYRe6hqacRZ;GLRa}#g#Xdt;T~4xE&QtDsL9>U zqTaASw*zzl)d@i)8I1l<05uNE@plCM!2N_4N?hgjTzfizD0rV~`9G%qkYakP`b5;x zOp|FYt1XRYbrs4lTnW+tYFKDh7RNH0x;z8qp^Y*->ANiR~$S=m<$60=xR*el{ zOypAp7(l^+-m?Od8$z##(w}-gg#I-8!vO+j2qdU!{!ALO>#3zm(q#YvBpsnJUZ-Wg zc|86g{Yah>#_S`{AmBCu>S6p5n>Acbp!U^)0OG6Sa>3aJji+&4K1`O>(=-kc00D;y z(2wu~`t0G~ci4<0>OHZXyCXS+QgZ~0z zo(wgRtQUa?9H0gGcp~>f@?@$9ynzcK5E25sUTS+w*o-4L`8C4`Abu@5yZo-KKV7XO zM(R}OFAfj@0e1*&mx5NX#%v?*>B|R5qEmpNt4c@3CkofUy4_KGK|ZFT8`voU-}QHLBQt()Ib$| z*3e7+vc{EB+-SlwO3JTt6%A)o@OX>dNAU1#8NzR$6X4J3TZUd9wBVr%1vL<#-!4j5LhOA_dbJNF>ptc>%KpYCM@3t1l%PM z_~4;%q6XsbP34?5+N?^Yba_S?SN1NX(!>g*M;l*h*-RmcEh{OZ?3~%5C|TDeZgbyQQL!;x8ylO07=fq zyH_gK9?iL?*3voFyH3J05C{!{I;rpd77JPMP^F6|EWh33RrQxp@K{Z5BY60&?BH(@ z2|O(iN}LhK;Gqf=Yhn52maf{=SFq3TqKyN;>lc^dCkO4}K@Fty&EcI-^VjqfB5xs$vbw-KPQeop2sMEO zH6c%DbB5g;Y6=}Pj{wpEhG*VZG!)U^0gsYDM|j33I06FUCa|4?#|0ET7Ka-{=iH+S zt22-6f2yp%ih{>0=h!^gGs)k}N+nTHn_kug{0R%vxt%LYc{h-8U z?6OgNz@ujZ2p~Okapx>GskD~n;(o^EH?!pe?1BIYbR)2f#_4{S%^Ci;ZdY)k8vz86Zb)1>QQ2@Rf&U9x z8tQ^Im;eC~FhoH9M{4|BHe>j^hQuL+wXj^h2UL?)Mg7$jJgUhqf`_Z{VGIOP5a8@@ zGNzXyc%)#|!zI)}dVu1{srjD{HfMS`k@pAK)?!tI1SOgk<8jRnO4?MqTumoQa#uS_0S0dAYh5W$27m=5}Mz!%u;G3>P7%- zVRb{2;fbn-YiX~57f1_&M}{bo90Y74AZewCN|_Tm7Qw?NqWjdK24XKy+T$_yCmzc^ znnre9NvhGuWH(>v0|9dcHb|lWK+YWbf98@w+6V#sz!{-rMW(9p>s-n|Qqo~ok|PZW zfIwyhXz+EEE7Ngv5d|{!it8ew8zKYlmfL86WAaM0%i%c)jGu9*=)vQL-uuAzw`` zh^rs~0tN`gY1HEao#*{Gn>A>s0eOhnNB{xEMwo_GmCFb#XqZLS?O#)xeumRQ5ClNr z00GY5l)Ai%Y8|T&oWwgf2_S&DNxQ49yBZG@y1b35-sg9{gwG%V0!ax_CprHm8#weX zb{K1&yb1T~?9gXamv^0fDd*P|a0;()GxVZ0@0(!wSJm`9=Wo zN@*-yox@rWxt3P#{cmy!&01b57x>{30`l)Po%}jlIrn{!I0QF{00M~DZiCw!qav4G zPczxCCf9m-%{}-70`3x^T>=@qj`r(#&Rx5(W`qC&h!INJkf~TVN^o%#eR(b*^H{Si z8)X2S*(9)q?)`GMW!OSmH8-A3M_@ZS0R)ibY`k%$vVJsY;%3I=nci>+UVwm;1mq67 z-xugPt-oc{kNC_pPs$mLsAl}TRJi#4Z5>SDI->D0EJDWLT ztDBZ#%Q^uB5bNyRPbq1Ip9KwPa@n@SVTt<=z#<6PPGC0$j(hZY?0>O4pzUlOSBL=O zIAVtke6iZE+A8;Chb%xX2slY#4^5<9sI&OZY}Tj^P8x!4=LjHxILFo<%~kc|xD+>_ zkHj6^u#z!>?NsBqU)R{JY{sxpGS&tKStWn~l2!aR8lT@(#)XsJ#%9x(^>WhO*JkZd2LdJu$hWBH zZXVk_{9z2oHHpp#;s_u^vN4uN~cowBU+>#C?0s(6T)cRC3b6m`0vxY9U zrUI$_KmY;652ciekL6OZXv0J<1kG$m^HwG(kv1)X4b;E!fUd>=z-EkEmsT1Az9)bH z;`<^rkpY^NC?x+OH7Q&~siG!QLKFm&5zv`r)l%?+lYu2=Cn90U+R!cor2bGdNc zp{$jkM`Q9;r~B8E3*rh0us!s^Q7xtZAZy6g+X zcHf~3g`F?rQ%DKWLWXh)b%;OAcIW&JLvTZS7bM9x0!R`tp+5b%)Y!)Q(`Yr|b19gd zLo5b|nn(CDuTxE9ISmDWnCUr>WAmo)-2g^&i~wSkR9NNID6WAGV}%rF(hthHgyD#= zq6m1t_t5myS7?v%=U7BMfmz}KFKwH$hX7&|$WW+Ji#b+(QYNH4o5IReS{e9w$~W#I z{}%pcD!k`(#-C-M=PW_~pDpr1uzSbHhp>n?g?77|DjAFnnq96>Rc{n1BbuH&f{v*9}W<(L;wNA5;i32MnE-n4Qe=n>3k~PRBC9jT5cX& z@^%B|>Iv%0kZ;rS|4S)g)H1#GB{uh{?XG$NV}T}s021ip&0^-%ToWzlB&Lh8^u#-k zJQ+)WsG&GpByq<1J6y_FDPYvGNKPG_K4gotCZRpd1Q0;NT>5V3eZePRW->uTh1oc% zvr_6kIhtIExa~%O?5(5Vyn?D3E2yTiiUP(P6fkxND1XR`#|a>Sc)WlD?cRNBd$G^O zXqFJi3$7nW{V2y#e$`+9KL;wM#Cn`ASUjp+#9msNG zhp>b^lrk}xNqz|Rx%8v=w7Hh-%Y+ypr9625XU5ev)nrCuNe?Bnlm4g$iEX7iP=aa( z+vzVxsdm!Hv%Orf-83(zg|2PUrBof)%@i26P%UZ;lkrWA_ibjg1}PmlKp-##{y$YA V@w%;YR<{5E002ovPDHLkV1noI;4lCH literal 0 HcmV?d00001 diff --git a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json index ffa8d43a6a5ab4..f74a0bb9e0102d 100755 --- a/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json +++ b/metadata-ingestion/tests/integration/openapi/openapi_mces_golden.json @@ -20,7 +20,7 @@ "com.linkedin.pegasus2avro.common.InstitutionalMemory": { "elements": [ { - "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/", + "url": "https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/", "description": "Link to call for the dataset.", "createStamp": { "time": 1586847600, @@ -96,7 +96,7 @@ "com.linkedin.pegasus2avro.common.InstitutionalMemory": { "elements": [ { - "url": "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/v2", + "url": "https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/v2", "description": "Link to call for the dataset.", "createStamp": { "time": 1586847600, @@ -183,4 +183,4 @@ "lastRunId": "no-run-id-provided" } } -] \ No newline at end of file +] diff --git a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml index 8575c23239461e..937ec54e9f1c27 100644 --- a/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml +++ b/metadata-ingestion/tests/integration/openapi/openapi_to_file.yml @@ -2,12 +2,11 @@ source: type: openapi config: name: test_openapi - url: https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/tests/ - swagger_file: v3.0/pass/api-with-examples.yaml + url: https://raw.githubusercontent.com/OAI/learn.openapis.org/refs/heads/main/examples/ + swagger_file: v3.0/api-with-examples.yaml sink: type: file config: filename: "/tmp/openapi_mces.json" - diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml index 2230d552ed4c0e..4c4a7c2183073c 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml @@ -417,7 +417,7 @@ name: mlflow displayName: MLflow type: OTHERS - logoUrl: "/assets/platforms/mlflowlogo.png" + logoUrl: "/assets/platforms/mlflowlogo2.png" - entityUrn: urn:li:dataPlatform:glue entityType: dataPlatform aspectName: dataPlatformInfo diff --git a/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js b/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js index a82ac125bf820b..10fcce58f8d2b9 100644 --- a/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js +++ b/smoke-test/tests/cypress/cypress/e2e/lineageV2/v2_lineage_column_level.js @@ -5,7 +5,7 @@ const DATASET_URN = const clickDownAndUpArrow = (asset, arrow) => { cy.contains(".react-flow__node-lineage-entity", asset) .find(`svg[data-testid="${arrow}"]`) - .click(); + .click({ force: true }); }; describe("column-level lineage graph test", () => { From ddb3db9c6fceda70b9156a04caf85ee53203e3a2 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 30 Jan 2025 15:27:20 +0530 Subject: [PATCH 5/5] docs: change heading of ingestion page (#12501) --- docs/ui-ingestion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ui-ingestion.md b/docs/ui-ingestion.md index 521d8db26011cf..1706d86c657f4c 100644 --- a/docs/ui-ingestion.md +++ b/docs/ui-ingestion.md @@ -3,7 +3,7 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Ingestion +# UI Based Ingestion / Managed Ingestion