diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py index c6d8fcb60d..ab065ddcc1 100644 --- a/cognite/client/data_classes/_base.py +++ b/cognite/client/data_classes/_base.py @@ -170,7 +170,10 @@ def to_pandas( if expand_metadata and "metadata" in dumped and isinstance(dumped["metadata"], dict): dumped.update({f"{metadata_prefix}{k}": v for k, v in dumped.pop("metadata").items()}) - return pd.Series(dumped).to_frame(name="value") + df = pd.Series(dumped).to_frame(name="value") + if TIME_ATTRIBUTES.intersection(dumped) == dumped.keys(): + df.value = df.value.astype("datetime64[ms]") + return df def _repr_html_(self) -> str: from cognite.client.utils._pandas_helpers import notebook_display_with_fallback diff --git a/cognite/client/utils/_pandas_helpers.py b/cognite/client/utils/_pandas_helpers.py index 25dc5f91fc..f5a078bf54 100644 --- a/cognite/client/utils/_pandas_helpers.py +++ b/cognite/client/utils/_pandas_helpers.py @@ -145,7 +145,7 @@ def convert_nullable_int_cols(df: pd.DataFrame) -> pd.DataFrame: def convert_timestamp_columns_to_datetime(df: pd.DataFrame) -> pd.DataFrame: to_convert = df.columns.intersection(TIME_ATTRIBUTES) - df[to_convert] = (1_000_000 * df[to_convert]).astype("datetime64[ns]") + df[to_convert] = df[to_convert].astype("datetime64[ms]") return df diff --git a/poetry.lock b/poetry.lock index 27e7ba3eb9..054c2de432 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3330,14 +3330,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "20.37.0" +version = "20.39.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "virtualenv-20.37.0-py3-none-any.whl", hash = "sha256:5d3951c32d57232ae3569d4de4cc256c439e045135ebf43518131175d9be435d"}, - {file = "virtualenv-20.37.0.tar.gz", hash = "sha256:6f7e2064ed470aa7418874e70b6369d53b66bcd9e9fd5389763e96b6c94ccb7c"}, + {file = "virtualenv-20.39.0-py3-none-any.whl", hash = "sha256:44888bba3775990a152ea1f73f8e5f566d49f11bbd1de61d426fd7732770043e"}, + {file = "virtualenv-20.39.0.tar.gz", hash = "sha256:a15f0cebd00d50074fd336a169d53422436a12dfe15149efec7072cfe817df8b"}, ] [package.dependencies] @@ -3346,10 +3346,6 @@ filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} platformdirs = ">=3.9.1,<5" typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} -[package.extras] -docs = ["furo (>=2023.7.26)", "pre-commit-uv (>=4.1.4)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinx-autodoc-typehints (>=3.6.2)", "sphinx-copybutton (>=0.5.2)", "sphinx-inline-tabs (>=2025.12.21.14)", "sphinxcontrib-mermaid (>=2)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "pytest-xdist (>=3.5)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] - [[package]] name = "wcwidth" version = "0.6.0" diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py index e0a9ae728a..fc1a4068b3 100644 --- a/tests/tests_unit/test_base.py +++ b/tests/tests_unit/test_base.py @@ -413,7 +413,14 @@ def test_to_pandas(self) -> None: class SomeResource(CogniteResource): def __init__( - self, a_list: list, ob: dict, metadata: dict, ob_ignore: dict, prim: str, prim_ignore: int + self, + a_list: list, + ob: dict, + metadata: dict, + ob_ignore: dict, + prim: str, + prim_ignore: int, + created_time: int, ) -> None: self.a_list = a_list self.ob = ob @@ -421,14 +428,15 @@ def __init__( self.ob_ignore = ob_ignore self.prim = prim self.prim_ignore = prim_ignore + self.created_time = created_time _load = None # type: ignore [assignment] expected_df = pd.DataFrame( - {"value": ["abc", [1, 2, 3], {"x": "y"}, "md_value"]}, - index=["prim", "aList", "ob", "md_key"], + {"value": ["abc", [1, 2, 3], {"x": "y"}, "md_value", pd.Timestamp(1, unit="ms")]}, + index=["prim", "aList", "ob", "md_key", "createdTime"], ) - res = SomeResource([1, 2, 3], {"x": "y"}, {"md_key": "md_value"}, {"bla": "bla"}, "abc", 1) + res = SomeResource([1, 2, 3], {"x": "y"}, {"md_key": "md_value"}, {"bla": "bla"}, "abc", 1, 1) actual_df = res.to_pandas( expand_metadata=True, metadata_prefix="", ignore=["primIgnore", "obIgnore"], camel_case=True ) @@ -468,6 +476,26 @@ def test_use_method_which_requires_cognite_client__client_not_set(self) -> None: with pytest.raises(CogniteMissingClientError): mr.use() + @pytest.mark.dsl + def test_to_pandas_time_dtype_if_only_timestamp_attributes(self) -> None: + import pandas as pd + + class SomeResource(CogniteResource): + def __init__(self, created_time: int = 1, last_updated_time: int = 1) -> None: + self.created_time = created_time + self.last_updated_time = last_updated_time + + _load = None # type: ignore [assignment] + + expected_df = pd.DataFrame( + {"value": [pd.Timestamp(1, unit="ms"), pd.Timestamp(1, unit="ms")]}, + index=["created_time", "last_updated_time"], + dtype="datetime64[ms]", + ) + + actual_df = SomeResource().to_pandas() + pd.testing.assert_frame_equal(expected_df, actual_df, check_like=True) + class TestCogniteResourceList: def test_dump(self) -> None: @@ -487,6 +515,8 @@ def test_to_pandas(self) -> None: "varB": [None, 3], }, ) + # NOTE: Pandas v3 is able to infer the datetime type without this explicit conversion + expected_df.lastUpdatedTime = expected_df.lastUpdatedTime.astype("datetime64[ms]") pd.testing.assert_frame_equal(resource_list.to_pandas(camel_case=True), expected_df) @pytest.mark.dsl diff --git a/tests/tests_unit/test_data_classes/test_data_models/test_typed_instances.py b/tests/tests_unit/test_data_classes/test_data_models/test_typed_instances.py index 4c61ca1e5d..4280ca3e3a 100644 --- a/tests/tests_unit/test_data_classes/test_data_models/test_typed_instances.py +++ b/tests/tests_unit/test_data_classes/test_data_models/test_typed_instances.py @@ -298,25 +298,25 @@ def test_to_pandas_list(self, person_read: PersonRead) -> None: persons = NodeList[PersonRead]([person_read]) df = persons.to_pandas(expand_properties=True) - pd.testing.assert_frame_equal( - df, - pd.DataFrame( - { - "space": ["sp_my_fixed_space"], - "external_id": ["my_external_id"], - "version": [1], - "last_updated_time": [pd.Timestamp("1970-01-01 00:00:00")], - "created_time": [pd.Timestamp("1970-01-01 00:00:00")], - "instance_type": ["node"], - "type": [{"space": "sp_model_space", "external_id": "person"}], - "name": ["John Doe"], - "birth_date": ["1990-01-01"], - "email": ["john@doe.com"], - "siblings": None, - "conflicting_with_reserved_property": None, - } - ), + expected_df = pd.DataFrame( + { + "space": ["sp_my_fixed_space"], + "external_id": ["my_external_id"], + "version": [1], + "last_updated_time": [pd.Timestamp("1970-01-01 00:00:00")], + "created_time": [pd.Timestamp("1970-01-01 00:00:00")], + "instance_type": ["node"], + "type": [{"space": "sp_model_space", "external_id": "person"}], + "name": ["John Doe"], + "birth_date": ["1990-01-01"], + "email": ["john@doe.com"], + "siblings": None, + "conflicting_with_reserved_property": None, + } ) + expected_df.created_time = expected_df.created_time.astype("datetime64[ms]") + expected_df.last_updated_time = expected_df.last_updated_time.astype("datetime64[ms]") + pd.testing.assert_frame_equal(df, expected_df) class TestTypedEdge: diff --git a/tests/tests_unit/test_data_classes/test_groups.py b/tests/tests_unit/test_data_classes/test_groups.py index 7ff3c43ce8..9e4dbcca8e 100644 --- a/tests/tests_unit/test_data_classes/test_groups.py +++ b/tests/tests_unit/test_data_classes/test_groups.py @@ -104,7 +104,7 @@ class TestGroupsList: @pytest.mark.parametrize( "convert_timestamps, expected", ( - (True, dict(data=[None, "1970-01-02 10:17:36.789", None], dtype="datetime64[ns]", name="deleted_time")), + (True, dict(data=[None, "1970-01-02 10:17:36.789", None], dtype="datetime64[ms]", name="deleted_time")), (False, dict(data=[-1, 123456789, None], dtype="Int64", name="deleted_time")), ), )