feat!: Change Pandas timestamp resolution to ms for timestamp attributes

cognite/client/data_classes/_base.py

@@ -170,7 +170,10 @@ def to_pandas(
         if expand_metadata and "metadata" in dumped and isinstance(dumped["metadata"], dict):
             dumped.update({f"{metadata_prefix}{k}": v for k, v in dumped.pop("metadata").items()})
 
-        return pd.Series(dumped).to_frame(name="value")
+        df = pd.Series(dumped).to_frame(name="value")
+        if TIME_ATTRIBUTES.intersection(dumped) == dumped.keys():
+            df.value = df.value.astype("datetime64[ms]")
+        return df
 
     def _repr_html_(self) -> str:
         from cognite.client.utils._pandas_helpers import notebook_display_with_fallback

cognite/client/utils/_pandas_helpers.py

@@ -145,7 +145,7 @@ def convert_nullable_int_cols(df: pd.DataFrame) -> pd.DataFrame:
 
 def convert_timestamp_columns_to_datetime(df: pd.DataFrame) -> pd.DataFrame:
     to_convert = df.columns.intersection(TIME_ATTRIBUTES)
-    df[to_convert] = (1_000_000 * df[to_convert]).astype("datetime64[ns]")
+    df[to_convert] = df[to_convert].astype("datetime64[ms]")
     return df
 
 

tests/tests_unit/test_base.py

@@ -413,22 +413,30 @@ def test_to_pandas(self) -> None:
 
         class SomeResource(CogniteResource):
             def __init__(
-                self, a_list: list, ob: dict, metadata: dict, ob_ignore: dict, prim: str, prim_ignore: int
+                self,
+                a_list: list,
+                ob: dict,
+                metadata: dict,
+                ob_ignore: dict,
+                prim: str,
+                prim_ignore: int,
+                created_time: int,
             ) -> None:
                 self.a_list = a_list
                 self.ob = ob
                 self.metadata = metadata
                 self.ob_ignore = ob_ignore
                 self.prim = prim
                 self.prim_ignore = prim_ignore
+                self.created_time = created_time
 
             _load = None  # type: ignore [assignment]
 
         expected_df = pd.DataFrame(
-            {"value": ["abc", [1, 2, 3], {"x": "y"}, "md_value"]},
-            index=["prim", "aList", "ob", "md_key"],
+            {"value": ["abc", [1, 2, 3], {"x": "y"}, "md_value", pd.Timestamp(1, unit="ms")]},
+            index=["prim", "aList", "ob", "md_key", "createdTime"],
         )
-        res = SomeResource([1, 2, 3], {"x": "y"}, {"md_key": "md_value"}, {"bla": "bla"}, "abc", 1)
+        res = SomeResource([1, 2, 3], {"x": "y"}, {"md_key": "md_value"}, {"bla": "bla"}, "abc", 1, 1)
         actual_df = res.to_pandas(
             expand_metadata=True, metadata_prefix="", ignore=["primIgnore", "obIgnore"], camel_case=True
         )
@@ -468,6 +476,26 @@ def test_use_method_which_requires_cognite_client__client_not_set(self) -> None:
         with pytest.raises(CogniteMissingClientError):
             mr.use()
 
+    @pytest.mark.dsl
+    def test_to_pandas_time_dtype_if_only_timestamp_attributes(self) -> None:
+        import pandas as pd
+
+        class SomeResource(CogniteResource):
+            def __init__(self, created_time: int = 1, last_updated_time: int = 1) -> None:
+                self.created_time = created_time
+                self.last_updated_time = last_updated_time
+
+            _load = None  # type: ignore [assignment]
+
+        expected_df = pd.DataFrame(
+            {"value": [pd.Timestamp(1, unit="ms"), pd.Timestamp(1, unit="ms")]},
+            index=["created_time", "last_updated_time"],
+            dtype="datetime64[ms]",
+        )
+
+        actual_df = SomeResource().to_pandas()
+        pd.testing.assert_frame_equal(expected_df, actual_df, check_like=True)
+
 
 class TestCogniteResourceList:
     def test_dump(self) -> None:
@@ -487,6 +515,8 @@ def test_to_pandas(self) -> None:
                 "varB": [None, 3],
             },
         )
+        # NOTE: Pandas v3 is able to infer the datetime type without this explicit conversion
+        expected_df.lastUpdatedTime = expected_df.lastUpdatedTime.astype("datetime64[ms]")
         pd.testing.assert_frame_equal(resource_list.to_pandas(camel_case=True), expected_df)
 
     @pytest.mark.dsl

tests/tests_unit/test_data_classes/test_data_models/test_typed_instances.py

@@ -298,25 +298,25 @@ def test_to_pandas_list(self, person_read: PersonRead) -> None:
         persons = NodeList[PersonRead]([person_read])
         df = persons.to_pandas(expand_properties=True)
 
-        pd.testing.assert_frame_equal(
-            df,
-            pd.DataFrame(
-                {
-                    "space": ["sp_my_fixed_space"],
-                    "external_id": ["my_external_id"],
-                    "version": [1],
-                    "last_updated_time": [pd.Timestamp("1970-01-01 00:00:00")],
-                    "created_time": [pd.Timestamp("1970-01-01 00:00:00")],
-                    "instance_type": ["node"],
-                    "type": [{"space": "sp_model_space", "external_id": "person"}],
-                    "name": ["John Doe"],
-                    "birth_date": ["1990-01-01"],
-                    "email": ["john@doe.com"],
-                    "siblings": None,
-                    "conflicting_with_reserved_property": None,
-                }
-            ),
+        expected_df = pd.DataFrame(
+            {
+                "space": ["sp_my_fixed_space"],
+                "external_id": ["my_external_id"],
+                "version": [1],
+                "last_updated_time": [pd.Timestamp("1970-01-01 00:00:00")],
+                "created_time": [pd.Timestamp("1970-01-01 00:00:00")],
+                "instance_type": ["node"],
+                "type": [{"space": "sp_model_space", "external_id": "person"}],
+                "name": ["John Doe"],
+                "birth_date": ["1990-01-01"],
+                "email": ["john@doe.com"],
+                "siblings": None,
+                "conflicting_with_reserved_property": None,
+            }
         )
+        expected_df.created_time = expected_df.created_time.astype("datetime64[ms]")
+        expected_df.last_updated_time = expected_df.last_updated_time.astype("datetime64[ms]")
+        pd.testing.assert_frame_equal(df, expected_df)
 
 
 class TestTypedEdge:

tests/tests_unit/test_data_classes/test_groups.py

@@ -104,7 +104,7 @@ class TestGroupsList:
     @pytest.mark.parametrize(
         "convert_timestamps, expected",
         (
-            (True, dict(data=[None, "1970-01-02 10:17:36.789", None], dtype="datetime64[ns]", name="deleted_time")),
+            (True, dict(data=[None, "1970-01-02 10:17:36.789", None], dtype="datetime64[ms]", name="deleted_time")),
             (False, dict(data=[-1, 123456789, None], dtype="Int64", name="deleted_time")),
         ),
     )