lincc-frameworks · dougbrn · Oct 2, 2025 · Sep 16, 2025 · Sep 16, 2025 · Sep 16, 2025
diff --git a/README.md b/README.md
@@ -46,7 +46,12 @@ Allowing powerful and straightforward operations, like:
 ```python
    # Compute the mean flux for each row of "object_nf"
    import numpy as np
-   object_nf.reduce(np.mean, "nested_sources.flux")
+
+   def mean_flux(row):
+   """Calculates the mean flux for each object"""
+       return np.mean(row["nested_sources.flux"])
+
+   object_nf.map_rows(mean_flux, output_names="mean_flux")
 ```
 
 <p align="center">

diff --git a/docs/gettingstarted/quickstart.ipynb b/docs/gettingstarted/quickstart.ipynb
@@ -282,9 +282,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Reduce Function\n",
+    "## The `map_rows` Function\n",
     "\n",
-    "Finally, we'll end with the flexible `reduce` function. `reduce` functions similarly to pandas' `apply` but flattens (reduces) the inputs from nested layers into array inputs to the given apply function. For example, let's find the mean flux for each dataframe in \"nested\":"
+    "Finally, we'll end with the flexible `map_rows` function. `map_rows` functions similarly to pandas' `apply` but applies row by row and flattens the inputs from nested layers into array inputs to the given apply function. For example, let's find the mean flux for each dataframe in \"nested\":"
    ]
   },
   {
@@ -297,7 +297,8 @@
     "\n",
     "# use hierarchical column names to access the flux column\n",
     "# passed as an array to np.mean\n",
-    "nf.reduce(np.mean, \"lightcurve.brightness\")"
+    "# row_container signals how to pass the data to the function, in this case as direct arguments\n",
+    "nf.map_rows(np.mean, \"lightcurve.brightness\", row_container=\"args\")"
    ]
   },
   {
@@ -313,15 +314,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "def show_inputs(*args):\n",
-    "    return args"
+    "def show_inputs(row):\n",
+    "    return row"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Applying some inputs via reduce, we see how it sends inputs to a given function.  The output frame `nf_inputs` consists of two columns containing the output of the “ra” column and the “lightcurve.time” column."
+    "Applying some inputs via `map_rows`, we see how it sends inputs to a given function.  The output frame `nf_inputs` consists of two columns containing the output of the “ra” column and the “lightcurve.time” column."
    ]
   },
   {
@@ -330,8 +331,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "nf_inputs = nf.reduce(show_inputs, \"ra\", \"lightcurve.time\")\n",
-    "nf_inputs"
+    "# row_container=\"dict\" passes the data as a dictionary to the function\n",
+    "nf_inputs = nf.map_rows(show_inputs, columns=[\"ra\", \"lightcurve.time\"], row_container=\"dict\")\n",
+    "nf_inputs\n",
+    "\n",
+    "# map_rows returns a dataframe view of the dicts, but the two columns can be accessed with show_inputs as\n",
+    "# row[\"ra\"] and row[\"lightcurve.time\"]"
    ]
   },
   {
@@ -343,6 +348,23 @@
     "nf_inputs.loc[0]"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# row_container=\"args\" passes the data as arguments to the function\n",
+    "\n",
+    "\n",
+    "def show_inputs(*args):\n",
+    "    return args\n",
+    "\n",
+    "\n",
+    "nf_inputs = nf.map_rows(show_inputs, columns=[\"ra\", \"lightcurve.time\"], row_container=\"args\")\n",
+    "nf_inputs"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/docs/pre_executed/nested_spectra.ipynb b/docs/pre_executed/nested_spectra.ipynb
@@ -280,7 +280,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -452,7 +452,7 @@
     }
    ],
    "source": [
-    "spec_ndf = xid_ndf.add_nested(flat_spec, \"coadd_spectrum\").set_index(\"objid\")\n",
+    "spec_ndf = xid_ndf.join_nested(flat_spec, \"coadd_spectrum\").set_index(\"objid\")\n",
     "spec_ndf"
    ]
   },

diff --git a/docs/pre_executed/performance.ipynb b/docs/pre_executed/performance.ipynb
@@ -98,7 +98,7 @@
     "# Read in parquet data\n",
     "# nesting sources into objects\n",
     "nf = npd.read_parquet(\"objects.parquet\")\n",
-    "nf = nf.add_nested(npd.read_parquet(\"ztf_sources.parquet\"), \"ztf_sources\")\n",
+    "nf = nf.join_nested(npd.read_parquet(\"ztf_sources.parquet\"), \"ztf_sources\")\n",
     "\n",
     "# Filter on object\n",
     "nf = nf.query(\"ra > 10.0\")\n",

diff --git a/docs/reference/accessor.rst b/docs/reference/accessor.rst
@@ -18,12 +18,9 @@ Functions
     NestSeriesAccessor.to_lists
     NestSeriesAccessor.to_flat
     NestSeriesAccessor.to_flatten_inner
-    NestSeriesAccessor.with_field
-    NestSeriesAccessor.with_flat_field
-    NestSeriesAccessor.with_list_field
-    NestSeriesAccessor.with_filled_field
-    NestSeriesAccessor.without_field
-    NestSeriesAccessor.query_flat
-    NestSeriesAccessor.get_flat_index
-    NestSeriesAccessor.get_flat_series
-    NestSeriesAccessor.get_list_series
+    NestSeriesAccessor.set_column
+    NestSeriesAccessor.set_flat_column
+    NestSeriesAccessor.set_list_column
+    NestSeriesAccessor.set_filled_column
+    NestSeriesAccessor.drop
+    NestSeriesAccessor.query
diff --git a/docs/reference/nesteddtype.rst b/docs/reference/nesteddtype.rst
@@ -17,6 +17,6 @@ Functions
 
     NestedDtype.construct_array_type
     NestedDtype.construct_from_string
-    NestedDtype.from_fields
+    NestedDtype.from_columns
     NestedDtype.from_pandas_arrow_dtype
     NestedDtype.to_pandas_arrow_dtype
diff --git a/docs/reference/nestedframe.rst b/docs/reference/nestedframe.rst
@@ -10,12 +10,21 @@ Constructor
 
    NestedFrame
 
+Helpful Properties
+~~~~~~~~~~~~~~~~~~
+.. autosummary::
+    :toctree: api/
+
+    NestedFrame.nested_columns
+    NestedFrame.base_columns
+    NestedFrame.all_columns
+
 Nesting
 ~~~~~~~~~
 .. autosummary::
     :toctree: api/
 
-    NestedFrame.add_nested
+    NestedFrame.join_nested
     NestedFrame.nest_lists
     NestedFrame.from_flat
     NestedFrame.from_lists
@@ -25,19 +34,21 @@ Extended Pandas.DataFrame Interface
 
 .. note:: 
    The NestedFrame extends the Pandas.DataFrame interface, so all methods
-   of Pandas.DataFrame are available. The following methods are extended
+   of Pandas.DataFrame are available. The following methods are a mix of
+   newly added methods and extended methods from Pandas DataFrame
    to support NestedFrame functionality. Please reference the Pandas
    documentation for more information.
    https://pandas.pydata.org/docs/reference/frame.html
 
 .. autosummary::
     :toctree: api/
 
+    NestedFrame.get_subcolumns
     NestedFrame.eval
     NestedFrame.query
     NestedFrame.dropna
     NestedFrame.sort_values
-    NestedFrame.reduce
+    NestedFrame.map_rows
     NestedFrame.drop
     NestedFrame.min
     NestedFrame.max

diff --git a/docs/reference/nestedseries.rst b/docs/reference/nestedseries.rst
@@ -16,4 +16,4 @@ Functions
     :toctree: api/
 
     NestedSeries.to_lists
-    NestedSeries.to_flat
+    NestedSeries.explode
diff --git a/docs/tutorials/data_loading_notebook.ipynb b/docs/tutorials/data_loading_notebook.ipynb
@@ -141,7 +141,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We can then create an additional pandas dataframes for the nested columns and pack them into our `NestedFrame` with `NestedFrame.add_nested()` function. `add_nested` will align the nest based on the index by default (a column may be selected instead via the `on` kwarg), as we see the `nested` `DataFrame` has a repeated index corresponding to the `nf` `NestedFrame`."
+    "We can then create an additional pandas dataframes for the nested columns and pack them into our `NestedFrame` with `NestedFrame.join_nested()` function. `join_nested` will align the nest based on the index by default (a column may be selected instead via the `on` kwarg), as we see the `nested` `DataFrame` has a repeated index corresponding to the `nf` `NestedFrame`."
    ]
   },
   {
@@ -158,7 +158,7 @@
     "    index=[0, 0, 0, 1, 1, 1, 2, 2, 2, 2],\n",
     ")\n",
     "\n",
-    "nf = nf.add_nested(nested, \"nested\")\n",
+    "nf = nf.join_nested(nested, \"nested\")\n",
     "nf"
    ]
   },
@@ -182,7 +182,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We could add other nested columns by creating new sub-tables and adding them with `add_nested()`. Note that while the tables added with each `add_nested()` must be rectangular, they do not need to have the same dimensions between calls. We could add another nested row with a different number of observations."
+    "We could add other nested columns by creating new sub-tables and adding them with `join_nested()`. Note that while the tables added with each `join_nested()` must be rectangular, they do not need to have the same dimensions between calls. We could add another nested row with a different number of observations."
    ]
   },
   {
@@ -199,7 +199,7 @@
     "    index=[0, 0, 1, 1, 1, 2],\n",
     ")\n",
     "\n",
-    "nf = nf.add_nested(nested, \"nested2\")\n",
+    "nf = nf.join_nested(nested, \"nested2\")\n",
     "nf"
    ]
   },

diff --git a/docs/tutorials/data_manipulation.ipynb b/docs/tutorials/data_manipulation.ipynb
@@ -105,13 +105,6 @@
     "## Adding or Replacing Nested Columns"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "> *A Note on Performance: These operations involve full reconstruction of the nested columns so expect impacted performance when doing this at scale. It may be appropriate to do these operations within reduce functions directly (e.g. subtracting a value from a column) if performance is key.*"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -210,7 +203,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This is functionally equivalent to using `add_nested`:"
+    "This is functionally equivalent to using `join_nested`:"
    ]
   },
   {
@@ -224,7 +217,7 @@
    },
    "outputs": [],
    "source": [
-    "ndf.add_nested(ndf[\"nested.band\"].to_frame(), \"bands_from_add_nested\")"
+    "ndf.join_nested(ndf[\"nested.band\"].to_frame(), \"bands_from_add_nested\")"
    ]
   },
   {
@@ -254,7 +247,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The above again being shorthand for the following `add_nested` call:"
+    "The above again being shorthand for the following `join_nested` call:"
    ]
   },
   {
@@ -263,7 +256,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ndf.add_nested(flat_df, \"example_from_add_nested\")"
+    "ndf.join_nested(flat_df, \"example_from_add_nested\")"
    ]
   },
   {

diff --git a/docs/tutorials/low_level.ipynb b/docs/tutorials/low_level.ipynb
@@ -111,7 +111,7 @@
    "id": "33d8caacf0bf042e",
    "metadata": {},
    "source": [
-    "You can also get a list of fields with `.fields` attribute:"
+    "You can also get a list of columns with the `.columns` attribute:"
    ]
   },
   {
@@ -126,7 +126,7 @@
    },
    "outputs": [],
    "source": [
-    "nested_series.nest.fields"
+    "nested_series.nest.columns"
    ]
   },
   {
@@ -205,23 +205,23 @@
     "new_series.nest[\"flux\"] = new_series.nest[\"flux\"] - new_series.nest[\"flux\"].mean()\n",
     "\n",
     "# Create a new series with a new column\n",
-    "new_series = new_series.nest.with_field(\"lsst_band\", \"lsst_\" + new_series.nest[\"band\"])\n",
+    "new_series = new_series.nest.set_column(\"lsst_band\", \"lsst_\" + new_series.nest[\"band\"])\n",
     "\n",
     "# Create a new series with a column removed, you can also pass a list of columns to remove\n",
-    "new_series = new_series.nest.without_field(\"band\")\n",
+    "new_series = new_series.nest.drop(\"band\")\n",
     "\n",
     "# Add a new column with a python list instead of a Series\n",
-    "new_series = new_series.nest.with_field(\n",
+    "new_series = new_series.nest.set_column(\n",
     "    \"new_column\",\n",
     "    [1, 2] * (new_series.nest.flat_length // 2),\n",
     ")\n",
     "\n",
     "# Add a new column repeating values for each nested element\n",
     "# It can be useful when you want to move some metadata to the nested data\n",
-    "new_series = new_series.nest.with_filled_field(\"index_mult_100\", new_series.index * 100)\n",
+    "new_series = new_series.nest.set_filled_column(\"index_mult_100\", new_series.index * 100)\n",
     "\n",
     "# Create a new series, with a column dtype changed\n",
-    "new_series = new_series.nest.with_field(\"t\", new_series.nest[\"t\"].astype(np.int8))\n",
+    "new_series = new_series.nest.set_column(\"t\", new_series.nest[\"t\"].astype(np.int8))\n",
     "\n",
     "new_series.nest.to_flat()"
    ]
@@ -293,7 +293,7 @@
    "source": [
     "# Adjust each time to be relative to the first observation\n",
     "dt = new_series.nest.to_lists()[\"t\"].apply(lambda t: t - t.min())\n",
-    "new_series = new_series.nest.with_list_field(\"dt\", dt)\n",
+    "new_series = new_series.nest.set_list_column(\"dt\", dt)\n",
     "new_series.nest.to_flat()"
    ]
   },
@@ -367,7 +367,7 @@
     "We have already seen how `.nest` accessor could be used to get different views on the nested data: \"flat\" dataframe, and list-array dataframe with columns of `pd.ArrowDtype`.\n",
     "\n",
     "This section is about converting nested Series to and from other data types.\n",
-    "If you just need to add a nested column to a `NestedFrame`, you can do it with `.add_nested()` method."
+    "If you just need to add a nested column to a `NestedFrame`, you can do it with `.join_nested()` method."
    ]
   },
   {
@@ -542,7 +542,7 @@
     "        {\"t\": [4, 5], \"flux\": [0.4, 0.5]},\n",
     "        None,\n",
     "    ],\n",
-    "    dtype=NestedDtype.from_fields({\"t\": pa.float64(), \"flux\": pa.float32()}),\n",
+    "    dtype=NestedDtype.from_columns({\"t\": pa.float64(), \"flux\": pa.float32()}),\n",
     ")\n",
     "series_from_pack"
    ]
@@ -588,7 +588,7 @@
     "        pd.DataFrame({\"t\": [1, 2, 3], \"band\": [\"g\", \"r\", \"r\"]}),\n",
     "        {\"t\": np.array([4, 5]), \"band\": [None, \"r\"]},\n",
     "    ],\n",
-    "    dtype=NestedDtype.from_fields({\"t\": pa.float64(), \"band\": pa.string()}),\n",
+    "    dtype=NestedDtype.from_columns({\"t\": pa.float64(), \"band\": pa.string()}),\n",
     ")\n",
     "series_from_dtype"
    ]

diff --git a/pyproject.toml b/pyproject.toml
@@ -22,7 +22,9 @@ dependencies = [
     # We use internal pd._libs.missing and experimental ArrowExtensionArray
     "pandas>=2.2.3,<2.4",
     "pyarrow>=16",  # remove struct_field_names() and struct_fields() when upgraded to 18+
-
+    "Deprecated>=1.2.0",
+    "wrapt>=1.12.1",
+
     # NOTE: package PINNED at <0.3.0, see https://github.com/astronomy-commons/lsdb/issues/1047
     "universal_pathlib>=0.2,<0.3.0",
 ]
@@ -43,6 +45,7 @@ dev = [
     "aiohttp",
     "requests",
     "s3fs",
+    "types-Deprecated", # Needed for mypy type checking of Deprecated package
 ]
 
 [build-system]