Skip to content

Commit

Permalink
add decision todos
Browse files Browse the repository at this point in the history 
Signed-off-by: Nitish Bharambe <[email protected]>
  • Loading branch information
@nitbharambe
nitbharambe committed Oct 21, 2024
1 parent b4f6353 commit 7672a16
Showing 1 changed file with 140 additions and 28 deletions.
168 changes: 140 additions & 28 deletions docs/examples/arrow_example.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,38 @@
"print(pgm_combined_schema(DatasetType.input, ComponentType.asym_load))"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"pyarrow.RecordBatch\n",
"id: int32\n",
"u_rated: double\n",
"----\n",
"id: [1,2,3]\n",
"u_rated: [10500,10500,10500]"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#TODO Decisions: Create from schema\n",
"pa.record_batch(\n",
" {\n",
" \"id\": [1, 2, 3],\n",
" \"u_rated\": [10500.0, 10500.0, 10500.0],\n",
" },\n",
" schema=pgm_schema(DatasetType.input, ComponentType.node),\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -248,7 +280,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"outputs": [
{
Expand All @@ -262,7 +294,7 @@
"u_rated: [10500,10500,10500]"
]
},
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -354,7 +386,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
Expand All @@ -364,7 +396,7 @@
" dtype={'names': ['id', 'u_rated'], 'formats': ['<i4', '<f8'], 'offsets': [0, 8], 'itemsize': 16, 'aligned': True})"
]
},
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -396,7 +428,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"outputs": [
{
Expand All @@ -405,7 +437,7 @@
"{'id': array([1, 2, 3]), 'u_rated': array([10500., 10500., 10500.])}"
]
},
"execution_count": 7,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -442,7 +474,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -474,7 +506,7 @@
" 'q_specified': array([0.5, 1.5])}}"
]
},
"execution_count": 8,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -492,7 +524,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -513,7 +545,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 11,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -620,7 +652,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 12,
"metadata": {},
"outputs": [
{
Expand All @@ -644,7 +676,7 @@
"q: [-3299418.661306348,-0.5000000701801947,-1.4999998507078594]"
]
},
"execution_count": 11,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -679,7 +711,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 13,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -716,7 +748,7 @@
"q_specified_c: [0.1,1500]"
]
},
"execution_count": 12,
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -766,7 +798,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand All @@ -777,7 +809,7 @@
" dtype={'names': ['id', 'node', 'status', 'type', 'p_specified', 'q_specified'], 'formats': ['<i4', '<i4', 'i1', 'i1', ('<f8', (3,)), ('<f8', (3,))], 'offsets': [0, 4, 8, 9, 16, 40], 'itemsize': 64, 'aligned': True})"
]
},
"execution_count": 13,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -826,7 +858,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 15,
"metadata": {},
"outputs": [
{
Expand All @@ -842,7 +874,7 @@
" [1.5e+00, 2.5e+00, 1.5e+03]])}"
]
},
"execution_count": 14,
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -897,7 +929,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 27,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -928,12 +960,15 @@
"q_specified: [[0.5,1500,0.1],[1.5,2.5,1500]]"
]
},
"execution_count": 15,
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# TODO Decisions: Asymmetric attributes should be fixed list array or individual phases or both?\n",
"\n",
"\n",
"asym_load_input_dtype = initialize_array(\"input\", \"asym_load\", 0).dtype\n",
"print(\"asym_load:\", asym_load_input_dtype)\n",
"# asym_float_type = pa.struct([(\"a\", pa.float64()), (\"b\", pa.float64()), (\"c\", pa.float64())])\n",
Expand All @@ -955,7 +990,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -972,7 +1007,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"outputs": [
{
Expand All @@ -988,7 +1023,7 @@
" [1.5e+00, 2.5e+00, 1.5e+03]])}"
]
},
"execution_count": 17,
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1036,7 +1071,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 19,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1095,7 +1130,7 @@
"2 -0.004338 -2.098733 2.090057"
]
},
"execution_count": 18,
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1129,7 +1164,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 20,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -1167,7 +1202,7 @@
"..."
]
},
"execution_count": 19,
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1206,7 +1241,84 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"pyarrow.Table\n",
"id: int32\n",
"energized: int8\n",
"u_pu_a: double\n",
"u_pu_b: double\n",
"u_pu_c: double\n",
"u_a: double\n",
"u_b: double\n",
"u_c: double\n",
"u_angle_a: double\n",
"u_angle_b: double\n",
"u_angle_c: double\n",
"p_a: double\n",
"p_b: double\n",
"p_c: double\n",
"q_a: double\n",
"q_b: double\n",
"q_c: double\n",
"----\n",
"id: [[1,2,3]]\n",
"energized: [[1,1,1]]\n",
"u_pu_a: [[1.0003248257977395,1.0028803762176164,1.0041143008174032]]\n",
"u_pu_b: [[1.0003243769486854,1.0028710993140406,1.0041033583077175]]\n",
"u_pu_c: [[1.00032436416241,1.0028730789021523,1.0041004935738533]]\n",
"u_a: [[6064.146978239599,6079.639179329456,6087.119449677845]]\n",
"u_b: [[6064.144257236815,6079.582941090301,6087.053114238262]]\n",
"u_c: [[6064.1441797241405,6079.594941705457,6087.035747712152]]\n",
"u_angle_a: [[-0.00006651848125694397,-0.0029298831864832267,-0.004337685507209373]]\n",
"u_angle_b: [[-2.094461573665813,-2.0973219974462594,-2.098732840554144]]\n",
"..."
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def numpy_columnar_to_arrow_combined(data: np.ndarray) -> pa.lib.table:\n",
" \"\"\"Convert NumPy data to Arrow data.\"\"\"\n",
" simple_data_types = []\n",
" multi_value_data_types = []\n",
"\n",
" for name, (dtype, _) in data.dtype.fields.items():\n",
" if len(dtype.shape) == 0:\n",
" simple_data_types.append(name)\n",
" else:\n",
" multi_value_data_types.append(name)\n",
"\n",
" result = pa.table(pd.DataFrame(data[simple_data_types]))\n",
"\n",
" phases = (\"a\", \"b\", \"c\")\n",
" for name in multi_value_data_types:\n",
" column = data[name]\n",
"\n",
" assert column.shape[1] == len(phases), \"Asymmetric data has 3 phase output\"\n",
"\n",
" for phase_index, phase in enumerate(phases):\n",
" sub_column = column[:, phase_index]\n",
" result = result.append_column(f\"{name}_{phase}\", [pd.Series(sub_column)])\n",
"\n",
" return result\n",
"\n",
"\n",
"pa_asym_node_result = numpy_to_arrow(asym_result[\"node\"])\n",
"\n",
"pa_asym_node_result"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
Expand Down

0 comments on commit 7672a16

Please sign in to comment.