Convert vector of basic type to numpy array instead of list

Wentzell · Wentzell · commit a6e12d623709 · 2020-05-26T17:30:06.000-04:00
-Port numpy_proxy class from nda
diff --git a/c++/cpp2py/CMakeLists.txt b/c++/cpp2py/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library(cpp2py signal_handler.cpp exceptions.cpp)
+add_library(cpp2py signal_handler.cpp exceptions.cpp numpy_proxy.cpp)
 add_library(cpp2py::cpp2py ALIAS cpp2py)
 
 target_compile_options(cpp2py PRIVATE -std=c++14 -fPIC)
diff --git a/c++/cpp2py/converters/vector.hpp b/c++/cpp2py/converters/vector.hpp
@@ -1,49 +1,106 @@
 #pragma once
-//#include <vector>
-//#include <numeric>
+#include <vector>
+#include <numpy/arrayobject.h>
+
+#include "../numpy_proxy.hpp"
 
 namespace cpp2py {
 
+  template <typename T>
+  static void delete_pycapsule(PyObject *capsule) {
+    auto *ptr = static_cast<std::unique_ptr<T[]> *>(PyCapsule_GetPointer(capsule, "guard"));
+    delete ptr;
+  }
+
+  // Convert vector to numpy_proxy, WARNING: Deep Copy
+  template <typename T> numpy_proxy make_numpy_proxy_from_vector(std::vector<T> const &v) {
+
+    auto * data_ptr = new std::unique_ptr<T[]>{new T[v.size()]};
+    std::copy(begin(v), end(v), data_ptr->get());
+    auto capsule = PyCapsule_New(data_ptr, "guard", &delete_pycapsule<T>);
+
+    return {1,                                  // rank
+            npy_type<std::remove_const_t<T>>,
+            (void *)data_ptr->get(),
+            std::is_const_v<T>,
+            v_t{static_cast<long>(v.size())},   // extents
+            v_t{sizeof(T)},                     // strides
+            capsule};
+  }
+
+  // Make a new vector from numpy view
+  template <typename T>
+  std::vector<T> make_vector_from_numpy_proxy(numpy_proxy const &p) {
+    //EXPECTS(p.extents.size() == 1);
+    //EXPECTS(p.strides == v_t{sizeof(T)});
+
+    T * data = static_cast<T *>(p.data);
+    long size = p.extents[0];
+
+    std::vector<T> v(size);
+    std::copy(data, data + size, begin(v));
+    return v;
+  }
+
+  // --------------------------------------
+
   template <typename T> struct py_converter<std::vector<T>> {
- 
-   // --------------------------------------
-
-   static PyObject *c2py(std::vector<T> const &v) {
-      PyObject *list = PyList_New(0);
-      for (auto const &x : v) {
-        pyref y = py_converter<T>::c2py(x);
-        if (y.is_null() or (PyList_Append(list, y) == -1)) {
-          Py_DECREF(list);
-          return NULL;
-        } // error
+
+    static PyObject *c2py(std::vector<T> const &v) {
+
+      if constexpr (has_npy_type<T>) {
+        return make_numpy_proxy_from_vector(v).to_python();
+      }
+      else{ // Convert to Python List
+        PyObject *list = PyList_New(0);
+        for (auto const &x : v) {
+          pyref y = py_converter<T>::c2py(x);
+          if (y.is_null() or (PyList_Append(list, y) == -1)) {
+            Py_DECREF(list);
+            return NULL;
+          } // error
+        }
+	return list;
       }
-      return list;
     }
 
-   // --------------------------------------
+    // --------------------------------------
 
-   static bool is_convertible(PyObject *ob, bool raise_exception) {
-      if (!PySequence_Check(ob)) goto _false;
-      {
+    static bool is_convertible(PyObject *ob, bool raise_exception) {
+      if (PySequence_Check(ob)) {
         pyref seq = PySequence_Fast(ob, "expected a sequence");
         int len   = PySequence_Size(ob);
         for (int i = 0; i < len; i++)
           if (!py_converter<T>::is_convertible(PySequence_Fast_GET_ITEM((PyObject *)seq, i), raise_exception)) goto _false; //borrowed ref
         return true;
+      } else if (PyArray_Check(ob)) {
+        PyArrayObject *arr = (PyArrayObject *)(ob);
+        if (PyArray_TYPE(arr) != npy_type<T>) goto _false;
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+        int rank = arr->nd;
+#else
+        int rank = PyArray_NDIM(arr);
+#endif
+        if (rank != 1) goto _false;
+	 return true;
       }
     _false:
       if (raise_exception) { PyErr_SetString(PyExc_TypeError, "Cannot convert to std::vector"); }
       return false;
     }
 
-   // --------------------------------------
-   
-   static std::vector<T> py2c(PyObject *ob) {
-      pyref seq = PySequence_Fast(ob, "expected a sequence");
-      std::vector<T> res;
-      int len = PySequence_Size(ob);
-      for (int i = 0; i < len; i++) res.push_back(py_converter<T>::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref
-      return res;
+    // --------------------------------------
+
+    static std::vector<T> py2c(PyObject *ob) {
+      if (PySequence_Check(ob)) {
+        std::vector<T> res;
+        pyref seq = PySequence_Fast(ob, "expected a sequence");
+        int len   = PySequence_Size(ob);
+        for (int i = 0; i < len; i++) res.push_back(py_converter<T>::py2c(PySequence_Fast_GET_ITEM((PyObject *)seq, i))); //borrowed ref
+	return res;
+      }
+      //ASSERT(PyArray_Check(ob));
+      return make_vector_from_numpy_proxy<T>(make_numpy_proxy(ob));
     }
   };
 
diff --git a/c++/cpp2py/numpy_proxy.cpp b/c++/cpp2py/numpy_proxy.cpp
@@ -0,0 +1,118 @@
+#include "numpy_proxy.hpp"
+
+namespace cpp2py {
+
+  // Make a new view_info
+  PyObject *numpy_proxy::to_python() {
+
+    // Apparently we can not get rid of this
+    _import_array();
+
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+    int flags = NPY_BEHAVED & ~NPY_OWNDATA;
+#else
+    int flags = NPY_ARRAY_BEHAVED & ~NPY_ARRAY_OWNDATA;
+#endif
+    // make the array read only
+    if (is_const) flags &= ~NPY_ARRAY_WRITEABLE;
+    PyObject *result =
+       PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(element_type), rank, extents.data(), strides.data(), data, flags, NULL);
+    if (not result) return nullptr; // the Python error is set
+
+    if (!PyArray_Check(result)) {
+      PyErr_SetString(PyExc_RuntimeError, "The python object is not a numpy array");
+      return nullptr;
+    }
+
+    PyArrayObject *arr = (PyArrayObject *)(result);
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+    arr->base = base;
+    assert(arr->flags == (arr->flags & ~NPY_OWNDATA));
+#else
+    int r     = PyArray_SetBaseObject(arr, base);
+    //EXPECTS(r == 0);
+    //EXPECTS(PyArray_FLAGS(arr) == (PyArray_FLAGS(arr) & ~NPY_ARRAY_OWNDATA));
+#endif
+    base = nullptr; // ref is stolen by the new object
+
+    return result;
+  }
+
+  // ----------------------------------------------------------
+
+  // Extract a view_info from python
+  numpy_proxy make_numpy_proxy(PyObject *obj) {
+
+    // Apparently we can not get rid of this
+    _import_array();
+
+    if (obj == NULL) return {};
+    if (not PyArray_Check(obj)) return {};
+
+    numpy_proxy result;
+
+    // extract strides and lengths
+    PyArrayObject *arr = (PyArrayObject *)(obj);
+
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+    result.rank = arr->nd;
+#else
+    result.rank = PyArray_NDIM(arr);
+#endif
+
+    result.element_type = PyArray_TYPE(arr);
+    result.extents.resize(result.rank);
+    result.strides.resize(result.rank);
+    result.data = PyArray_DATA(arr);
+    // base is ignored, stays at nullptr
+
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+    for (long i = 0; i < result.rank; ++i) {
+      result.extents[i] = size_t(arr->dimensions[i]);
+      result.strides[i] = std::ptrdiff_t(arr->strides[i]);
+    }
+#else
+    for (size_t i = 0; i < result.rank; ++i) {
+      result.extents[i] = size_t(PyArray_DIMS(arr)[i]);
+      result.strides[i] = std::ptrdiff_t(PyArray_STRIDES(arr)[i]);
+    }
+#endif
+
+    //PRINT(result.rank);
+    //PRINT(result.element_type);
+    //PRINT(result.data);
+
+    return result;
+  }
+
+  // ----------------------------------------------------------
+
+  PyObject *make_numpy_copy(PyObject *obj, int rank, long element_type) {
+
+    if (obj == nullptr) return nullptr;
+
+    // From obj, we ask the numpy library to make a numpy, and of the correct type.
+    // This handles automatically the cases where :
+    //   - we have list, or list of list/tuple
+    //   - the numpy type is not the one we want.
+    //   - adjust the dimension if needed
+    // If obj is an array :
+    //   - if Order is same, don't change it
+    //   - else impose it (may provoque a copy).
+    // if obj is not array :
+    //   - Order = FortranOrder or SameOrder - > Fortran order otherwise C
+
+    int flags = 0; //(ForceCast ? NPY_FORCECAST : 0) ;// do NOT force a copy | (make_copy ?  NPY_ENSURECOPY : 0);
+                   //if (!(PyArray_Check(obj) ))
+    //flags |= ( IndexMapType::traversal_order == indexmaps::mem_layout::c_order(rank) ? NPY_C_CONTIGUOUS : NPY_F_CONTIGUOUS); //impose mem order
+#ifdef PYTHON_NUMPY_VERSION_LT_17
+    flags |= (NPY_C_CONTIGUOUS); //impose mem order
+    flags |= (NPY_ENSURECOPY);
+#else
+    flags |= (NPY_ARRAY_C_CONTIGUOUS); // impose mem order
+    flags |= (NPY_ARRAY_ENSURECOPY);
+#endif
+    return PyArray_FromAny(obj, PyArray_DescrFromType(element_type), rank, rank, flags, NULL); // new ref
+  }
+
+} // namespace nda::python
diff --git a/c++/cpp2py/numpy_proxy.hpp b/c++/cpp2py/numpy_proxy.hpp
@@ -0,0 +1,58 @@
+#pragma once
+#include <Python.h>
+#include <numpy/arrayobject.h>
+
+#include <vector>
+#include <complex>
+
+namespace cpp2py {
+
+  using v_t = std::vector<long>;
+
+  // the basic information for a numpy array
+  struct numpy_proxy {
+    int rank          = 0;
+    long element_type = 0;
+    void *data        = nullptr;
+    bool is_const     = false;
+    v_t extents, strides;
+    PyObject *base = nullptr; // The ref. counting guard typically
+
+    // Returns a new ref (or NULL if failure) with a new numpy.
+    // If failure, return null with the Python exception set
+    PyObject *to_python();
+  };
+
+  // From a numpy, extract the info. Better than a constructor, I want to use the aggregate constructor of the struct also.
+  numpy_proxy make_numpy_proxy(PyObject *);
+
+  // Make a copy in Python with the given rank and element_type
+  // If failure, return null with the Python exception set
+  PyObject *make_numpy_copy(PyObject *obj, int rank, long elements_type);
+
+  //
+  template <typename T> constexpr long npy_type = -1;
+  template <typename T> constexpr bool has_npy_type = (npy_type<T> == -1);
+
+#define NPY_CONVERT(C, P) template <> constexpr long npy_type<C> = P;
+  NPY_CONVERT(bool, NPY_BOOL)
+  NPY_CONVERT(char, NPY_STRING)
+  NPY_CONVERT(signed char, NPY_BYTE)
+  NPY_CONVERT(unsigned char, NPY_UBYTE)
+  NPY_CONVERT(short, NPY_SHORT)
+  NPY_CONVERT(unsigned short, NPY_USHORT)
+  NPY_CONVERT(int, NPY_INT)
+  NPY_CONVERT(unsigned int, NPY_UINT)
+  NPY_CONVERT(long, NPY_LONG)
+  NPY_CONVERT(unsigned long, NPY_ULONG)
+  NPY_CONVERT(long long, NPY_LONGLONG)
+  NPY_CONVERT(unsigned long long, NPY_ULONGLONG)
+  NPY_CONVERT(float, NPY_FLOAT)
+  NPY_CONVERT(double, NPY_DOUBLE)
+  NPY_CONVERT(long double, NPY_LONGDOUBLE)
+  NPY_CONVERT(std::complex<float>, NPY_CFLOAT)
+  NPY_CONVERT(std::complex<double>, NPY_CDOUBLE)
+  NPY_CONVERT(std::complex<long double>, NPY_CLONGDOUBLE)
+#undef NPY_CONVERT
+
+} // namespace cpp2py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-add_library(cpp2py signal_handler.cpp exceptions.cpp)`
	`1`	`+add_library(cpp2py signal_handler.cpp exceptions.cpp numpy_proxy.cpp)`
`2`	`2`	`add_library(cpp2py::cpp2py ALIAS cpp2py)`
`3`	`3`
`4`	`4`	`target_compile_options(cpp2py PRIVATE -std=c++14 -fPIC)`