implement endian-aware reader support

pdal/io/NumpyReader.cpp

@@ -41,6 +41,7 @@
 #include <pdal/util/ProgramArgs.hpp>
 #include <pdal/util/FileUtils.hpp>
 #include <pdal/util/Algorithm.hpp>
+#include <pdal/util/Extractor.hpp>
 
 #include "../plang/Environment.hpp"
 
@@ -407,10 +408,13 @@ void NumpyReader::createFields(PointLayoutPtr layout)
             offset = PyLong_AsLong(offset_o);
 
             // Get type.
-            type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0),
-                name);
+            PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0);
+            type = getType(dt, name);
+
+            char byteorder = dt->byteorder;
+            int elsize = dt->elsize;
             id = registerDim(layout, name, type);
-            m_fields.push_back({id, type, offset});
+            m_fields.push_back({id, type, offset, byteorder, elsize});
         }
     }
 }
@@ -510,6 +514,12 @@ void NumpyReader::ready(PointTableRef table)
     for (npy_intp i = 0; i < m_ndims; ++i)
         log()->get(LogLevel::Debug) << "numpy shape dimension number '" <<
             i << "' is '" << m_shape[i] <<"'" << std::endl;
+
+    PointLayoutPtr layout = table.layout();
+    MetadataNode m = layout->toMetadata();
+
+    pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3));
+
 }
 
 bool NumpyReader::nextPoint()
@@ -530,12 +540,93 @@ bool NumpyReader::nextPoint()
 }
 
 
+
+
 bool NumpyReader::loadPoint(PointRef& point, point_count_t position)
 {
     using namespace Dimension;
 
+    pdal::SwitchableExtractor extractor(p_data, *m_strideptr);
+
+    std::vector<char> buf(*m_strideptr,0);
+
+    float flt(0.0);
+    double dbl(0.0);
+    uint8_t u8(0);
+    uint16_t u16(0);
+    uint32_t u32(0);
+    uint64_t u64(0);
+    int8_t i8(0);
+    int16_t i16(0);
+    int32_t i32(0);
+    int64_t i64(0);
+
     for (const Field& f : m_fields)
-        point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset));
+    {
+        if (f.m_byteorder == '>')
+            extractor.switchToBigEndian();
+        else
+            extractor.switchToLittleEndian();
+
+        switch (f.m_type)
+        {
+            case Dimension::Type::Signed8:
+                extractor >> i8;
+                point.setField(f.m_id, i8);
+                break;
+
+            case Dimension::Type::Signed16:
+                extractor >> i16;
+                point.setField(f.m_id, i16);
+                break;
+
+            case Dimension::Type::Signed32:
+                extractor >> i32;
+                point.setField(f.m_id, i32);
+                break;
+
+            case Dimension::Type::Signed64:
+                extractor >> i64;
+                point.setField(f.m_id, i64);
+                break;
+
+            case Dimension::Type::Unsigned8:
+                extractor >> u8;
+                point.setField(f.m_id, u8);
+                break;
+
+            case Dimension::Type::Unsigned16:
+                extractor >> u16;
+                point.setField(f.m_id, u16);
+                break;
+
+            case Dimension::Type::Unsigned32:
+                extractor >> u32;
+                point.setField(f.m_id, u32);
+                break;
+
+            case Dimension::Type::Unsigned64:
+                extractor >> u64;
+                point.setField(f.m_id, u64);
+                break;
+
+            case Dimension::Type::Float:
+                extractor >> flt;
+                point.setField(f.m_id, flt);
+                break;
+
+            case Dimension::Type::Double:
+                extractor >> dbl;
+                point.setField(f.m_id, dbl);
+                break;
+
+            default:
+                // skip it
+                extractor.skip(f.m_elsize);
+
+        }
+
+    }
 
     if (m_storeXYZ)
     {

pdal/io/NumpyReader.hpp

@@ -120,6 +120,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable
         Dimension::Id m_id;
         Dimension::Type m_type;
         int m_offset;
+        char m_byteorder;
+        int m_elsize;
     };
     std::vector<Field> m_fields;
     point_count_t m_index;

pdal/test/NumpyReaderTest.cpp

@@ -236,3 +236,100 @@ TEST(NumpyReaderTest, threedim_col)
     }
 }
 
+
+TEST(NumpyReaderTest, threedim_row_script)
+{
+    StageFactory f;
+
+    Options opts;
+    opts.add("filename", Support::datapath("threedim.py"));
+    opts.add("order", "row");
+    opts.add("function", "load");
+    opts.add("module", "threedim");
+    opts.add("fargs",  Support::datapath("threedim.npy"));
+
+    NumpyReader reader;
+    reader.setOptions(opts);
+
+    PointTable table;
+
+    reader.prepare(table);
+
+    PointViewSet viewSet = reader.execute(table);
+    PointViewPtr view = *viewSet.begin();
+    PointLayoutPtr layout = view->layout();
+    Dimension::Id health = layout->findDim("Health");
+    Dimension::Id foo = layout->findDim("Foo");
+
+    EXPECT_EQ(view->size(), 24u);
+    int ex = 0;
+    int ey = 0;
+    int ez = 0;
+    for (PointId id = 0; id < view->size(); ++id)
+    {
+        int f = view->getFieldAs<int>(foo, id);
+        int h = view->getFieldAs<int>(health, id);
+        int i = view->getFieldAs<int>(Dimension::Id::Intensity, id);
+        int x = view->getFieldAs<int>(Dimension::Id::X, id);
+        int y = view->getFieldAs<int>(Dimension::Id::Y, id);
+        int z = view->getFieldAs<int>(Dimension::Id::Z, id);
+        EXPECT_EQ(f, ((int)id / 12) % 2);
+        EXPECT_EQ(h, ((int)id / 4) % 3);
+        EXPECT_EQ(i, (int)id % 4);
+        EXPECT_EQ(x, ex);
+        EXPECT_EQ(y, ey);
+        EXPECT_EQ(z, ez);
+        // Array is 2 x 3 x 4
+        ez++;
+        if (ez == 4)
+        {
+            ez = 0;
+            ey++;
+            if (ey == 3)
+            {
+                ey = 0;
+                ex++;
+            }
+        }
+    }
+}
+
+TEST(NumpyReaderTest, four_floats_script)
+{
+    StageFactory f;
+
+    Options opts;
+    opts.add("filename", Support::datapath("four-floats.py"));
+    opts.add("order", "row");
+    opts.add("function", "load");
+    opts.add("module", "four_floats");
+    opts.add("fargs",  Support::datapath("four-floats-be.bin"));
+
+    NumpyReader reader;
+    reader.setOptions(opts);
+
+    PointTable table;
+
+    reader.prepare(table);
+
+    PointViewSet viewSet = reader.execute(table);
+    PointViewPtr view = *viewSet.begin();
+    PointLayoutPtr layout = view->layout();
+
+    EXPECT_EQ(view->size(), 10u);
+    int ex = 0;
+    int ey = 0;
+    int ez = 0;
+    int factor = 10;
+    for (PointId id = 0; id < view->size(); ++id)
+    {
+        int t = view->getFieldAs<int>(Dimension::Id::GpsTime, id);
+        int x = view->getFieldAs<int>(Dimension::Id::X, id);
+        int y = view->getFieldAs<int>(Dimension::Id::Y, id);
+        int z = view->getFieldAs<int>(Dimension::Id::Z, id);
+        EXPECT_EQ(x, (int)id * 1 * factor);
+        EXPECT_EQ(y, (int)id * 2 * factor);
+        EXPECT_EQ(z, (int)id * 3 * factor);
+        EXPECT_EQ(t, (int)id * 4 * factor);
+    }
+}

pdal/test/data/four-floats.py

@@ -0,0 +1,10 @@
+import numpy as np
+
+
+dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', '<f4'), ('GPSTime', '>f4')])
+
+def load(filename):
+    data = open(filename, 'rb').read()
+    array = np.frombuffer(data, dtype=dtype)
+
+    return array

pdal/test/data/threedim.py

@@ -0,0 +1,5 @@
+import numpy as np
+
+def load(filename):
+    array = np.load(filename)
+    return array