From 1b5a51b244cf7f171749f487bee9c40d729c2446 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Tue, 19 May 2020 19:59:55 -0500 Subject: [PATCH 1/3] implement endian-aware reader support --- pdal/io/NumpyReader.cpp | 107 ++++++++++++++++++++++++++++-- pdal/io/NumpyReader.hpp | 2 + pdal/test/NumpyReaderTest.cpp | 99 ++++++++++++++++++++++++++- pdal/test/data/four-floats-be.bin | Bin 0 -> 160 bytes pdal/test/data/four-floats.py | 10 +++ pdal/test/data/threedim.py | 5 ++ 6 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 pdal/test/data/four-floats-be.bin create mode 100644 pdal/test/data/four-floats.py create mode 100644 pdal/test/data/threedim.py diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp index 47992d8f..f2c7e8fd 100644 --- a/pdal/io/NumpyReader.cpp +++ b/pdal/io/NumpyReader.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include "../plang/Environment.hpp" @@ -407,10 +408,13 @@ void NumpyReader::createFields(PointLayoutPtr layout) offset = PyLong_AsLong(offset_o); // Get type. - type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0), - name); + PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); + type = getType(dt, name); + + char byteorder = dt->byteorder; + int elsize = dt->elsize; id = registerDim(layout, name, type); - m_fields.push_back({id, type, offset}); + m_fields.push_back({id, type, offset, byteorder, elsize}); } } } @@ -510,6 +514,12 @@ void NumpyReader::ready(PointTableRef table) for (npy_intp i = 0; i < m_ndims; ++i) log()->get(LogLevel::Debug) << "numpy shape dimension number '" << i << "' is '" << m_shape[i] <<"'" << std::endl; + + PointLayoutPtr layout = table.layout(); + MetadataNode m = layout->toMetadata(); + + pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3)); + } bool NumpyReader::nextPoint() @@ -530,12 +540,101 @@ bool NumpyReader::nextPoint() } + + bool NumpyReader::loadPoint(PointRef& point, point_count_t position) { using namespace Dimension; + pdal::SwitchableExtractor extractor(p_data, *m_strideptr); + + std::vector buf(*m_strideptr,0); + + float flt(0.0); + double dbl(0.0); + uint8_t uint8(0); + uint16_t uint16(0); + uint32_t uint32(0); + uint64_t uint64(0); + int8_t int8(0); + int16_t int16(0); + int32_t int32(0); + int64_t int64(0); + for (const Field& f : m_fields) - point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset)); + { + if (f.m_byteorder == '>') + extractor.switchToBigEndian(); + else + extractor.switchToLittleEndian(); + + if (Dimension::base(f.m_type) == Dimension::BaseType::Floating) + { + if (f.m_elsize == 4) + { + extractor >> flt; + point.setField(f.m_id, flt); + } + else if (f.m_elsize == 8) + { + extractor >> dbl; + point.setField(f.m_id, dbl); + + } + } + else if (Dimension::base(f.m_type) == Dimension::BaseType::Signed) + { + if (f.m_elsize == 1) + { + extractor >> int8; + point.setField(f.m_id, int8); + } + if (f.m_elsize == 2) + { + extractor >> int16; + point.setField(f.m_id, int16); + } + if (f.m_elsize == 4) + { + extractor >> int32; + point.setField(f.m_id, int32); + } + if (f.m_elsize == 8) + { + extractor >> int64; + point.setField(f.m_id, int64); + } + } + else if (Dimension::base(f.m_type) == Dimension::BaseType::Unsigned) + { + if (f.m_elsize == 1) + { + extractor >> uint8; + point.setField(f.m_id, uint8); + } + if (f.m_elsize == 2) + { + extractor >> uint16; + point.setField(f.m_id, uint16); + } + if (f.m_elsize == 4) + { + extractor >> uint32; + point.setField(f.m_id, uint32); + } + if (f.m_elsize == 8) + { + extractor >> int64; + point.setField(f.m_id, uint64); + } + } + else + { + // skip it + extractor.skip(f.m_elsize); + } + + } if (m_storeXYZ) { diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp index 1b8ae8dd..2c62a1d5 100644 --- a/pdal/io/NumpyReader.hpp +++ b/pdal/io/NumpyReader.hpp @@ -120,6 +120,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable Dimension::Id m_id; Dimension::Type m_type; int m_offset; + char m_byteorder; + int m_elsize; }; std::vector m_fields; point_count_t m_index; diff --git a/pdal/test/NumpyReaderTest.cpp b/pdal/test/NumpyReaderTest.cpp index 048e600c..26d5e9b7 100644 --- a/pdal/test/NumpyReaderTest.cpp +++ b/pdal/test/NumpyReaderTest.cpp @@ -91,7 +91,7 @@ TEST(NumpyReaderTest, NumpyReaderTest_read_array) EXPECT_EQ(view->layout()->pointSize(), 24u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::Intensity,5000), - 0.5); + 0.0); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::X,5000), 50u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::X,5023), 50u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::Y,5023), 23u); @@ -236,3 +236,100 @@ TEST(NumpyReaderTest, threedim_col) } } + +TEST(NumpyReaderTest, threedim_row_script) +{ + StageFactory f; + + Options opts; + opts.add("filename", Support::datapath("threedim.py")); + opts.add("order", "row"); + opts.add("function", "load"); + opts.add("module", "threedim"); + opts.add("fargs", Support::datapath("threedim.npy")); + + NumpyReader reader; + reader.setOptions(opts); + + PointTable table; + + reader.prepare(table); + + PointViewSet viewSet = reader.execute(table); + PointViewPtr view = *viewSet.begin(); + PointLayoutPtr layout = view->layout(); + Dimension::Id health = layout->findDim("Health"); + Dimension::Id foo = layout->findDim("Foo"); + + EXPECT_EQ(view->size(), 24u); + int ex = 0; + int ey = 0; + int ez = 0; + for (PointId id = 0; id < view->size(); ++id) + { + int f = view->getFieldAs(foo, id); + int h = view->getFieldAs(health, id); + int i = view->getFieldAs(Dimension::Id::Intensity, id); + int x = view->getFieldAs(Dimension::Id::X, id); + int y = view->getFieldAs(Dimension::Id::Y, id); + int z = view->getFieldAs(Dimension::Id::Z, id); + EXPECT_EQ(f, ((int)id / 12) % 2); + EXPECT_EQ(h, ((int)id / 4) % 3); + EXPECT_EQ(i, (int)id % 4); + EXPECT_EQ(x, ex); + EXPECT_EQ(y, ey); + EXPECT_EQ(z, ez); + // Array is 2 x 3 x 4 + ez++; + if (ez == 4) + { + ez = 0; + ey++; + if (ey == 3) + { + ey = 0; + ex++; + } + } + } +} + +TEST(NumpyReaderTest, four_floats_script) +{ + StageFactory f; + + Options opts; + opts.add("filename", Support::datapath("four-floats.py")); + opts.add("order", "row"); + opts.add("function", "load"); + opts.add("module", "four_floats"); + opts.add("fargs", Support::datapath("four-floats-be.bin")); + + NumpyReader reader; + reader.setOptions(opts); + + PointTable table; + + reader.prepare(table); + + PointViewSet viewSet = reader.execute(table); + PointViewPtr view = *viewSet.begin(); + PointLayoutPtr layout = view->layout(); + + EXPECT_EQ(view->size(), 10u); + int ex = 0; + int ey = 0; + int ez = 0; + int factor = 10; + for (PointId id = 0; id < view->size(); ++id) + { + int t = view->getFieldAs(Dimension::Id::GpsTime, id); + int x = view->getFieldAs(Dimension::Id::X, id); + int y = view->getFieldAs(Dimension::Id::Y, id); + int z = view->getFieldAs(Dimension::Id::Z, id); + EXPECT_EQ(x, (int)id * 1 * factor); + EXPECT_EQ(y, (int)id * 2 * factor); + EXPECT_EQ(z, (int)id * 3 * factor); + EXPECT_EQ(t, (int)id * 4 * factor); + } +} diff --git a/pdal/test/data/four-floats-be.bin b/pdal/test/data/four-floats-be.bin new file mode 100644 index 0000000000000000000000000000000000000000..de91178ca57f660c4d60e2fe6a25a578be9a5b0b GIT binary patch literal 160 zcmZQzKn0Ep3=ED7fcOIggA@q0mLVOY%w4Pu|aA< kawb3wV)p=P9v}_^Vjvr&7o-m)=L|9jXf9Z7JCFvl0k={bp#T5? literal 0 HcmV?d00001 diff --git a/pdal/test/data/four-floats.py b/pdal/test/data/four-floats.py new file mode 100644 index 00000000..c968ad9c --- /dev/null +++ b/pdal/test/data/four-floats.py @@ -0,0 +1,10 @@ +import numpy as np + + +dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', '>f4'), ('GPSTime', '>f4')]) + +def load(filename): + data = open(filename, 'rb').read() + array = np.frombuffer(data, dtype=dtype) + + return array diff --git a/pdal/test/data/threedim.py b/pdal/test/data/threedim.py new file mode 100644 index 00000000..95f9a065 --- /dev/null +++ b/pdal/test/data/threedim.py @@ -0,0 +1,5 @@ +import numpy as np + +def load(filename): + array = np.load(filename) + return array From 218ef8df81b36485516d25211b5a1ea530e6673d Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Wed, 20 May 2020 21:14:48 -0500 Subject: [PATCH 2/3] switch to switch statement as suggested. Provide a multi-endian test scenario --- pdal/io/NumpyReader.cpp | 129 ++++++++++++++---------------- pdal/test/NumpyReaderTest.cpp | 2 +- pdal/test/data/four-floats-be.bin | Bin 160 -> 160 bytes pdal/test/data/four-floats.py | 2 +- 4 files changed, 63 insertions(+), 70 deletions(-) diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp index f2c7e8fd..48711c11 100644 --- a/pdal/io/NumpyReader.cpp +++ b/pdal/io/NumpyReader.cpp @@ -552,86 +552,79 @@ bool NumpyReader::loadPoint(PointRef& point, point_count_t position) float flt(0.0); double dbl(0.0); - uint8_t uint8(0); - uint16_t uint16(0); - uint32_t uint32(0); - uint64_t uint64(0); - int8_t int8(0); - int16_t int16(0); - int32_t int32(0); - int64_t int64(0); + uint8_t u8(0); + uint16_t u16(0); + uint32_t u32(0); + uint64_t u64(0); + int8_t i8(0); + int16_t i16(0); + int32_t i32(0); + int64_t i64(0); for (const Field& f : m_fields) { + log()->get(LogLevel::Debug) << "endian: " << f.m_byteorder << std::endl; if (f.m_byteorder == '>') extractor.switchToBigEndian(); else extractor.switchToLittleEndian(); - if (Dimension::base(f.m_type) == Dimension::BaseType::Floating) + switch (f.m_type) { - if (f.m_elsize == 4) - { + case Dimension::Type::Signed8: + extractor >> i8; + point.setField(f.m_id, i8); + break; + + case Dimension::Type::Signed16: + extractor >> i16; + point.setField(f.m_id, i16); + break; + + case Dimension::Type::Signed32: + extractor >> i32; + point.setField(f.m_id, i32); + break; + + case Dimension::Type::Signed64: + extractor >> i64; + point.setField(f.m_id, i64); + break; + + case Dimension::Type::Unsigned8: + extractor >> u8; + point.setField(f.m_id, u8); + break; + + case Dimension::Type::Unsigned16: + extractor >> u16; + point.setField(f.m_id, u16); + break; + + case Dimension::Type::Unsigned32: + extractor >> u32; + point.setField(f.m_id, u32); + break; + + case Dimension::Type::Unsigned64: + extractor >> u64; + point.setField(f.m_id, u64); + break; + + case Dimension::Type::Float: extractor >> flt; - point.setField(f.m_id, flt); - } - else if (f.m_elsize == 8) - { + point.setField(f.m_id, flt); + break; + + case Dimension::Type::Double: extractor >> dbl; - point.setField(f.m_id, dbl); + point.setField(f.m_id, dbl); + break; + + default: + // skip it + extractor.skip(f.m_elsize); - } - } - else if (Dimension::base(f.m_type) == Dimension::BaseType::Signed) - { - if (f.m_elsize == 1) - { - extractor >> int8; - point.setField(f.m_id, int8); - } - if (f.m_elsize == 2) - { - extractor >> int16; - point.setField(f.m_id, int16); - } - if (f.m_elsize == 4) - { - extractor >> int32; - point.setField(f.m_id, int32); - } - if (f.m_elsize == 8) - { - extractor >> int64; - point.setField(f.m_id, int64); - } - } - else if (Dimension::base(f.m_type) == Dimension::BaseType::Unsigned) - { - if (f.m_elsize == 1) - { - extractor >> uint8; - point.setField(f.m_id, uint8); - } - if (f.m_elsize == 2) - { - extractor >> uint16; - point.setField(f.m_id, uint16); - } - if (f.m_elsize == 4) - { - extractor >> uint32; - point.setField(f.m_id, uint32); - } - if (f.m_elsize == 8) - { - extractor >> int64; - point.setField(f.m_id, uint64); - } - } - else - { - // skip it - extractor.skip(f.m_elsize); } } diff --git a/pdal/test/NumpyReaderTest.cpp b/pdal/test/NumpyReaderTest.cpp index 26d5e9b7..bd930093 100644 --- a/pdal/test/NumpyReaderTest.cpp +++ b/pdal/test/NumpyReaderTest.cpp @@ -91,7 +91,7 @@ TEST(NumpyReaderTest, NumpyReaderTest_read_array) EXPECT_EQ(view->layout()->pointSize(), 24u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::Intensity,5000), - 0.0); + 0.5); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::X,5000), 50u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::X,5023), 50u); EXPECT_EQ(view->getFieldAs(pdal::Dimension::Id::Y,5023), 23u); diff --git a/pdal/test/data/four-floats-be.bin b/pdal/test/data/four-floats-be.bin index de91178ca57f660c4d60e2fe6a25a578be9a5b0b..032167dfec85d2a1c222c8ca9982f981a573e045 100644 GIT binary patch literal 160 zcmY+6F%iHp38?wC=kvcK2l!VjKpWYu#*Z_B vP7c2@q0mLVOY%w4Pu|aA< kawb3wV)p=P9v}_^Vjvr&7o-m)=L|9jXf9Z7JCFvl0k={bp#T5? diff --git a/pdal/test/data/four-floats.py b/pdal/test/data/four-floats.py index c968ad9c..38408a9c 100644 --- a/pdal/test/data/four-floats.py +++ b/pdal/test/data/four-floats.py @@ -1,7 +1,7 @@ import numpy as np -dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', '>f4'), ('GPSTime', '>f4')]) +dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', 'f4')]) def load(filename): data = open(filename, 'rb').read() From 5a779417130ac3623ad42667eb477234114fb383 Mon Sep 17 00:00:00 2001 From: Howard Butler Date: Thu, 21 May 2020 08:07:25 -0500 Subject: [PATCH 3/3] remove lint --- pdal/io/NumpyReader.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp index 48711c11..8983340e 100644 --- a/pdal/io/NumpyReader.cpp +++ b/pdal/io/NumpyReader.cpp @@ -563,7 +563,6 @@ bool NumpyReader::loadPoint(PointRef& point, point_count_t position) for (const Field& f : m_fields) { - log()->get(LogLevel::Debug) << "endian: " << f.m_byteorder << std::endl; if (f.m_byteorder == '>') extractor.switchToBigEndian(); else