fromExcel Enhancements and add dataset.toDict

.gitignore

@@ -1,3 +1,5 @@
 .idea/
 **/build/
 .gradle/
+
+local.properties

common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt

@@ -1,17 +1,16 @@
 package org.imdc.extensions.common
 
 import com.inductiveautomation.ignition.common.Dataset
+import com.inductiveautomation.ignition.common.PyUtilities
 import com.inductiveautomation.ignition.common.TypeUtilities
 import com.inductiveautomation.ignition.common.script.PyArgParser
 import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs
 import com.inductiveautomation.ignition.common.script.hints.ScriptArg
 import com.inductiveautomation.ignition.common.script.hints.ScriptFunction
 import com.inductiveautomation.ignition.common.util.DatasetBuilder
 import com.inductiveautomation.ignition.common.xmlserialization.ClassNameResolver
-import org.apache.poi.ss.usermodel.CellType.BOOLEAN
-import org.apache.poi.ss.usermodel.CellType.FORMULA
-import org.apache.poi.ss.usermodel.CellType.NUMERIC
-import org.apache.poi.ss.usermodel.CellType.STRING
+import org.apache.poi.ss.usermodel.Cell
+import org.apache.poi.ss.usermodel.CellType
 import org.apache.poi.ss.usermodel.DateUtil
 import org.apache.poi.ss.usermodel.WorkbookFactory
 import org.python.core.Py
@@ -20,15 +19,19 @@ import org.python.core.PyBoolean
 import org.python.core.PyFloat
 import org.python.core.PyFunction
 import org.python.core.PyInteger
+import org.python.core.PyList
 import org.python.core.PyLong
 import org.python.core.PyObject
 import org.python.core.PyString
+import org.python.core.PyStringMap
 import org.python.core.PyType
 import org.python.core.PyUnicode
 import java.io.File
 import java.math.BigDecimal
 import java.util.Date
+import kotlin.jvm.optionals.getOrElse
 import kotlin.math.max
+import kotlin.streams.asSequence
 
 object DatasetExtensions {
     @Suppress("unused")
@@ -220,8 +223,40 @@ object DatasetExtensions {
     @Suppress("unused")
     @ScriptFunction(docBundlePrefix = "DatasetExtensions")
     @KeywordArgs(
-        names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn"],
-        types = [ByteArray::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class],
+        names = ["dataset", "filterNull"],
+        types = [Dataset::class, Boolean::class],
+    )
+    fun toDict(args: Array<PyObject>, keywords: Array<String>): PyStringMap {
+        val parsedArgs = PyArgParser.parseArgs(
+            args,
+            keywords,
+            arrayOf("dataset", "filterNull"),
+            Array(2) { Any::class.java },
+            "toDict",
+        )
+        val dataset = parsedArgs.requirePyObject("dataset").toJava<Dataset>()
+        val filterNull = parsedArgs.getBoolean("filterNull").orElse(false)
+        return PyStringMap(
+            dataset.columnIndices.associate { col ->
+                dataset.getColumnName(col) to PyList(
+                    buildList {
+                        for (row in dataset.rowIndices) {
+                            val value = dataset[row, col]
+                            if (value != null || !filterNull) {
+                                add(value)
+                            }
+                        }
+                    },
+                )
+            },
+        )
+    }
+
+    @Suppress("unused")
+    @ScriptFunction(docBundlePrefix = "DatasetExtensions")
+    @KeywordArgs(
+        names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn", "typeOverrides"],
+        types = [ByteArray::class, Int::class, Int::class, Int::class, Int::class, Int::class, Int::class, PyStringMap::class],
     )
     fun fromExcel(args: Array<PyObject>, keywords: Array<String>): Dataset {
         val parsedArgs = PyArgParser.parseArgs(
@@ -235,11 +270,20 @@ object DatasetExtensions {
                 "lastRow",
                 "firstColumn",
                 "lastColumn",
+                "typeOverrides",
             ),
-            Array(7) { Any::class.java },
+            Array(8) { Any::class.java },
             "fromExcel",
         )
 
+        val typeOverrides = parsedArgs.getPyObject("typeOverrides")
+            .map(PyUtilities::streamEntries)
+            .map { stream ->
+                stream.asSequence().associate { (pyKey, value) ->
+                    pyKey.asIndex() to value.asJavaClass()
+                }
+            }.getOrElse { emptyMap() }
+
         when (val input = parsedArgs.requirePyObject("input").toJava<Any>()) {
             is String -> WorkbookFactory.create(File(input))
             is ByteArray -> WorkbookFactory.create(input.inputStream().buffered())
@@ -262,7 +306,8 @@ object DatasetExtensions {
             }
 
             val columnRow = sheet.getRow(if (headerRow >= 0) headerRow else firstRow)
-            val firstColumn = parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() }
+            val firstColumn =
+                parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() }
             val lastColumn =
                 parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() }
             if (firstColumn >= lastColumn) {
@@ -292,11 +337,11 @@ object DatasetExtensions {
 
                 val row = sheet.getRow(i)
 
-                val rowValues = Array<Any?>(columnCount) { j ->
-                    val cell = row.getCell(j + firstColumn)
+                val rowValues = Array(columnCount) { j ->
+                    val cell: Cell? = row.getCell(j + firstColumn)
 
-                    when (cell?.cellType.takeUnless { it == FORMULA } ?: cell.cachedFormulaResultType) {
-                        NUMERIC -> {
+                    val actualValue: Any? = when (cell?.cellType) {
+                        CellType.NUMERIC -> {
                             if (DateUtil.isCellDateFormatted(cell)) {
                                 if (!typesSet) {
                                     columnTypes.add(Date::class.java)
@@ -318,14 +363,14 @@ object DatasetExtensions {
                             }
                         }
 
-                        STRING -> {
+                        CellType.STRING -> {
                             if (!typesSet) {
                                 columnTypes.add(String::class.java)
                             }
                             cell.stringCellValue
                         }
 
-                        BOOLEAN -> {
+                        CellType.BOOLEAN -> {
                             if (!typesSet) {
                                 columnTypes.add(Boolean::class.javaObjectType)
                             }
@@ -339,6 +384,19 @@ object DatasetExtensions {
                             null
                         }
                     }
+                    val typeOverride = typeOverrides[j]
+                    if (typeOverride != null) {
+                        if (!typesSet) {
+                            columnTypes[j] = typeOverride
+                        }
+                        try {
+                            TypeUtilities.coerceGeneric(actualValue, typeOverride)
+                        } catch (e: ClassCastException) {
+                            throw Py.TypeError(e.message)
+                        }
+                    } else {
+                        actualValue
+                    }
                 }
 
                 if (!typesSet) {
@@ -415,8 +473,6 @@ object DatasetExtensions {
         }
     }
 
-    private val classNameResolver = ClassNameResolver.createBasic()
-
     @ScriptFunction(docBundlePrefix = "DatasetExtensions")
     @KeywordArgs(
         names = ["**columns"],
@@ -435,7 +491,9 @@ object DatasetExtensions {
         return DatasetBuilder.newBuilder().colNames(colNames).colTypes(colTypes)
     }
 
-    fun PyObject.asJavaClass(): Class<*>? = when (this) {
+    private val classNameResolver = ClassNameResolver.createBasic()
+
+    internal fun PyObject.asJavaClass(): Class<*>? = when (this) {
         is PyBaseString -> classNameResolver.classForName(asString())
         !is PyType -> throw ClassCastException()
         PyString.TYPE, PyUnicode.TYPE -> String::class.java

common/src/main/kotlin/org/imdc/extensions/common/Utilities.kt

@@ -53,3 +53,5 @@ inline fun <reified T> BundleUtil.addPropertyBundle() {
 inline fun <reified T : Annotation> Method.getAnnotation(): T? {
     return getAnnotation(T::class.java)
 }
+
+inline fun <reified T> Class<*>.isAssignableFrom(): Boolean = this.isAssignableFrom(T::class.java)

common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties

@@ -15,6 +15,11 @@ print.param.output=The output destination. Defaults to sys.stdout.
 print.param.includeTypes=If True, includes the type of the column in the first row. Defaults to False.
 print.returns=None.
 
+toDict.desc=Converts a dataset to dict
+toDict.param.dataset=The dataset to convert. Must not be null
+toDict.param.filterNull=Gives the option to filter Null/None
+toDict.returns=dict of dataset
+
 fromExcel.desc=Creates a dataset by reading select cells from an Excel spreadsheet.
 fromExcel.param.input=The Excel document to read - either the path to a file on disk, or a byte array with the contents of a file.
 fromExcel.param.headerRow=The row number to use for the column names in the output dataset.
@@ -23,6 +28,7 @@ fromExcel.param.firstRow=The first row (zero-indexed) in the Excel document to r
 fromExcel.param.lastRow=The last row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty row will be used.
 fromExcel.param.firstColumn=The first column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty column will be used.
 fromExcel.param.lastColumn=The last column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty column will be used.
+fromExcel.param.typeOverrides=A dictionary of column index: column types, using the same semantics as system.dataset.builder for types.
 fromExcel.returns=A Dataset created from the Excel document. Types are assumed based on the first row of input data.
 
 equals.desc=Compares two datasets for structural equality.
@@ -45,4 +51,4 @@ columnsEqual.returns=True if the two datasets have the same columns, per additio
 builder.desc=Creates a new dataset using supplied column names and types.
 builder.param.**columns=Optional. Keyword arguments can be supplied to predefine column names and types. The value of the argument should be string "typecode" (see system.dataset.fromCSV) or a Java or Python class instance.
 builder.returns=A DatasetBuilder object. Use <code>addRow(value, ...)</code> to add new values, and <code>build()</code> to construct the final dataset. \
-  If keyword arguments were not supplied, column names and types can be manually declared using <code>colNames()</code> and <code>colTypes()</code>.
+  If keyword arguments were not supplied, column names and types can be manually declared using <code>colNames()</code> and <code>colTypes()</code>.

common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt

@@ -4,13 +4,12 @@ import com.inductiveautomation.ignition.common.BasicDataset
 import com.inductiveautomation.ignition.common.Dataset
 import com.inductiveautomation.ignition.common.util.DatasetBuilder
 import io.kotest.assertions.withClue
+import io.kotest.engine.spec.tempfile
 import io.kotest.matchers.shouldBe
 import org.imdc.extensions.common.DatasetExtensions.printDataset
 import org.python.core.Py
 import java.awt.Color
 import java.util.Date
-import kotlin.io.path.createTempFile
-import kotlin.io.path.writeBytes
 
 @Suppress("PyUnresolvedReferences", "PyInterpreter")
 class DatasetExtensionsTests : JythonTest(
@@ -26,18 +25,24 @@ class DatasetExtensionsTests : JythonTest(
 
         val excelSample =
             DatasetExtensionsTests::class.java.getResourceAsStream("sample.xlsx")!!.readAllBytes()
-        val tempXlsx = createTempFile(suffix = "xlsx").also {
+        val tempXlsx = tempfile(suffix = "xlsx").also {
             it.writeBytes(excelSample)
-            it.toFile().deleteOnExit()
         }
         globals["xlsxBytes"] = excelSample
         globals["xlsxFile"] = tempXlsx.toString()
 
+        val excelSample2 =
+            DatasetExtensionsTests::class.java.getResourceAsStream("sample2.xlsx")!!.readAllBytes()
+        val tempXlsx2 = tempfile(suffix = "xlsx").also {
+            it.writeBytes(excelSample2)
+        }
+        globals["xlsxFile2"] = tempXlsx2.toString()
+        globals["xlsxBytes2"] = excelSample2
+
         val xlsSample =
             DatasetExtensionsTests::class.java.getResourceAsStream("sample.xls")!!.readAllBytes()
-        val tempXls = createTempFile(suffix = "xls").also {
+        val tempXls = tempfile(suffix = "xls").also {
             it.writeBytes(xlsSample)
-            it.toFile().deleteOnExit()
         }
         globals["xlsBytes"] = xlsSample
         globals["xlsFile"] = tempXls.toString()
@@ -239,6 +244,48 @@ class DatasetExtensionsTests : JythonTest(
                     )
                 }
             }
+            test("With String Override") {
+                eval<Dataset>("utils.fromExcel(xlsxBytes2, headerRow=0, typeOverrides={2: 'str', 4: 'str'})").asClue {
+                    it.rowCount shouldBe 99
+                    it.columnCount shouldBe 16
+                    it.columnNames shouldBe listOf(
+                        "Segment",
+                        "Country",
+                        "Product",
+                        "Discount Band",
+                        "Units Sold",
+                        "Manufacturing Price",
+                        "Sale Price",
+                        "Gross Sales",
+                        "Discounts",
+                        "Sales",
+                        "COGS",
+                        "Profit",
+                        "Date",
+                        "Month Number",
+                        "Month Name",
+                        "Year",
+                    )
+                    it.columnTypes shouldBe listOf(
+                        String::class.java,
+                        Any::class.java,
+                        String::class.java,
+                        String::class.java,
+                        String::class.java,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Int::class.javaObjectType,
+                        Date::class.java,
+                        Int::class.javaObjectType,
+                        String::class.java,
+                        String::class.java,
+                    )
+                }
+            }
             test("First row") {
                 eval<Dataset>("utils.fromExcel(xlsxBytes, headerRow=0, firstRow=50)").asClue {
                     it.rowCount shouldBe 50

common/src/test/kotlin/org/imdc/extensions/common/JythonTest.kt

@@ -19,15 +19,15 @@ import org.python.core.PyStringMap
 import org.python.core.PyType
 
 @Suppress("PyInterpreter")
-abstract class JythonTest(init: (globals: PyStringMap) -> Unit) : FunSpec() {
+abstract class JythonTest(init: FunSpec.(globals: PyStringMap) -> Unit) : FunSpec() {
     protected var globals: PyStringMap = PyStringMap()
 
     init {
         extension(
             object : BeforeEachListener {
                 override suspend fun beforeEach(testCase: TestCase) {
                     globals.clear()
-                    init(globals)
+                    init(this@JythonTest, globals)
                 }
             },
         )