diff --git a/.gitignore b/.gitignore index 6dd5458..1a00f40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea/ **/build/ .gradle/ + +local.properties diff --git a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt index 8aaa091..5b31d60 100644 --- a/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt +++ b/common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt @@ -1,6 +1,7 @@ package org.imdc.extensions.common import com.inductiveautomation.ignition.common.Dataset +import com.inductiveautomation.ignition.common.PyUtilities import com.inductiveautomation.ignition.common.TypeUtilities import com.inductiveautomation.ignition.common.script.PyArgParser import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs @@ -8,10 +9,8 @@ import com.inductiveautomation.ignition.common.script.hints.ScriptArg import com.inductiveautomation.ignition.common.script.hints.ScriptFunction import com.inductiveautomation.ignition.common.util.DatasetBuilder import com.inductiveautomation.ignition.common.xmlserialization.ClassNameResolver -import org.apache.poi.ss.usermodel.CellType.BOOLEAN -import org.apache.poi.ss.usermodel.CellType.FORMULA -import org.apache.poi.ss.usermodel.CellType.NUMERIC -import org.apache.poi.ss.usermodel.CellType.STRING +import org.apache.poi.ss.usermodel.Cell +import org.apache.poi.ss.usermodel.CellType import org.apache.poi.ss.usermodel.DateUtil import org.apache.poi.ss.usermodel.WorkbookFactory import org.python.core.Py @@ -20,15 +19,19 @@ import org.python.core.PyBoolean import org.python.core.PyFloat import org.python.core.PyFunction import org.python.core.PyInteger +import org.python.core.PyList import org.python.core.PyLong import org.python.core.PyObject import org.python.core.PyString +import org.python.core.PyStringMap import org.python.core.PyType import org.python.core.PyUnicode import java.io.File import java.math.BigDecimal import java.util.Date +import kotlin.jvm.optionals.getOrElse import kotlin.math.max +import kotlin.streams.asSequence object DatasetExtensions { @Suppress("unused") @@ -220,8 +223,40 @@ object DatasetExtensions { @Suppress("unused") @ScriptFunction(docBundlePrefix = "DatasetExtensions") @KeywordArgs( - names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn"], - types = [ByteArray::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class], + names = ["dataset", "filterNull"], + types = [Dataset::class, Boolean::class], + ) + fun toDict(args: Array, keywords: Array): PyStringMap { + val parsedArgs = PyArgParser.parseArgs( + args, + keywords, + arrayOf("dataset", "filterNull"), + Array(2) { Any::class.java }, + "toDict", + ) + val dataset = parsedArgs.requirePyObject("dataset").toJava() + val filterNull = parsedArgs.getBoolean("filterNull").orElse(false) + return PyStringMap( + dataset.columnIndices.associate { col -> + dataset.getColumnName(col) to PyList( + buildList { + for (row in dataset.rowIndices) { + val value = dataset[row, col] + if (value != null || !filterNull) { + add(value) + } + } + }, + ) + }, + ) + } + + @Suppress("unused") + @ScriptFunction(docBundlePrefix = "DatasetExtensions") + @KeywordArgs( + names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn", "typeOverrides"], + types = [ByteArray::class, Int::class, Int::class, Int::class, Int::class, Int::class, Int::class, PyStringMap::class], ) fun fromExcel(args: Array, keywords: Array): Dataset { val parsedArgs = PyArgParser.parseArgs( @@ -235,11 +270,20 @@ object DatasetExtensions { "lastRow", "firstColumn", "lastColumn", + "typeOverrides", ), - Array(7) { Any::class.java }, + Array(8) { Any::class.java }, "fromExcel", ) + val typeOverrides = parsedArgs.getPyObject("typeOverrides") + .map(PyUtilities::streamEntries) + .map { stream -> + stream.asSequence().associate { (pyKey, value) -> + pyKey.asIndex() to value.asJavaClass() + } + }.getOrElse { emptyMap() } + when (val input = parsedArgs.requirePyObject("input").toJava()) { is String -> WorkbookFactory.create(File(input)) is ByteArray -> WorkbookFactory.create(input.inputStream().buffered()) @@ -262,7 +306,8 @@ object DatasetExtensions { } val columnRow = sheet.getRow(if (headerRow >= 0) headerRow else firstRow) - val firstColumn = parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() } + val firstColumn = + parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() } val lastColumn = parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() } if (firstColumn >= lastColumn) { @@ -292,11 +337,11 @@ object DatasetExtensions { val row = sheet.getRow(i) - val rowValues = Array(columnCount) { j -> - val cell = row.getCell(j + firstColumn) + val rowValues = Array(columnCount) { j -> + val cell: Cell? = row.getCell(j + firstColumn) - when (cell?.cellType.takeUnless { it == FORMULA } ?: cell.cachedFormulaResultType) { - NUMERIC -> { + val actualValue: Any? = when (cell?.cellType) { + CellType.NUMERIC -> { if (DateUtil.isCellDateFormatted(cell)) { if (!typesSet) { columnTypes.add(Date::class.java) @@ -318,14 +363,14 @@ object DatasetExtensions { } } - STRING -> { + CellType.STRING -> { if (!typesSet) { columnTypes.add(String::class.java) } cell.stringCellValue } - BOOLEAN -> { + CellType.BOOLEAN -> { if (!typesSet) { columnTypes.add(Boolean::class.javaObjectType) } @@ -339,6 +384,19 @@ object DatasetExtensions { null } } + val typeOverride = typeOverrides[j] + if (typeOverride != null) { + if (!typesSet) { + columnTypes[j] = typeOverride + } + try { + TypeUtilities.coerceGeneric(actualValue, typeOverride) + } catch (e: ClassCastException) { + throw Py.TypeError(e.message) + } + } else { + actualValue + } } if (!typesSet) { @@ -415,8 +473,6 @@ object DatasetExtensions { } } - private val classNameResolver = ClassNameResolver.createBasic() - @ScriptFunction(docBundlePrefix = "DatasetExtensions") @KeywordArgs( names = ["**columns"], @@ -435,7 +491,9 @@ object DatasetExtensions { return DatasetBuilder.newBuilder().colNames(colNames).colTypes(colTypes) } - fun PyObject.asJavaClass(): Class<*>? = when (this) { + private val classNameResolver = ClassNameResolver.createBasic() + + internal fun PyObject.asJavaClass(): Class<*>? = when (this) { is PyBaseString -> classNameResolver.classForName(asString()) !is PyType -> throw ClassCastException() PyString.TYPE, PyUnicode.TYPE -> String::class.java diff --git a/common/src/main/kotlin/org/imdc/extensions/common/Utilities.kt b/common/src/main/kotlin/org/imdc/extensions/common/Utilities.kt index 16981f0..efee39c 100644 --- a/common/src/main/kotlin/org/imdc/extensions/common/Utilities.kt +++ b/common/src/main/kotlin/org/imdc/extensions/common/Utilities.kt @@ -53,3 +53,5 @@ inline fun BundleUtil.addPropertyBundle() { inline fun Method.getAnnotation(): T? { return getAnnotation(T::class.java) } + +inline fun Class<*>.isAssignableFrom(): Boolean = this.isAssignableFrom(T::class.java) diff --git a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties index 990ae7d..b7a5530 100644 --- a/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties +++ b/common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties @@ -15,6 +15,11 @@ print.param.output=The output destination. Defaults to sys.stdout. print.param.includeTypes=If True, includes the type of the column in the first row. Defaults to False. print.returns=None. +toDict.desc=Converts a dataset to dict +toDict.param.dataset=The dataset to convert. Must not be null +toDict.param.filterNull=Gives the option to filter Null/None +toDict.returns=dict of dataset + fromExcel.desc=Creates a dataset by reading select cells from an Excel spreadsheet. fromExcel.param.input=The Excel document to read - either the path to a file on disk, or a byte array with the contents of a file. fromExcel.param.headerRow=The row number to use for the column names in the output dataset. @@ -23,6 +28,7 @@ fromExcel.param.firstRow=The first row (zero-indexed) in the Excel document to r fromExcel.param.lastRow=The last row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty row will be used. fromExcel.param.firstColumn=The first column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty column will be used. fromExcel.param.lastColumn=The last column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty column will be used. +fromExcel.param.typeOverrides=A dictionary of column index: column types, using the same semantics as system.dataset.builder for types. fromExcel.returns=A Dataset created from the Excel document. Types are assumed based on the first row of input data. equals.desc=Compares two datasets for structural equality. @@ -45,4 +51,4 @@ columnsEqual.returns=True if the two datasets have the same columns, per additio builder.desc=Creates a new dataset using supplied column names and types. builder.param.**columns=Optional. Keyword arguments can be supplied to predefine column names and types. The value of the argument should be string "typecode" (see system.dataset.fromCSV) or a Java or Python class instance. builder.returns=A DatasetBuilder object. Use addRow(value, ...) to add new values, and build() to construct the final dataset. \ - If keyword arguments were not supplied, column names and types can be manually declared using colNames() and colTypes(). + If keyword arguments were not supplied, column names and types can be manually declared using colNames() and colTypes(). \ No newline at end of file diff --git a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt index 2cda029..b7b2dce 100644 --- a/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt +++ b/common/src/test/kotlin/org/imdc/extensions/common/DatasetExtensionsTests.kt @@ -4,13 +4,12 @@ import com.inductiveautomation.ignition.common.BasicDataset import com.inductiveautomation.ignition.common.Dataset import com.inductiveautomation.ignition.common.util.DatasetBuilder import io.kotest.assertions.withClue +import io.kotest.engine.spec.tempfile import io.kotest.matchers.shouldBe import org.imdc.extensions.common.DatasetExtensions.printDataset import org.python.core.Py import java.awt.Color import java.util.Date -import kotlin.io.path.createTempFile -import kotlin.io.path.writeBytes @Suppress("PyUnresolvedReferences", "PyInterpreter") class DatasetExtensionsTests : JythonTest( @@ -26,18 +25,24 @@ class DatasetExtensionsTests : JythonTest( val excelSample = DatasetExtensionsTests::class.java.getResourceAsStream("sample.xlsx")!!.readAllBytes() - val tempXlsx = createTempFile(suffix = "xlsx").also { + val tempXlsx = tempfile(suffix = "xlsx").also { it.writeBytes(excelSample) - it.toFile().deleteOnExit() } globals["xlsxBytes"] = excelSample globals["xlsxFile"] = tempXlsx.toString() + val excelSample2 = + DatasetExtensionsTests::class.java.getResourceAsStream("sample2.xlsx")!!.readAllBytes() + val tempXlsx2 = tempfile(suffix = "xlsx").also { + it.writeBytes(excelSample2) + } + globals["xlsxFile2"] = tempXlsx2.toString() + globals["xlsxBytes2"] = excelSample2 + val xlsSample = DatasetExtensionsTests::class.java.getResourceAsStream("sample.xls")!!.readAllBytes() - val tempXls = createTempFile(suffix = "xls").also { + val tempXls = tempfile(suffix = "xls").also { it.writeBytes(xlsSample) - it.toFile().deleteOnExit() } globals["xlsBytes"] = xlsSample globals["xlsFile"] = tempXls.toString() @@ -239,6 +244,48 @@ class DatasetExtensionsTests : JythonTest( ) } } + test("With String Override") { + eval("utils.fromExcel(xlsxBytes2, headerRow=0, typeOverrides={2: 'str', 4: 'str'})").asClue { + it.rowCount shouldBe 99 + it.columnCount shouldBe 16 + it.columnNames shouldBe listOf( + "Segment", + "Country", + "Product", + "Discount Band", + "Units Sold", + "Manufacturing Price", + "Sale Price", + "Gross Sales", + "Discounts", + "Sales", + "COGS", + "Profit", + "Date", + "Month Number", + "Month Name", + "Year", + ) + it.columnTypes shouldBe listOf( + String::class.java, + Any::class.java, + String::class.java, + String::class.java, + String::class.java, + Int::class.javaObjectType, + Int::class.javaObjectType, + Int::class.javaObjectType, + Int::class.javaObjectType, + Int::class.javaObjectType, + Int::class.javaObjectType, + Int::class.javaObjectType, + Date::class.java, + Int::class.javaObjectType, + String::class.java, + String::class.java, + ) + } + } test("First row") { eval("utils.fromExcel(xlsxBytes, headerRow=0, firstRow=50)").asClue { it.rowCount shouldBe 50 diff --git a/common/src/test/kotlin/org/imdc/extensions/common/JythonTest.kt b/common/src/test/kotlin/org/imdc/extensions/common/JythonTest.kt index 9144e89..6ffc4ed 100644 --- a/common/src/test/kotlin/org/imdc/extensions/common/JythonTest.kt +++ b/common/src/test/kotlin/org/imdc/extensions/common/JythonTest.kt @@ -19,7 +19,7 @@ import org.python.core.PyStringMap import org.python.core.PyType @Suppress("PyInterpreter") -abstract class JythonTest(init: (globals: PyStringMap) -> Unit) : FunSpec() { +abstract class JythonTest(init: FunSpec.(globals: PyStringMap) -> Unit) : FunSpec() { protected var globals: PyStringMap = PyStringMap() init { @@ -27,7 +27,7 @@ abstract class JythonTest(init: (globals: PyStringMap) -> Unit) : FunSpec() { object : BeforeEachListener { override suspend fun beforeEach(testCase: TestCase) { globals.clear() - init(globals) + init(this@JythonTest, globals) } }, ) diff --git a/common/src/test/resources/org/imdc/extensions/common/sample2.xlsx b/common/src/test/resources/org/imdc/extensions/common/sample2.xlsx new file mode 100644 index 0000000..292d262 Binary files /dev/null and b/common/src/test/resources/org/imdc/extensions/common/sample2.xlsx differ