Package python bindings

.github/workflows/pytest.yml

@@ -0,0 +1,133 @@
+name: Python application
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: [ self-hosted, "${{ matrix.archconfig }}" ]
+    strategy:
+      matrix:
+        archconfig: [ x86_64, aarch64 ]
+        #build_type: [Debug, Release]
+      fail-fast: false
+
+    steps:
+    - name: Cleanup previous jobs
+      run: |
+        echo "Cleaning up previous runs"
+        sudo rm -rf ${{ github.workspace }}/*
+        sudo rm -rf ${{ github.workspace }}/.??*
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Find SHA
+      run: |
+        if [[ "${{github.event.pull_request.head.sha}}" != "" ]]
+        then
+          echo "ARTIFACT_SHA=$(echo ${{github.event.pull_request.head.sha}})" >> $GITHUB_ENV
+        else
+          echo "ARTIFACT_SHA=$(echo ${{github.sha}})" >> $GITHUB_ENV
+        fi
+
+    - name: Create python isolated env
+      run: |
+        python3 -m venv .venv
+
+    - name: Install python dependencies
+      run: |
+        ./.venv/bin/python3 -m pip install --upgrade pip
+        # FIXME: add requirements instead of pip-installing them here
+        ./.venv/bin/python3 -m pip install flake8 build setuptools cffi pytest pytest-cov datestamp cmake_build_extension
+        #if [ -f requirements.txt ]; then python3 -m pip install -r requirements.txt; fi
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        ./.venv/bin/python3 -m flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.venv
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        ./.venv/bin/python3 -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=.venv
+
+    # FIXME: fetch deb from s3 instead of building it locally
+    - name: Build vaccelrt
+      run: |
+        git clone https://github.com/cloudkernels/vaccelrt --recursive
+        cd vaccelrt
+        mkdir build
+        cd build
+        cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_PLUGIN_NOOP=ON -DBUILD_EXAMPLES=ON
+        make
+        cpack
+
+    - name: Install vaccelrt
+      run: |
+        sudo dpkg -i vaccelrt/build/*.deb
+        # FIXME: make sure rundir exists
+        sudo mkdir -p /run/user/1001
+        sudo chown runner -R /run/user/1001
+
+    - name: Test with pytest
+      env:
+        VACCEL_BACKENDS: /usr/local/lib/libvaccel-noop.so
+        LD_LIBRARY_PATH: /usr/local/lib
+        PYTHONPATH: .
+      run: |
+        #python3 -m build
+        ./.venv/bin/python3 builder.py
+        ./.venv/bin/python3 -m pytest
+
+    - name: Test with coverage
+      env:
+        VACCEL_BACKENDS: /usr/local/lib/libvaccel-noop.so
+        LD_LIBRARY_PATH: /usr/local/lib
+        PYTHONPATH: .
+      run: |
+        ./.venv/bin/python3 -m pytest --cov=vaccel tests/
+
+    - name: Build pip package
+      env:
+        VACCEL_BACKENDS: /usr/local/lib/libvaccel-noop.so
+        LD_LIBRARY_PATH: /usr/local/lib
+      run: |
+        ./.venv/bin/python3 -m build
+        VERSION=$(./.venv/bin/python3 setup.py --version)
+        PYVER=$(./.venv/bin/python3 --version | awk '{print $2}' | awk -F\. '{print $1 $2 }')
+        echo "wheel_name=vaccel-${VERSION}-${PYVER}-${PYVER}m_Linux_${{ matrix.archconfig }}" >> $GITHUB_ENV
+        echo "built: ${{ env.wheel_name }}"
+
+    # FIXME: Add step to upload package to s3
+
+    - name: Upload artifact to s3
+      uses: cloudkernels/minio-upload@master
+      with:
+        url: https://s3.nubificus.co.uk
+        access-key: ${{ secrets.AWS_ACCESS_KEY }}
+        secret-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        #local-path: dist/${{ env.wheel_name }}
+        local-path: dist/
+        remote-path: nbfc-assets/github/python-vaccel/${{ env.ARTIFACT_SHA }}/${{ matrix.archconfig }}/
+
+     # FIXME: add pypi publish step
+     #- name: Publish package
+     #  if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+     #  uses: pypa/gh-action-pypi-publish@release/v1
+     #  with:
+     #    password: ${{ secrets.PYPI_ANANOS_TOKEN }}
+
+    - name: Cleanup
+      run: |
+        sudo apt remove -y vaccel
+        # FIXME: remove pip package if installed
+        #pip3 uninstall vaccel
+        echo "built: ${{ env.wheel_name }}"
+        rm -rf ./.venv

.gitignore

@@ -5,3 +5,6 @@ build
 **.so
 **.o
 __pycache__/
+.pytest_cache/
+env/
+.coverage

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vaccelrt"]
-	path = libs/vaccelrt
-	url = https://github.com/cloudkernels/vaccelrt.git

README.md

@@ -62,3 +62,16 @@ The module should be ready. To test run:
 ```bash
 PYTHON_VACCEL_PLUGIN=./libs/vaccelrt/build/plugins/noop/libvaccel-noop.so LD_LIBRARY_PATH=./libs/install/lib PYTHONPATH=$PYTHONPATH:. python3 vaccel/test.py
 ```
+
+
+## Test
+
+To run the tests:
+
+```bash
+# Run tests
+PYTHON_VACCEL_PLUGIN=./libs/vaccelrt/build/plugins/noop/libvaccel-noop.so LD_LIBRARY_PATH=./libs/install/lib PYTHONPATH=$PYTHONPATH:. pytest
+
+# Test coverage
+PYTHON_VACCEL_PLUGIN=./libs/vaccelrt/build/plugins/noop/libvaccel-noop.so LD_LIBRARY_PATH=./libs/install/lib PYTHONPATH=$PYTHONPATH:. pytest --cov=vaccel tests/
+```

builder.py

@@ -4,9 +4,6 @@
 
 ffibuilder = FFI()
 
-incdirs=['libs/install/include']
-libdirs=['libs/install/lib']
-
 ffibuilder.set_source(
         "vaccel._vaccel",
         r"""
@@ -18,10 +15,7 @@
         #include <ops/image.h>
         #include <plugin.h>
         """,
-        include_dirs=incdirs,
-        library_dirs=libdirs,
         libraries=['vaccel-python', 'dl'],
-        extra_link_args=["-Wl,-rpath,{}".format(libdirs[0])]
 )
 
 # Session API

pyproject.toml

@@ -1,2 +1,4 @@
 [build-system]
-requires = ["setuptools", "wheel", "datestamp", "cffi"]
+requires = ["setuptools", "wheel", "datestamp", "cffi", "cmake_build_extension"]
+build-backend = "setuptools.build_meta"
+

setup.py

@@ -10,7 +10,8 @@
     _ = f'"(package.title())" only supported to run on Linux'
     raise RuntimeError(_)
 
-setup(url = 'https://github.com/nubificus/python-vaccel',
+setup(name = 'vaccel',
+        url = 'https://github.com/nubificus/python-vaccel',
         author = 'Nubificus Ltd.',
         version = stamp(package),
         packages = find_packages(),

tests/test_general.py

@@ -0,0 +1,25 @@
+from vaccel.session import Session
+from vaccel.noop import Noop
+from vaccel.genop import Genop, VaccelArg
+
+def test_session():
+    ses_a = Session(flags=0)
+    ses_b = Session(flags=1)
+    assert ses_a.id() != ses_b.id()
+
+def test_noop():
+    ses = Session(flags=0)
+    res = Noop.noop(ses)
+    assert res == 0
+
+##def test_genop():
+#    """
+#    should work, but it doesn't because the sanity check on vAccel
+#    regarding the number of arguments is not correct
+#    """
+#    ses = Session(flags=1)
+#    arg_read = []
+#    arg_read.append(VaccelArg(data=0))
+#    arg_write = []
+#    res = Genop.genop(ses, arg_read, arg_write)
+#    assert res == 22 #should this be 22?

tests/test_image.py

@@ -0,0 +1,32 @@
+from vaccel.session import Session
+from vaccel.image import ImageClassify, ImageDetect, ImageSegment, ImagePose, ImageDepth
+
+imgsource = "/usr/local/share/images/example.jpg"
+
+def test_image_classify():
+   ses = Session(flags=3)
+   res = ImageClassify.classify_from_filename(session=ses, source=imgsource)
+   assert res == "This is a dummy classification tag!"
+
+
+def test_image_detect():
+   ses = Session(flags=3)
+   res = ImageDetect.detect_from_filename(session=ses, source=imgsource)
+   assert res == 0
+
+
+def test_image_segment():
+   ses = Session(flags=3)
+   res = ImageSegment.segment_from_filename(session=ses, source=imgsource)
+   assert res == 0
+
+
+def test_image_pose():
+   ses = Session(flags=3)
+   res = ImagePose.pose_from_filename(session=ses, source=imgsource)
+   assert res == 0 # is this supposed to be 95?
+
+def test_image_depth():
+   ses = Session(flags=3)
+   res = ImageDepth.depth_from_filename(session=ses, source=imgsource)
+   assert res == 0 # is this supposed to be 95?

tests/test_image_genop.py

@@ -0,0 +1,25 @@
+from vaccel import image_genop as genimg
+
+imgsource = "/usr/local/share/images/example.jpg"
+
+def test_image_class_genop():
+    res = genimg.ImageClassify.classify(image=imgsource)
+    assert res == ("This is a dummy classification tag!")
+
+def test_image_detect_genop():
+    res = genimg.ImageDetect.detect(image=imgsource)
+    assert res == ("")
+
+def test_image_segme_genop():
+    res = genimg.ImageSegment.segment(image=imgsource)
+    assert res == ("")
+
+def test_image_pose_genop():
+    res = genimg.ImagePose.pose(image=imgsource)
+    print(res)
+    assert res == ("")
+
+def test_image_depth_genop():
+    res = genimg.ImageDepth.depth(image=imgsource)
+    assert res == ("")
+

vaccel/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2021.6.28'
+__version__ = '2022.11.2'
 __vaccel__ = '0.1.0'
 
 import os, sys, ctypes

vaccel/error.py

@@ -1,3 +1,5 @@
+import errno
+
 class VaccelError(RuntimeError):
     """Exception raised when a vAccel runtime error occurs"""
 
@@ -6,5 +8,5 @@ def __init__(self, err_val, message):
         self.message = message
 
     def __str__(self):
-        return "{}: {}".format(self.message, self.error)
+        return "[errno {}] {}: {}".format(self.error, errno.errorcode[self.error], self.message)
 

vaccel/genop.py

@@ -140,6 +140,9 @@ def genop(self, session: Session, arg_read: List[VaccelArg], arg_write: List[Vac
         vaccel_args_read = VaccelArgList(arg_read).to_cffi()
         vaccel_args_write = VaccelArgList(arg_write).to_cffi()
 
-        a = lib.vaccel_genop(csession, vaccel_args_read, nr_read,
+        ret = lib.vaccel_genop(csession, vaccel_args_read, nr_read,
                              vaccel_args_write, nr_write)
-        return a
+        #if ret != 0:
+        #    raise VaccelError(ret, "Could not execute generic operation")
+
+        return ret