diff --git a/User_Guide.md b/User_Guide.md deleted file mode 100644 index c9e32e3..0000000 --- a/User_Guide.md +++ /dev/null @@ -1,1093 +0,0 @@ -# vAccel Python bindings - -This document briefly describes the process to use vAccel from Python programs. - -- [Build from Source](#build-from-source) or [get the binary packages](#get-the-binary-packages) [currently only for Debian/Ubuntu variants and pip] -- Run a [simple example](#simple-example) [using the `noop` plugin] -- Run a more [elaborate example](#jetson-example) [using the `jetson-inference` plugin] - -## Get the binary packages - -We provide freshly built packages, based on the current development of the -vAccel components. - -### Get vAccelRT deb - -You can install vAccelRT (in `/usr/local`) using the following commands: - -```bash -wget https://s3.nbfc.io/nbfc-assets/github/vaccelrt/master/x86_64/Release-deb/vaccel-0.5.0-Linux.deb -sudo dpkg -i vaccel-0.5.0-Linux.deb -``` - -### Get python bindings - -#### Prerequisites - -In Ubuntu-based systems, you need to have the following packages installed: - -```bash -sudo apt-get install -y python3-dev python3-venv python3-pip -``` - -#### Install the Python package - -To install the python bindings we use a whl package. Create a fresh virtual environment and install the package there: - -```bash -python3 -m venv .vaccel-venv -.vaccel-venv/bin/pip3 install https://s3.nbfc.io/nbfc-assets/github/python-vaccel/main/x86_64/vaccel-python-0.0.1.tar.gz -``` - -## Build from Source - -### Prerequisites for building - -In Ubuntu-based systems, you need to have the following packages to build `vaccelrt`: - -- cmake -- build-essential -- python3-dev -- python3-venv - -You can install them using the following command: - -```bash -sudo apt-get install -y cmake build-essential python3-dev python3-venv -``` - -### Get the source code - -Get the source code for **python-vaccel** and **vaccelrt**: - -```bash -git clone https://github.com/nubificus/python-vaccel.git -cd python-vaccel -mkdir libs -cd libs -git clone https://github.com/cloudkernels/vaccelrt --recursive -``` - -### Build and install vaccelrt - -Build vaccelrt and package it in a **.deb** file: - -```bash -cd vaccelrt -mkdir build -cd build -cmake ../ -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_PLUGIN_NOOP=ON -DBUILD_EXAMPLES=ON -make -cpack -``` - -Install **vaccelrt**: - -```bash -sudo dpkg -i vaccel*.deb -``` - -*** - -### Build the Python package - -We will create a virtual environment to install the **python-vaccel** package inside the root directory of **python-vaccel**. - -```bash -cd ../../.. -python3 -m venv venv -``` - -Now, go ahead and activate the newly created environment: - -```bash -. venv/bin/activate -``` - -Update pip and install Python's dependencies: - -```bash -python3 -m pip install --upgrade pip -python3 -m pip install wheel -python3 -m pip install flake8 build setuptools \ - cffi pytest pytest-cov datestamp cmake_build_extension -``` - -Now let's build the package: - -```bash -export VACCEL_BACKENDS=/usr/local/lib/libvaccel-noop.so -export LD_LIBRARY_PATH=/usr/local/lib -export PYTHONPATH=. -python3 builder.py -python3 setup.py install -``` - -[Optional] Run the tests to make sure everything was build correctly: - -```bash -python3 -m pytest -``` - -## Do It Yourself - -To see python vAccel bindings in action, let's try the following example: - -### Simple Example - -Download an adorable kitten photo: - -```bash -wget https://i.imgur.com/aSuOWgU.jpeg -O cat.jpeg -``` - -*** - -Create a new python file called **cat.py** and add the following lines: - -```python -from vaccel.session import Session -from vaccel.image import ImageClassify - -source = "cat.jpeg" - -def main(): - ses = Session(flags=3) - print(f'Session id is {ses.id()}') - res = ImageClassify.classify_from_filename(session=ses, source=source) - print(res) - -if __name__=="__main__": - main() -``` - -Now, when you run that python file, you can see the dummy classification tag for that image: - -```bash -export VACCEL_BACKENDS=/usr/local/lib/libvaccel-noop.so -export LD_LIBRARY_PATH=/usr/local/lib -export PYTHONPATH=. -python3 cat.py -# Loading libvaccel -# Loading plugins -# Loading plugin: /usr/local/lib/libvaccel-noop.so -# Loaded plugin noop from /usr/local/lib/libvaccel-noop.so -# Session id is 1 -# [noop] Calling Image classification for session 1 -# [noop] Dumping arguments for Image classification: -# [noop] len_img: 54372 -# [noop] will return a dummy result -# This is a dummy classification tag! -# Shutting down vAccel -``` - -### Jetson example - -To use vAccel on a more real-life example we'll use the jetson-inference framework. This way we will be able to perform image inference on a GPU and get something more useful than a dummy classification tag ;-) - -Let's re-use the python program from the [simple example](#simple-example) above. - -#### `x86_64` - -We will need to use a host with an NVIDIA GPU (our's is just an `RTX 2060 -SUPER`) and jetson-inference installed. To facilitate dependency resolving we -use a [container image](#appendix-i-build-a-jetson-inference-container-image) -on a host with nvidia-container-runtime installed. - -so, assuming our code is in `/data/code` let's spawn our container and see this in action: - -```bash -docker run --gpus 0 --rm -it -v/data/code:/data/ -w /data nubificus/jetson-inference-updated:x86_64 /bin/bash -``` - -Afterwards, the steps are more or less the same as above. Install the vAccelRT package: - -```bash -root@32e90efe86b9:/data/code# wget https://s3.nbfc.io/nbfc-assets/github/vaccelrt/master/x86_64/Release-deb/vaccel-0.5.0-Linux.deb ---2022-11-05 13:43:43-- https://s3.nbfc.io/nbfc-assets/github/vaccelrt/master/x86_64/Release-deb/vaccel-0.5.0-Linux.deb -Resolving s3.nbfc.io (s3.nbfc.io)... 84.254.1.240 -Connecting to s3.nbfc.io (s3.nbfc.io)|84.254.1.240|:443... connected. -HTTP request sent, awaiting response... 200 OK -Length: 2124230 (2.0M) [application/x-debian-package] -Saving to: 'vaccel-0.5.0-Linux.deb' - -vaccel-0.5.0-Linux.deb 100%[=======================================================>] 2.03M --.-KB/s in 0.06s - -2022-11-05 13:43:43 (33.8 MB/s) - 'vaccel-0.5.0-Linux.deb' saved [2124230/2124230] - -root@32e90efe86b9:/data/code# dpkg -i vaccel-0.5.0-Linux.deb -Selecting previously unselected package vaccel. -(Reading database ... 60677 files and directories currently installed.) -Preparing to unpack vaccel-0.5.0-Linux.deb ... -Unpacking vaccel (0.5.0) ... -Setting up vaccel (0.5.0) ... -``` - -Get and install the jetson plugin: - -```bash -root@32e90efe86b9:/data/code# wget https://s3.nubificus.co.uk/nbfc-assets/github/vaccelrt/plugins/jetson_inference/master/x86_64/vaccelrt-plugin-jetson-0.1-Linux.deb ---2022-11-05 14:45:53-- https://s3.nubificus.co.uk/nbfc-assets/github/vaccelrt/plugins/jetson_inference/master/x86_64/vaccelrt-plugin-jetson-0.1-Linux.deb -Resolving s3.nubificus.co.uk (s3.nubificus.co.uk)... 84.254.1.240 -Connecting to s3.nubificus.co.uk (s3.nubificus.co.uk)|84.254.1.240|:443... connected. -HTTP request sent, awaiting response... 200 OK -Length: 13146 (13K) [application/x-debian-package] -Saving to: 'vaccelrt-plugin-jetson-0.1-Linux.deb' - -vaccelrt-plugin-jetson-0.1-Linu 100%[=======================================================>] 12.84K --.-KB/s in 0.001s - -2022-11-05 14:45:53 (8.39 MB/s) - 'vaccelrt-plugin-jetson-0.1-Linux.deb' saved [13146/13146] - -root@32e90efe86b9:/data/code# dpkg -i vaccelrt-plugin-jetson-0.1-Linux.deb -(Reading database ... 60748 files and directories currently installed.) -Preparing to unpack vaccelrt-plugin-jetson-0.1-Linux.deb ... -Unpacking vaccelrt-plugin-jetson (0.1) over (0.1) ... -Setting up vaccelrt-plugin-jetson (0.1) ... -``` - -Install the bindings: - -```bash -root@32e90efe86b9:/data/code# .vaccel-venv/bin/pip3 install https://s3.nbfc.io/nbfc-assets/github/python-vaccel/main/x86_64/vaccel-2022.11.5.tar.gz -Collecting https://s3.nbfc.io/nbfc-assets/github/python-vaccel/main/x86_64/vaccel-2022.11.5.tar.gz - Downloading https://s3.nbfc.io/nbfc-assets/github/python-vaccel/main/x86_64/vaccel-2022.11.5.tar.gz (23 kB) - Installing build dependencies ... done - Getting requirements to build wheel ... done - Preparing wheel metadata ... done -Collecting cffi>=1.0.0 - Using cached cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (442 kB) -Collecting pycparser - Using cached pycparser-2.21-py2.py3-none-any.whl (118 kB) -Building wheels for collected packages: vaccel - Building wheel for vaccel (PEP 517) ... done - Created wheel for vaccel: filename=vaccel-2022.11.5-cp38-cp38-linux_x86_64.whl size=44494 sha256=a63bd263ba219e985821fc34416dc8f12ced508eb8e265b78a896ac2ed375f72 - Stored in directory: /root/.cache/pip/wheels/a6/e6/1c/4c91a42c1cad7e5e4ca86acd006bcded10cba25e85268e81ef -Successfully built vaccel -Installing collected packages: pycparser, cffi, vaccel -Successfully installed cffi-1.15.1 pycparser-2.21 vaccel-2022.11.5 -``` - -Now let's go ahead and run the example! - -```bash -root@32e90efe86b9:/data/code# export LD_LIBRARY_PATH=/usr/local/lib/ -root@32e90efe86b9:/data/code# export VACCEL_BACKENDS=/usr/local/lib/libvaccel-jetson.so -root@32e90efe86b9:/data/code# export VACCEL_IMAGENET_NETWORKS=/data/code/networks -root@32e90efe86b9:/data/code# .vaccel-venv/bin/python3 cat.py -Loading libvaccel -Loading plugins -Loading plugin: /usr/local/lib/libvaccel-jetson.so -Loaded plugin jetson-inference from ./libvaccel-jetson.so - -imageNet -- loading classification network model from: - -- prototxt ./local_net//googlenet.prototxt - -- model ./local_net//bvlc_googlenet.caffemodel - -- class_labels ./local_net//ilsvrc12_synset_words.txt - -- input_blob 'data' - -- output_blob 'prob' - -- batch_size 1 - -[TRT] TensorRT version 8.5.1 -[TRT] loading NVIDIA plugins... -[TRT] Registered plugin creator - ::BatchedNMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::BatchedNMS_TRT version 1 -[TRT] Registered plugin creator - ::BatchTilePlugin_TRT version 1 -[TRT] Registered plugin creator - ::Clip_TRT version 1 -[TRT] Registered plugin creator - ::CoordConvAC version 1 -[TRT] Registered plugin creator - ::CropAndResizeDynamic version 1 -[TRT] Registered plugin creator - ::CropAndResize version 1 -[TRT] Registered plugin creator - ::DecodeBbox3DPlugin version 1 -[TRT] Registered plugin creator - ::DetectionLayer_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Explicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Implicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_ONNX_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_TRT version 1 -[TRT] Could not register plugin creator - ::FlattenConcat_TRT version 1 -[TRT] Registered plugin creator - ::GenerateDetection_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchor_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchorRect_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 2 -[TRT] Registered plugin creator - ::LReLU_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelCropAndResize_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelProposeROI_TRT version 1 -[TRT] Registered plugin creator - ::MultiscaleDeformableAttnPlugin_TRT version 1 -[TRT] Registered plugin creator - ::NMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::NMS_TRT version 1 -[TRT] Registered plugin creator - ::Normalize_TRT version 1 -[TRT] Registered plugin creator - ::PillarScatterPlugin version 1 -[TRT] Registered plugin creator - ::PriorBox_TRT version 1 -[TRT] Registered plugin creator - ::ProposalDynamic version 1 -[TRT] Registered plugin creator - ::ProposalLayer_TRT version 1 -[TRT] Registered plugin creator - ::Proposal version 1 -[TRT] Registered plugin creator - ::PyramidROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::Region_TRT version 1 -[TRT] Registered plugin creator - ::Reorg_TRT version 1 -[TRT] Registered plugin creator - ::ResizeNearest_TRT version 1 -[TRT] Registered plugin creator - ::ROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::RPROI_TRT version 1 -[TRT] Registered plugin creator - ::ScatterND version 1 -[TRT] Registered plugin creator - ::SpecialSlice_TRT version 1 -[TRT] Registered plugin creator - ::Split version 1 -[TRT] Registered plugin creator - ::VoxelGeneratorPlugin version 1 -[TRT] detected model format - caffe (extension '.caffemodel') -[TRT] desired precision specified for GPU: FASTEST -[TRT] requested fasted precision for device GPU without providing valid calibrator, disabling INT8 -[TRT] [MemUsageChange] Init CUDA: CPU +307, GPU +0, now: CPU 320, GPU 223 (MiB) -[TRT] Trying to load shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] Loaded shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] [MemUsageChange] Init builder kernel library: CPU +262, GPU +74, now: CPU 636, GPU 297 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] native precisions detected for GPU: FP32, FP16, INT8 -[TRT] selecting fastest native precision for GPU: FP16 -[TRT] attempting to open engine cache file ./local_net//bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] loading network plan from engine cache... ./local_net//bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] device GPU, loaded ./local_net//bvlc_googlenet.caffemodel -[TRT] Loaded engine size: 15 MiB -[TRT] Trying to load shared library libcudnn.so.8 -[TRT] Loaded shared library libcudnn.so.8 -[TRT] Using cuDNN as plugin tactic source -[TRT] Using cuDNN as core library tactic source -[TRT] [MemUsageChange] Init cuDNN: CPU +576, GPU +236, now: CPU 977, GPU 477 (MiB) -[TRT] Deserialization required 488590 microseconds. -[TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +13, now: CPU 0, GPU 13 (MiB) -[TRT] Trying to load shared library libcudnn.so.8 -[TRT] Loaded shared library libcudnn.so.8 -[TRT] Using cuDNN as plugin tactic source -[TRT] Using cuDNN as core library tactic source -[TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 977, GPU 477 (MiB) -[TRT] Total per-runner device persistent memory is 94720 -[TRT] Total per-runner host persistent memory is 147088 -[TRT] Allocated activation device memory of size 3612672 -[TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +3, now: CPU 0, GPU 16 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] -[TRT] CUDA engine context initialized on device GPU: -[TRT] -- layers 72 -[TRT] -- maxBatchSize 1 -[TRT] -- deviceMemory 3612672 -[TRT] -- bindings 2 -[TRT] binding 0 - -- index 0 - -- name 'data' - -- type FP32 - -- in/out INPUT - -- # dims 3 - -- dim #0 3 - -- dim #1 224 - -- dim #2 224 -[TRT] binding 1 - -- index 1 - -- name 'prob' - -- type FP32 - -- in/out OUTPUT - -- # dims 3 - -- dim #0 1000 - -- dim #1 1 - -- dim #2 1 -[TRT] -[TRT] binding to input 0 data binding index: 0 -[TRT] binding to input 0 data dims (b=1 c=3 h=224 w=224) size=602112 -[TRT] binding to output 0 prob binding index: 1 -[TRT] binding to output 0 prob dims (b=1 c=1000 h=1 w=1) size=4000 -[TRT] -[TRT] device GPU, ./local_net//bvlc_googlenet.caffemodel initialized. -[TRT] imageNet -- loaded 1000 class info entries -[TRT] imageNet -- ./local_net//bvlc_googlenet.caffemodel initialized. -class 0281 - 0.219604 (tabby, tabby cat) -class 0282 - 0.062927 (tiger cat) -class 0283 - 0.018173 (Persian cat) -class 0284 - 0.017746 (Siamese cat, Siamese) -class 0285 - 0.483398 (Egyptian cat) -class 0287 - 0.180664 (lynx, catamount) -imagenet: 48.33984% class #285 (Egyptian cat) -imagenet: attempting to save output image -imagenet: completed saving -imagenet: shutting down... -48.340% Egyptian cat -Shutting down vAccel -``` - -#### `aarch64` - -For aarch64 things are more or less the same. We run the example on a Jetson -Xavier AGX, so jetson-inference and the nvidia stack is included in the Jetson -Linux variant (L4T). - -The steps to take only refer to installing jetson-inference libs, vAccel and -the python bindings so assuming there's a Jetson Linux distro with Jetpack -installed: - -- install jetson-inference: - -```bash -git clone --recursive https://github.com/dusty-nv/jetson-inference -cd jetson-inference -mkdir build -cd build -cmake ../ -make install -``` - -- install vAccelRT: - -```bash -wget https://s3.nubificus.co.uk/nbfc-assets/github/vaccelrt/master/aarch64/Release-deb/vaccel-0.5.0-Linux.deb -dpkg -i vaccel-0.5.0-Linux.deb -``` - -- install the jetson plugin: - -```bash -wget https://s3.nubificus.co.uk/nbfc-assets/github/vaccelrt/plugins/jetson_inference/master/aarch64/vaccelrt-plugin-jetson-0.1-Linux.deb -dpkg -i vaccelrt-plugin-jetson-0.1-Linux.deb -``` - -- install python bindings in a virtual env: - -```bash -python3 -m venv .vaccel-venv -.vaccel-venv/bin/pip3 install https://s3.nbfc.io/nbfc-assets/github/python-vaccel/main/aarch64/vaccel-2022.11.5.tar.gz -``` - -- run the example: - -```bash -# .vaccel-venv/bin/python3 cat.py -Loading libvaccel -2022.11.05-20:25:12.79 - Initializing vAccel -2022.11.05-20:25:12.79 - Created top-level rundir: /run/user/0/vaccel.G2ZhVr -Loading plugins -Loading plugin: /usr/local/lib/libvaccel-jetson.so -2022.11.05-20:25:12.91 - Registered plugin jetson-inference -2022.11.05-20:25:12.91 - Registered function image classification from plugin jetson-inference -2022.11.05-20:25:12.91 - Registered function image detection from plugin jetson-inference -2022.11.05-20:25:12.91 - Registered function image segmentation from plugin jetson-inference -Loaded plugin jetson-inference from /usr/local/lib/libvaccel-jetson.so -2022.11.05-20:25:12.96 - session:1 New session -Session id is 1 -2022.11.05-20:25:12.96 - session:1 Looking for plugin implementing image classification -2022.11.05-20:25:12.96 - Found implementation in jetson-inference plugin - -imageNet -- loading classification network model from: - -- prototxt /home/ananos/develop/jetson-inference/data/networks/googlenet.prototxt - -- model /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel - -- class_labels /home/ananos/develop/jetson-inference/data/networks/ilsvrc12_synset_words.txt - -- input_blob 'data' - -- output_blob 'prob' - -- batch_size 1 - -[TRT] TensorRT version 8.4.1 -[TRT] loading NVIDIA plugins... -[TRT] Registered plugin creator - ::GridAnchor_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchorRect_TRT version 1 -[TRT] Registered plugin creator - ::NMS_TRT version 1 -[TRT] Registered plugin creator - ::Reorg_TRT version 1 -[TRT] Registered plugin creator - ::Region_TRT version 1 -[TRT] Registered plugin creator - ::Clip_TRT version 1 -[TRT] Registered plugin creator - ::LReLU_TRT version 1 -[TRT] Registered plugin creator - ::PriorBox_TRT version 1 -[TRT] Registered plugin creator - ::Normalize_TRT version 1 -[TRT] Registered plugin creator - ::ScatterND version 1 -[TRT] Registered plugin creator - ::RPROI_TRT version 1 -[TRT] Registered plugin creator - ::BatchedNMS_TRT version 1 -[TRT] Registered plugin creator - ::BatchedNMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::BatchTilePlugin_TRT version 1 -[TRT] Could not register plugin creator - ::FlattenConcat_TRT version 1 -[TRT] Registered plugin creator - ::CropAndResize version 1 -[TRT] Registered plugin creator - ::CropAndResizeDynamic version 1 -[TRT] Registered plugin creator - ::DetectionLayer_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_ONNX_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Explicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Implicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::ProposalDynamic version 1 -[TRT] Registered plugin creator - ::Proposal version 1 -[TRT] Registered plugin creator - ::ProposalLayer_TRT version 1 -[TRT] Registered plugin creator - ::PyramidROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::ResizeNearest_TRT version 1 -[TRT] Registered plugin creator - ::Split version 1 -[TRT] Registered plugin creator - ::SpecialSlice_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 2 -[TRT] Registered plugin creator - ::CoordConvAC version 1 -[TRT] Registered plugin creator - ::DecodeBbox3DPlugin version 1 -[TRT] Registered plugin creator - ::GenerateDetection_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelCropAndResize_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelProposeROI_TRT version 1 -[TRT] Registered plugin creator - ::NMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::PillarScatterPlugin version 1 -[TRT] Registered plugin creator - ::VoxelGeneratorPlugin version 1 -[TRT] Registered plugin creator - ::MultiscaleDeformableAttnPlugin_TRT version 1 -[TRT] detected model format - caffe (extension '.caffemodel') -[TRT] desired precision specified for GPU: FASTEST -[TRT] requested fasted precision for device GPU without providing valid calibrator, disabling INT8 -[TRT] [MemUsageChange] Init CUDA: CPU +187, GPU +0, now: CPU 211, GPU 3925 (MiB) -[TRT] [MemUsageChange] Init builder kernel library: CPU +131, GPU +123, now: CPU 361, GPU 4067 (MiB) -[TRT] native precisions detected for GPU: FP32, FP16, INT8 -[TRT] selecting fastest native precision for GPU: FP16 -[TRT] found engine cache file /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel.1.1.8401.GPU.FP16.engine -[TRT] found model checksum /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel.sha256sum -[TRT] echo "$(cat /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel.sha256sum) /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel" | sha256sum --check --status -[TRT] model matched checksum /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel.sha256sum -[TRT] loading network plan from engine cache... /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel.1.1.8401.GPU.FP16.engine -[TRT] device GPU, loaded /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel -[TRT] [MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 245, GPU 4080 (MiB) -[TRT] Loaded engine size: 14 MiB -[TRT] Deserialization required 15317 microseconds. -[TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +13, now: CPU 0, GPU 13 (MiB) -[TRT] Total per-runner device persistent memory is 75776 -[TRT] Total per-runner host persistent memory is 110304 -[TRT] Allocated activation device memory of size 5218304 -[TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +5, now: CPU 0, GPU 18 (MiB) -[TRT] -[TRT] CUDA engine context initialized on device GPU: -[TRT] -- layers 72 -[TRT] -- maxBatchSize 1 -[TRT] -- deviceMemory 5218304 -[TRT] -- bindings 2 -[TRT] binding 0 - -- index 0 - -- name 'data' - -- type FP32 - -- in/out INPUT - -- # dims 3 - -- dim #0 3 - -- dim #1 224 - -- dim #2 224 -[TRT] binding 1 - -- index 1 - -- name 'prob' - -- type FP32 - -- in/out OUTPUT - -- # dims 3 - -- dim #0 1000 - -- dim #1 1 - -- dim #2 1 -[TRT] -[TRT] binding to input 0 data binding index: 0 -[TRT] binding to input 0 data dims (b=1 c=3 h=224 w=224) size=602112 -[TRT] binding to output 0 prob binding index: 1 -[TRT] binding to output 0 prob dims (b=1 c=1000 h=1 w=1) size=4000 -[TRT] -[TRT] device GPU, /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel initialized. -[TRT] loaded 1000 class labels -[TRT] imageNet -- /home/ananos/develop/jetson-inference/data/networks/bvlc_googlenet.caffemodel initialized. -class 0281 - 0.222134 (tabby, tabby cat) -class 0282 - 0.063147 (tiger cat) -class 0283 - 0.018521 (Persian cat) -class 0284 - 0.018234 (Siamese cat, Siamese) -class 0285 - 0.477663 (Egyptian cat) -class 0287 - 0.182722 (lynx, catamount) -imagenet: 47.76627% class #285 (Egyptian cat) -imagenet: attempting to save output image -imagenet: completed saving -imagenet: shutting down... -47.766% Egyptian cat -2022.11.05-20:25:16.39 - session:1 Free session -Shutting down vAccel -2022.11.05-20:25:16.45 - Shutting down vAccel -2022.11.05-20:25:16.45 - Cleaning up plugins -2022.11.05-20:25:16.45 - Unregistered plugin jetson-inference -``` - -### Appendix I: Build a jetson-inference container image - -[Jetson-inference](https://github.com/dusty-nv/jetson-inference) is an suite of -tools built around TensorRT to expose an image inference API. Installing this -suite on x86 used to be a real challenge (kind of still is), but things are -getting better! - -We use a container file to capture the individual steps to install jetson inference. Assuming the host is debian-based (we tried that on Ubuntu 20.04), and has a recent NVIDIA driver (`520.61.05`) we follow the steps below: - -- clone the container repo: - -```bash -git clone https://github.com/nubificus/jetson-inference-container -``` - -- build the container image: - -```bash -docker build -t nubificus/jetson-inference-updated:x86_64 -f Dockerfile . -``` - -or just get the one we've built (could take some time, i'ts 12GB...): - -```bash -docker pull nubificus/jetson-inference-updated:x86_64 -``` - -- run the `jetson-inference` example: - -Run the container: - -```bash -# docker run --gpus all --rm -it -v/data/code:/data/code -w $PWD nubificus/jetson-inference-updated:x86_64 /bin/bash -root@9f5224cb28cc:/data/code# -``` - -Use pre-installed example images and models to do image inference: - -```bash -root@9f5224cb28cc:/data/code# ln -s /usr/local/data/images . -root@9f5224cb28cc:/data/code# ln -s /usr/local/data/networks . - -root@9f5224cb28cc:/data/code# imagenet-console images/dog_0.jpg -[video] created imageLoader from file:///data/code/images/dog_0.jpg ------------------------------------------------- -imageLoader video options: ------------------------------------------------- - -- URI: file:///data/code/images/dog_0.jpg - - protocol: file - - location: images/dog_0.jpg - - extension: jpg - -- deviceType: file - -- ioType: input - -- codec: unknown - -- width: 0 - -- height: 0 - -- frameRate: 0.000000 - -- bitRate: 0 - -- numBuffers: 4 - -- zeroCopy: true - -- flipMethod: none - -- loop: 0 - -- rtspLatency 2000 ------------------------------------------------- -[video] videoOptions -- failed to parse output resource URI (null) -[video] videoOutput -- failed to parse command line options -imagenet: failed to create output stream - -imageNet -- loading classification network model from: - -- prototxt networks/googlenet.prototxt - -- model networks/bvlc_googlenet.caffemodel - -- class_labels networks/ilsvrc12_synset_words.txt - -- input_blob 'data' - -- output_blob 'prob' - -- batch_size 1 - -[TRT] TensorRT version 8.5.1 -[TRT] loading NVIDIA plugins... -[TRT] Registered plugin creator - ::BatchedNMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::BatchedNMS_TRT version 1 -[TRT] Registered plugin creator - ::BatchTilePlugin_TRT version 1 -[TRT] Registered plugin creator - ::Clip_TRT version 1 -[TRT] Registered plugin creator - ::CoordConvAC version 1 -[TRT] Registered plugin creator - ::CropAndResizeDynamic version 1 -[TRT] Registered plugin creator - ::CropAndResize version 1 -[TRT] Registered plugin creator - ::DecodeBbox3DPlugin version 1 -[TRT] Registered plugin creator - ::DetectionLayer_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Explicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Implicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_ONNX_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_TRT version 1 -[TRT] Could not register plugin creator - ::FlattenConcat_TRT version 1 -[TRT] Registered plugin creator - ::GenerateDetection_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchor_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchorRect_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 2 -[TRT] Registered plugin creator - ::LReLU_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelCropAndResize_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelProposeROI_TRT version 1 -[TRT] Registered plugin creator - ::MultiscaleDeformableAttnPlugin_TRT version 1 -[TRT] Registered plugin creator - ::NMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::NMS_TRT version 1 -[TRT] Registered plugin creator - ::Normalize_TRT version 1 -[TRT] Registered plugin creator - ::PillarScatterPlugin version 1 -[TRT] Registered plugin creator - ::PriorBox_TRT version 1 -[TRT] Registered plugin creator - ::ProposalDynamic version 1 -[TRT] Registered plugin creator - ::ProposalLayer_TRT version 1 -[TRT] Registered plugin creator - ::Proposal version 1 -[TRT] Registered plugin creator - ::PyramidROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::Region_TRT version 1 -[TRT] Registered plugin creator - ::Reorg_TRT version 1 -[TRT] Registered plugin creator - ::ResizeNearest_TRT version 1 -[TRT] Registered plugin creator - ::ROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::RPROI_TRT version 1 -[TRT] Registered plugin creator - ::ScatterND version 1 -[TRT] Registered plugin creator - ::SpecialSlice_TRT version 1 -[TRT] Registered plugin creator - ::Split version 1 -[TRT] Registered plugin creator - ::VoxelGeneratorPlugin version 1 -[TRT] detected model format - caffe (extension '.caffemodel') -[TRT] desired precision specified for GPU: FASTEST -[TRT] requested fasted precision for device GPU without providing valid calibrator, disabling INT8 -[TRT] [MemUsageChange] Init CUDA: CPU +298, GPU +0, now: CPU 321, GPU 223 (MiB) -[TRT] Trying to load shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] Loaded shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] [MemUsageChange] Init builder kernel library: CPU +262, GPU +76, now: CPU 637, GPU 299 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] native precisions detected for GPU: FP32, FP16, INT8 -[TRT] selecting fastest native precision for GPU: FP16 -[TRT] attempting to open engine cache file networks/bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] loading network plan from engine cache... networks/bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] device GPU, loaded networks/bvlc_googlenet.caffemodel -[TRT] Loaded engine size: 15 MiB -[TRT] Trying to load shared library libcudnn.so.8 -[TRT] Loaded shared library libcudnn.so.8 -[TRT] Using cuDNN as plugin tactic source -[TRT] Using cuDNN as core library tactic source -[TRT] [MemUsageChange] Init cuDNN: CPU +576, GPU +236, now: CPU 978, GPU 477 (MiB) -[TRT] Deserialization required 506628 microseconds. -[TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +13, now: CPU 0, GPU 13 (MiB) -[TRT] Trying to load shared library libcudnn.so.8 -[TRT] Loaded shared library libcudnn.so.8 -[TRT] Using cuDNN as plugin tactic source -[TRT] Using cuDNN as core library tactic source -[TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +8, now: CPU 979, GPU 477 (MiB) -[TRT] Total per-runner device persistent memory is 94720 -[TRT] Total per-runner host persistent memory is 147808 -[TRT] Allocated activation device memory of size 3612672 -[TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +3, now: CPU 0, GPU 16 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] -[TRT] CUDA engine context initialized on device GPU: -[TRT] -- layers 75 -[TRT] -- maxBatchSize 1 -[TRT] -- deviceMemory 3612672 -[TRT] -- bindings 2 -[TRT] binding 0 - -- index 0 - -- name 'data' - -- type FP32 - -- in/out INPUT - -- # dims 3 - -- dim #0 3 - -- dim #1 224 - -- dim #2 224 -[TRT] binding 1 - -- index 1 - -- name 'prob' - -- type FP32 - -- in/out OUTPUT - -- # dims 3 - -- dim #0 1000 - -- dim #1 1 - -- dim #2 1 -[TRT] -[TRT] binding to input 0 data binding index: 0 -[TRT] binding to input 0 data dims (b=1 c=3 h=224 w=224) size=602112 -[TRT] binding to output 0 prob binding index: 1 -[TRT] binding to output 0 prob dims (b=1 c=1000 h=1 w=1) size=4000 -[TRT] -[TRT] device GPU, networks/bvlc_googlenet.caffemodel initialized. -[TRT] imageNet -- loaded 1000 class info entries -[TRT] imageNet -- networks/bvlc_googlenet.caffemodel initialized. -[image] loaded 'images/dog_0.jpg' (500x375, 3 channels) -class 0248 - 0.229980 (Eskimo dog, husky) -class 0249 - 0.605469 (malamute, malemute, Alaskan malamute) -class 0250 - 0.160400 (Siberian husky) -imagenet: 60.54688% class #249 (malamute, malemute, Alaskan malamute) - -[TRT] ------------------------------------------------ -[TRT] Timing Report networks/bvlc_googlenet.caffemodel -[TRT] ------------------------------------------------ -[TRT] Pre-Process CPU 0.02498ms CUDA 0.21392ms -[TRT] Network CPU 1.04583ms CUDA 0.86154ms -[TRT] Post-Process CPU 0.02265ms CUDA 0.02288ms -[TRT] Total CPU 1.09346ms CUDA 1.09834ms -[TRT] ------------------------------------------------ - -[TRT] note -- when processing a single image, run 'sudo jetson_clocks' before - to disable DVFS for more accurate profiling/timing measurements - -[image] imageLoader -- End of Stream (EOS) has been reached, stream has been closed -imagenet: shutting down... -imagenet: shutdown complete. -``` - -**Note**: _The first time the engine needs to do some autotuning, so it will take some time and drop output similar to the one below_: - - - - -```bash -... -[TRT] Tactic: 0x89c2d153627e52ba Time: 0.0134678 -[TRT] loss3/classifier Set Tactic Name: volta_h884cudnn_256x128_ldg8_relu_exp_small_nhwc_tn_v1 Tactic: 0xc110e19c9f5aa36e -[TRT] Tactic: 0xc110e19c9f5aa36e Time: 0.0752844 -[TRT] loss3/classifier Set Tactic Name: turing_h1688cudnn_256x128_ldg8_relu_exp_medium_nhwc_tn_v1 Tactic: 0xdc1c841ef1cd3e8e -[TRT] Tactic: 0xdc1c841ef1cd3e8e Time: 0.0422516 -[TRT] loss3/classifier Set Tactic Name: sm70_xmma_fprop_implicit_gemm_f16f16_f16f16_f16_nhwckrsc_nhwc_tilesize128x64x32_stage1_warpsize2x2x1_g1_tensor8x8x4_t1r1s1 Tactic: 0x4c17dc9d992e6a1d -[TRT] Tactic: 0x4c17dc9d992e6a1d Time: 0.0231476 -[TRT] loss3/classifier Set Tactic Name: sm75_xmma_fprop_implicit_gemm_f16f16_f16f16_f16_nhwckrsc_nhwc_tilesize128x128x64_stage1_warpsize2x2x1_g1_tensor16x8x8_t1r1s1 Tactic: 0xc399fdbffdc34032 -[TRT] Tactic: 0xc399fdbffdc34032 Time: 0.0262451 -[TRT] loss3/classifier Set Tactic Name: turing_h1688cudnn_256x64_sliced1x2_ldg8_relu_exp_interior_nhwc_tn_v1 Tactic: 0x105f56cf03ee5549 -[TRT] Tactic: 0x105f56cf03ee5549 Time: 0.0241811 -[TRT] Fastest Tactic: 0x017a89ce2d82b850 Time: 0.00878863 -[TRT] >>>>>>>>>>>>>>> Chose Runner Type: CaskGemmConvolution Tactic: 0x0000000000020318 -[TRT] =============== Computing costs for -[TRT] *************** Autotuning format combination: Float(1000,1,1,1) -> Float(1000,1,1,1) *************** -[TRT] --------------- Timing Runner: prob (CudaSoftMax) -[TRT] Tactic: 0x00000000000003ea Time: 0.00364102 -[TRT] Fastest Tactic: 0x00000000000003ea Time: 0.00364102 -[TRT] --------------- Timing Runner: prob (CaskSoftMax) -[TRT] CaskSoftMax has no valid tactics for this config, skipping -[TRT] >>>>>>>>>>>>>>> Chose Runner Type: CudaSoftMax Tactic: 0x00000000000003ea -[TRT] *************** Autotuning format combination: Half(1000,1,1,1) -> Half(1000,1,1,1) *************** -[TRT] --------------- Timing Runner: prob (CudaSoftMax) -[TRT] Tactic: 0x00000000000003ea Time: 0.00370118 -[TRT] Fastest Tactic: 0x00000000000003ea Time: 0.00370118 -[TRT] --------------- Timing Runner: prob (CaskSoftMax) -[TRT] CaskSoftMax has no valid tactics for this config, skipping -[TRT] >>>>>>>>>>>>>>> Chose Runner Type: CudaSoftMax Tactic: 0x00000000000003ea -[TRT] *************** Autotuning format combination: Half(500,1:2,1,1) -> Half(500,1:2,1,1) *************** -[TRT] --------------- Timing Runner: prob (CudaSoftMax) -[TRT] Tactic: 0x0000000000000012 Time: 0.00347464 -[TRT] Fastest Tactic: 0x0000000000000012 Time: 0.00347464 -[TRT] >>>>>>>>>>>>>>> Chose Runner Type: CudaSoftMax Tactic: 0x0000000000000012 -[TRT] Adding reformat layer: Reformatted Input Tensor 0 to conv1/7x7_s2 + conv1/relu_7x7 (data) from Float(150528,50176,224,1) to Half(100352,50176:2,224,1) -[TRT] Adding reformat layer: Reformatted Input Tensor 0 to conv2/3x3_reduce + conv2/relu_3x3_reduce (pool1/norm1) from Half(100352,3136:2,56,1) to Half(25088,1:8,448,8) -[TRT] Adding reformat layer: Reformatted Input Tensor 0 to conv2/norm2 (conv2/3x3) from Half(75264,1:8,1344,24) to Half(301056,3136:2,56,1) -[TRT] Adding reformat layer: Reformatted Output Tensor 0 to pool2/3x3_s2 (pool2/3x3_s2) from Half(75264,784:2,28,1) to Half(18816,1:8,672,24) -[TRT] Adding reformat layer: Reformatted Output Tensor 0 to inception_4a/1x1 + inception_4a/relu_1x1 || inception_4a/3x3_reduce + inception_4a/relu_3x3_reduce || inception_4a/5x5_reduce + inception_4a/relu_5x5_reduce (inception_4a/1x1 + inception_4a/relu_1x1 || inception_4a/3x3_reduce + inception_4a/relu_3x3_reduce || inception_4a/5x5_reduce + inception_4a/relu_5x5_reduce) from Half(7448,1:8,532,38) to Half(29792,196:2,14,1) -... -``` - - - -**Note**: _If you want to avoid that everytime you run the container, keep the networks folder outside the container and bind mount it (eg. in the `/data/code` path). That is, instead of doing `ln -s /usr/local/data/networks .` do a `cp -avf /usr/local/data/networks .`. Thus, every time you re-run the example using this folder, the auto-tuned engine will be there._ - -## Appendix I Build a jetson-inference container image - -We only need this for `x86_64` Hosts, as for the `aarch64` case, -`nvidia-container-toolkit` is bound to the `l4t` distribution so running in a -container with different `cuda` and `cudnn`/`tensorrt` version is really -tricky. - -The following Dockerfile includes the steps to setup and install the `jetson-inference` suite on an `x86_64` host: - -```Dockerfile -FROM ubuntu:20.04 - -ARG DEBIAN_FRONTEND=noninteractive - -### Install dependencies -RUN apt update && TZ=Etc/UTC apt install -y build-essential \ - git \ - cmake \ - python3 \ - python3-venv \ - libpython3-dev \ - python3-numpy \ - gcc-8 \ - g++-8 \ - lsb-release \ - wget \ - software-properties-common && \ - rm -rf /var/cache/apt/archives /var/lib/apt/lists && \ - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 8 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 8 && \ - update-alternatives --set gcc /usr/bin/gcc-8 - -RUN git clone https://github.com/dusty-nv/jetson-inference --recursive - -### Install NVIDIA CUDA, CUDNN and TENSORRT -ARG OS=ubuntu2004 -RUN wget http://developer.download.nvidia.com/compute/machine-learning/repos/${OS}/x86_64/nvidia-machine-learning-repo-${OS}_1.0.0-1_amd64.deb && \ - dpkg -i nvidia-machine-learning-repo-${OS}_1.0.0-1_amd64.deb && \ - apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/machine-learning/repos/${OS}/x86_64/7fa2af80.pub && \ - wget https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/cuda-${OS}.pin && \ - mv cuda-${OS}.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/3bf863cc.pub && \ - add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/${OS}/x86_64/ /" && \ - apt-get update && \ - apt-get install -y libcudnn8 libcudnn8-dev tensorrt nvidia-cuda-toolkit && \ - rm -rf /var/cache/apt/archives /var/lib/apt/lists - - -### Build jetson-inference and download models -RUN cd jetson-inference && \ - git checkout x86 && \ - git submodule update --init && \ - mkdir build && \ - cd build && \ - cmake ../ && \ - make -j$(nproc) install && \ - cd /jetson-inference/tools && \ - ./download-models.sh 0 && \ - cd / && cp -avf /jetson-inference/data /usr/local/data && \ - cp -avf /jetson-inference/utils/image/stb /usr/local/include && \ - rm -rf /jetson-inference -``` - -Essentially, the pain points are the following: - -- Add custom NVIDIA repos for `cuda`, `cudnn` and `tensorrt` -- Use `gcc-8` as the default compiler - -Other than that, the steps are straightforward. From the jetson-inference build -dir, we need to keep `data/networks` as it holds the models for inference. We -also keep `image/stb` include files for the building of the vAccelRT jetson -plugin. - -Build using the following command: - -```bash -docker build -t jetson-inference:x86_64 -f Dockerfile . -``` - -expect a lot of output, and about ~15' on a generic machine (6 cores, 16GB of RAM). - -and run an example: - -```console -# docker run --rm -it --gpus all nubificus/jetson-inference-updated:x86_64 /bin/bash -root@f741113696a8:/# cd /usr/local/data/ -root@f741113696a8:/usr/local/data# export LD_LIBRARY_PATH=/usr/local/lib/ -root@f741113696a8:/usr/local/data# imagenet-console images/dog_0.jpg -[video] created imageLoader from file:///usr/local/data/images/dog_0.jpg ------------------------------------------------- -imageLoader video options: ------------------------------------------------- - -- URI: file:///usr/local/data/images/dog_0.jpg - - protocol: file - - location: images/dog_0.jpg - - extension: jpg - -- deviceType: file - -- ioType: input - -- codec: unknown - -- width: 0 - -- height: 0 - -- frameRate: 0.000000 - -- bitRate: 0 - -- numBuffers: 4 - -- zeroCopy: true - -- flipMethod: none - -- loop: 0 - -- rtspLatency 2000 ------------------------------------------------- -[video] videoOptions -- failed to parse output resource URI (null) -[video] videoOutput -- failed to parse command line options -imagenet: failed to create output stream - -imageNet -- loading classification network model from: - -- prototxt networks/googlenet.prototxt - -- model networks/bvlc_googlenet.caffemodel - -- class_labels networks/ilsvrc12_synset_words.txt - -- input_blob 'data' - -- output_blob 'prob' - -- batch_size 1 - -[TRT] TensorRT version 8.5.1 -[TRT] loading NVIDIA plugins... -[TRT] Registered plugin creator - ::BatchedNMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::BatchedNMS_TRT version 1 -[TRT] Registered plugin creator - ::BatchTilePlugin_TRT version 1 -[TRT] Registered plugin creator - ::Clip_TRT version 1 -[TRT] Registered plugin creator - ::CoordConvAC version 1 -[TRT] Registered plugin creator - ::CropAndResizeDynamic version 1 -[TRT] Registered plugin creator - ::CropAndResize version 1 -[TRT] Registered plugin creator - ::DecodeBbox3DPlugin version 1 -[TRT] Registered plugin creator - ::DetectionLayer_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Explicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_Implicit_TF_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_ONNX_TRT version 1 -[TRT] Registered plugin creator - ::EfficientNMS_TRT version 1 -[TRT] Could not register plugin creator - ::FlattenConcat_TRT version 1 -[TRT] Registered plugin creator - ::GenerateDetection_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchor_TRT version 1 -[TRT] Registered plugin creator - ::GridAnchorRect_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 1 -[TRT] Registered plugin creator - ::InstanceNormalization_TRT version 2 -[TRT] Registered plugin creator - ::LReLU_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelCropAndResize_TRT version 1 -[TRT] Registered plugin creator - ::MultilevelProposeROI_TRT version 1 -[TRT] Registered plugin creator - ::MultiscaleDeformableAttnPlugin_TRT version 1 -[TRT] Registered plugin creator - ::NMSDynamic_TRT version 1 -[TRT] Registered plugin creator - ::NMS_TRT version 1 -[TRT] Registered plugin creator - ::Normalize_TRT version 1 -[TRT] Registered plugin creator - ::PillarScatterPlugin version 1 -[TRT] Registered plugin creator - ::PriorBox_TRT version 1 -[TRT] Registered plugin creator - ::ProposalDynamic version 1 -[TRT] Registered plugin creator - ::ProposalLayer_TRT version 1 -[TRT] Registered plugin creator - ::Proposal version 1 -[TRT] Registered plugin creator - ::PyramidROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::Region_TRT version 1 -[TRT] Registered plugin creator - ::Reorg_TRT version 1 -[TRT] Registered plugin creator - ::ResizeNearest_TRT version 1 -[TRT] Registered plugin creator - ::ROIAlign_TRT version 1 -[TRT] Registered plugin creator - ::RPROI_TRT version 1 -[TRT] Registered plugin creator - ::ScatterND version 1 -[TRT] Registered plugin creator - ::SpecialSlice_TRT version 1 -[TRT] Registered plugin creator - ::Split version 1 -[TRT] Registered plugin creator - ::VoxelGeneratorPlugin version 1 -[TRT] detected model format - caffe (extension '.caffemodel') -[TRT] desired precision specified for GPU: FASTEST -[TRT] requested fasted precision for device GPU without providing valid calibrator, disabling INT8 -[TRT] [MemUsageChange] Init CUDA: CPU +298, GPU +0, now: CPU 321, GPU 223 (MiB) -[TRT] Trying to load shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] Loaded shared library libnvinfer_builder_resource.so.8.5.1 -[TRT] [MemUsageChange] Init builder kernel library: CPU +262, GPU +76, now: CPU 637, GPU 299 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] native precisions detected for GPU: FP32, FP16, INT8 -[TRT] selecting fastest native precision for GPU: FP16 -[TRT] attempting to open engine cache file networks/bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] loading network plan from engine cache... networks/bvlc_googlenet.caffemodel.1.1.8501.GPU.FP16.engine -[TRT] device GPU, loaded networks/bvlc_googlenet.caffemodel -[TRT] Loaded engine size: 15 MiB -[TRT] Deserialization required 7510 microseconds. -[TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +13, now: CPU 0, GPU 13 (MiB) -[TRT] Total per-runner device persistent memory is 94720 -[TRT] Total per-runner host persistent memory is 149472 -[TRT] Allocated activation device memory of size 3612672 -[TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +3, now: CPU 0, GPU 16 (MiB) -[TRT] CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars -[TRT] -[TRT] CUDA engine context initialized on device GPU: -[TRT] -- layers 71 -[TRT] -- maxBatchSize 1 -[TRT] -- deviceMemory 3612672 -[TRT] -- bindings 2 -[TRT] binding 0 - -- index 0 - -- name 'data' - -- type FP32 - -- in/out INPUT - -- # dims 3 - -- dim #0 3 - -- dim #1 224 - -- dim #2 224 -[TRT] binding 1 - -- index 1 - -- name 'prob' - -- type FP32 - -- in/out OUTPUT - -- # dims 3 - -- dim #0 1000 - -- dim #1 1 - -- dim #2 1 -[TRT] -[TRT] binding to input 0 data binding index: 0 -[TRT] binding to input 0 data dims (b=1 c=3 h=224 w=224) size=602112 -[TRT] binding to output 0 prob binding index: 1 -[TRT] binding to output 0 prob dims (b=1 c=1000 h=1 w=1) size=4000 -[TRT] -[TRT] device GPU, networks/bvlc_googlenet.caffemodel initialized. -[TRT] imageNet -- loaded 1000 class info entries -[TRT] imageNet -- networks/bvlc_googlenet.caffemodel initialized. -[image] loaded 'images/dog_0.jpg' (500x375, 3 channels) -class 0248 - 0.230449 (Eskimo dog, husky) -class 0249 - 0.607152 (malamute, malemute, Alaskan malamute) -class 0250 - 0.158385 (Siberian husky) -imagenet: 60.71516% class #249 (malamute, malemute, Alaskan malamute) - -[TRT] ------------------------------------------------ -[TRT] Timing Report networks/bvlc_googlenet.caffemodel -[TRT] ------------------------------------------------ -[TRT] Pre-Process CPU 0.02392ms CUDA 0.21290ms -[TRT] Network CPU 1.04193ms CUDA 0.85664ms -[TRT] Post-Process CPU 0.02108ms CUDA 0.02154ms -[TRT] Total CPU 1.08694ms CUDA 1.09107ms -[TRT] ------------------------------------------------ - -[TRT] note -- when processing a single image, run 'sudo jetson_clocks' before - to disable DVFS for more accurate profiling/timing measurements - -[image] imageLoader -- End of Stream (EOS) has been reached, stream has been closed -imagenet: shutting down... -[TRT] 3: [engine.cpp::~Engine::307] Error Code 3: API Usage Error (Parameter check failed at: runtime/api/engine.cpp::~Engine::307, condition: mObjectCounter.use_count() == 1. Destroying an engine object before destroying objects it created leads to undefined behavior. -) -imagenet: shutdown complete. -``` diff --git a/build_ffi.py b/build_ffi.py index 6f505af..348c494 100644 --- a/build_ffi.py +++ b/build_ffi.py @@ -56,50 +56,100 @@ def handle_atomic_declaration(line: str) -> str | None: return None -def handle_deprecated_function(line: str, *, in_function: bool) -> (bool, bool): - """Handle removal of deprecated function declarations.""" - deprecated_re = re.compile(r"__attribute__\(\(deprecated\b") - - if in_function: - if line.strip().endswith("}"): - in_function = False - return (in_function, True) - if deprecated_re.search(line): - return (True, True) - - return (in_function, False) - - -def handle_inline_function( - line: str, brace_depth: int, *, in_function: bool -) -> (bool, int, str): - """Handle removal of inline function declarations.""" - inline_re = re.compile(r"\bstatic\s+inline\b") - - if in_function: - brace_depth += line.count("{") - line.count("}") - if brace_depth <= 0: - in_function = False - brace_depth = 0 - return in_function, brace_depth, None - - match = inline_re.search(line) - if match: - prefix = line[: match.start()].rstrip() - if prefix: - # Still check for atomics in that prefix - atomic_result = handle_atomic_declaration(prefix) - result_line = atomic_result if atomic_result else prefix - brace_depth = line.count("{") - line.count("}") - if brace_depth > 0: - in_function = True - return in_function, brace_depth, result_line - - return in_function, brace_depth, line +def remove_deprecated_functions(src: str) -> str: + """Remove deprecated function declarations.""" + out, i, n = [], 0, len(src) + pat = re.compile(r'__attribute__\s*\(\(deprecated\b') + + while i < n: + m = pat.search(src, i) + if not m: + out.append(src[i:]) + break + + # Go backward to find the start of the function declaration + start = src.rfind('\n', 0, m.start()) + start = start + 1 if start != -1 else 0 + out.append(src[i:start]) + + # Move forward to skip the function body + j = src.find('{', m.end()) + if j == -1: + i = m.end() + # Malformed, skip the attribute + continue + + brace = 1 + j += 1 + while j < n and brace: + if src[j] == '{': + brace += 1 + elif src[j] == '}': + brace -= 1 + j += 1 + + # Skip trailing characters (semicolons, whitespace) + while j < n and src[j] in " \t\n;": + j += 1 + + i = j + + return ''.join(out) + + +def remove_static_inline_functions(source: str) -> str: + """Remove 'static inline' functions.""" + static_inline_re = re.compile(r'\bstatic\s+inline\b') + result = [] + i = 0 + n = len(source) + + while i < n: + match = static_inline_re.search(source, i) + if not match: + # No more static inlines, copy the rest + result.append(source[i:]) + break + + start = match.start() + + # Copy anything before the match + if start > i: + result.append(source[i:start]) + + # Find the function body using brace matching + brace_count = 0 + body_start = source.find('{', match.end()) + if body_start == -1: + # Malformed inline without a body, treat as normal text + result.append(source[match.start():match.end()]) + i = match.end() + continue + + i = body_start + brace_count = 1 + i += 1 + + while i < n and brace_count > 0: + if source[i] == '{': + brace_count += 1 + elif source[i] == '}': + brace_count -= 1 + i += 1 + + # If there is a semicolon mid-line, skip it + while i < n and source[i] in ' \t\n;': + i += 1 + + return ''.join(result) def sanitize_cdef(cdef: str, pkg: str) -> str: """Sanitize cdef by removing unsupported declarations.""" + # Parse multi-line patterns first + cdef = remove_static_inline_functions(cdef) + cdef = remove_deprecated_functions(cdef) + output_lines = cdef.splitlines() filtered_lines = [] @@ -107,11 +157,10 @@ def sanitize_cdef(cdef: str, pkg: str) -> str: current_file = None pkg_lines = True - in_deprecated_function = False - in_inline_function = False - brace_depth = 0 + # Parse single-line patterns next for line in output_lines: + # Parse and ignore GCC comments if line.startswith("#"): match = line_re.match(line) if match: @@ -119,7 +168,7 @@ def sanitize_cdef(cdef: str, pkg: str) -> str: pkg_lines = pkg in current_file continue - # Remove non-package declarations + # Ignore non-package declarations if not pkg_lines: continue @@ -133,21 +182,6 @@ def sanitize_cdef(cdef: str, pkg: str) -> str: filtered_lines.append(atomic_result) continue - # Handle deprecated functions - (in_deprecated_function, skip) = handle_deprecated_function( - line, in_function=in_deprecated_function - ) - if skip: - continue - - # Handle inline functions - (in_inline_function, brace_depth, result_line) = handle_inline_function( - line, brace_depth, in_function=in_inline_function - ) - if result_line is not None: - filtered_lines.append(result_line) - continue - filtered_lines.append(line) return "\n".join(filtered_lines) diff --git a/tests/test_tf.py b/tests/test_tf.py index d8853b2..4187ae3 100644 --- a/tests/test_tf.py +++ b/tests/test_tf.py @@ -100,6 +100,8 @@ def test_tf(test_nodes, test_tensor, test_model): assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + session.tf_model_unload(model) + def test_tf_with_run_options(test_nodes, test_tensor, test_model): session = Session() @@ -127,6 +129,8 @@ def test_tf_with_run_options(test_nodes, test_tensor, test_model): assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + session.tf_model_unload(model) + def test_tf_from_buffer(test_nodes, test_tensor, test_model): session = Session() @@ -154,6 +158,8 @@ def test_tf_from_buffer(test_nodes, test_tensor, test_model): assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + session.tf_model_unload(model) + def test_tf_from_numpy(test_nodes, test_tensor, test_model): session = Session() @@ -176,3 +182,5 @@ def test_tf_from_numpy(test_nodes, test_tensor, test_model): assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + + session.tf_model_unload(model) diff --git a/tests/test_tflite.py b/tests/test_tflite.py index 8ae67a9..06654cc 100644 --- a/tests/test_tflite.py +++ b/tests/test_tflite.py @@ -81,6 +81,8 @@ def test_tflite(test_tensor, test_model): assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + session.tflite_model_unload(model) + def test_tflite_from_buffer(test_tensor, test_model): session = Session() @@ -103,6 +105,8 @@ def test_tflite_from_buffer(test_tensor, test_model): assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + session.tflite_model_unload(model) + def test_tflite_from_numpy(test_tensor, test_model): session = Session() @@ -120,3 +124,5 @@ def test_tflite_from_numpy(test_tensor, test_model): assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data assert out_tensors[0].to_bytes() == in_tensors[0].to_bytes() + + session.tflite_model_unload(model) diff --git a/tests/test_torch.py b/tests/test_torch.py index c574ac6..b060ff7 100644 --- a/tests/test_torch.py +++ b/tests/test_torch.py @@ -99,11 +99,13 @@ def test_torch(test_tensor, test_model): model = Resource(test_model, ResourceType.MODEL) model.register(session) + session.torch_model_load(model) + in_tensors = [ Tensor(test_tensor["dims"], test_tensor["type"], test_tensor["data"]) ] - out_tensors = session.torch_jitload_forward(model, in_tensors) + out_tensors = session.torch_model_run(model, in_tensors) assert out_tensors[0].dims == in_tensors[0].dims assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data @@ -119,13 +121,15 @@ def test_torch_with_run_options(test_tensor, test_model): model = Resource(test_model, ResourceType.MODEL) model.register(session) + session.torch_model_load(model) + in_tensors = [ Tensor(test_tensor["dims"], test_tensor["type"], test_tensor["data"]) ] run_options = Buffer(b"none") - out_tensors = session.torch_jitload_forward( + out_tensors = session.torch_model_run( model, in_tensors, run_options=run_options ) assert out_tensors[0].dims == in_tensors[0].dims @@ -143,13 +147,15 @@ def test_torch_from_buffer(test_tensor, test_model): model = Resource(test_model, ResourceType.MODEL) model.register(session) + session.torch_model_load(model) + in_tensors = [ Tensor.from_buffer( test_tensor["dims"], test_tensor["type"], test_tensor["data_bytes"] ) ] - out_tensors = session.torch_jitload_forward(model, in_tensors) + out_tensors = session.torch_model_run(model, in_tensors) assert out_tensors[0].dims == in_tensors[0].dims assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data @@ -165,9 +171,11 @@ def test_torch_from_numpy(test_tensor, test_model): model = Resource(test_model, ResourceType.MODEL) model.register(session) + session.torch_model_load(model) + in_tensors = [Tensor.from_numpy(test_tensor["data_np"])] - out_tensors = session.torch_jitload_forward(model, in_tensors) + out_tensors = session.torch_model_run(model, in_tensors) assert out_tensors[0].dims == in_tensors[0].dims assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data @@ -184,9 +192,11 @@ def test_torch_from_torch(test_tensor, test_model): model = Resource(test_model, ResourceType.MODEL) model.register(session) + session.torch_model_load(model) + in_tensors = [Tensor.from_torch(test_tensor["data_torch"])] - out_tensors = session.torch_jitload_forward(model, in_tensors) + out_tensors = session.torch_model_run(model, in_tensors) assert out_tensors[0].dims == in_tensors[0].dims assert out_tensors[0].data_type == in_tensors[0].data_type assert out_tensors[0].data == in_tensors[0].data diff --git a/vaccel/ops/tf.py b/vaccel/ops/tf.py deleted file mode 100644 index 5ecf998..0000000 --- a/vaccel/ops/tf.py +++ /dev/null @@ -1,662 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -"""Tensorflow operations.""" - -import logging -from typing import Any, ClassVar - -from vaccel._c_types import CBytes, CList, CStr, CType -from vaccel._c_types.utils import CEnumBuilder -from vaccel._libvaccel import ffi, lib -from vaccel.error import FFIError, NullPointerError -from vaccel.resource import Resource - -logger = logging.getLogger(__name__) - -enum_builder = CEnumBuilder(lib) -TensorType = enum_builder.from_prefix("TensorType", "VACCEL_TF_") - - -class Node(CType): - """Wrapper for the `struct vaccel_tf_node` C object. - - Manages the creation and initialization of a C `struct vaccel_tf_node` and - provides access to it through Python properties. - - Inherits: - CType: Abstract base class for defining C data types. - - Attributes: - _name (str): The name of the node. - _id (int): The ID of the node. - _c_obj_ptr (ffi.CData): A double pointer to the underlying - `struct vaccel_tf_node` C object. - """ - - def __init__(self, name: str, id_: int): - """Initializes a new `Node` object. - - Args: - name: The name to be passed to the C struct. - id_: The ID to be passed to the C struct. - """ - self._name = name - self._id = id_ - self._c_obj_ptr = ffi.NULL - super().__init__() - - def _init_c_obj(self): - """Initializes the underlying C `struct vaccel_tf_node` object. - - Raises: - FFIError: If node initialization fails. - """ - self._c_obj_ptr = ffi.new("struct vaccel_tf_node **") - - ret = lib.vaccel_tf_node_new( - self._c_obj_ptr, self._name.encode(), self._id - ) - if ret != 0: - raise FFIError(ret, "Could not initialize node") - - self._c_obj = self._c_obj_ptr[0] - self._c_size = ffi.sizeof("struct vaccel_tf_node") - - @property - def value(self) -> ffi.CData: - """Returns the value of the underlying C struct. - - Returns: - The dereferenced 'struct vaccel_tf_node` - """ - return self._c_ptr_or_raise[0] - - def _del_c_obj(self): - """Deletes the underlying `struct vaccel_tf_node` C object. - - Raises: - FFIError: If node deletion fails. - """ - ret = lib.vaccel_tf_node_delete(self._c_ptr_or_raise) - if ret != 0: - raise FFIError(ret, "Could not delete node") - - def __del__(self): - try: - self._del_c_obj() - except NullPointerError: - pass - except FFIError: - logger.exception("Failed to clean up Node") - - @property - def name(self) -> str: - """The node name. - - Returns: - The node's name. - """ - return CStr.from_c_obj(self._c_ptr_or_raise.name).value - - @property - def id(self) -> int: - """The node identifier. - - Returns: - The node's ID. - """ - return int(self._c_ptr_or_raise.id[0]) - - def __repr__(self): - try: - c_ptr = ( - f"0x{int(ffi.cast('uintptr_t', self._c_obj)):x}" - if self._c_obj != ffi.NULL - else "NULL" - ) - node_name = self.name - node_id = self.id - except (AttributeError, TypeError, NullPointerError): - return f"<{self.__class__.__name__} (uninitialized or invalid)>" - return ( - f"<{self.__class__.__name__} name={node_name!r} " - f"id={node_id} " - f"at {c_ptr}>" - ) - - -class Tensor(CType): - """Wrapper for the `struct vaccel_tf_tensor` C object. - - Manages the creation and initialization of a C `struct vaccel_tf_tensor` and - provides access to it through Python properties. - - Inherits: - CType: Abstract base class for defining C data types. - - Attributes: - _dims (list[int]): The dims of the tensor. - _data (list[Any]): The data of the tensor. - _data_type (TensorType): The type of the tensor. - _c_obj_ptr (ffi.CData): A double pointer to the underlying - `struct vaccel_tf_tensor` C object. - _c_obj_data (ffi.CData): A pointer to the data of the underlying - `struct vaccel_tf_tensor` C object. - - """ - - _size_dict: ClassVar[dict[TensorType, (int, str)]] = { - TensorType.FLOAT: (ffi.sizeof("float"), "float"), - TensorType.DOUBLE: (ffi.sizeof("double"), "double"), - TensorType.INT32: (ffi.sizeof("int32_t"), "int32_t"), - TensorType.UINT8: (ffi.sizeof("uint8_t"), "uint8_t"), - TensorType.INT16: (ffi.sizeof("int16_t"), "int16_t"), - TensorType.INT8: (ffi.sizeof("int8_t"), "int8_t"), - TensorType.INT64: (ffi.sizeof("int64_t"), "int64_t"), - TensorType.BOOL: (ffi.sizeof("bool"), "bool"), - TensorType.UINT16: (ffi.sizeof("uint16_t"), "uint16_t"), - TensorType.UINT32: (ffi.sizeof("uint32_t"), "uint32_t"), - TensorType.UINT64: (ffi.sizeof("uint64_t"), "uint64_t"), - } - - def __init__(self, dims: list[int], data_type: TensorType, data: list[Any]): - """Initializes a new `Tensor` object. - - Args: - dims: The dims to be passed to the C struct. - data_type: The data_type to be passed to the C struct. - data: The data to be passed to the C struct. - """ - self._dims = dims - self._data = data - self._data_type = data_type - self._c_obj_ptr = ffi.NULL - self._c_obj_data = ffi.NULL - super().__init__() - - def _init_c_obj(self): - """Initializes the underlying C `struct vaccel_tf_tensor` object. - - Raises: - FFIError: If tensor initialization fails. - """ - nr_dims = len(self._dims) - c_dims = ffi.new(f"int64_t[{nr_dims}]", self._dims) - - self._c_obj_ptr = ffi.new("struct vaccel_tf_tensor **") - ret = lib.vaccel_tf_tensor_new( - self._c_obj_ptr, nr_dims, c_dims, self._data_type - ) - if ret != 0: - FFIError(ret, "Could not initialize tensor") - - self._c_obj = self._c_obj_ptr[0] - self._c_size = ffi.sizeof("struct vaccel_tf_tensor") - - self._c_obj_data = ffi.new( - f"{self._c_data_type}[{len(self._data)}]", self._data - ) - - ret = lib.vaccel_tf_tensor_set_data( - self._c_obj, self._c_obj_data, self._c_data_size * len(self._data) - ) - if ret != 0: - raise FFIError(ret, "Could not set tensor data") - - @property - def value(self) -> ffi.CData: - """Returns the value of the underlying C struct. - - Returns: - The dereferenced 'struct vaccel_tf_tensor` - """ - return self._c_ptr_or_raise[0] - - def _del_c_obj(self): - """Deletes the underlying `struct vaccel_tf_tensor` C object. - - Raises: - FFIError: If tensor deletion fails. - """ - ret = lib.vaccel_tf_tensor_delete(self._c_ptr_or_raise) - if ret != 0: - raise FFIError(ret, "Could not delete tensor") - - def __del__(self): - try: - self._del_c_obj() - except NullPointerError: - pass - except FFIError: - logger.exception("Failed to clean up Tensor") - - @property - def _c_data_size(self): - return self._size_dict[self.data_type][0] - - @property - def _c_data_type(self): - return self._size_dict[self.data_type][1] - - @property - def dims(self) -> list[int]: - """The tensor dims. - - Returns: - The dims of the tensor. - """ - return [ - int(self._c_ptr_or_raise.dims[i]) - for i in range(self._c_ptr_or_raise.nr_dims) - ] - - @property - def data(self) -> list | str: - """The tensor data. - - Returns: - The data of the tensor. - """ - typed_c_data = ffi.cast( - f"{self._c_data_type} *", self._c_ptr_or_raise.data - ) - return ffi.unpack( - typed_c_data, int(self._c_ptr_or_raise.size / self._c_data_size) - ) - - @property - def data_type(self) -> TensorType: - """The tensor data type. - - Returns: - The data type of the tensor. - """ - return TensorType(self._c_ptr_or_raise.data_type) - - @classmethod - def from_c_obj(cls, c_obj: ffi.CData) -> "Tensor": - """Initializes a new `Tensor` object from an existing C struct. - - Args: - c_obj: A pointer to a `struct vaccel_tf_tensor` C object. - - Returns: - A new `Tensor` object - """ - type_str = ffi.getctype(ffi.typeof(c_obj)) - if type_str != "struct vaccel_tf_tensor *": - msg = f"Expected 'struct vaccel_tf_tensor *', got '{type_str}'" - raise TypeError(msg) - - inst = cls.__new__(cls) - inst._c_obj = c_obj - inst._c_size = ffi.sizeof(inst._c_obj) - inst._dims = None - inst._data = None - inst._data_type = None - inst._c_obj_ptr = ffi.NULL - return inst - - @classmethod - def empty(cls) -> "Tensor": - """Initializes a new empty `Tensor` object. - - The object a NULL pointer in place of the C struct. - - Returns: - A new `Tensor` object - """ - inst = cls.__new__(cls) - inst._c_obj_ptr = ffi.new("struct vaccel_tf_tensor **") - inst._c_obj = inst._c_obj_ptr[0] - inst._c_obj_ptr[0] = ffi.NULL - inst._c_size = ffi.sizeof("struct vaccel_tf_tensor *") - return inst - - def __repr__(self): - try: - c_ptr = ( - f"0x{int(ffi.cast('uintptr_t', self._c_obj)):x}" - if self._c_obj != ffi.NULL - else "NULL" - ) - dims = self.dims - data_type = self.data_type.name - except (AttributeError, TypeError, NullPointerError): - return f"<{self.__class__.__name__} (uninitialized or invalid)>" - return ( - f"<{self.__class__.__name__} dims={dims} " - f"data_type={data_type} " - f"at {c_ptr}>" - ) - - -class Status(CType): - """Wrapper for the `struct vaccel_tf_status` C object. - - Manages the creation and initialization of a C `struct vaccel_tf_status` and - provides access to it through Python properties. - - Inherits: - CType: Abstract base class for defining C data types. - - Attributes: - _error_code (int): The status's error code. - _message (str): The status' message. - _c_obj_ptr (ffi.CData): A double pointer to the underlying - `struct vaccel_tf_status` C object. - """ - - def __init__(self, error_code: int = 0, message: str = ""): - """Initializes a new `Status` object. - - Args: - error_code: The error code to be passed to the C struct. Defaults - to 0. - message: The message to be passed to the C struct. Defaults to "". - """ - self._error_code = error_code - self._message = message - self._c_obj_ptr = ffi.NULL - super().__init__() - - def _init_c_obj(self): - """Initializes the underlying C `struct vaccel_tf_status` object. - - Raises: - FFIError: If status initialization fails. - """ - self._c_obj_ptr = ffi.new("struct vaccel_tf_status **") - ret = lib.vaccel_tf_status_new( - self._c_obj_ptr, self._error_code, self._message.encode() - ) - if ret != 0: - raise FFIError(ret, "Could not initialize status") - - self._c_obj = self._c_obj_ptr[0] - self._c_size = ffi.sizeof("struct vaccel_tf_status") - - @property - def value(self) -> ffi.CData: - """Returns the value of the underlying C struct. - - Returns: - The dereferenced 'struct vaccel_tf_status` - """ - return self._c_ptr_or_raise[0] - - def _del_c_obj(self): - """Deletes the underlying `struct vaccel_tf_status` C object. - - Raises: - FFIError: If status deletion fails. - """ - ret = lib.vaccel_tf_status_delete(self._c_ptr_or_raise) - if ret != 0: - raise FFIError(ret, "Could not delete status") - - def __del__(self): - try: - self._del_c_obj() - except NullPointerError: - pass - except FFIError: - logger.exception("Failed to clean up Status") - - @property - def code(self) -> int: - """The status error code. - - Returns: - The code of the status. - """ - return int(self._c_ptr_or_raise.error_code) - - @property - def message(self) -> str: - """The status message. - - Returns: - The message of the status. - """ - return CStr.from_c_obj(self._c_ptr_or_raise.message).value - - def __repr__(self): - try: - c_ptr = ( - f"0x{int(ffi.cast('uintptr_t', self._c_obj)):x}" - if self._c_obj != ffi.NULL - else "NULL" - ) - code = self.code - message = self.message - except (AttributeError, TypeError, NullPointerError): - return f"<{self.__class__.__name__} (uninitialized or invalid)>" - return ( - f"<{self.__class__.__name__} code={code} " - f"message={message!r} " - f"at {c_ptr}>" - ) - - -class Buffer(CType): - """Wrapper for the `struct vaccel_tf_buffer` C object. - - Manages the creation and initialization of a C `struct vaccel_tf_buffer` and - provides access to it through Python properties. - - Inherits: - CType: Abstract base class for defining C data types. - - Attributes: - _data (bytes | bytearray): The data of the buffer. - _c_data (CBytes): The encapsulated buffer data passed to the C struct. - _c_obj_ptr (ffi.CData): A double pointer to the underlying - `struct vaccel_tf_buffer` C object. - """ - - def __init__(self, data: bytes | bytearray): - """Initializes a new `Buffer` object. - - Args: - data: The buffer data to be passed to the C struct. - """ - self._data = data - self._c_data = None - self._c_obj_ptr = ffi.NULL - super().__init__() - - def _init_c_obj(self): - """Initializes the underlying C `struct vaccel_tf_buffer` object. - - Raises: - FFIError: If buffer initialization fails. - """ - self._c_data = CBytes(self._data) - self._c_obj_ptr = ffi.new("struct vaccel_tf_buffer **") - - ret = lib.vaccel_tf_buffer_new( - self._c_obj_ptr, self._c_data._c_ptr, len(self._c_data) - ) - if ret != 0: - raise FFIError(ret, "Could not initialize buffer") - - self._c_obj = self._c_obj_ptr[0] - self._c_size = ffi.sizeof("struct vaccel_tf_buffer") - - @property - def value(self) -> ffi.CData: - """Returns the value of the underlying C struct. - - Returns: - The dereferenced 'struct vaccel_tf_buffer` - """ - return self._c_ptr_or_raise[0] - - def _del_c_obj(self): - """Deletes the underlying `struct vaccel_tf_buffer` C object. - - Raises: - FFIError: If buffer deletion fails. - """ - c_data = ffi.new("void **") - c_size = ffi.new("size_t *") - ret = lib.vaccel_tf_buffer_take_data( - self._c_ptr_or_raise, c_data, c_size - ) - if ret != 0: - raise FFIError(ret, "Failed to take ownership of buffer data") - - ret = lib.vaccel_tf_buffer_delete(self._c_ptr_or_raise) - if ret != 0: - raise FFIError(ret, "Could not delete buffer") - - def __del__(self): - try: - self._del_c_obj() - except NullPointerError: - pass - except FFIError: - logger.exception("Failed to clean up Buffer") - - @property - def size(self) -> int: - """The buffer size. - - Returns: - The size of the buffer. - """ - return int(self._c_ptr_or_raise.size) - - @property - def data(self) -> bytes: - """The buffer data. - - Returns: - The data of the buffer. - """ - if not self._obj.data or self.size == 0: - return b"" - return ffi.buffer(self._c_ptr_or_raise.data, self.size)[:] - - def __repr__(self): - try: - c_ptr = ( - f"0x{int(ffi.cast('uintptr_t', self._c_obj)):x}" - if self._c_obj != ffi.NULL - else "NULL" - ) - size = self.size - except (AttributeError, TypeError, NullPointerError): - return f"<{self.__class__.__name__} (uninitialized or invalid)>" - return f"<{self.__class__.__name__} size={size} at {c_ptr}>" - - -class TFMixin: - """Mixin providing Tensorflow operations for a `Session`. - - This mixin is intended to be used in combination with `BaseSession` and - should not be instantiated on its own. - - Intended usage: - class Session(BaseSession, TensorflowMixin): - ... - """ - - def tf_model_load(self, resource: Resource) -> Status: - """Performs the Tensorflow model loading operation. - - Wraps the `vaccel_tf_session_load()` C operation. - - Args: - resource: A resource with the model to load. - - Returns: - The status of the operation execution. - - Raises: - FFIError: If the C operation fails. - """ - status = Status() - ret = lib.vaccel_tf_session_load( - self._c_ptr_or_raise, resource._c_ptr, status._c_ptr - ) - if ret != 0: - raise FFIError(ret, "Tensorflow model loading failed") - return status - - def tf_model_run( - self, - resource: Resource, - in_nodes: list[Node], - in_tensors: list[Tensor], - out_nodes: list[Node], - run_options: Buffer | None = None, - ) -> (list[Tensor], Status): - """Performs the Tensorflow model run operation. - - Wraps the `vaccel_tf_session_run()` C operation. - - Args: - resource: A resource with the model to run. - in_nodes: The input nodes for the inference. - in_tensors: The input tensors for the inference. - out_nodes: The output nodes for the inference. - run_options: The inference options. - - Returns: - A tuple containing: - - The output tensors - - The status of the operation execution. - - Raises: - FFIError: If the C operation fails. - """ - run_options_ptr = ( - ffi.NULL if run_options is None else run_options._c_ptr - ) - c_in_nodes = CList(in_nodes) - c_in_tensors = CList.from_ptrs(in_tensors) - c_out_nodes = CList(out_nodes) - c_out_tensors = CList.from_ptrs([Tensor.empty()] * len(c_out_nodes)) - status = Status() - - ret = lib.vaccel_tf_session_run( - self._c_ptr_or_raise, - resource._c_ptr, - run_options_ptr, - c_in_nodes._c_ptr, - c_in_tensors._c_ptr, - len(c_in_nodes), - c_out_nodes._c_ptr, - c_out_tensors._c_ptr, - len(c_out_nodes), - status._c_ptr, - ) - if ret != 0: - raise FFIError(ret, "Tensorflow model run failed") - - out_tensors = [Tensor.from_c_obj(t) for t in c_out_tensors.value] - return (out_tensors, status) - - def tf_model_delete(self, resource: Resource) -> Status: - """Performs the Tensorflow model deletion operation. - - Wraps the `vaccel_tf_session_delete()` C operation. - - Args: - resource: A resource with the model to unload. - - Returns: - The status of the operation execution. - - Raises: - FFIError: If the C operation fails. - """ - status = Status() - ret = lib.vaccel_tf_session_delete( - self._c_ptr_or_raise, resource._c_ptr, status._c_ptr - ) - if ret != 0: - raise FFIError(ret, "Tensorflow model delete failed") - return status diff --git a/vaccel/ops/tf/lite/mixin.py b/vaccel/ops/tf/lite/mixin.py index 53a10c4..6d6549b 100644 --- a/vaccel/ops/tf/lite/mixin.py +++ b/vaccel/ops/tf/lite/mixin.py @@ -22,9 +22,9 @@ class Session(BaseSession, TensorflowLiteMixin): """ def tflite_model_load(self, resource: Resource) -> None: - """Performs the Tensorflow Lite model loading operation. + """Performs the Tensorflow Lite model load operation. - Wraps the `vaccel_tflite_session_load()` C operation. + Wraps the `vaccel_tflite_model_load()` C operation. Args: resource: A resource with the model to load. @@ -32,11 +32,31 @@ def tflite_model_load(self, resource: Resource) -> None: Raises: FFIError: If the C operation fails. """ - ret = lib.vaccel_tflite_session_load( + ret = lib.vaccel_tflite_model_load( self._c_ptr_or_raise, resource._c_ptr ) if ret != 0: - raise FFIError(ret, "Tensorflow Lite model loading failed") + raise FFIError(ret, "Tensorflow Lite model load failed") + + def tflite_model_unload(self, resource: Resource) -> None: + """Performs the Tensorflow Lite model unload operation. + + Wraps the `vaccel_tflite_model_unload()` C operation. + + Args: + resource: A resource with the model to unload. + + Returns: + The status of the operation execution. + + Raises: + FFIError: If the C operation fails. + """ + ret = lib.vaccel_tflite_model_unload( + self._c_ptr_or_raise, resource._c_ptr + ) + if ret != 0: + raise FFIError(ret, "Tensorflow Lite model unload failed") def tflite_model_run( self, @@ -46,7 +66,7 @@ def tflite_model_run( ) -> (list[Tensor], int): """Performs the Tensorflow Lite model run operation. - Wraps the `vaccel_tflite_session_run()` C operation. + Wraps the `vaccel_tflite_model_run()` C operation. Args: resource: A resource with the model to run. @@ -65,7 +85,7 @@ def tflite_model_run( c_out_tensors = CList.from_ptrs([Tensor.empty()] * nr_out_tensors) status = CInt(0, "uint8_t") - ret = lib.vaccel_tflite_session_run( + ret = lib.vaccel_tflite_model_run( self._c_ptr_or_raise, resource._c_ptr, c_in_tensors._c_ptr, @@ -79,23 +99,3 @@ def tflite_model_run( out_tensors = [Tensor.from_c_obj(t) for t in c_out_tensors.value] return (out_tensors, status.value) - - def tflite_model_delete(self, resource: Resource) -> None: - """Performs the Tensorflow Lite model deletion operation. - - Wraps the `vaccel_tflite_session_delete()` C operation. - - Args: - resource: A resource with the model to unload. - - Returns: - The status of the operation execution. - - Raises: - FFIError: If the C operation fails. - """ - ret = lib.vaccel_tflite_session_delete( - self._c_ptr_or_raise, resource._c_ptr - ) - if ret != 0: - raise FFIError(ret, "Tensorflow Lite model delete failed") diff --git a/vaccel/ops/tf/mixin.py b/vaccel/ops/tf/mixin.py index e38b554..f892f7e 100644 --- a/vaccel/ops/tf/mixin.py +++ b/vaccel/ops/tf/mixin.py @@ -25,9 +25,9 @@ class Session(BaseSession, TensorflowMixin): """ def tf_model_load(self, resource: Resource) -> Status: - """Performs the Tensorflow model loading operation. + """Performs the Tensorflow model load operation. - Wraps the `vaccel_tf_session_load()` C operation. + Wraps the `vaccel_tf_model_load()` C operation. Args: resource: A resource with the model to load. @@ -39,11 +39,33 @@ def tf_model_load(self, resource: Resource) -> Status: FFIError: If the C operation fails. """ status = Status() - ret = lib.vaccel_tf_session_load( + ret = lib.vaccel_tf_model_load( self._c_ptr_or_raise, resource._c_ptr, status._c_ptr ) if ret != 0: - raise FFIError(ret, "Tensorflow model loading failed") + raise FFIError(ret, "Tensorflow model load failed") + return status + + def tf_model_unload(self, resource: Resource) -> Status: + """Performs the Tensorflow model unload operation. + + Wraps the `vaccel_tf_model_unload()` C operation. + + Args: + resource: A resource with the model to unload. + + Returns: + The status of the operation execution. + + Raises: + FFIError: If the C operation fails. + """ + status = Status() + ret = lib.vaccel_tf_model_unload( + self._c_ptr_or_raise, resource._c_ptr, status._c_ptr + ) + if ret != 0: + raise FFIError(ret, "Tensorflow model unload failed") return status def tf_model_run( @@ -56,7 +78,7 @@ def tf_model_run( ) -> (list[Tensor], Status): """Performs the Tensorflow model run operation. - Wraps the `vaccel_tf_session_run()` C operation. + Wraps the `vaccel_tf_model_run()` C operation. Args: resource: A resource with the model to run. @@ -82,7 +104,7 @@ def tf_model_run( c_out_tensors = CList.from_ptrs([Tensor.empty()] * len(c_out_nodes)) status = Status() - ret = lib.vaccel_tf_session_run( + ret = lib.vaccel_tf_model_run( self._c_ptr_or_raise, resource._c_ptr, run_options_ptr, @@ -99,25 +121,3 @@ def tf_model_run( out_tensors = [Tensor.from_c_obj(t) for t in c_out_tensors.value] return (out_tensors, status) - - def tf_model_delete(self, resource: Resource) -> Status: - """Performs the Tensorflow model deletion operation. - - Wraps the `vaccel_tf_session_delete()` C operation. - - Args: - resource: A resource with the model to unload. - - Returns: - The status of the operation execution. - - Raises: - FFIError: If the C operation fails. - """ - status = Status() - ret = lib.vaccel_tf_session_delete( - self._c_ptr_or_raise, resource._c_ptr, status._c_ptr - ) - if ret != 0: - raise FFIError(ret, "Tensorflow model delete failed") - return status diff --git a/vaccel/ops/torch/mixin.py b/vaccel/ops/torch/mixin.py index 31c5b27..4850863 100644 --- a/vaccel/ops/torch/mixin.py +++ b/vaccel/ops/torch/mixin.py @@ -22,16 +22,31 @@ class Session(BaseSession, TorchMixin): ... """ - def torch_jitload_forward( + def torch_model_load(self, resource: Resource) -> None: + """Performs the Torch model load operation. + + Wraps the `vaccel_torch_model_load()` C operation. + + Args: + resource: A resource with the model to load. + + Raises: + FFIError: If the C operation fails. + """ + ret = lib.vaccel_torch_model_load(self._c_ptr_or_raise, resource._c_ptr) + if ret != 0: + raise FFIError(ret, "Torch model load failed") + + def torch_model_run( self, resource: Resource, in_tensors: list[Tensor], nr_out_tensors: int = 1, run_options: Buffer | None = None, ) -> list[Tensor]: - """Performs the Torch jitload forward operation. + """Performs the Torch model run operation. - Wraps the `vaccel_torch_jitload_forward()` C operation. + Wraps the `vaccel_torch_model_run()` C operation. Args: resource: A resource with the model to run. @@ -51,7 +66,7 @@ def torch_jitload_forward( c_in_tensors = CList.from_ptrs(in_tensors) c_out_tensors = CList.from_ptrs([Tensor.empty()] * nr_out_tensors) - ret = lib.vaccel_torch_jitload_forward( + ret = lib.vaccel_torch_model_run( self._c_ptr_or_raise, resource._c_ptr, run_options_ptr,