Added tests checking filters compression

.github/workflows/ci.yml

@@ -29,6 +29,7 @@ jobs:
             python-version: 3.6
             BUILD_COMMAND: sdist
             H5PY_OLDER_VERSION: 2.6.0
+            NUMPY_OLDER_VERSION: 1.11.3
 
           - name-suffix: "bdist_wheel-h5py_2.8.0"
             os: ubuntu-latest
@@ -113,6 +114,9 @@ jobs:
 
       - name: Run the tests with oldest available wheel of h5py
         run: |
+          if [ -n "${{ matrix.NUMPY_OLDER_VERSION }}" ]; then
+              pip install --upgrade numpy==${{ matrix.NUMPY_OLDER_VERSION }};
+          fi
           pip install --upgrade h5py==${{ matrix.H5PY_OLDER_VERSION }}
           pip list
           python test/test.py

src/hdf5plugin/test.py

@@ -43,15 +43,20 @@ def setUpClass(cls):
     def tearDownClass(cls):
         shutil.rmtree(cls.tempdir)
 
-    def _test(self, filter_name, dtype=numpy.int32, lossless=True, **options):
+    def _test(self,
+              filter_name,
+              dtype=numpy.int32,
+              lossless=True,
+              compressed=True,
+              **options):
         """Run test for a particular filter
 
         :param str filter_name: The name of the filter to use
         :param Union[None,tuple(int)] options:
             create_dataset's compression_opts argument
         :return: The tuple describing the filter
         """
-        data = numpy.arange(100, dtype=dtype)
+        data = numpy.ones((1000,), dtype=dtype).reshape(100, 10)
         filename = os.path.join(self.tempdir, "test_" + filter_name + ".h5")
 
         args = {"blosc": hdf5plugin.Blosc,
@@ -64,16 +69,24 @@ def _test(self, filter_name, dtype=numpy.int32, lossless=True, **options):
 
         # Write
         f = h5py.File(filename, "w")
-        f.create_dataset("data", data=data, **args)
+        f.create_dataset("data", data=data, chunks=data.shape, **args)
         f.close()
 
         # Read
-        f = h5py.File(filename, "r")
-        saved = f['data'][()]
-        plist = f['data'].id.get_create_plist()
-        filters = [plist.get_filter(i) for i in range(plist.get_nfilters())]
-        f.close()
-
+        with h5py.File(filename, "r") as f:
+            saved = f['data'][()]
+            plist = f['data'].id.get_create_plist()
+            filters = [plist.get_filter(i) for i in range(plist.get_nfilters())]
+
+            if h5py.version.version_tuple >= (2, 10):  # Need read_direct_chunk
+                # Read chunk raw (compressed) data
+                chunk = f['data'].id.read_direct_chunk((0,) * data.ndim)[1]
+
+                if compressed:  # Check if chunk is actually compressed
+                    self.assertLess(len(chunk), data.nbytes)
+                else:
+                    self.assertEqual(len(chunk), data.nbytes)
+
         if lossless:
             self.assertTrue(numpy.array_equal(saved, data))
         else:
@@ -91,13 +104,12 @@ def testBitshuffle(self):
         self._test('bshuf')  # Default options
 
         # Specify options
-        # numpy.int8, int16, int32, int64
-        for dtype in (numpy.int8, numpy.int16, numpy.int32, numpy.int64):
-            for x in range(1, 3):
-                filter_ = self._test('bshuf', dtype, nelems=1024*x, lz4=False)
-                self.assertEqual(filter_[2][3:], (1024*x, 0))
-                filter_ = self._test('bshuf', dtype, nelems=1024*x, lz4=True)
-                self.assertEqual(filter_[2][3:], (1024*x, 2))
+        for lz4 in (False, True):
+            for dtype in (numpy.int8, numpy.int16, numpy.int32, numpy.int64):
+                for nelems in (1024, 2048):
+                    with self.subTest(lz4=lz4, dtype=dtype, nelems=nelems):
+                        filter_ = self._test('bshuf', dtype, compressed=lz4, nelems=nelems, lz4=lz4)
+                        self.assertEqual(filter_[2][3:], (nelems, 2 if lz4 else 0))
 
     def testBlosc(self):
         """Write/read test with blosc filter plugin"""
@@ -108,13 +120,20 @@ def testBlosc(self):
                     hdf5plugin.Blosc.SHUFFLE,
                     hdf5plugin.Blosc.BITSHUFFLE)
         compress = 'blosclz', 'lz4', 'lz4hc', 'snappy', 'zlib', 'zstd'
-        for clevel in range(10):
+        for compression_id, cname in enumerate(compress):
             for shuffle in shuffles:
-                for compression_id, cname in enumerate(compress):
-                    filter_ = self._test(
-                        'blosc', cname=cname, clevel=clevel, shuffle=shuffle)
-                    self.assertEqual(
-                        filter_[2][4:], (clevel, shuffle, compression_id))
+                for clevel in range(10):
+                    with self.subTest(compression=cname,
+                                      shuffle=shuffle,
+                                      clevel=clevel):
+                        filter_ = self._test(
+                            'blosc',
+                            compressed=clevel!=0,  # No compression for clevel=0
+                            cname=cname,
+                            clevel=clevel,
+                            shuffle=shuffle)
+                        self.assertEqual(
+                            filter_[2][4:], (clevel, shuffle, compression_id))
 
     def testLZ4(self):
         """Write/read test with lz4 filter plugin"""
@@ -128,7 +147,10 @@ def testLZ4(self):
                          "FCIDECOMP filter not available")
     def testFciDecomp(self):
         """Write/read test with fcidecomp filter plugin"""
-        self._test('fcidecomp')
+        # Test with supported datatypes
+        for dtype in (numpy.uint8, numpy.uint16, numpy.int8, numpy.int16):
+            with self.subTest(dtype=dtype):
+                self._test('fcidecomp', dtype=dtype)
 
     @unittest.skipUnless(h5py.h5z.filter_avail(hdf5plugin.ZFP_ID),
                          "ZFP filter not available")
@@ -143,9 +165,10 @@ def testZfp(self):
             # Expert: with default parameters
             {'lossless': False, 'minbits': 1, 'maxbits': 16657, 'maxprec': 64, 'minexp': -1074},
             ]
-        for kwargs in tests:
+        for options in tests:
             for dtype in (numpy.float32, numpy.float64):
-                self._test('zfp', dtype=dtype, **kwargs)
+                with self.subTest(options=options, dtype=dtype):
+                    self._test('zfp', dtype=dtype, **options)
 
         self._test('zfp', dtype=numpy.int32, reversible=True)