From b024304a3c0fce5ee5d8141caed8bd6ea8dfe937 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 09:44:13 +0200 Subject: [PATCH 1/6] Remove HDF5Plugin-Zstandard source --- lib/HDF5Plugin-Zstandard/.gitignore | 33 ---- lib/HDF5Plugin-Zstandard/CMakeLists.txt | 31 ---- lib/HDF5Plugin-Zstandard/LICENSE | 201 ----------------------- lib/HDF5Plugin-Zstandard/README.md | 36 ---- lib/HDF5Plugin-Zstandard/zstd_h5plugin.c | 81 --------- lib/HDF5Plugin-Zstandard/zstd_h5plugin.h | 22 --- 6 files changed, 404 deletions(-) delete mode 100644 lib/HDF5Plugin-Zstandard/.gitignore delete mode 100644 lib/HDF5Plugin-Zstandard/CMakeLists.txt delete mode 100644 lib/HDF5Plugin-Zstandard/LICENSE delete mode 100644 lib/HDF5Plugin-Zstandard/README.md delete mode 100644 lib/HDF5Plugin-Zstandard/zstd_h5plugin.c delete mode 100644 lib/HDF5Plugin-Zstandard/zstd_h5plugin.h diff --git a/lib/HDF5Plugin-Zstandard/.gitignore b/lib/HDF5Plugin-Zstandard/.gitignore deleted file mode 100644 index f805e810..00000000 --- a/lib/HDF5Plugin-Zstandard/.gitignore +++ /dev/null @@ -1,33 +0,0 @@ -# Object files -*.o -*.ko -*.obj -*.elf - -# Precompiled Headers -*.gch -*.pch - -# Libraries -*.lib -*.a -*.la -*.lo - -# Shared objects (inc. Windows DLLs) -*.dll -*.so -*.so.* -*.dylib - -# Executables -*.exe -*.out -*.app -*.i*86 -*.x86_64 -*.hex - -# Debug files -*.dSYM/ -*.su diff --git a/lib/HDF5Plugin-Zstandard/CMakeLists.txt b/lib/HDF5Plugin-Zstandard/CMakeLists.txt deleted file mode 100644 index f8a85d7a..00000000 --- a/lib/HDF5Plugin-Zstandard/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -cmake_minimum_required(VERSION 2.8.10) -project(zstd_hdf5) - -# options -set(PLUGIN_INSTALL_PATH "/usr/local/hdf5/lib/plugin" CACHE PATH - "Where to install the dynamic HDF5-plugin") - -# sources -set(SOURCES zstd_h5plugin.c) -set(PLUGIN_SOURCES zstd_h5plugin.c) - -# dependencies -if(MSVC) - # FindHDF5.cmake does not find Windows installations. Try to - # use an environment variable instead until the official "find" - # file can be updated for Windows. - # - # Note that you have to set this environment variable by hand. - file(TO_CMAKE_PATH "$ENV{HDF5_DIR}" HDF5_HINT) - set(HDF5_DIR ${HDF5_HINT} CACHE STRING "Path to HDF5 CMake config directory.") - find_package(HDF5 REQUIRED HINTS ${HDF5_DIR}) -else(MSVC) - find_package(HDF5 REQUIRED) -endif(MSVC) -include_directories(${HDF5_INCLUDE_DIRS}) - -# HDF5 plugin as shared library -add_library(zstd_h5_plugin_shared SHARED ${PLUGIN_SOURCES}) -set_target_properties(zstd_h5_plugin_shared PROPERTIES OUTPUT_NAME H5Zzstd) -target_link_libraries(zstd_h5_plugin_shared zstd ${HDF5_LIBRARIES}) -install(TARGETS zstd_h5_plugin_shared DESTINATION ${PLUGIN_INSTALL_PATH} COMPONENT HDF5_FILTER_DEV) diff --git a/lib/HDF5Plugin-Zstandard/LICENSE b/lib/HDF5Plugin-Zstandard/LICENSE deleted file mode 100644 index 8dada3ed..00000000 --- a/lib/HDF5Plugin-Zstandard/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/lib/HDF5Plugin-Zstandard/README.md b/lib/HDF5Plugin-Zstandard/README.md deleted file mode 100644 index 43a8fcd2..00000000 --- a/lib/HDF5Plugin-Zstandard/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# HDF5Plugin-Zstandard - -*HDF5* is a data model, library, and file format for storing and -managing data. It supports an unlimited variety of datatypes, and is -designed for flexible and efficient I/O and for high volume and -complex data. HDF5 is portable and is extensible, allowing -applications to evolve in their use of HDF5. The HDF5 Technology suite -includes tools and applications for managing, manipulating, viewing, -and analyzing data in the HDF5 format. - -https://support.hdfgroup.org/HDF5/ - ---- - -*Zstandard* is a real-time compression algorithm, providing high -compression ratios. It offers a very wide range of compression/speed -trade-off, while being backed by a very fast decoder. - -http://www.zstd.net - ---- - -This repository provides an implementation of Zstandard compression -filter plugin for HDF5 with the assigned filter code 32015. - ---- - -## Build - -This plugin can be built with cmake and installed as shared library to `/usr/local/hdf5/lib/plugin` (or a custom path). - -```bash -cmake . -make -sudo make install -``` diff --git a/lib/HDF5Plugin-Zstandard/zstd_h5plugin.c b/lib/HDF5Plugin-Zstandard/zstd_h5plugin.c deleted file mode 100644 index 5a3dfd38..00000000 --- a/lib/HDF5Plugin-Zstandard/zstd_h5plugin.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include "zstd_h5plugin.h" -#include "zstd.h" - -#define ZSTD_FILTER 32015 - -DLL_EXPORT size_t zstd_filter(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf) -{ - void *outbuf = NULL; /* Pointer to new output buffer */ - void *inbuf = NULL; /* Pointer to input buffer */ - inbuf = *buf; - - size_t ret_value; - size_t origSize = nbytes; /* Number of bytes for output (compressed) buffer */ - - if (flags & H5Z_FLAG_REVERSE) - { - size_t decompSize = ZSTD_getDecompressedSize(*buf, origSize); - if (NULL == (outbuf = malloc(decompSize))) - goto error; - - decompSize = ZSTD_decompress(outbuf, decompSize, inbuf, origSize); - - free(*buf); - *buf = outbuf; - outbuf = NULL; - ret_value = (size_t)decompSize; - } - else - { - int aggression; - if (cd_nelmts > 0) - aggression = (int)cd_values[0]; - else - aggression = ZSTD_CLEVEL_DEFAULT; - if (aggression < 1 /*ZSTD_minCLevel()*/) - aggression = 1 /*ZSTD_minCLevel()*/; - else if (aggression > ZSTD_maxCLevel()) - aggression = ZSTD_maxCLevel(); - - size_t compSize = ZSTD_compressBound(origSize); - if (NULL == (outbuf = malloc(compSize))) - goto error; - - compSize = ZSTD_compress(outbuf, compSize, inbuf, origSize, aggression); - - free(*buf); - *buf = outbuf; - *buf_size = compSize; - outbuf = NULL; - ret_value = compSize; - } - if (outbuf != NULL) - free(outbuf); - return ret_value; - -error: - return 0; -} - -const H5Z_class_t zstd_H5Filter = -{ - H5Z_CLASS_T_VERS, - (H5Z_filter_t)(ZSTD_FILTER), - 1, 1, - "Zstandard compression: http://www.zstd.net", - NULL, NULL, - (H5Z_func_t)(zstd_filter) -}; - -DLL_EXPORT H5PL_type_t H5PLget_plugin_type(void) -{ - return H5PL_TYPE_FILTER; -} - -DLL_EXPORT const void* H5PLget_plugin_info(void) -{ - return &zstd_H5Filter; -} diff --git a/lib/HDF5Plugin-Zstandard/zstd_h5plugin.h b/lib/HDF5Plugin-Zstandard/zstd_h5plugin.h deleted file mode 100644 index e29c0689..00000000 --- a/lib/HDF5Plugin-Zstandard/zstd_h5plugin.h +++ /dev/null @@ -1,22 +0,0 @@ -#include "hdf5.h" - -#if defined(_MSC_VER) - #define DLL_EXPORT __declspec(dllexport) -#else - #define DLL_EXPORT -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -DLL_EXPORT size_t zstd_filter(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf); - -DLL_EXPORT H5PL_type_t H5PLget_plugin_type(void); -DLL_EXPORT const void* H5PLget_plugin_info(void); - -#ifdef __cplusplus -} -#endif From 8a4c7b12d494ac425569b1532b7d0c6fa7a380d6 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 09:44:54 +0200 Subject: [PATCH 2/6] Add ZSTD from https://github.com/HDFGroup/hdf5_plugins commit 5573db8 --- .../ZSTD/Additional_Legal/LICENSE | 201 ++++++++++++++++++ .../PyTables_Copyrights_and_Licenses.txt | 36 ++++ lib/hdf5_plugins/ZSTD/README.txt | 22 ++ lib/hdf5_plugins/ZSTD/src/H5Zzstd.c | 123 +++++++++++ 4 files changed, 382 insertions(+) create mode 100644 lib/hdf5_plugins/ZSTD/Additional_Legal/LICENSE create mode 100644 lib/hdf5_plugins/ZSTD/Additional_Legal/PyTables_Copyrights_and_Licenses.txt create mode 100644 lib/hdf5_plugins/ZSTD/README.txt create mode 100644 lib/hdf5_plugins/ZSTD/src/H5Zzstd.c diff --git a/lib/hdf5_plugins/ZSTD/Additional_Legal/LICENSE b/lib/hdf5_plugins/ZSTD/Additional_Legal/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/lib/hdf5_plugins/ZSTD/Additional_Legal/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/hdf5_plugins/ZSTD/Additional_Legal/PyTables_Copyrights_and_Licenses.txt b/lib/hdf5_plugins/ZSTD/Additional_Legal/PyTables_Copyrights_and_Licenses.txt new file mode 100644 index 00000000..558344cf --- /dev/null +++ b/lib/hdf5_plugins/ZSTD/Additional_Legal/PyTables_Copyrights_and_Licenses.txt @@ -0,0 +1,36 @@ +PyTables Copyright Statement +============================ + +Copyright Notice and Statement for PyTables Software Library and Utilities: + +Copyright (c) 2002, 2003, 2004 Francesc Altet +Copyright (c) 2005, 2006, 2007 Carabos Coop. V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +c. Neither the name of the Carabos Coop. V. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/hdf5_plugins/ZSTD/README.txt b/lib/hdf5_plugins/ZSTD/README.txt new file mode 100644 index 00000000..0a78ae77 --- /dev/null +++ b/lib/hdf5_plugins/ZSTD/README.txt @@ -0,0 +1,22 @@ +Building this filter/example requires knowledge of the hdf5 and the zstd +installation. + +For autotools configure, you must supply these using + --with-hdf5 and --with-zstdlib. +Example (in the build folder): + ../configure --with-hdf5=/temp/hdf5 --with-zstdlib=/temp/zstd + +For CMake, you must supply the location of the cmake configuration files + in environment variables. + In addition, CMake options "H5PL_BUILD_EXAMPLES" and "H5PL_BUILD_TESTING" must + be set "ON" in order to build the example and run the tests. +Example: + set(ENV{HDF5_ROOT} "/temp/hdf5/") + set(ENV{ZSTD_ROOT} "/temp/zstd/") + set(ENV{LD_LIBRARY_PATH} "/temp/zstd/lib:/temp/hdf5/lib") + set(ADD_BUILD_OPTIONS "-DH5PL_BUILD_EXAMPLES:BOOL=ON -DH5PL_BUILD_TESTING:BOOL=ON") + + For non-cmake built hdf5 or zstd, use the location of the include/lib + folders: + set(ENV{HDF5_ROOT} "/temp/hdf5") + set(ENV{ZSTD_ROOT} "/temp/zstd") diff --git a/lib/hdf5_plugins/ZSTD/src/H5Zzstd.c b/lib/hdf5_plugins/ZSTD/src/H5Zzstd.c new file mode 100644 index 00000000..b96fa84c --- /dev/null +++ b/lib/hdf5_plugins/ZSTD/src/H5Zzstd.c @@ -0,0 +1,123 @@ +/* + * ZSTD HDF5 filter + * + * Author: Mark Rivers + * Created: 2019 + * + * + */ + +#include +#include +#include +#include +#include + +#include "H5PLextern.h" + +#include "zstd.h" + +static size_t H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], + size_t nbytes, size_t *buf_size, void **buf); + +#define H5Z_FILTER_ZSTD 32015 + +#define PUSH_ERR(func, minor, str) \ + H5Epush(H5E_DEFAULT, __FILE__, func, __LINE__, H5E_ERR_CLS, H5E_PLINE, minor, str) + +const H5Z_class2_t H5Z_ZSTD[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)H5Z_FILTER_ZSTD, /* Filter id number */ +#ifdef FILTER_DECODE_ONLY + 0, /* encoder_present flag (false is not available) */ +#else + 1, /* encoder_present flag (set to true) */ +#endif + 1, /* decoder_present flag (set to true) */ + "HDF5 zstd filter; see " + "https://github.com/HDFGroup/hdf5_plugins/blob/master/docs/RegisteredFilterPlugins.md", + /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_zstd, /* The actual filter function */ +}}; + +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} +const void * +H5PLget_plugin_info(void) +{ + return H5Z_ZSTD; +} + +static size_t +H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + size_t buf_size_out = 0; + size_t origSize = nbytes; /* Number of bytes for output (compressed) buffer */ + void *outbuf = NULL; + void *inbuf = NULL; /* Pointer to input buffer */ + inbuf = *buf; + + if (flags & H5Z_FLAG_REVERSE) { + /* We're decompressing */ + size_t decompSize = ZSTD_getFrameContentSize(*buf, origSize); + if (NULL == (outbuf = malloc(decompSize))) + goto error; + + decompSize = ZSTD_decompress(outbuf, decompSize, inbuf, origSize); + +#ifdef ZSTD_DEBUG + fprintf(stderr, " decompressing nbytes: %ld\n", decompSize); +#endif + + buf_size_out = decompSize; + } + else { + /* We're compressing */ + /* + * cd_values[0] = aggression + * + * As of Zstandard v1.5.7 + * ZSTD_minCLevel() == -1<<17 == -131072 + * ZSTD_maxCLevel() == 22 + * + * Negative compression levels are faster at the cost of compression + * aggression >= 20 require more memory + */ + int aggression; + if (cd_nelmts > 0) + aggression = (int)cd_values[0]; + else + aggression = ZSTD_CLEVEL_DEFAULT; + if (aggression < ZSTD_minCLevel()) + aggression = ZSTD_minCLevel(); + else if (aggression > ZSTD_maxCLevel()) + aggression = ZSTD_maxCLevel(); + + size_t compSize = ZSTD_compressBound(origSize); + if (NULL == (outbuf = malloc(compSize))) + goto error; + + compSize = ZSTD_compress(outbuf, compSize, inbuf, origSize, aggression); + +#ifdef ZSTD_DEBUG + fprintf(stderr, " compressing nbytes: %ld\n", compSize); +#endif + + buf_size_out = compSize; + } + free(*buf); + *buf = outbuf; + *buf_size = buf_size_out; + return buf_size_out; + +error: + if (outbuf) + free(outbuf); + return 0; +} From c37af676f8c544326b07aecf794809f5624fe852 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 09:51:32 +0200 Subject: [PATCH 3/6] use zstd filter source from hdf5_plugins --- setup.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 79e9752b..e943ae51 100644 --- a/setup.py +++ b/setup.py @@ -1243,15 +1243,13 @@ def _get_blosc2_plugin(): PLUGIN_LIB_DEPENDENCIES["blosc2"] = "lz4", "zlib", "zstd" -def _get_zstandard_plugin(): - """HDF5Plugin-Zstandard plugin build config""" - zstandard_dir = "lib/HDF5Plugin-Zstandard" - +def _get_zstd_plugin(): + """Zstd plugin build config""" return HDF5PluginExtension( "hdf5plugin.plugins.libh5zstd", - sources=[f"{zstandard_dir}/zstd_h5plugin.c"], + sources=["lib/hdf5_plugins/ZSTD/src/H5Zzstd.c"], extra_objects=get_clib_config("zstd", "extra_objects"), - include_dirs=[zstandard_dir] + get_clib_config("zstd", "include_dirs"), + include_dirs=get_clib_config("zstd", "include_dirs"), extra_link_args=get_clib_config("zstd", "extra_link_args"), libraries=get_clib_config("zstd", "libraries"), ) @@ -1504,7 +1502,7 @@ def _get_sperr_plugin(): "sz": _get_sz_plugin, "sz3": _get_sz3_plugin, "zfp": _get_h5zfp_plugin, - "zstd": _get_zstandard_plugin, + "zstd": _get_zstd_plugin, } PLUGIN_NAMES = set(_EMBEDDED_PLUGIN_EXTENSIONS.keys()) From b0fe53f4621859cc78923f1c2991fa8c68c5938c Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 09:52:19 +0200 Subject: [PATCH 4/6] update zstd filter upstream documentation --- doc/information.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/information.rst b/doc/information.rst index 38a1e3aa..d81ea7d7 100644 --- a/doc/information.rst +++ b/doc/information.rst @@ -83,8 +83,8 @@ HDF5 compression filters and compression libraries sources were obtained from: (commit `4bbe9df7e4bcb `_) using `SZ3 `_ and ZStd. * `HDF5-ZFP plugin `_ (v1.1.1) using ZFP. -* `HDF5Plugin-Zstandard `_ - (commit `d5afdb5 `_) using ZStd. +* `ZStd plugin `_ + (commit `5573db8 `_) using ZStd. Sources of compression libraries shared accross multiple filters were obtained from: From 42e85a980cb804ab589492c565b8e87a881df1f5 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 10:28:38 +0200 Subject: [PATCH 5/6] Add support for Zstd's negative compression ratio --- src/hdf5plugin/_filters.py | 24 +++++++++++++++++------- src/hdf5plugin/test.py | 18 ++++++++++++++---- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/hdf5plugin/_filters.py b/src/hdf5plugin/_filters.py index a60b6071..52be8c3a 100644 --- a/src/hdf5plugin/_filters.py +++ b/src/hdf5plugin/_filters.py @@ -1270,25 +1270,35 @@ class Zstd(FilterBase): compression=hdf5plugin.Zstd(clevel=22)) f.close() - :param clevel: Compression level from 1 (lowest compression) to 22 (maximum compression). - Ultra compression extends from 20 through 22. Default: 3. + :param clevel: Compression level from -131072 (lowest compression) to 22 (maximum compression). + Negative compression levels offer faster compression and decompression speed at the cost of compression ratio. + Compression levels from 20 to 22 offer better compression ratio at the expense of requiring more memory. + Default: 3. """ filter_name = "zstd" filter_id = ZSTD_ID + # As of Zstandard v1.5.7: ZSTD_minCLevel() -> -1<<17 = -131072 + _ZSTD_MIN_CLEVEL = -131072 + _ZSTD_MAX_CLEVEL = 22 + def __init__(self, clevel: int = 3): - if not 1 <= clevel <= 22: - raise ValueError("clevel must be in the range [1, 22]") + if not self._ZSTD_MIN_CLEVEL <= clevel <= self._ZSTD_MAX_CLEVEL: + raise ValueError( + f"clevel must be in the range [{self._ZSTD_MIN_CLEVEL}, {self._ZSTD_MAX_CLEVEL}]" + ) + clevel_uint32 = struct.unpack("I", struct.pack("i", clevel))[0] super().__init__( - filter_options=(clevel,), + filter_options=(clevel_uint32,), config={"clevel": clevel}, ) @property def clevel(self) -> int: - """Compression level from 1 (lowest compression) to 22 (maximum compression)""" - return self.filter_options[0] + """Compression level from -131072 (lowest compression) to 22 (maximum compression)""" + clevel = int(struct.unpack("i", struct.pack("I", self.filter_options[0]))[0]) + return clevel @classmethod def _from_filter_options(cls, filter_options: tuple[int, ...]) -> Zstd: diff --git a/src/hdf5plugin/test.py b/src/hdf5plugin/test.py index bbfc0f61..2848fb3c 100644 --- a/src/hdf5plugin/test.py +++ b/src/hdf5plugin/test.py @@ -406,11 +406,21 @@ def testZfp(self): def testZstd(self): """Write/read test with Zstd filter plugin""" self._test("zstd") - tests = [{"clevel": 3}, {"clevel": 22}] - for options in tests: + tests: list[tuple[int, str | bool]] = [ + (-131072, "nocheck"), # Do not check if data is compressed for faster mode + (-1, True), + (3, True), + (22, True), + ] + for clevel, compressed in tests: for dtype in (numpy.float32, numpy.float64): - with self.subTest(options=options, dtype=dtype): - self._test("zstd", dtype=dtype, options=options) + with self.subTest(clevel=clevel, dtype=dtype): + self._test( + "zstd", + dtype=dtype, + options={"clevel": clevel}, + compressed=compressed, + ) class TestStrings(unittest.TestCase): From 58afed305cad59c3b13b9b19f23ab16ea7f011f2 Mon Sep 17 00:00:00 2001 From: Thomas VINCENT Date: Tue, 23 Jun 2026 16:42:21 +0200 Subject: [PATCH 6/6] use helper function to handle int<->uint32 conversion --- src/hdf5plugin/_filters.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/hdf5plugin/_filters.py b/src/hdf5plugin/_filters.py index 52be8c3a..eca6f3ab 100644 --- a/src/hdf5plugin/_filters.py +++ b/src/hdf5plugin/_filters.py @@ -1258,6 +1258,19 @@ def _from_filter_options(cls, filter_options: tuple[int, ...]) -> SZ3: raise ValueError(f"Unsupported sz_mode: {sz_mode}") +def _to_uint32(value: int) -> int: + """Cast an integer to a uint32 and store it as a Python int + + Useful to pass negative integers as hdf5 filter options (uint32) + """ + return int(struct.unpack("I", struct.pack("i", value))[0]) + + +def _from_uint32(value: int) -> int: + """Cast value stored as uint32 to a signed int""" + return int(struct.unpack("i", struct.pack("I", value))[0]) + + class Zstd(FilterBase): """``h5py.Group.create_dataset``'s compression arguments for using Zstd filter. @@ -1288,7 +1301,7 @@ def __init__(self, clevel: int = 3): raise ValueError( f"clevel must be in the range [{self._ZSTD_MIN_CLEVEL}, {self._ZSTD_MAX_CLEVEL}]" ) - clevel_uint32 = struct.unpack("I", struct.pack("i", clevel))[0] + clevel_uint32 = _to_uint32(clevel) super().__init__( filter_options=(clevel_uint32,), config={"clevel": clevel}, @@ -1297,7 +1310,7 @@ def __init__(self, clevel: int = 3): @property def clevel(self) -> int: """Compression level from -131072 (lowest compression) to 22 (maximum compression)""" - clevel = int(struct.unpack("i", struct.pack("I", self.filter_options[0]))[0]) + clevel = _from_uint32(self.filter_options[0]) return clevel @classmethod