Hi,

Im getting the following error while trying to build_ext.

(venv) [arul@arul-desktop-01 test_1t]$ gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC -I/usr/local/cuda-11.7/targets/x86_64-linux/include/ -I/home/arul/PycharmProjects/test_1t/venv/include -I/usr/local/include/python3.8 -c ./src/nvcomp.cpp -o build/temp.linux-x86_64-3.8/./src/nvcomp.o
./src/nvcomp.cpp:1031:11: error: ‘LZ4Compressor’ in namespace ‘nvcomp’ does not name a type
nvcomp::LZ4Compressor c;
^~~~~~~~~~~~~
./src/nvcomp.cpp:1045:11: error: ‘LZ4Decompressor’ in namespace ‘nvcomp’ does not name a type
nvcomp::LZ4Decompressor d;
^~~~~~~~~~~~~~~
./src/nvcomp.cpp:1058:11: error: ‘CascadedCompressor’ in namespace ‘nvcomp’ does not name a type
nvcomp::CascadedCompressor c;
^~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp:1058:3: note: suggested alternative: ‘CascadedManager’
nvcomp::CascadedCompressor c;
^~~~~~
CascadedManager
./src/nvcomp.cpp:1072:11: error: ‘CascadedDecompressor’ in namespace ‘nvcomp’ does not name a type
nvcomp::CascadedDecompressor d;
^~~~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp:1072:3: note: suggested alternative: ‘CascadedManager’
nvcomp::CascadedDecompressor d;
^~~~~~
CascadedManager
./src/nvcomp.cpp:1630:74: error: use of enum ‘nvcompError_t’ without previous declaration
static CYTHON_INLINE PyObject __Pyx_PyInt_From_enum__nvcompError_t(enum nvcompError_t value);
^~~~~~~~~~~~~
./src/nvcomp.cpp: In function ‘int pyx_pf_6nvcomp_13LZ4Compressor___cinit(__pyx_obj_6nvcomp_LZ4Compressor, size_t)’:
./src/nvcomp.cpp:2432:11: error: ‘LZ4Compressor’ is not a member of ‘nvcomp’
nvcomp::LZ4Compressor __pyx_t_1;
^~~~~~~~~~~~~
./src/nvcomp.cpp:2432:26: error: ‘__pyx_t_1’ was not declared in this scope
nvcomp::LZ4Compressor __pyx_t_1;
^~~~~~~~~
./src/nvcomp.cpp:2432:26: note: suggested alternative: ‘__pyx_k_v’
nvcomp::LZ4Compressor __pyx_t_1;
^~~~~~~~~
__pyx_k_v
./src/nvcomp.cpp:2446:21: error: expected type-specifier
__pyx_t_1 = new nvcomp::LZ4Compressor(__pyx_v_chunk_size);
^~~~~~
./src/nvcomp.cpp:2451:17: error: ‘struct __pyx_obj_6nvcomp_LZ4Compressor’ has no member named ‘c’
__pyx_v_self->c = __pyx_t_1;
^
./src/nvcomp.cpp: In function ‘void pyx_pf_6nvcomp_13LZ4Compressor_2__dealloc(__pyx_obj_6nvcomp_LZ4Compressor)’:
./src/nvcomp.cpp:2502:24: error: ‘struct __pyx_obj_6nvcomp_LZ4Compressor’ has no member named ‘c’
delete __pyx_v_self->c;
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_13LZ4Compressor_4configure(__pyx_obj_6nvcomp_LZ4Compressor, PyObject, PyObject, PyObject)’:
./src/nvcomp.cpp:2655:19: error: ‘struct __pyx_obj_6nvcomp_LZ4Compressor’ has no member named ‘c’
__pyx_v_self->c->configure(((size_t)__pyx_t_3), ((size_t )__pyx_v_temp_bytes_ptr), ((size_t )__pyx_v_out_bytes_ptr));
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_13LZ4Compressor_6compress_async(__pyx_obj_6nvcomp_LZ4Compressor, PyObject, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, uintptr_t)’:
./src/nvcomp.cpp:2910:19: error: ‘struct __pyx_obj_6nvcomp_LZ4Compressor’ has no member named ‘c’
__pyx_v_self->c->compress_async(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_3), ((void )__pyx_v_temp_ptr), ((size_t)__pyx_t_4), ((void )__pyx_v_out_ptr), ((size_t )__pyx_v_out_bytes_ptr), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘int pyx_pf_6nvcomp_15LZ4Decompressor___cinit(__pyx_obj_6nvcomp_LZ4Decompressor)’:
./src/nvcomp.cpp:3077:11: error: ‘LZ4Decompressor’ is not a member of ‘nvcomp’
nvcomp::LZ4Decompressor __pyx_t_1;
^~~~~~~~~~~~~~~
./src/nvcomp.cpp:3077:28: error: ‘__pyx_t_1’ was not declared in this scope
nvcomp::LZ4Decompressor __pyx_t_1;
^~~~~~~~~
./src/nvcomp.cpp:3077:28: note: suggested alternative: ‘__pyx_k_v’
nvcomp::LZ4Decompressor __pyx_t_1;
^~~~~~~~~
__pyx_k_v
./src/nvcomp.cpp:3091:21: error: expected type-specifier
__pyx_t_1 = new nvcomp::LZ4Decompressor();
^~~~~~
./src/nvcomp.cpp:3096:17: error: ‘struct __pyx_obj_6nvcomp_LZ4Decompressor’ has no member named ‘d’
__pyx_v_self->d = __pyx_t_1;
^
./src/nvcomp.cpp: In function ‘void pyx_pf_6nvcomp_15LZ4Decompressor_2__dealloc(__pyx_obj_6nvcomp_LZ4Decompressor)’:
./src/nvcomp.cpp:3147:24: error: ‘struct __pyx_obj_6nvcomp_LZ4Decompressor’ has no member named ‘d’
delete __pyx_v_self->d;
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_f_6nvcomp_15LZ4Decompressor_configure(__pyx_obj_6nvcomp_LZ4Decompressor, PyObject, PyObject, PyObject, PyObject*, int, __pyx_opt_args_6nvcomp_15LZ4Decompressor_configure*)’:
./src/nvcomp.cpp:3341:19: error: ‘struct __pyx_obj_6nvcomp_LZ4Decompressor’ has no member named ‘d’
__pyx_v_self->d->configure(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_9), ((size_t )__pyx_v_temp_bytes_ptr), ((size_t )__pyx_v_out_bytes_ptr), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_15LZ4Decompressor_6decompress_async(__pyx_obj_6nvcomp_LZ4Decompressor, PyObject, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, uintptr_t)’:
./src/nvcomp.cpp:3730:19: error: ‘struct __pyx_obj_6nvcomp_LZ4Decompressor’ has no member named ‘d’
__pyx_v_self->d->decompress_async(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_3), ((void )__pyx_v_temp_ptr), ((size_t)__pyx_t_4), ((void )__pyx_v_out_ptr), ((size_t)__pyx_t_5), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘int pyx_pf_6nvcomp_18CascadedCompressor___cinit(__pyx_obj_6nvcomp_CascadedCompressor, nvcompType_t, int, int, bool)’:
./src/nvcomp.cpp:3968:11: error: ‘CascadedCompressor’ is not a member of ‘nvcomp’
nvcomp::CascadedCompressor __pyx_t_2;
^~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp:3968:11: note: suggested alternative: ‘CascadedManager’
nvcomp::CascadedCompressor __pyx_t_2;
^~~~~~~~~~~~~~~~~~
CascadedManager
./src/nvcomp.cpp:3968:31: error: ‘__pyx_t_2’ was not declared in this scope
nvcomp::CascadedCompressor __pyx_t_2;
^~~~~~~~~
./src/nvcomp.cpp:3968:31: note: suggested alternative: ‘__pyx_t_1’
nvcomp::CascadedCompressor __pyx_t_2;
^~~~~~~~~
__pyx_t_1
./src/nvcomp.cpp:3984:21: error: expected type-specifier
__pyx_t_2 = new nvcomp::CascadedCompressor(__pyx_v_t, __pyx_v_num_RLEs, __pyx_v_num_deltas, __pyx_t_1);
^~~~~~
./src/nvcomp.cpp:3990:17: error: ‘struct __pyx_obj_6nvcomp_CascadedCompressor’ has no member named ‘c’
__pyx_v_self->c = __pyx_t_2;
^
./src/nvcomp.cpp: In function ‘void pyx_pf_6nvcomp_18CascadedCompressor_2__dealloc(__pyx_obj_6nvcomp_CascadedCompressor)’:
./src/nvcomp.cpp:4042:24: error: ‘struct __pyx_obj_6nvcomp_CascadedCompressor’ has no member named ‘c’
delete __pyx_v_self->c;
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_18CascadedCompressor_4configure(__pyx_obj_6nvcomp_CascadedCompressor, PyObject, PyObject, PyObject)’:
./src/nvcomp.cpp:4195:19: error: ‘struct __pyx_obj_6nvcomp_CascadedCompressor’ has no member named ‘c’
__pyx_v_self->c->configure(((size_t)__pyx_t_3), ((size_t )__pyx_v_temp_bytes_ptr), ((size_t )__pyx_v_out_bytes_ptr));
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_18CascadedCompressor_6compress_async(__pyx_obj_6nvcomp_CascadedCompressor, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, uintptr_t)’:
./src/nvcomp.cpp:4450:19: error: ‘struct __pyx_obj_6nvcomp_CascadedCompressor’ has no member named ‘c’
__pyx_v_self->c->compress_async(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_3), ((void )__pyx_v_temp_ptr), ((size_t)__pyx_t_4), ((void )__pyx_v_out_ptr), ((size_t )__pyx_v_out_bytes_ptr), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘int pyx_pf_6nvcomp_20CascadedDecompressor___cinit(__pyx_obj_6nvcomp_CascadedDecompressor)’:
./src/nvcomp.cpp:4617:11: error: ‘CascadedDecompressor’ is not a member of ‘nvcomp’
nvcomp::CascadedDecompressor __pyx_t_1;
^~~~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp:4617:11: note: suggested alternative: ‘CascadedManager’
nvcomp::CascadedDecompressor __pyx_t_1;
^~~~~~~~~~~~~~~~~~~~
CascadedManager
./src/nvcomp.cpp:4617:33: error: ‘__pyx_t_1’ was not declared in this scope
nvcomp::CascadedDecompressor __pyx_t_1;
^~~~~~~~~
./src/nvcomp.cpp:4617:33: note: suggested alternative: ‘__pyx_k_v’
nvcomp::CascadedDecompressor __pyx_t_1;
^~~~~~~~~
__pyx_k_v
./src/nvcomp.cpp:4631:21: error: expected type-specifier
__pyx_t_1 = new nvcomp::CascadedDecompressor();
^~~~~~
./src/nvcomp.cpp:4636:17: error: ‘struct __pyx_obj_6nvcomp_CascadedDecompressor’ has no member named ‘d’
__pyx_v_self->d = __pyx_t_1;
^
./src/nvcomp.cpp: In function ‘void pyx_pf_6nvcomp_20CascadedDecompressor_2__dealloc(__pyx_obj_6nvcomp_CascadedDecompressor)’:
./src/nvcomp.cpp:4687:24: error: ‘struct __pyx_obj_6nvcomp_CascadedDecompressor’ has no member named ‘d’
delete __pyx_v_self->d;
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_f_6nvcomp_20CascadedDecompressor_configure(__pyx_obj_6nvcomp_CascadedDecompressor, PyObject, PyObject, PyObject, PyObject, int, __pyx_opt_args_6nvcomp_20CascadedDecompressor_configure*)’:
./src/nvcomp.cpp:4881:19: error: ‘struct __pyx_obj_6nvcomp_CascadedDecompressor’ has no member named ‘d’
__pyx_v_self->d->configure(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_9), ((size_t )__pyx_v_temp_bytes_ptr), ((size_t )__pyx_v_out_bytes_ptr), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_pf_6nvcomp_20CascadedDecompressor_6decompress_async(__pyx_obj_6nvcomp_CascadedDecompressor, PyObject, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, uintptr_t)’:
./src/nvcomp.cpp:5270:19: error: ‘struct __pyx_obj_6nvcomp_CascadedDecompressor’ has no member named ‘d’
__pyx_v_self->d->decompress_async(((void )__pyx_v_in_ptr), ((size_t)__pyx_t_3), ((void )__pyx_v_temp_ptr), ((size_t)__pyx_t_4), ((void )__pyx_v_out_ptr), ((size_t)__pyx_t_5), ((cudaStream_t)__pyx_v_stream));
^
./src/nvcomp.cpp: In function ‘PyObject __pyx_f_6nvcomp_batchedLZ4CompressGetTempSize(PyObject, PyObject, PyObject*, int)’:
./src/nvcomp.cpp:5471:84: error: could not convert ‘(size_t*)__pyx_v_temp_bytes_ptr’ from ‘size_t*’ {aka ‘long unsigned int*’} to ‘nvcompBatchedLZ4Opts_t’
nvcompBatchedLZ4CompressGetTempSize(((size_t)__pyx_t_3), ((size_t)__pyx_t_4), ((size_t )__pyx_v_temp_bytes_ptr));
~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp: In function ‘PyObject __pyx_f_6nvcomp_batchedLZ4CompressGetMaxOutputChunkSize(PyObject*, PyObject*, int)’:
./src/nvcomp.cpp:5650:73: error: could not convert ‘(size_t*)__pyx_v_max_compressed_bytes_ptr’ from ‘size_t*’ {aka ‘long unsigned int*’} to ‘nvcompBatchedLZ4Opts_t’
nvcompBatchedLZ4CompressGetMaxOutputChunkSize(((size_t)__pyx_t_3), ((size_t )__pyx_v_max_compressed_bytes_ptr));
~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./src/nvcomp.cpp: In function ‘PyObject __pyx_f_6nvcomp_batchedLZ4CompressAsync(PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, int, __pyx_opt_args_6nvcomp_batchedLZ4CompressAsync*)’:
./src/nvcomp.cpp:5890:271: error: could not convert ‘(cudaStream_t)__pyx_v_stream’ from ‘cudaStream_t’ {aka ‘CUstream_st*’} to ‘nvcompBatchedLZ4Opts_t’
3), ((void *)__pyx_v_temp_ptr), ((size_t)__pyx_t_4), ((void **)__pyx_v_device_out_ptrs_ptr), ((size_t *)__pyx_v_device_out_bytes_ptr), ((cudaStream_t)__pyx_v_stream));
~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~

./src/nvcomp.cpp: In function ‘PyObject* __pyx_f_6nvcomp_batchedLZ4DecompressAsync(PyObject*, PyObject*, PyObject*, PyObject*, int, __pyx_opt_args_6nvcomp_batchedLZ4DecompressAsync*)’:
./src/nvcomp.cpp:6178:249: error: cannot convert ‘cudaStream_t’ {aka ‘CUstream_st*’} to ‘nvcompStatus_t*’
r), ((size_t *)__pyx_v__unused_ptr), 0, ((size_t)__pyx_t_3), ((void *)__pyx_v__unused_ptr), 0, ((void **)__pyx_v_device_out_ptrs_ptr), ((cudaStream_t)__pyx_v_stream));
~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~

In file included from ./src/nvcomp.cpp:748:
/usr/local/cuda-11.7/targets/x86_64-linux/include/nvcomp/lz4.h:217:21: note: initializing argument 9 of ‘nvcompStatus_t nvcompBatchedLZ4DecompressAsync(const void* const*, const size_t*, const size_t*, size_t*, size_t, void*, size_t, void* const*, nvcompStatus_t*, cudaStream_t)’
nvcompStatus_t* device_statuses,
~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~
./src/nvcomp.cpp: At global scope:
./src/nvcomp.cpp:13678:74: error: use of enum ‘nvcompError_t’ without previous declaration
static CYTHON_INLINE PyObject* __Pyx_PyInt_From_enum__nvcompError_t(enum nvcompError_t value) {
^~~~~~~~~~~~~
./src/nvcomp.cpp: In function ‘PyObject* __Pyx_PyInt_From_enum__nvcompError_t(int)’:
./src/nvcomp.cpp:13683:16: error: use of enum ‘nvcompError_t’ without previous declaration
const enum nvcompError_t neg_one = (enum nvcompError_t) -1, const_zero = (enum nvcompError_t) 0;
^~~~~~~~~~~~~
./src/nvcomp.cpp:13683:46: error: use of enum ‘nvcompError_t’ without previous declaration
const enum nvcompError_t neg_one = (enum nvcompError_t) -1, const_zero = (enum nvcompError_t) 0;
^~~~~~~~~~~~~
./src/nvcomp.cpp:13687:39: error: ‘const_zero’ was not declared in this scope
const int is_unsigned = neg_one > const_zero;
^~~~~~~~~~
./src/nvcomp.cpp:13687:39: note: suggested alternative: ‘constexpr’
const int is_unsigned = neg_one > const_zero;
^~~~~~~~~~
constexpr
./src/nvcomp.cpp:13689:25: error: use of enum ‘nvcompError_t’ without previous declaration
if (sizeof(enum nvcompError_t) < sizeof(long)) {
^~~~~~~~~~~~~
./src/nvcomp.cpp:13691:32: error: use of enum ‘nvcompError_t’ without previous declaration
} else if (sizeof(enum nvcompError_t) <= sizeof(unsigned long)) {
^~~~~~~~~~~~~
./src/nvcomp.cpp:13694:32: error: use of enum ‘nvcompError_t’ without previous declaration
} else if (sizeof(enum nvcompError_t) <= sizeof(unsigned PY_LONG_LONG)) {
^~~~~~~~~~~~~
./src/nvcomp.cpp:13699:25: error: use of enum ‘nvcompError_t’ without previous declaration
if (sizeof(enum nvcompError_t) <= sizeof(long)) {
^~~~~~~~~~~~~
./src/nvcomp.cpp:13702:32: error: use of enum ‘nvcompError_t’ without previous declaration
} else if (sizeof(enum nvcompError_t) <= sizeof(PY_LONG_LONG)) {
^~~~~~~~~~~~~
./src/nvcomp.cpp:13710:57: error: use of enum ‘nvcompError_t’ without previous declaration
return _PyLong_FromByteArray(bytes, sizeof(enum nvcompError_t),
^~~~~~~~~~~~~
./src/nvcomp.cpp:13711:47: error: in argument to unary !
little, !is_unsigned);
^~~~~~~~~~~

@carsonswope thanks again for adding the license. After dealing with some internal issues we were able to start integrating the bindings into a new library just for I/O. The PR is here: rapidsai/kvikio#24

We decided on this library rather than cuDF because we found other folks who wanted a much smaller library to get python bindings for nvcomp or just python bindings for GDS but not all of cuDF.

For some LZ4 decompression I am calling "decompressStream" from the low level interface inside a cuda kernel. This could be due my incompetence in C++ (my code is here).

Are you planning to interface the low level parts too? I can't really see how my case could work without a python interface to set up the batching.

Obviously, it will be better for me if I can call your interface instead of copy+paste of the kernel code. Thanks for sharing your work!

Hi,

I'm trying to build it in RHEL 8.6 / Python 3.9 / Cuda 11.7

facing issue,

(venv) cat setup.py

from distutils.core import setup, Extension

from setuptools import Extension, setup
from Cython.Build import cythonize

NVCOMP_INCLUDE_DIR = '/home/arul/Downloads/nvcomp/include'
NVCOMP_LIB_NAME = 'libnvcomp.so'
NVCOMP_LIB_DIR = '/home/arul/Downloads/nvcomp/lib'

CUDA_INCLUDE_DIR = '/usr/local/cuda-11.7/targets/x86_64-linux/include/'
CUDA_LIB_NAME = 'libcudart.so'
CUDA_LIB_DIR = '/usr/local/cuda-11.7/targets/x86_64-linux/lib/'

extensions = [Extension(
'nvcomp',
['./src/nvcomp.pyx'],
language='c++',
include_dirs=[NVCOMP_INCLUDE_DIR, CUDA_INCLUDE_DIR],
libraries=[NVCOMP_LIB_NAME, CUDA_LIB_NAME],
library_dirs=[NVCOMP_LIB_DIR, CUDA_LIB_DIR],
)]

setup(name='nvcomp', ext_modules=cythonize(extensions, compiler_directives={'language_level' : '3'}))

(venv) python3.9 setup.py build_ext --inplace
running build_ext
x86_64-unknown-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -O2 -g -pipe -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -fno-omit-frame-pointer -O2 -g -pipe -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -fno-omit-frame-pointer -fPIC -I/home/arul/Downloads/nvcomp/include -I/usr/local/cuda-11.7/targets/x86_64-linux/include/ -I/home/arul/PycharmProjects/py_turbo/venv/include -I/usr/include/python3.9 -c ./src/nvcomp.cpp -o build/temp.linux-x86_64-3.9/./src/nvcomp.o
./src/nvcomp.cpp:741:10: fatal error: cuda_runtime.h: No such file or directory
741 | #include "cuda_runtime.h"
| ^~~~~~~~~~~~~~~~
compilation terminated.
error: command '/usr/bin/x86_64-unknown-linux-gnu-gcc' failed with exit code 1

Please help.

Thanks,
Arul

We are interested in using part of these Python bindings in https://github.com/rapidsai/cudf .

1. @carsonswope are you ok with us doing this with proper attributions to you ?
2. If 1) is a yes, would you be willing to add a permissible license such as bsd-3 or apache ?