vaexio / vaex-talks Goto Github PK

1- vaex_df = vaex.from_pandas(cap_file, copy_index=False)
2 - print (vaex_df)
3 - vaex_df.export_hdf5(path='/Users/p0p029i/Downloads/cap_monthly_20210625_2.hdf5', progress=True)
4 - df = vaex.open('/Users/p0p029i/Downloads/cap_monthly_20210625_2.hdf5')

above top three lines are working fine but, when i am executing 4th line, it is not working. I have checked my download folder also, file cap_monthly_20210625_2.hdf5 has been created also.

Error which line 4 is giving:

ERROR:MainThread:vaex:error opening '/Users/p0p029i/Downloads/cap_monthly_20210625_2.hdf5'
Traceback (most recent call last):
File "/Users/p0p029i/PycharmProjects/datavalidation/main.py", line 85, in
df = vaex.open('/Users/p0p029i/Downloads/cap_monthly_20210625_2.hdf5')
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/init.py", line 223, in open
ds = vaex.dataset.open(path, fs_options=fs_options, fs=fs, **kwargs)
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/dataset.py", line 70, in open
return opener.open(path, fs_options=fs_options, fs=fs, *args, **kwargs)
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/dataset.py", line 1447, in open
return cls(path, *args, **kwargs)
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/hdf5/dataset.py", line 71, in init
self._load()
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/hdf5/dataset.py", line 194, in _load
self._load_columns(self.h5file["/table"])
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/vaex/hdf5/dataset.py", line 337, in _load_columns
data = column if self._version == 1 else column['data']
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "/Users/p0p029i/PycharmProjects/datavalidation/venv/lib/python3.9/site-packages/h5py/_hl/group.py", line 305, in getitem
oid = h5o.open(self.id, self._e(name), lapl=self._lapl)
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "h5py/h5o.pyx", line 190, in h5py.h5o.open
KeyError: "Unable to open object (object 'data' doesn't exist)"

Code:
import vaex as vx
df = vx.open("/Users/admin/Documents/Data/HOT/CA/ca_18_19.parquet")

Error message:
ERROR:MainThread:vaex:error opening '/Users/admin/Documents/Data/HOT/CA/ca_18_19.parquet'

TypeError Traceback (most recent call last)
in
----> 1 df = vx.open("/Users/admin/Documents/Data/HOT/CA/ca_18_19.parquet")

/anaconda3/lib/python3.7/site-packages/vaex/init.py in open(path, convert, shuffle, copy_index, *args, **kwargs)
189 ds = from_csv(path, copy_index=copy_index, **kwargs)
190 else:
--> 191 ds = vaex.file.open(path, *args, **kwargs)
192 if convert:
193 ds.export_hdf5(filename_hdf5, shuffle=shuffle)

/anaconda3/lib/python3.7/site-packages/vaex/file/init.py in open(path, *args, **kwargs)
28 for opener in opener_classes:
29 if opener.can_open(path, *args, **kwargs):
---> 30 return opener.open(path, *args, **kwargs)
31 if hdf5:
32 openers.extend(hdf5.dataset.dataset_type_map.items())

/anaconda3/lib/python3.7/site-packages/vaex_arrow/opener.py in open(path, *args, **kwargs)
19 def open(path, *args, **kwargs):
20 from .dataset import DatasetParquet
---> 21 return DatasetParquet(path, *args, **kwargs)
22
23 def register_opener():

/anaconda3/lib/python3.7/site-packages/vaex_arrow/dataset.py in init(self, filename, table, write)
18 self._write = write
19 if table is None:
---> 20 self._load()
21 else:
22 self._load_table(table)

/anaconda3/lib/python3.7/site-packages/vaex_arrow/dataset.py in _load(self)
59 # do mmapping later on
60 table = pq.read_table(self.path)
---> 61 self._load_table(table)
62
63 vaex.file.other.dataset_type_map["arrow"] = DatasetArrow

/anaconda3/lib/python3.7/site-packages/vaex_arrow/dataset.py in _load_table(self, table)
35 # TODO: keep the arrow columns, and support and test chunks
36 arrow_array = col.data.chunks[0]
---> 37 column = column_from_arrow_array(arrow_array)
38
39 self.columns[name] = column

/anaconda3/lib/python3.7/site-packages/vaex_arrow/convert.py in column_from_arrow_array(arrow_array)
36 null_bitmap = np.frombuffer(bitmap_buffer, 'uint8', len(bitmap_buffer))
37 offsets = np.frombuffer(offsets, np.int32, len(offsets)//4)
---> 38 string_bytes = np.frombuffer(string_bytes, 'S1', len(string_bytes))
39 column = ColumnStringArrow(offsets, string_bytes, len(arrow_array), null_bitmap=null_bitmap)
40 return column

TypeError: object of type 'NoneType' has no len()