1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
diff --git a/nanoparquet/src/Makevars b/nanoparquet/src/Makevars
index 8ccbd10..abccbbd 100644
--- a/nanoparquet/src/Makevars
+++ b/nanoparquet/src/Makevars
@@ -5,26 +5,10 @@ OBJECTS= \
dictionary-encoding.o test.o \
lib/ParquetFile.o lib/ParquetOutFile.o lib/RleBpDecoder.o \
parquet/parquet_types.o \
- thrift/protocol/TProtocol.o thrift/transport/TTransportException.o \
- thrift/transport/TBufferTransports.o \
fastpforlib/bitpacking.o \
- snappy/snappy.o snappy/snappy-sinksource.o \
- miniz/miniz.o \
- zstd/common/entropy_common.o zstd/common/error_private.o \
- zstd/common/fse_decompress.o zstd/common/xxhash.o \
- zstd/common/zstd_common.o zstd/decompress/huf_decompress.o \
- zstd/decompress/zstd_ddict.o zstd/decompress/zstd_decompress.o \
- zstd/decompress/zstd_decompress_block.o \
- zstd/compress/fse_compress.o zstd/compress/hist.o \
- zstd/compress/huf_compress.o zstd/compress/zstd_compress.o \
- zstd/compress/zstd_compress_literals.o \
- zstd/compress/zstd_compress_sequences.o \
- zstd/compress/zstd_compress_superblock.o \
- zstd/compress/zstd_double_fast.o zstd/compress/zstd_fast.o \
- zstd/compress/zstd_lazy.o zstd/compress/zstd_ldm.o \
- zstd/compress/zstd_opt.o
+ miniz/miniz.o
-PKG_CPPFLAGS = -Ithrift -I. -Izstd/include
+PKG_CPPFLAGS = -I/usr/include/thrift -I.
# PKG_CFLAGS = -DR_NO_REMAP
PKG_CXXFLAGS = -DR_NO_REMAP
@@ -42,3 +26,4 @@ PKG_CXX29FLAGS = -DR_NO_REMAP
PKG_CXX30FLAGS = -DR_NO_REMAP
# PKG_LIBS = -lws2_32
+PKG_LIBS := -lsnappy -lthrift -lzstd
diff --git a/nanoparquet/src/lib/ParquetFile.cpp b/nanoparquet/src/lib/ParquetFile.cpp
index ae0d65e..6d2753c 100644
--- a/nanoparquet/src/lib/ParquetFile.cpp
+++ b/nanoparquet/src/lib/ParquetFile.cpp
@@ -1097,14 +1097,14 @@ void ParquetFile::scan_column(ScanState &state, ResultColumn &result_col) {
decompressed_buf.resize(cs.page_header.uncompressed_page_size + 1 + xdef);
memcpy(decompressed_buf.ptr, chunk_buf.ptr + xrep, xdef);
- size_t res = zstd::ZSTD_decompress(
+ size_t res = ZSTD_decompress(
decompressed_buf.ptr + xdef,
cs.page_header.uncompressed_page_size - xrep - xdef,
chunk_buf.ptr + xrep + xdef,
cs.page_header.compressed_page_size - xrep - xdef
);
- if (zstd::ZSTD_isError(res) ||
+ if (ZSTD_isError(res) ||
res != (size_t) cs.page_header.uncompressed_page_size - xrep -xdef) {
std::stringstream ss;
ss << "Zstd decompression failure, possibly corrupt Parquet file '"
diff --git a/nanoparquet/src/lib/ParquetOutFile.cpp b/nanoparquet/src/lib/ParquetOutFile.cpp
index e18d3f2..e8ab57d 100644
--- a/nanoparquet/src/lib/ParquetOutFile.cpp
+++ b/nanoparquet/src/lib/ParquetOutFile.cpp
@@ -395,16 +395,16 @@ size_t ParquetOutFile::compress(
mzs.Compress(src.ptr, src_size, tgt.ptr, &tgt_size);
return tgt_size;
} else if (codec == CompressionCodec::ZSTD) {
- size_t tgt_size_est = zstd::ZSTD_compressBound(src_size);
+ size_t tgt_size_est = ZSTD_compressBound(src_size);
tgt.reset(tgt_size_est);
- size_t tgt_size = zstd::ZSTD_compress(
+ size_t tgt_size = ZSTD_compress(
tgt.ptr,
tgt_size_est,
src.ptr,
src_size,
ZSTD_CLEVEL_DEFAULT
);
- if (zstd::ZSTD_isError(tgt_size)) {
+ if (ZSTD_isError(tgt_size)) {
std::stringstream ss;
ss << "Zstd compression failure" << "' @ " << __FILE__ << ":"
<< __LINE__;
diff --git a/nanoparquet/src/snappy.cpp b/nanoparquet/src/snappy.cpp
index 8f39d20..f20e370 100644
--- a/nanoparquet/src/snappy.cpp
+++ b/nanoparquet/src/snappy.cpp
@@ -69,9 +69,9 @@ SEXP gzip_uncompress_raw(SEXP x, SEXP ucl) {
SEXP zstd_compress_raw(SEXP x) {
R_xlen_t data_size = Rf_xlength(x);
miniz::MiniZStream mzs;
- size_t maxcsize = zstd::ZSTD_compressBound(data_size);
+ size_t maxcsize = ZSTD_compressBound(data_size);
SEXP res = PROTECT(Rf_allocVector(RAWSXP, maxcsize));
- size_t tgt_size = zstd::ZSTD_compress(
+ size_t tgt_size = ZSTD_compress(
(char*) RAW(res),
maxcsize,
(const char*) RAW(x),
@@ -87,7 +87,7 @@ SEXP zstd_uncompress_raw(SEXP x, SEXP ucl) {
R_xlen_t data_size = Rf_xlength(x);
size_t cusl = INTEGER(ucl)[0];
SEXP res = PROTECT(Rf_allocVector(RAWSXP, cusl));
- zstd::ZSTD_decompress(RAW(res), cusl, RAW(x), data_size);
+ ZSTD_decompress(RAW(res), cusl, RAW(x), data_size);
UNPROTECT(1);
return res;
}
|