SWC-DB  v0.5.12 C++ documentations
SWC-DB© (Super Wide Column Database) - High Performance Scalable Database (https://github.com/kashirin-alex/swc-db)
Encoder.cc
Go to the documentation of this file.
1 /*
2  * SWC-DB© Copyright since 2019 Alex Kashirin <kashirin.alex@gmail.com>
3  * License details at <https://github.com/kashirin-alex/swc-db/#license>
4  */
5 
6 
7 #include "swcdb/core/Exception.h"
8 #include "swcdb/core/Encoder.h"
9 
10 #include <snappy.h>
11 #include <zlib.h>
12 #include <zstd.h>
13 
14 #undef Z_NULL
15 #define Z_NULL nullptr
16 
17 
18 namespace SWC { namespace Core {
19 
20 
21 namespace {
22  const char Encoder_DEFAULT[] = "DEFAULT";
23  const char Encoder_PLAIN[] = "PLAIN";
24  const char Encoder_ZLIB[] = "ZLIB";
25  const char Encoder_SNAPPY[] = "SNAPPY";
26  const char Encoder_ZSTD[] = "ZSTD";
27  const char Encoder_UNKNOWN[] = "UNKNOWN";
28 }
29 
30 
31 const char* Encoder::to_string(Encoder::Type typ) noexcept {
32  switch(typ) {
33  case Encoder::Type::DEFAULT:
34  return Encoder_DEFAULT;
35  case Encoder::Type::PLAIN:
36  return Encoder_PLAIN;
37  case Encoder::Type::ZLIB:
38  return Encoder_ZLIB;
39  case Encoder::Type::SNAPPY:
40  return Encoder_SNAPPY;
41  case Encoder::Type::ZSTD:
42  return Encoder_ZSTD;
43  case Encoder::Type::UNKNOWN:
44  return Encoder_UNKNOWN;
45  default:
46  return "UNKNOWN";
47  }
48 }
49 
50 Encoder::Type Encoder::encoding_from(const std::string& typ) noexcept {
51  switch(typ.length()) {
52  case 1: {
53  switch(*typ.data()) {
54  case '0':
55  return Encoder::Type::DEFAULT;
56  case '1':
57  return Encoder::Type::PLAIN;
58  case '2':
59  return Encoder::Type::ZLIB;
60  case '3':
61  return Encoder::Type::SNAPPY;
62  case '4':
63  return Encoder::Type::ZSTD;
64  default:
65  break;
66  }
67  break;
68  }
69  case 4: {
70  if(Condition::str_case_eq(typ.data(), Encoder_ZLIB, 4))
71  return Encoder::Type::ZLIB;
72  if(Condition::str_case_eq(typ.data(), Encoder_ZSTD, 4))
73  return Encoder::Type::ZSTD;
74  break;
75  }
76  case 5: {
77  if(Condition::str_case_eq(typ.data(), Encoder_PLAIN, 5))
78  return Encoder::Type::PLAIN;
79  break;
80  }
81  case 6: {
82  if(Condition::str_case_eq(typ.data(), Encoder_SNAPPY, 6))
83  return Encoder::Type::SNAPPY;
84  break;
85  }
86  case 7: {
87  if(Condition::str_case_eq(typ.data(), Encoder_DEFAULT, 7))
88  return Encoder::Type::DEFAULT;
89  break;
90  }
91  default:
92  break;
93  }
94  return Encoder::Type::UNKNOWN;
95 }
96 
97 
98 
99 void Encoder::decode(
100  int& err, Encoder::Type encoder,
101  const uint8_t* src, size_t sz_enc,
102  uint8_t *dst, size_t sz) {
103 
104  switch(encoder) {
105  case Encoder::Type::ZLIB: {
106  z_stream strm;
107  memset(&strm, 0, sizeof(z_stream));
108  strm.zalloc = Z_NULL;
109  strm.zfree = Z_NULL;
110  strm.opaque = Z_NULL;
111  strm.avail_in = 0;
112  strm.next_in = Z_NULL;
113  if(::inflateInit(&strm) == Z_OK) {
114  strm.avail_in = sz_enc;
115  strm.next_in = const_cast<Bytef*>(src);
116  strm.avail_out = sz;
117  strm.next_out = dst;
118  if(::inflate(&strm, Z_NO_FLUSH) != Z_STREAM_END || strm.avail_out)
119  err = Error::ENCODER_DECODE;
120  } else {
121  err = Error::ENCODER_DECODE;
122  }
123  ::inflateEnd(&strm);
124  return;
125  }
126 
127  case Encoder::Type::SNAPPY: {
128  if(!snappy::RawUncompress(
129  reinterpret_cast<const char*>(src), sz_enc,
130  reinterpret_cast<char*>(dst)))
131  err = Error::ENCODER_DECODE;
132  return;
133  }
134 
135  case Encoder::Type::ZSTD: {
136  if(ZSTD_decompress(
137  static_cast<void*>(dst), sz,
138  static_cast<void*>(const_cast<uint8_t*>(src)), sz_enc) != sz)
139  err = Error::ENCODER_DECODE;
140  return;
141  }
142 
143  default: {
144  //SWC_ASSERT(encoder==Encoder::Type::PLAIN);
145  break;
146  }
147  }
148 }
149 
150 
151 
152 namespace {
153 
154 SWC_SHOULD_NOT_INLINE
155 bool encode_zlib(const uint8_t* src, size_t src_sz,
156  size_t* sz_enc, DynamicBuffer& output,
157  uint32_t reserve, bool ok_more) {
158  z_stream strm;
159  memset(&strm, 0, sizeof(z_stream));
160  strm.zalloc = Z_NULL;
161  strm.zfree = Z_NULL;
162  strm.opaque = Z_NULL;
163  if(::deflateInit(&strm, Z_DEFAULT_COMPRESSION) == Z_OK) {
164 
165  uint32_t avail_out = src_sz + 6 + (((src_sz / 16000) + 1) * 5);
166  output.ensure(reserve + avail_out);
167  output.ptr += reserve;
168 
169  strm.avail_out = avail_out;
170  strm.next_out = output.ptr;
171  strm.avail_in = src_sz;
172  strm.next_in = const_cast<Bytef*>(src);
173  if(::deflate(&strm, Z_FINISH) == Z_STREAM_END)
174  *sz_enc = avail_out - strm.avail_out;
175  }
176  ::deflateEnd(&strm);
177  if(*sz_enc && (ok_more || *sz_enc < src_sz)) {
178  output.ptr += *sz_enc;
179  return true;
180  }
181  return false;
182 }
183 
184 SWC_SHOULD_NOT_INLINE
185 bool encode_snappy(const uint8_t* src, size_t src_sz,
186  size_t* sz_enc, DynamicBuffer& output,
187  uint32_t reserve, bool ok_more) {
188  output.ensure(reserve + snappy::MaxCompressedLength(src_sz));
189  output.ptr += reserve;
190  snappy::RawCompress(
191  reinterpret_cast<const char*>(src), src_sz,
192  reinterpret_cast<char*>(output.ptr), sz_enc);
193  if(*sz_enc && (ok_more || *sz_enc < src_sz)) {
194  output.ptr += *sz_enc;
195  return true;
196  }
197  return false;
198 }
199 
200 SWC_SHOULD_NOT_INLINE
201 bool encode_zstd(const uint8_t* src, size_t src_sz,
202  size_t* sz_enc, DynamicBuffer& output,
203  uint32_t reserve, bool ok_more) {
204  size_t const avail_out = ZSTD_compressBound(src_sz);
205  output.ensure(reserve + avail_out);
206  output.ptr += reserve;
207 
208  *sz_enc = ZSTD_compress(
209  static_cast<void*>(output.ptr), avail_out,
210  static_cast<void*>(const_cast<uint8_t*>(src)), src_sz,
211  ZSTD_CLEVEL_DEFAULT
212  );
213  if(*sz_enc && !ZSTD_isError(*sz_enc) && (ok_more || *sz_enc < src_sz)) {
214  output.ptr += *sz_enc;
215  return true;
216  }
217  return false;
218 }
219 
220 }
221 
222 void Encoder::encode(
223  int&, Encoder::Type encoder,
224  const uint8_t* src, size_t src_sz,
225  size_t* sz_enc, DynamicBuffer& output,
226  uint32_t reserve, bool no_plain_out, bool ok_more) {
227  switch(encoder) {
228  case Encoder::Type::ZLIB: {
229  if(encode_zlib(src, src_sz, sz_enc, output, reserve, ok_more))
230  return;
231  break;
232  }
233  case Encoder::Type::SNAPPY: {
234  if(encode_snappy(src, src_sz, sz_enc, output, reserve, ok_more))
235  return;
236  break;
237  }
238  case Encoder::Type::ZSTD: {
239  if(encode_zstd(src, src_sz, sz_enc, output, reserve, ok_more))
240  return;
241  break;
242  }
243  default:
244  break;
245  }
246  *sz_enc = 0;
247  if(no_plain_out)
248  return;
249 
250  output.free();
251  output.ensure(reserve + src_sz);
252  output.ptr += reserve;
253  if(src_sz)
254  output.add_unchecked(src, src_sz);
255 }
256 
257 
258 }}
SWC::Core::BufferDyn::ptr
value_type * ptr
Definition: Buffer.h:293
SWC::Core::Encoder::Type
Type
Definition: Encoder.h:28
SWC::Core::DynamicBuffer
BufferDyn< StaticBuffer > DynamicBuffer
Definition: Buffer.h:299
SWC::Core::BufferDyn::ensure
SWC_CAN_INLINE void ensure(size_t len)
Definition: Buffer.h:212
Z_NULL
#define Z_NULL
Definition: Encoder.cc:15
SWC::Core::Encoder::encoding_from
Type SWC_PURE_FUNC encoding_from(const std::string &typ) noexcept
Definition: Encoder.cc:50
SWC::Core::BufferDyn::add_unchecked
SWC_CAN_INLINE value_type * add_unchecked(const value_type *data, size_t len) noexcept
Definition: Buffer.h:240
SWC::Error::ENCODER_DECODE
@ ENCODER_DECODE
Definition: Error.h:77
SWC::Core::Encoder::Type::DEFAULT
@ DEFAULT
encoder
Core::Encoder::Type encoder
Buffer Encoder.
Definition: HeaderBufferInfo.h:50
Exception.h
SWC::Core::Encoder::decode
void decode(int &err, Type encoder, const uint8_t *src, size_t sz_enc, uint8_t *dst, size_t sz)
Definition: Encoder.cc:99
SWC
The SWC-DB C++ namespace 'SWC'.
Definition: main.cc:12
SWC::Core::BufferDyn< StaticBuffer >
SWC::Core::BufferDyn::free
SWC_CAN_INLINE void free()
Definition: Buffer.h:171
SWC::Core::Encoder::to_string
const char *SWC_CONST_FUNC to_string(Type typ) noexcept
Definition: Encoder.cc:31
SWC::Core::Encoder::encode
void encode(int &err, Type encoder, const uint8_t *src, size_t src_sz, size_t *sz_enc, DynamicBuffer &output, uint32_t reserve, bool no_plain_out=false, bool ok_more=false)
Definition: Encoder.cc:222
Encoder.h
SWC::Condition::str_case_eq
bool str_case_eq(const char *s1, const char *s2, size_t count) noexcept SWC_ATTRIBS((SWC_ATTRIB_O3))
Definition: Comparators_basic.h:257