9 #ifndef PQXX_H_ENCODINGS
10 #define PQXX_H_ENCODINGS
14 #include <string_view>
45 template<
char... NEEDLE>
48 std::size_t here = 0u)
50 auto const sz{std::size(haystack)};
51 auto const data{std::data(haystack)};
54 auto next{scanner(data, sz, here)};
58 if ((... or (data[here] == NEEDLE)))
80 template<
typename CALLABLE>
83 std::size_t buffer_len, std::size_t start = 0)
86 for (std::size_t here = start, next; here < buffer_len; here = next)
88 next = scan(buffer, buffer_len, here);
90 callback(buffer + here, buffer + next);
99 get_byte(
char const buffer[], std::size_t offset) noexcept
101 return static_cast<unsigned char>(buffer[offset]);
105 [[noreturn]]
PQXX_COLD void throw_for_encoding_error(
106 char const *encoding_name,
char const buffer[], std::size_t start,
110 s <<
"Invalid byte sequence for encoding " << encoding_name <<
" at byte "
111 << start <<
": " << std::hex << std::setw(2) << std::setfill(
'0');
112 for (std::size_t i{0}; i < count; ++i)
114 s <<
"0x" <<
static_cast<unsigned int>(get_byte(buffer, start + i));
124 between_inc(
unsigned char value,
unsigned bottom,
unsigned top)
126 return value >= bottom and value <= top;
141 call(
char const buffer[], std::size_t buffer_len, std::size_t start);
156 find_ascii_char(std::string_view haystack, std::size_t here)
160 static_assert((... and ((NEEDLE & 0x80) == 0)));
162 auto const sz{std::size(haystack)};
163 auto const data{std::data(haystack)};
187 if ((... or (data[here] == NEEDLE)))
209 static_assert((... and ((NEEDLE >> 7) == 0)));
211 auto const sz{std::size(haystack)};
212 auto const data{std::data(haystack)};
216 while ((... and (data[here] != NEEDLE)))
229 call(
char const [], std::size_t buffer_len, std::size_t start)
232 if (start >= buffer_len)
244 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
246 if (start >= buffer_len)
249 auto const byte1{get_byte(buffer, start)};
253 if (not between_inc(byte1, 0x81, 0xfe) or (start + 2 > buffer_len))
255 throw_for_encoding_error(
"BIG5", buffer, start, 1);
257 auto const byte2{get_byte(buffer, start + 1)};
259 not between_inc(byte2, 0x40, 0x7e) and
260 not between_inc(byte2, 0xa1, 0xfe))
262 throw_for_encoding_error(
"BIG5", buffer, start, 2);
284 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
286 if (start >= buffer_len)
287 return std::string::npos;
289 auto const byte1{get_byte(buffer, start)};
293 if (not between_inc(byte1, 0xa1, 0xf7) or start + 2 > buffer_len)
295 throw_for_encoding_error(
"EUC_CN", buffer, start, 1);
297 auto const byte2{get_byte(buffer, start + 1)};
298 if (not between_inc(byte2, 0xa1, 0xfe))
300 throw_for_encoding_error(
"EUC_CN", buffer, start, 2);
315 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
317 if (start >= buffer_len)
318 return std::string::npos;
320 auto const byte1{get_byte(buffer, start)};
324 if (start + 2 > buffer_len)
326 throw_for_encoding_error(
"EUC_JP", buffer, start, 1);
328 auto const byte2{get_byte(buffer, start + 1)};
331 if (not between_inc(byte2, 0xa1, 0xfe))
333 throw_for_encoding_error(
"EUC_JP", buffer, start, 2);
338 if (between_inc(byte1, 0xa1, 0xfe))
340 if (not between_inc(byte2, 0xa1, 0xfe))
342 throw_for_encoding_error(
"EUC_JP", buffer, start, 2);
347 if (byte1 == 0x8f and start + 3 <= buffer_len)
349 auto const byte3{get_byte(buffer, start + 2)};
351 not between_inc(byte2, 0xa1, 0xfe) or
352 not between_inc(byte3, 0xa1, 0xfe))
354 throw_for_encoding_error(
"EUC_JP", buffer, start, 3);
359 throw_for_encoding_error(
"EUC_JP", buffer, start, 1);
368 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
370 if (start >= buffer_len)
373 auto const byte1{get_byte(buffer, start)};
377 if (not between_inc(byte1, 0xa1, 0xfe) or start + 2 > buffer_len)
379 throw_for_encoding_error(
"EUC_KR", buffer, start, 1);
381 auto const byte2{get_byte(buffer, start + 1)};
382 if (not between_inc(byte2, 0xa1, 0xfe))
384 throw_for_encoding_error(
"EUC_KR", buffer, start, 1);
395 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
397 if (start >= buffer_len)
399 return std::string::npos;
401 auto const byte1{get_byte(buffer, start)};
405 if (start + 2 > buffer_len)
407 throw_for_encoding_error(
"EUC_KR", buffer, start, 1);
409 auto const byte2{get_byte(buffer, start + 1)};
410 if (between_inc(byte1, 0xa1, 0xfe))
412 if (not between_inc(byte2, 0xa1, 0xfe))
414 throw_for_encoding_error(
"EUC_KR", buffer, start, 2);
419 if (byte1 != 0x8e or start + 4 > buffer_len)
421 throw_for_encoding_error(
"EUC_KR", buffer, start, 1);
424 between_inc(byte2, 0xa1, 0xb0) and
425 between_inc(get_byte(buffer, start + 2), 0xa1, 0xfe) and
426 between_inc(get_byte(buffer, start + 3), 0xa1, 0xfe))
430 throw_for_encoding_error(
"EUC_KR", buffer, start, 4);
439 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
441 if (start >= buffer_len)
444 auto const byte1{get_byte(buffer, start)};
448 throw_for_encoding_error(
"GB18030", buffer, start, buffer_len - start);
450 if (start + 2 > buffer_len)
452 throw_for_encoding_error(
"GB18030", buffer, start, buffer_len - start);
454 auto const byte2{get_byte(buffer, start + 1)};
455 if (between_inc(byte2, 0x40, 0xfe))
459 throw_for_encoding_error(
"GB18030", buffer, start, 2);
464 if (start + 4 > buffer_len)
466 throw_for_encoding_error(
"GB18030", buffer, start, buffer_len - start);
469 between_inc(byte2, 0x30, 0x39) and
470 between_inc(get_byte(buffer, start + 2), 0x81, 0xfe) and
471 between_inc(get_byte(buffer, start + 3), 0x30, 0x39))
475 throw_for_encoding_error(
"GB18030", buffer, start, 4);
484 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
486 if (start >= buffer_len)
489 auto const byte1{get_byte(buffer, start)};
493 if (start + 2 > buffer_len)
495 throw_for_encoding_error(
"GBK", buffer, start, 1);
497 auto const byte2{get_byte(buffer, start + 1)};
499 (between_inc(byte1, 0xa1, 0xa9) and between_inc(byte2, 0xa1, 0xfe)) or
500 (between_inc(byte1, 0xb0, 0xf7) and between_inc(byte2, 0xa1, 0xfe)) or
501 (between_inc(byte1, 0x81, 0xa0) and between_inc(byte2, 0x40, 0xfe) and
503 (between_inc(byte1, 0xaa, 0xfe) and between_inc(byte2, 0x40, 0xa0) and
505 (between_inc(byte1, 0xa8, 0xa9) and between_inc(byte2, 0x40, 0xa0) and
507 (between_inc(byte1, 0xaa, 0xaf) and between_inc(byte2, 0xa1, 0xfe)) or
508 (between_inc(byte1, 0xf8, 0xfe) and between_inc(byte2, 0xa1, 0xfe)) or
509 (between_inc(byte1, 0xa1, 0xa7) and between_inc(byte2, 0x40, 0xa0) and
514 throw_for_encoding_error(
"GBK", buffer, start, 2);
531 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
533 if (start >= buffer_len)
536 auto const byte1{get_byte(buffer, start)};
540 if (start + 2 > buffer_len)
542 throw_for_encoding_error(
"JOHAB", buffer, start, 1);
544 auto const byte2{get_byte(buffer, start)};
546 (between_inc(byte1, 0x84, 0xd3) and
547 (between_inc(byte2, 0x41, 0x7e) or between_inc(byte2, 0x81, 0xfe))) or
548 ((between_inc(byte1, 0xd8, 0xde) or between_inc(byte1, 0xe0, 0xf9)) and
549 (between_inc(byte2, 0x31, 0x7e) or between_inc(byte2, 0x91, 0xfe))))
553 throw_for_encoding_error(
"JOHAB", buffer, start, 2);
568 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
570 if (start >= buffer_len)
573 auto const byte1{get_byte(buffer, start)};
577 if (start + 2 > buffer_len)
579 throw_for_encoding_error(
"MULE_INTERNAL", buffer, start, 1);
581 auto const byte2{get_byte(buffer, start + 1)};
582 if (between_inc(byte1, 0x81, 0x8d) and byte2 >= 0xa0)
585 if (start + 3 > buffer_len)
587 throw_for_encoding_error(
"MULE_INTERNAL", buffer, start, 2);
590 ((byte1 == 0x9a and between_inc(byte2, 0xa0, 0xdf)) or
591 (byte1 == 0x9b and between_inc(byte2, 0xe0, 0xef)) or
592 (between_inc(byte1, 0x90, 0x99) and byte2 >= 0xa0)) and
596 if (start + 4 > buffer_len)
598 throw_for_encoding_error(
"MULE_INTERNAL", buffer, start, 3);
601 ((byte1 == 0x9c and between_inc(byte2, 0xf0, 0xf4)) or
602 (byte1 == 0x9d and between_inc(byte2, 0xf5, 0xfe))) and
603 get_byte(buffer, start + 2) >= 0xa0 and
604 get_byte(buffer, start + 4) >= 0xa0)
608 throw_for_encoding_error(
"MULE_INTERNAL", buffer, start, 4);
625 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
627 if (start >= buffer_len)
628 return std::string::npos;
630 auto const byte1{get_byte(buffer, start)};
631 if (byte1 < 0x80 or between_inc(byte1, 0xa1, 0xdf))
635 not between_inc(byte1, 0x81, 0x9f) and
636 not between_inc(byte1, 0xe0, 0xfc))
638 throw_for_encoding_error(
"SJIS", buffer, start, 1);
640 if (start + 2 > buffer_len)
642 throw_for_encoding_error(
"SJIS", buffer, start, buffer_len - start);
644 auto const byte2{get_byte(buffer, start + 1)};
647 throw_for_encoding_error(
"SJIS", buffer, start, 2);
649 if (between_inc(byte2, 0x40, 0x9e) or between_inc(byte2, 0x9f, 0xfc))
653 throw_for_encoding_error(
"SJIS", buffer, start, 2);
662 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
664 if (start >= buffer_len)
667 auto const byte1{get_byte(buffer, start)};
671 if (start + 2 > buffer_len)
673 throw_for_encoding_error(
"UHC", buffer, start, buffer_len - start);
675 auto const byte2{get_byte(buffer, start + 1)};
676 if (between_inc(byte1, 0x80, 0xc6))
679 between_inc(byte2, 0x41, 0x5a) or between_inc(byte2, 0x61, 0x7a) or
680 between_inc(byte2, 0x80, 0xfe))
684 throw_for_encoding_error(
"UHC", buffer, start, 2);
687 if (between_inc(byte1, 0xa1, 0xfe))
689 if (not between_inc(byte2, 0xa1, 0xfe))
691 throw_for_encoding_error(
"UHC", buffer, start, 2);
696 throw_for_encoding_error(
"UHC", buffer, start, 1);
705 call(
char const buffer[], std::size_t buffer_len, std::size_t start)
707 if (start >= buffer_len)
710 auto const byte1{get_byte(buffer, start)};
714 if (start + 2 > buffer_len)
716 throw_for_encoding_error(
"UTF8", buffer, start, buffer_len - start);
718 auto const byte2{get_byte(buffer, start + 1)};
719 if (between_inc(byte1, 0xc0, 0xdf))
721 if (not between_inc(byte2, 0x80, 0xbf))
723 throw_for_encoding_error(
"UTF8", buffer, start, 2);
728 if (start + 3 > buffer_len)
730 throw_for_encoding_error(
"UTF8", buffer, start, buffer_len - start);
732 auto const byte3{get_byte(buffer, start + 2)};
733 if (between_inc(byte1, 0xe0, 0xef))
735 if (between_inc(byte2, 0x80, 0xbf) and between_inc(byte3, 0x80, 0xbf))
739 throw_for_encoding_error(
"UTF8", buffer, start, 3);
742 if (start + 4 > buffer_len)
744 throw_for_encoding_error(
"UTF8", buffer, start, buffer_len - start);
746 if (between_inc(byte1, 0xf0, 0xf7))
749 between_inc(byte2, 0x80, 0xbf) and between_inc(byte3, 0x80, 0xbf) and
750 between_inc(get_byte(buffer, start + 3), 0x80, 0xbf))
754 throw_for_encoding_error(
"UTF8", buffer, start, 4);
758 throw_for_encoding_error(
"UTF8", buffer, start, 1);
807 template<
char... NEEDLE>
815 return pqxx::internal::find_ascii_char<
832 "Unexpected encoding group: ",
833 static_cast<std::underlying_type_t<encoding_group>
>(as_if),
835 static_cast<std::underlying_type_t<encoding_group>
>(enc),
").")};
844 template<
char... NEEDLE>
870 "Unexpected encoding group: ",
871 static_cast<std::underlying_type_t<encoding_group>
>(as_if),
873 static_cast<std::underlying_type_t<encoding_group>
>(enc),
").")};
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:705
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Find the next glyph in buffer after position start.
std::string concat(TYPE...item)
Efficiently combine a bunch of items into one big string.
Definition: concat.hxx:31
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:568
encoding_group
Definition: encoding_group.hxx:18
void for_glyphs(encoding_group enc, CALLABLE callback, char const buffer[], std::size_t buffer_len, std::size_t start=0)
Iterate over the glyphs in a buffer.
Definition: encodings.hxx:81
Internal items for libpqxx' own use. Do not use these yourself.
Definition: encodings.cxx:32
PQXX_PURE constexpr char_finder_func * get_char_finder(encoding_group enc)
Look up a character search function for an encoding group.
Definition: encodings.hxx:809
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:662
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:531
PQXX_PURE constexpr char_finder_func * get_s_char_finder(encoding_group enc)
Look up a "sentry" character search function for an encoding group.
Definition: encodings.hxx:846
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:315
Wrapper struct template for "find next glyph" functions.
Definition: encodings.hxx:136
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:625
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:484
static PQXX_PURE constexpr std::size_t call(char const [], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:229
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:368
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:395
Invalid argument passed to libpqxx, similar to std::invalid_argument.
Definition: except.hxx:265
std::size_t(char const buffer[], std::size_t buffer_len, std::size_t start) glyph_scanner_func
Function type: "find the end of the current glyph.".
Definition: encoding_group.hxx:53
constexpr encoding_group map_ascii_search_group(encoding_group enc) noexcept
Just for searching an ASCII character, what encoding can we use here?
Definition: encodings.hxx:779
std::size_t find_char(glyph_scanner_func *scanner, std::string_view haystack, std::size_t here=0u)
Find any of the ASCII characters NEEDLE in haystack.
Definition: encodings.hxx:46
std::size_t(std::string_view haystack, std::size_t start) char_finder_func
Function type: "find first occurrence of specific any of ASCII characters.".
Definition: encoding_group.hxx:71
pqxx::internal::encoding_group enc_group(std::string_view encoding_name)
Convert libpq encoding name to its libpqxx encoding group.
Definition: encodings.cxx:35
Internal error in libpqxx library.
Definition: except.hxx:241
PQXX_PURE std::size_t find_s_ascii_char(std::string_view haystack, std::size_t here)
Find first of NEEDLE ASCII chars in haystack.
Definition: encodings.hxx:205
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:439
PQXX_LIBEXPORT glyph_scanner_func * get_glyph_scanner(encoding_group)
Look up the glyph scanner function for a given encoding group.
PQXX_PURE char const * name_encoding(int encoding_id)
Return PostgreSQL's name for encoding enum value.
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:244
static PQXX_PURE std::size_t call(char const buffer[], std::size_t buffer_len, std::size_t start)
Definition: encodings.hxx:284