Skip to content

Commit a1db99a

Browse files
danielaparkerdanielaparker
committed
is_legal_utf8
1 parent 92fc852 commit a1db99a

1 file changed

Lines changed: 68 additions & 56 deletions

File tree

‎include/jsoncons/utility/unicode_traits.hpp‎

Lines changed: 68 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -299,36 +299,34 @@ namespace unicode_traits {
299299

300300
// utf8
301301

302-
template <typename CharT>
303-
typename std::enable_if<ext_traits::is_char8<CharT>::value, conv_errc>::type
304-
is_legal_utf8(const CharT* bytes, std::size_t length)
302+
inline conv_errc is_legal_utf8(const uint8_t* first, std::size_t length) noexcept
305303
{
306-
uint8_t a;
307-
const uint8_t* first = reinterpret_cast<const uint8_t*>(bytes);
308304
const uint8_t* last = first+length;
305+
uint8_t byte;
306+
309307
switch (length) {
310308
default:
311309
return conv_errc::over_long_utf8_sequence;
312310
case 4:
313-
if (((a = (*--last))& 0xC0) != 0x80)
311+
if (((byte = (*--last))& 0xC0) != 0x80)
314312
return conv_errc::expected_continuation_byte;
315313
JSONCONS_FALLTHROUGH;
316314
case 3:
317-
if (((a = (*--last))& 0xC0) != 0x80)
315+
if (((byte = (*--last))& 0xC0) != 0x80)
318316
return conv_errc::expected_continuation_byte;
319317
JSONCONS_FALLTHROUGH;
320318
case 2:
321-
if (((a = (*--last))& 0xC0) != 0x80)
319+
if (((byte = (*--last))& 0xC0) != 0x80)
322320
return conv_errc::expected_continuation_byte;
323321

324322
switch (static_cast<uint8_t>(*first))
325323
{
326324
// no fall-through in this inner switch
327-
case 0xE0: if (a < 0xA0) return conv_errc::source_illegal; break;
328-
case 0xED: if (a > 0x9F) return conv_errc::source_illegal; break;
329-
case 0xF0: if (a < 0x90) return conv_errc::source_illegal; break;
330-
case 0xF4: if (a > 0x8F) return conv_errc::source_illegal; break;
331-
default: if (a < 0x80) return conv_errc::source_illegal;
325+
case 0xE0: if (byte < 0xA0) return conv_errc::source_illegal; break;
326+
case 0xED: if (byte > 0x9F) return conv_errc::source_illegal; break;
327+
case 0xF0: if (byte < 0x90) return conv_errc::source_illegal; break;
328+
case 0xF4: if (byte > 0x8F) return conv_errc::source_illegal; break;
329+
default: if (byte < 0x80) return conv_errc::source_illegal;
332330
}
333331

334332
JSONCONS_FALLTHROUGH;
@@ -382,53 +380,56 @@ namespace unicode_traits {
382380
template <typename CharT,typename CodepointT>
383381
typename std::enable_if<ext_traits::is_char8<CharT>::value && ext_traits::is_char32<CodepointT>::value,
384382
convert_result<CharT>>::type
385-
to_codepoint(const CharT* first, const CharT* last,
383+
to_codepoint(const CharT* begin, const CharT* end,
386384
CodepointT& ch,
387385
conv_flags flags = conv_flags::strict) noexcept
388386
{
387+
const uint8_t* first = reinterpret_cast<const uint8_t*>(begin);
388+
const uint8_t* last = reinterpret_cast<const uint8_t*>(end);
389+
389390
ch = 0;
390391
if (first >= last)
391392
{
392-
return convert_result<CharT>{first, conv_errc::source_exhausted};
393+
return convert_result<CharT>{begin, conv_errc::source_exhausted};
393394
}
394395
conv_errc result = conv_errc();
395396

396-
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[static_cast<uint8_t>(*first)];
397+
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[*first];
397398
if (extra_bytes_to_read >= last - first)
398399
{
399400
result = conv_errc::source_exhausted;
400-
return convert_result<CharT>{first, result};
401+
return convert_result<CharT>{begin, result};
401402
}
402403
// Do this check whether lenient or strict
403404
if ((result=is_legal_utf8(first, extra_bytes_to_read+1)) != conv_errc())
404405
{
405-
return convert_result<CharT>{first, result};
406+
return convert_result<CharT>{begin, result};
406407
}
407408
// The cases all fall through. See "Note A" below.
408409
switch (extra_bytes_to_read)
409410
{
410411
case 5:
411-
ch += static_cast<uint8_t>(*first++);
412+
ch += (*first++);
412413
ch <<= 6;
413414
JSONCONS_FALLTHROUGH;
414415
case 4:
415-
ch += static_cast<uint8_t>(*first++);
416+
ch += (*first++);
416417
ch <<= 6;
417418
JSONCONS_FALLTHROUGH;
418419
case 3:
419-
ch += static_cast<uint8_t>(*first++);
420+
ch += (*first++);
420421
ch <<= 6;
421422
JSONCONS_FALLTHROUGH;
422423
case 2:
423-
ch += static_cast<uint8_t>(*first++);
424+
ch += (*first++);
424425
ch <<= 6;
425426
JSONCONS_FALLTHROUGH;
426427
case 1:
427-
ch += static_cast<uint8_t>(*first++);
428+
ch += (*first++);
428429
ch <<= 6;
429430
JSONCONS_FALLTHROUGH;
430431
case 0:
431-
ch += static_cast<uint8_t>(*first++);
432+
ch += (*first++);
432433
break;
433434
}
434435
ch -= offsets_from_utf8[extra_bytes_to_read];
@@ -444,7 +445,7 @@ namespace unicode_traits {
444445
{
445446
first -= (extra_bytes_to_read+1); // return to the illegal value itself
446447
result = conv_errc::source_illegal;
447-
return convert_result<CharT>{first, result};
448+
return convert_result<CharT>{begin, result};
448449
}
449450
else
450451
{
@@ -458,7 +459,7 @@ namespace unicode_traits {
458459
ch = replacement_char;
459460
}
460461

461-
return convert_result<CharT>{first,result} ;
462+
return convert_result<CharT>{begin,result} ;
462463
}
463464

464465
template <typename CharT,typename CodepointT>
@@ -557,15 +558,18 @@ namespace unicode_traits {
557558
&& ext_traits::is_back_insertable<Container>::value
558559
&& ext_traits::is_char8<typename Container::value_type>::value,
559560
convert_result<CharT>>::type
560-
convert(const CharT* data, std::size_t length, Container& target, conv_flags flags=conv_flags::strict)
561+
convert(const CharT* bytes, std::size_t length, Container& target, conv_flags flags=conv_flags::strict)
561562
{
562563
(void)flags;
563564

565+
const uint8_t* data = reinterpret_cast<const uint8_t*>(bytes);
566+
const uint8_t* last = data + length;
567+
564568
conv_errc result = conv_errc();
565-
const CharT* last = data + length;
569+
//const CharT* last = data + length;
566570
while (data != last)
567571
{
568-
std::size_t len = trailing_bytes_for_utf8[static_cast<uint8_t>(*data)] + 1;
572+
std::size_t len = trailing_bytes_for_utf8[*data] + 1;
569573
if (len > (std::size_t)(last - data))
570574
{
571575
return convert_result<CharT>{data, conv_errc::source_exhausted};
@@ -576,13 +580,13 @@ namespace unicode_traits {
576580
}
577581

578582
switch (len) {
579-
case 4: target.push_back(static_cast<uint8_t>(*data++));
583+
case 4: target.push_back(*data++);
580584
JSONCONS_FALLTHROUGH;
581-
case 3: target.push_back(static_cast<uint8_t>(*data++));
585+
case 3: target.push_back(*data++);
582586
JSONCONS_FALLTHROUGH;
583-
case 2: target.push_back(static_cast<uint8_t>(*data++));
587+
case 2: target.push_back(*data++);
584588
JSONCONS_FALLTHROUGH;
585-
case 1: target.push_back(static_cast<uint8_t>(*data++));
589+
case 1: target.push_back(*data++);
586590
}
587591
}
588592
return convert_result<CharT>{data,result} ;
@@ -593,16 +597,19 @@ namespace unicode_traits {
593597
&& ext_traits::is_back_insertable<Container>::value
594598
&& ext_traits::is_char16<typename Container::value_type>::value,
595599
convert_result<CharT>>::type
596-
convert(const CharT* data, std::size_t length,
600+
convert(const CharT* bytes, std::size_t length,
597601
Container& target,
598602
conv_flags flags = conv_flags::strict)
599603
{
604+
const uint8_t* data = reinterpret_cast<const uint8_t*>(bytes);
605+
const uint8_t* last = data + length;
606+
600607
conv_errc result = conv_errc();
601608

602-
const CharT* last = data + length;
609+
//const CharT* last = data + length;
603610
while (data != last)
604611
{
605-
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[static_cast<uint8_t>(*data)];
612+
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[*data];
606613
if (extra_bytes_to_read >= last - data)
607614
{
608615
result = conv_errc::source_exhausted;
@@ -618,17 +625,17 @@ namespace unicode_traits {
618625
*/
619626
uint32_t ch = 0;
620627
switch (extra_bytes_to_read) {
621-
case 5: ch += static_cast<uint8_t>(*data++); ch <<= 6; /* remember, illegal UTF-8 */
628+
case 5: ch += *data++; ch <<= 6; /* remember, illegal UTF-8 */
622629
JSONCONS_FALLTHROUGH;
623-
case 4: ch += static_cast<uint8_t>(*data++); ch <<= 6; /* remember, illegal UTF-8 */
630+
case 4: ch += *data++; ch <<= 6; /* remember, illegal UTF-8 */
624631
JSONCONS_FALLTHROUGH;
625-
case 3: ch += static_cast<uint8_t>(*data++); ch <<= 6;
632+
case 3: ch += *data++; ch <<= 6;
626633
JSONCONS_FALLTHROUGH;
627-
case 2: ch += static_cast<uint8_t>(*data++); ch <<= 6;
634+
case 2: ch += *data++; ch <<= 6;
628635
JSONCONS_FALLTHROUGH;
629-
case 1: ch += static_cast<uint8_t>(*data++); ch <<= 6;
636+
case 1: ch += *data++; ch <<= 6;
630637
JSONCONS_FALLTHROUGH;
631-
case 0: ch += static_cast<uint8_t>(*data++);
638+
case 0: ch += *data++;
632639
break;
633640
}
634641
ch -= offsets_from_utf8[extra_bytes_to_read];
@@ -670,17 +677,20 @@ namespace unicode_traits {
670677
&& ext_traits::is_back_insertable<Container>::value
671678
&& ext_traits::is_char32<typename Container::value_type>::value,
672679
convert_result<CharT>>::type
673-
convert(const CharT* data, std::size_t length,
680+
convert(const CharT* bytes, std::size_t length,
674681
Container& target,
675682
conv_flags flags = conv_flags::strict)
676683
{
684+
const uint8_t* data = reinterpret_cast<const uint8_t*>(bytes);
685+
const uint8_t* last = data + length;
686+
677687
conv_errc result = conv_errc();
678688

679689
const CharT* last = data + length;
680690
while (data < last)
681691
{
682692
uint32_t ch = 0;
683-
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[static_cast<uint8_t>(*data)];
693+
unsigned short extra_bytes_to_read = trailing_bytes_for_utf8[*data];
684694
if (extra_bytes_to_read >= last - data)
685695
{
686696
result = conv_errc::source_exhausted;
@@ -697,27 +707,27 @@ namespace unicode_traits {
697707
switch (extra_bytes_to_read)
698708
{
699709
case 5:
700-
ch += static_cast<uint8_t>(*data++);
710+
ch += *data++;
701711
ch <<= 6;
702712
JSONCONS_FALLTHROUGH;
703713
case 4:
704-
ch += static_cast<uint8_t>(*data++);
714+
ch += *data++;
705715
ch <<= 6;
706716
JSONCONS_FALLTHROUGH;
707717
case 3:
708-
ch += static_cast<uint8_t>(*data++);
718+
ch += *data++;
709719
ch <<= 6;
710720
JSONCONS_FALLTHROUGH;
711721
case 2:
712-
ch += static_cast<uint8_t>(*data++);
722+
ch += *data++;
713723
ch <<= 6;
714724
JSONCONS_FALLTHROUGH;
715725
case 1:
716-
ch += static_cast<uint8_t>(*data++);
726+
ch += *data++;
717727
ch <<= 6;
718728
JSONCONS_FALLTHROUGH;
719729
case 0:
720-
ch += static_cast<uint8_t>(*data++);
730+
ch += *data++;
721731
break;
722732
}
723733
ch -= offsets_from_utf8[extra_bytes_to_read];
@@ -1133,24 +1143,26 @@ namespace unicode_traits {
11331143
template <typename CharT>
11341144
typename std::enable_if<ext_traits::is_char8<CharT>::value,
11351145
convert_result<CharT>>::type
1136-
validate(const CharT* data, std::size_t length) noexcept
1146+
validate(const CharT* bytes, std::size_t length) noexcept
11371147
{
1148+
const uint8_t* data = reinterpret_cast<const uint8_t*>(bytes);
1149+
const uint8_t* last = data + length;
1150+
11381151
conv_errc result = conv_errc();
1139-
const CharT* last = data + length;
11401152
while (data != last)
11411153
{
1142-
std::size_t len = static_cast<std::size_t>(trailing_bytes_for_utf8[static_cast<uint8_t>(*data)]) + 1;
1154+
std::size_t len = static_cast<std::size_t>(trailing_bytes_for_utf8[*data]) + 1;
11431155
if (len > (std::size_t)(last - data))
11441156
{
1145-
return convert_result<CharT>{data, conv_errc::source_exhausted};
1157+
return convert_result<CharT>{bytes, conv_errc::source_exhausted};
11461158
}
11471159
if ((result=is_legal_utf8(data, len)) != conv_errc())
11481160
{
1149-
return convert_result<CharT>{data,result} ;
1161+
return convert_result<CharT>{bytes,result} ;
11501162
}
11511163
data += len;
11521164
}
1153-
return convert_result<CharT>{data,result} ;
1165+
return convert_result<CharT>{bytes,result} ;
11541166
}
11551167

11561168
// utf16

0 commit comments

Comments
 (0)