catchorg / catchorg/Catch2

Better stringification of std::u16string_view, std::u32string_view and friends

Open
#1,995 2 comments 0 reactions 0 assignees View on GitHub
Query
Dominant language
C++
Stars
21.5k
Forks
3.5k
Avg merge
3d 16h
Merged PRs (30d)
2

Description

**Description**
I'm writing a general unicode decoder, currently testing a UTF-32 file containing the following:
```
The quick brown fox jumped over the lazy dog

ሰማይ አይታረስ ንጉሥ አይከሰስ።
ብላ ካለኝ እንደአባቴ በቆመጠኝ።
ጌጥ ያለቤቱ ቁምጥና ነው።
ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው።
የአፍ ወለምታ በቅቤ አይታሽም።
አይጥ በበላ ዳዋ ተመታ።
ሲተረጉሙ ይደረግሙ።
ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል።
ድር ቢያብር አንበሳ ያስር።
ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም።
እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም።
የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ።
ሥራ ከመፍታት ልጄን ላፋታት።
ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል።
የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ።
ተንጋሎ ቢተፉ ተመልሶ ባፉ።
ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው።
እግርህን በፍራሽህ ልክ ዘርጋ።
```

My test reads this from a UTF-32 file, stores it in a `std::vector`, and makes a string_view from that, which gets piped into the decoder. The output is then compared against a separate `std::u32string_view` known to contain what the decoded file should be, akin to
```cpp
CHECK(output == UTF_TEST_TEXT_EXPECTED);
```
where `output` and `UTF_TEST_TEXT_EXPECTED` are both `std::u32string_view`. In the fail case Catch2 spits out this:
```sh
D:\Repositories\muu\tests\strings.cpp(82): FAILED:
CHECK( output == UR"(The quick brown fox jumped over the lazy dog)" U"\n" U"\n" UR"(
with expansion:
{ 65,279, 84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114,
111, 119, 110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 101, 100,
32, 111, 118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121,
32, 100, 111, 103, 10, 10, 4,656, 4,635, 4,845, 32, 4,768, 4,845,
4,723, 4,648, 4,661, 32, 4,757, 4,873, 4,645, 32, 4,768, 4,845, 4,
776, 4,656, 4,661, 4,962, 10, 4,709, 4,619, 32, 4,779, 4,616, 4,
765, 32, 4,773, 4,757, 4,848, 4,768, 4,707, 4,724, 32, 4,704, 4,
678, 4,632, 4,896, 4,765, 4,962, 10, 4,876, 4,901, 32, 4,843, 4,
616, 4,708, 4,721, 32, 4,673, 4,637, 4,901, 4,755, 32, 4,752, 4,
813, 4,962, 10, 4,848, 4,608, 32, 4,704, 4,629, 4,621, 4,633, 32,
4,677, 4,708, 32, 4,707, 4,845, 4,896, 4,899, 32, 4,757, 4,899, 4,
725, 32, 4,704, 4,872, 4,848, 4,616, 4,813, 4,962, 10, 4,840, 4,
768, 4,941, 32, 4,808, 4,616, 4,637, 4,723, 32, 4,704, 4,677, 4,
708, 32, 4,768, 4,845, 4,723, 4,669, 4,637, 4,962, 10, 4,768, 4,
845, 4,901, 32, 4,704, 4,704, 4,619, 32, 4,851, 4,811, 32, 4,720,
4,632, 4,723, 4,962, 10, 4,658, 4,720, 4,648, 4,873, 4,633, 32, 4,
845, 4,848, 4,648, 4,877, 4,633, 4,962, 10, 4,672, 4,661, 32, 4,
704, 4,672, 4,661, 4,965, 32, 4,821, 4,757, 4,673, 4,619, 4,621,
32, 4,704, 4,773, 4,877, 4,649, 32, 4,845, 4,612, 4,851, 4,621, 4,
962, 10, 4,853, 4,653, 32, 4,706, 4,843, 4,709, 4,653, 32, 4,768,
4,757, 4,704, 4,659, 32, 4,843, 4,661, 4,653, 4,962, 10, 4,656, 4,
813, 32, 4,773, 4,757, 4,848, 4,708, 4,721, 32, 4,773, 4,757, 4,
869, 32, 4,773, 4,757, 4,848, 32, 4,873, 4,648, 4,708, 4,721, 32,
4,768, 4,845, 4,720, 4,851, 4,848, 4,653, 4,637, 4,962, 10, 4,773,
4,877, 4,828, 4,653, 32, 4,840, 4,776, 4,936, 4,720, 4,813, 4,757,
32, 4,873, 4,654, 4,654, 32, 4,659, 4,845, 4,824, 4,875, 4,813, 32,
4,768, 4,845, 4,853, 4,653, 4,637, 4,962, 10, 4,840, 4,878, 4,648,
4,708, 4,725, 32, 4,620, 4,707, 4,965, 32, 4,706, 4,843, 4,841, 4,
725, 32, 4,845, 4,661, 4,677, 32, 4,707, 4,843, 4,841, 4,725, 32,
4,843, 4,896, 4,621, 4,677, 4,962, 10, 4,645, 4,651, 32, 4,776, 4,
632, 4,941, 4,723, 4,725, 32, 4,621, 4,868, 4,757, 32, 4,619, 4,
939, 4,723, 4,725, 4,962, 10, 4,819, 4,707, 4,845, 32, 4,635, 4,
848, 4,650, 4,843, 32, 4,840, 4,616, 4,813, 4,965, 32, 4,877, 4,
757, 4,853, 32, 4,845, 4,830, 32, 4,845, 4,830, 4,651, 4,621, 4,
962, 10, 4,840, 4,773, 4,661, 4,619, 4,637, 32, 4,768, 4,872, 4,
649, 32, 4,632, 4,779, 32, 4,840, 4,768, 4,638, 4,651, 32, 4,768,
4,872, 4,649, 32, 4,811, 4,653, 4,779, 4,962, 10, 4,720, 4,757, 4,
875, 4,622, 32, 4,706, 4,720, 4,937, 32, 4,720, 4,632, 4,621, 4,
662, 32, 4,707, 4,937, 4,962, 10, 4,808, 4,851, 4,869, 4,613, 32,
4,635, 4,653, 32, 4,706, 4,614, 4,757, 32, 4,904, 4,653, 4,661, 4,
613, 32, 4,768, 4,725, 4,619, 4,656, 4,813, 4,962, 10, 4,773, 4,
877, 4,653, 4,613, 4,757, 32, 4,704, 4,941, 4,651, 4,669, 4,613,
32, 4,621, 4,781, 32, 4,824, 4,653, 4,875, 4,962, 10 }
==
{ 84, 104, 101, 32, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119,
110, 32, 102, 111, 120, 32, 106, 117, 109, 112, 101, 100, 32, 111,
118, 101, 114, 32, 116, 104, 101, 32, 108, 97, 122, 121, 32, 100,
111, 103, 10, 10, 4,656, 4,635, 4,845, 32, 4,768, 4,845, 4,723, 4,
648, 4,661, 32, 4,757, 4,873, 4,645, 32, 4,768, 4,845, 4,776, 4,
656, 4,661, 4,962, 10, 4,709, 4,619, 32, 4,779, 4,616, 4,765, 32,
4,773, 4,757, 4,848, 4,768, 4,707, 4,724, 32, 4,704, 4,678, 4,632,
4,896, 4,765, 4,962, 10, 4,876, 4,901, 32, 4,843, 4,616, 4,708, 4,
721, 32, 4,673, 4,637, 4,901, 4,755, 32, 4,752, 4,813, 4,962, 10,
4,848, 4,608, 32, 4,704, 4,629, 4,621, 4,633, 32, 4,677, 4,708, 32,
4,707, 4,845, 4,896, 4,899, 32, 4,757, 4,899, 4,725, 32, 4,704, 4,
872, 4,848, 4,616, 4,813, 4,962, 10, 4,840, 4,768, 4,941, 32, 4,
808, 4,616, 4,637, 4,723, 32, 4,704, 4,677, 4,708, 32, 4,768, 4,
845, 4,723, 4,669, 4,637, 4,962, 10, 4,768, 4,845, 4,901, 32, 4,
704, 4,704, 4,619, 32, 4,851, 4,811, 32, 4,720, 4,632, 4,723, 4,
962, 10, 4,658, 4,720, 4,648, 4,873, 4,633, 32, 4,845, 4,848, 4,
648, 4,877, 4,633, 4,962, 10, 4,672, 4,661, 32, 4,704, 4,672, 4,
661, 4,965, 32, 4,821, 4,757, 4,673, 4,619, 4,621, 32, 4,704, 4,
773, 4,877, 4,649, 32, 4,845, 4,612, 4,851, 4,621, 4,962, 10, 4,
853, 4,653, 32, 4,706, 4,843, 4,709, 4,653, 32, 4,768, 4,757, 4,
704, 4,659, 32, 4,843, 4,661, 4,653, 4,962, 10, 4,656, 4,813, 32,
4,773, 4,757, 4,848, 4,708, 4,721, 32, 4,773, 4,757, 4,869, 32, 4,
773, 4,757, 4,848, 32, 4,873, 4,648, 4,708, 4,721, 32, 4,768, 4,
845, 4,720, 4,851, 4,848, 4,653, 4,637, 4,962, 10, 4,773, 4,877, 4,
828, 4,653, 32, 4,840, 4,776, 4,936, 4,720, 4,813, 4,757, 32, 4,
873, 4,654, 4,654, 32, 4,659, 4,845, 4,824, 4,875, 4,813, 32, 4,
768, 4,845, 4,853, 4,653, 4,637, 4,962, 10, 4,840, 4,878, 4,648, 4,
708, 4,725, 32, 4,620, 4,707, 4,965, 32, 4,706, 4,843, 4,841, 4,
725, 32, 4,845, 4,661, 4,677, 32, 4,707, 4,843, 4,841, 4,725, 32,
4,843, 4,896, 4,621, 4,677, 4,962, 10, 4,645, 4,651, 32, 4,776, 4,
632, 4,941, 4,723, 4,725, 32, 4,621, 4,868, 4,757, 32, 4,619, 4,
939, 4,723, 4,725, 4,962, 10, 4,819, 4,707, 4,845, 32, 4,635, 4,
848, 4,650, 4,843, 32, 4,840, 4,616, 4,813, 4,965, 32, 4,877, 4,
757, 4,853, 32, 4,845, 4,830, 32, 4,845, 4,830, 4,651, 4,621, 4,
962, 10, 4,840, 4,773, 4,661, 4,619, 4,637, 32, 4,768, 4,872, 4,
649, 32, 4,632, 4,779, 32, 4,840, 4,768, 4,638, 4,651, 32, 4,768,
4,872, 4,649, 32, 4,811, 4,653, 4,779, 4,962, 10, 4,720, 4,757, 4,
875, 4,622, 32, 4,706, 4,720, 4,937, 32, 4,720, 4,632, 4,621, 4,
662, 32, 4,707, 4,937, 4,962, 10, 4,808, 4,851, 4,869, 4,613, 32,
4,635, 4,653, 32, 4,706, 4,614, 4,757, 32, 4,904, 4,653, 4,661, 4,
613, 32, 4,768, 4,725, 4,619, 4,656, 4,813, 4,962, 10, 4,773, 4,
877, 4,653, 4,613, 4,757, 32, 4,704, 4,941, 4,651, 4,669, 4,613,
32, 4,621, 4,781, 32, 4,824, 4,653, 4,875, 4,962, 10 }
with message:
Decoding data/unicode_test_file_utf32le_bom.bin as UTF-32
(char32_t)
```
which is pretty verbose. Obviously the actual string content would be better, but encodings, consoles, text is a nightmare, etc... I get why that might be difficult. At the very least changing the individual code units of the text to be rendered as hex would go a long way, since that's typically how code units are depicted (decimal integers are less useful in this context).

One 'workaround' is to consider that on some platforms `sizeof(wchar_t)` == `sizeof(char32_t)` and on others `sizeof(wchar_t)` == `sizeof(char16_t)`; in either case you could 'solve' this by treating the `std::uXXstring_view` as the corresponding `std::wstring_view` and handling it however you currently do for wide-strings.

**Additional context**
Nada.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.