gmh5225 / algorithm-uni-algo Goto Github PK

// The module doesn't require Unicode data so it can be used as header-only
#include "cpp_uni_convert.h"

// Lenient conversion (cannot fail) "\xD800" is unpaired high surrogate in UTF-16
std::string str8 = uni::utf16to8(u"Te\xD800st");
assert(str8 == "Te\xEF\xBF\xBDst"); // "\xEF\xBF\xBD" is replacement character U+FFFD in UTF-8

// Strict conversion
uni::error error;
std::string str8 = uni::strict::utf16to8(u"Te\xD800st", error);
assert(str8.empty() && error && error.pos() == 2);

// Template functions:

// Lenient conversion (cannot fail) "\xC2" is truncated sequence in UTF-8
std::u16string str16 = uni::utf8to16<char, char16_t>("Te\xC2st");
assert(str16 == u"Te\xFFFDst"); // "\xFFFD" is replacement character U+FFFD in UTF-16

// Strict conversion
uni::error error;
std::u16string str16 = uni::strict::utf8to16<char, char16_t>("Te\xC2st", error);
assert(str16.empty() && error && error.pos() == 2);

// Note that most of the functions in the library have template versions because the low-level
// of the library is type-agnostic it only requires that UTF-16 type is enough to store 16-bit values
// and UTF-32 is enough to store 32-bit aside from that the low-level doesn't care about the types,
// so with the template functions you can do some strange stuff like this:

std::basic_string<long long> str16 = uni::utf8to16<char, long long>("😺");
std::string str8 = uni::utf16to8<long long, char>(str16);
assert(str8 == "😺");

// This works perfectly fine this is allowed and tested. Of course it is not needed in most
// situations but it can be used to handle std::wstring and wchar_t properly because the only
// portable way of using std::wstring is to use it only to store UTF-16 internally and for example
// on Linux convert it to the real UTF-32 std::wstring when it's needed. It can be done like this:

// Works on Linux, static_assert on Windows
std::wstring str32 = uni::utf16to32<wchar_t, wchar_t>(L"Test");

// It can be helpful if you work with a library that actually does this
// for example: https://github.com/taglib/taglib
// Aside from such corner cases you should use non-template functions because they are just shorter.

// It's not like only wchar_t is broken char has the same problem you never know what it stores
// so most of functions in the library start with utf8/utf16/utf32 it shows with what data
// a function works with, types are irrelevant.

// Note that there are no short functions for std::u8string at the moment
// but template functions will work perfectly fine with it.

#include "cpp_uni_case.h" // and compile "cpp_uni_data.cpp"

std::cout << uni::cases::utf8_upper("Straße") << '\n';
std::cout << uni::cases::utf8_lower("ДВА") << '\n';
std::cout << uni::cases::utf8_fold("Ligature ffi") << '\n';
std::cout << uni::cases::utf8_title("word WoRd wOrD") << '\n';

// Output:
// STRASSE
// два
// ligature ffi
// Word Word Word

// With Greek locale, removes diacritics etc.
std::cout << uni::cases::utf8_upper("Ἀριστοτέλης", uni::locale("el")) << '\n';
// With Turkish locale, maps i to İ etc.
std::cout << uni::cases::utf8_upper("istanbul", uni::locale("tr")) << '\n';
// With Dutch locale, maps ij to IJ at the start of a word
std::cout << uni::cases::utf8_title("ijsland", uni::locale("nl")) << '\n';

// Output:
// ΑΡΙΣΤΟΤΕΛΗΣ
// İSTANBUL
// IJsland

assert(uni::caseless::utf8_compare("ffi", "FFI") == 0);

uni::search found = uni::caseless::utf8_search("Ligature fi test", "FI");
assert(found && found.pos() == 9 && found.end_pos() == 12);

// The module provides a very simple collation function too

// Use the Makedonian alphabet for example
std::u32string str32 = U"абвгдѓежзѕијклљмнњопрстќуфхцчџшАБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ";

std::vector<std::string> vec8;
for (char32_t c : str32) // Convert the code points to a vector of UTF-8 code units
    vec8.emplace_back(uni::utf32to8(std::u32string(1, c)));
std::shuffle(vec8.begin(), vec8.end(), std::mt19937(42)); // Shuffle them just in case

// For example sort them with the binary comparison first
std::sort(vec8.begin(), vec8.end());

// Output: ЃЅЈЉЊЌЏАБВГДЕЖЗИКЛМНОПРСТУФХЦЧШабвгдежзиклмнопрстуфхцчшѓѕјљњќџ
// Everything is out of place

// Sort them with uni::casesens::utf8_collate
std::sort(vec8.begin(), vec8.end(), [](auto a, auto b) {
    return uni::casesens::utf8_collate(a, b) < 0;
});

std::for_each(vec8.begin(), vec8.end(), [](auto s) { std::cout << s; });
std::cout << '\n';

// Output: аАбБвВгГдДѓЃеЕжЖзЗѕЅиИјЈкКлЛљЉмМнНњЊоОпПрРсСтТќЌуУфФхХцЦчЧџЏшШ
// This is the correct order for the Makedonian alphabet

// Group them too
auto it = std::unique(vec8.begin(), vec8.end(), [](auto a, auto b) {
    return uni::caseless::utf8_collate(a, b) == 0;
});
vec8.erase(it, vec8.end());

// Output: абвгдѓежзѕијклљмнњопрстќуфхцчџш

// Be aware that this module will be redesigned in favour of std::ranges

// The module doesn't require Unicode data so it can be used as header-only
#include "cpp_uni_iterator.h"

// 3 UTF-8 code points, these emojis use 1 code point
std::string str8 = "😺😼🙀";

// Make UTF-8 iterator adaptors from std::string iterators
uni::iter::utf8 begin{str8.cbegin(), str8.cend()};
uni::iter::utf8 end{str8.cend(), str8.cend()};

// Count code points
std::cout << "Number of code points: " << std::distance(begin, end) << '\n';

// Print code points
for (auto it = begin; it != end; ++it)
    std::cout << "Code point: " << std::to_string(*it) << " at: " << it - begin << '\n';

// Skip 2 code points
if (auto it = std::next(begin, 2); it != end)
    std::cout << "3rd code point at: " << it - begin << '\n';

// Find code point 128572
if (auto it = std::find(begin, end, 128572); it != end)
    std::cout << "Code point 128572 at: " << it - begin << '\n';

std::cout << "Reverse order:" << '\n';
{
    uni::iter::utf8 begin{str8.cbegin(), str8.cend(), str8.cend()};
    uni::iter::utf8 end{str8.cbegin(), str8.cbegin()};

    for (auto it = begin; it != end;)
    {
        --it;
        std::cout << "Code point: " << std::to_string(*it) << " at: " << it - end << '\n';
    }
}

// Output:
// Number of code points: 3
// Code point: 128570 at: 0
// Code point: 128572 at: 4
// Code point: 128576 at: 8
// 3rd code point at: 8
// Code point 128572 at: 4
// Reverse order:
// Code point: 128576 at: 8
// Code point: 128572 at: 4
// Code point: 128570 at: 0

// Be aware that this module will be redesigned in favour of std::ranges

#include "cpp_uni_break.h" // and compile "cpp_uni_data.cpp"

// Breaks are similar to iterators but they don't return anything
// so operator* isn't defined and operator- must be used to determine a break point.
// Breaks use default grapheme/word boundary rules as described in Unicode Standard UAX #29

std::string str8 = "Алберт Ајнштајн";

// Replace it with uni::breaks::grapheme::utf8 for grapheme breaks
uni::breaks::word::utf8 begin{str8.cbegin(), str8.cend()};
uni::breaks::word::utf8 end{str8.cend(), str8.cend()};

// Count words/graphemes (note that a space/punctuation is a word too according to rules)
std::cout << "Number of words: " << std::distance(begin, end) << '\n';

// Print positions of the words/graphemes
for (auto it = begin; it != end; ++it)
    std::cout << "Word at: " << it - begin << '\n';

// Word/grapheme tokenizer
std::cout << "Words:" << '\n';
for (auto it = begin, prev = it; it != end; prev = it)
{
    ++it;
    // If you use word breaks you can use this helper function to determine that there is
    // an actual word and not a space or something on the left side of the boundary.
    if (it.word_on_left())
        std::cout << str8.substr(prev - begin, it - prev) << '\n';
}

// Output:
// Number of words: 3
// Word at: 0
// Word at: 14
// Word at: 15
// Words:
// Алберт
// Ајнштајн

// For example you need to append or insert a code point or grapheme to a string.

// For append a code point to an UTF-8 string all you need is Convert module.
std::string str8 = "🏴󠁧󠁢󠁷󠁬󠁳󠁿👨‍👩‍👧🧙‍♀️"; // These emojis use multiple code points
str8 += uni::utf32to8(std::u32string{U'😺'}); // This emoji use 1 code point

// But if you need to insert a code point you need Break module too.
uni::breaks::grapheme::utf8 it{str8.cbegin(), str8.cend()};
// Insert a code point after 2nd grapheme. This emoji use 1 code point.
str8.insert(std::next(it, 2) - it, uni::utf32to8(std::u32string{U'😼'}));

// The same way a grapheme can be appended or inserted and of course you don't even need
// Convert module if your grapheme or code point are already in UTF-8 encoding.

// Output: 🏴󠁧󠁢󠁷󠁬󠁳󠁿👨‍👩‍👧😼🧙‍♀️😺

#include "cpp_uni_norm.h" // and compile "cpp_uni_data.cpp"

// "Z" with acute == "Ź"
assert(uni::norm::utf8_nfc("\x5A\xCC\x81") == "\xC5\xB9");

assert(uni::norm::utf8_is_nfc("\x5A\xCC\x81") == false);
assert(uni::norm::utf8_is_nfc("\xC5\xB9") == true);

// Note that the normalization algorithm in the library supports streams
// so you can use the same function to normalize a file with 0 allocations

std::ifstream input{"random_file.txt", std::ios::binary};
std::ofstream output{"random_file_normalized.txt", std::ios::binary};

std::istreambuf_iterator<char> it{input.rdbuf()};
std::istreambuf_iterator<char> end;

std::ostreambuf_iterator<char> out{output.rdbuf()};

uni::norm::utf8_nfc(it, end, out);

// The function always produces well-formed normalized UTF-8 text in Stream-Safe Text Format
// even if an input was a binary file, of course in such degenerate case the output will be
// some random UTF-8 code points with a bunch of U+FFFD (replacement characters).
// It just means it is safe to use the function with everything.

// Note that the same result can be achieved with iterator chains.

// Be aware that this will be redesigned in favour of std::ranges

// For example we need to remove accents that mark the stressed vowels in a text.
// The algorithm is simple: NFD -> Remove grave and acute accents (U+0300 and U+0301) -> NFC
// Note that the library has uni::norm::utf8_unaccent function but it won't produce
// the same result because it removes all accents not specific ones,
// but it is possible to implement this algorithm using multiple iterators in a chain like this.

#include "cpp_uni_iterator.h"
#include "cpp_uni_norm.h"
// Note that we include Normalization module because some modules
// provide not only functions but iterators too

std::string remove_grave_and_acute_accents(std::string_view str)
{
    // Chain iterators: UTF-8 -> NFD -> Filter -> NFC -> UTF-8 Output
    // Note that sentinels are used here instead of end iterators it makes the code cleaner

    uni::iter::utf8 begin{str.cbegin(), str.cend()};

    uni::iter::norm::nfd nfd_begin{begin, uni::sentinel};

    struct func { bool operator()(char32_t c) { return c != 0x0300 && c != 0x0301; } };
    uni::iter::func::filter func_begin{func{}, nfd_begin, uni::sentinel};

    // Or you can just use a lambda function:
    //auto func = [](char32_t c) { return c != 0x0300 && c != 0x0301; };
    //uni::iter::func::filter func_begin{func, nfd_begin, uni::sentinel};

    uni::iter::norm::nfc nfc_begin{func_begin, uni::sentinel};

    std::string result;
    uni::iter::output::utf8 out{std::back_inserter(result)};

    for (auto it = nfc_begin; it != uni::sentinel; ++it)
        out = *it;

    return result;

    // This syntax is planned in the future
    // It will be compatible with C++20 std::ranges::views
    // so uni::views::filter can be replaced with std::views::filter
    std::string result = str
         | uni::views::utf8
         | uni::views::norm::nfd
         | uni::views::filter([](char32_t c) { return c != 0x0300 && c != 0x0301; })
         | uni::views::norm::nfc
         | uni::ranges::to_utf8<std::string>();
}

std::cout << remove_grave_and_acute_accents("bótte bòtte") << '\n';
// Output: botte botte