#pragma once #include "globalincs/pstypes.h" #include "mod_table/mod_table.h" #include #include #include #if !HAVE_CHAR32_T // Older compilers don't have this as a built-in type so we define it for those typedef std::uint_least32_t char32_t; #endif #if !HAVE_UNICODE_CHAR_LITERAL #define UNICODE_CHAR(c) (char32_t) c #else /** * @brief Compatibility macro for compilers which don't support U'' char literals. * * @warning This can only handle standard ASCII character values since older compilers still use standard character * literals */ #define UNICODE_CHAR(c) U##c #endif namespace unicode { /** * @brief A standard unicode codepoint */ typedef char32_t codepoint_t; /** * @brief An invalid and ignorable character, equivalent to -1 */ constexpr codepoint_t invalid_char = static_cast(-1); /** * @brief Substitute for malformed UTF-8 so that a bad decode can degrade gracefully */ constexpr codepoint_t replacement_char = 0xFFFD; constexpr codepoint_t comment_char = static_cast(COMMENT_CHAR); constexpr codepoint_t eoln = static_cast(EOLN); constexpr codepoint_t carriage_return = static_cast(CARRIAGE_RETURN); class text_iterator { const char* current_byte = nullptr; const char* range_end_byte = nullptr; const char* range_start_byte = nullptr; bool is_from_same_range(const text_iterator& other) const; public: explicit text_iterator(const char* current_byte, const char* range_start_byte, const char* range_end_byte = nullptr); using difference_type = std::ptrdiff_t; using value_type = codepoint_t; using pointer = codepoint_t const *; using reference = codepoint_t const &; using iterator_category = std::bidirectional_iterator_tag; const char* pos() const; text_iterator& operator++(); text_iterator& operator--(); text_iterator operator++(int); text_iterator operator--(int); value_type operator*() const; bool operator==(const text_iterator& rhs) const; bool operator!=(const text_iterator& rhs) const; bool operator<(const text_iterator& rhs) const; bool operator>(const text_iterator& rhs) const; bool operator<=(const text_iterator& rhs) const; bool operator>=(const text_iterator& rhs) const; text_iterator operator+(ptrdiff_t diff) const; text_iterator operator-(ptrdiff_t diff) const; }; /** * @brief Represents a range of unicode codepoints that can be iterated over * * @note This class can be used in range based for loops */ class codepoint_range { const char* start = nullptr; const char* end_ptr = nullptr; public: /** * @brief Creates a codepoint range based on an UTF-8 encoded string * @param start The start of the encoded string * @param end The end of the encoded string. May be @c nullptr in which case @c start is assumed to be null-terminated */ explicit codepoint_range(const char* start, const char* end = nullptr); /** * @brief Retrieves an iterator for the start of the codepoint range * @return An iterator which is located at the start of the range */ text_iterator begin(); /** * @brief Retrieves an iterator for the end of the codepoint range * @return An iterator which is located at the end of the range */ text_iterator end(); }; /** * @brief Computes the byte size the given codepoint would have if it were encoded using the standard encoding * * The standard encoding is UTF-8 in Unicode mode and ASCII otherwise. * * @param cp The codepoint to determine the size for. * @return The number of bytes required for encoding this code point. */ size_t encoded_size(codepoint_t cp); /** * @brief Appends the given code point to the byte range specified by buffer. * * @c buffer can be a pointer to a character buffer or an output iterator. If the engine is in Unicode mode then the * codepoint will be encoded using UTF-8. Otherwise the codepoint will be truncated to fit into the ASCII encoding. * * @tparam octet_iterator The type of the output sequence. * @param cp The codepoint to encode * @param buffer The buffer to write the encoded data to. * @return The value of the iterator after appending all bytes to the sequence. */ template octet_iterator encode(codepoint_t cp, octet_iterator buffer) { if (Unicode_text_mode) { try { return utf8::append(cp, buffer); } catch(const std::exception& e) { Error(LOCATION, "Exception while encoding Unicode code point %" PRIu32 ": %s", (uint32_t)cp, e.what()); return buffer; } } else { // In the legacy mode every code point is exactly one char *(buffer++) = (char)cp; return buffer; } } /** * @brief Counts the number of code points in the specified byte sequence * * @note This respects the Unicode mod setting and should be used where the text contains Unicode characters. * * @tparam octet_iterator The type of the byte sequence. * @param start An iterator located at the start of the sequence * @param end An iterator which signals the end of the sequence * @return The number of codepoints found between @c start and @c end */ template size_t num_codepoints(octet_iterator start, octet_iterator end) { if (Unicode_text_mode) { try { return static_cast(utf8::distance(start, end)); } catch(const std::exception& e) { Error(LOCATION, "Exception while counting Unicode code points: %s", e.what()); return 0; } } else { return static_cast(std::distance(start, end)); } } template void advance(octet_iterator& start, size_t n, octet_iterator end) { if (Unicode_text_mode) { utf8::advance(start, n, end); } else { start = std::min(start + n, end); } } enum class Encoding { Encoding_current, Encoding_utf8, Encoding_iso8859_1 }; bool string_is_ascii_only(const char* str, size_t len); const char* get_encoding_string(Encoding encoding); bool convert_encoding(SCP_string& buffer, const char* src, Encoding encoding_src, Encoding encoding_dest = Encoding::Encoding_current); }