From 26b1e3373774133f882c7e152887ed764488886a Mon Sep 17 00:00:00 2001 From: Dan Minor Date: Fri, 23 Jul 2021 14:58:00 +0000 Subject: [PATCH] Bug 1715595 - Use char rather than uint8_t for utf-8 in unified components r=platform-i18n-reviewers,gregtatum Differential Revision: https://phabricator.services.mozilla.com/D120508 --- intl/components/gtest/TestBuffer.h | 6 ++-- intl/components/gtest/TestCalendar.cpp | 6 ++-- intl/components/gtest/TestDateTimeFormat.cpp | 34 +++++++++--------- intl/components/gtest/TestNumberFormat.cpp | 36 ++++++++++---------- intl/components/src/NumberFormat.h | 6 ++-- intl/l10n/FluentBundle.cpp | 6 ++-- 6 files changed, 46 insertions(+), 48 deletions(-) diff --git a/intl/components/gtest/TestBuffer.h b/intl/components/gtest/TestBuffer.h index 30b079b6b09c..5a5cd0c97add 100644 --- a/intl/components/gtest/TestBuffer.h +++ b/intl/components/gtest/TestBuffer.h @@ -60,10 +60,8 @@ class TestBuffer { /** * Get a string view into the buffer, which is useful for test assertions. */ - template - std::basic_string_view get_string_view() { - return std::basic_string_view(reinterpret_cast(data()), - length()); + std::basic_string_view get_string_view() { + return std::basic_string_view(data(), length()); } Vector mBuffer{}; diff --git a/intl/components/gtest/TestCalendar.cpp b/intl/components/gtest/TestCalendar.cpp index aaf0206bc63f..1b5c249be36f 100644 --- a/intl/components/gtest/TestCalendar.cpp +++ b/intl/components/gtest/TestCalendar.cpp @@ -5,7 +5,7 @@ #include "mozilla/intl/Calendar.h" #include "mozilla/Span.h" -#include "./TestBuffer.h" +#include "TestBuffer.h" namespace mozilla::intl { @@ -95,12 +95,12 @@ TEST(IntlCalendar, GetCanonicalTimeZoneID) // Providing a canonical time zone results in the same string at the end. Calendar::GetCanonicalTimeZoneID(MakeStringSpan(u"America/Chicago"), buffer) .unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"America/Chicago"); + ASSERT_EQ(buffer.get_string_view(), u"America/Chicago"); // Providing an alias will result in the canonical representation. Calendar::GetCanonicalTimeZoneID(MakeStringSpan(u"Europe/Belfast"), buffer) .unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"Europe/London"); + ASSERT_EQ(buffer.get_string_view(), u"Europe/London"); // An unknown time zone results in an error. ASSERT_TRUE(Calendar::GetCanonicalTimeZoneID( diff --git a/intl/components/gtest/TestDateTimeFormat.cpp b/intl/components/gtest/TestDateTimeFormat.cpp index 82413e443ca6..3911f97ce8ee 100644 --- a/intl/components/gtest/TestDateTimeFormat.cpp +++ b/intl/components/gtest/TestDateTimeFormat.cpp @@ -7,7 +7,7 @@ #include "mozilla/intl/DateTimeFormat.h" #include "mozilla/intl/DateTimePatternGenerator.h" #include "mozilla/Span.h" -#include "./TestBuffer.h" +#include "TestBuffer.h" namespace mozilla::intl { @@ -29,10 +29,10 @@ TEST(IntlDateTimeFormat, Style_enUS_utf8) { auto dtFormat = testStyle("en-US", DateTimeStyle::Medium, DateTimeStyle::Medium); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM"); } TEST(IntlDateTimeFormat, Style_enUS_utf16) @@ -42,16 +42,16 @@ TEST(IntlDateTimeFormat, Style_enUS_utf16) TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"Sep 23, 2002, 8:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), u"Sep 23, 2002, 8:07:30 PM"); } TEST(IntlDateTimeFormat, Style_ar_utf8) { auto dtFormat = testStyle("ar", DateTimeStyle::Medium, DateTimeStyle::None); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "٨:٠٧:٣٠ م"); + ASSERT_EQ(buffer.get_string_view(), "٨:٠٧:٣٠ م"); } TEST(IntlDateTimeFormat, Style_ar_utf16) @@ -60,16 +60,16 @@ TEST(IntlDateTimeFormat, Style_ar_utf16) TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"٨:٠٧:٣٠ م"); + ASSERT_EQ(buffer.get_string_view(), u"٨:٠٧:٣٠ م"); } TEST(IntlDateTimeFormat, Style_enUS_fallback_to_default_styles) { auto dtFormat = testStyle("en-US", DateTimeStyle::None, DateTimeStyle::None); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM"); } TEST(IntlDateTimeFormat, Skeleton_enUS_utf8_in) @@ -78,10 +78,10 @@ TEST(IntlDateTimeFormat, Skeleton_enUS_utf8_in) DateTimeFormat::TryCreateFromSkeleton( "en-US", MakeStringSpan("yMdhhmmss"), Some(MakeStringSpan("GMT+3"))) .unwrap(); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM"); } TEST(IntlDateTimeFormat, Skeleton_enUS_utf16_in) @@ -90,10 +90,10 @@ TEST(IntlDateTimeFormat, Skeleton_enUS_utf16_in) DateTimeFormat::TryCreateFromSkeleton( "en-US", MakeStringSpan(u"yMdhhmmss"), Some(MakeStringSpan(u"GMT+3"))) .unwrap(); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM"); } TEST(IntlDateTimeFormat, Time_zone_IANA_identifier) @@ -103,9 +103,9 @@ TEST(IntlDateTimeFormat, Time_zone_IANA_identifier) MakeStringSpan("en-US"), DateTimeStyle::Medium, DateTimeStyle::Medium, Some(MakeStringSpan(u"America/Chicago"))) .unwrap(); - TestBuffer buffer; + TestBuffer buffer; dtFormat->TryFormat(DATE, buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 12:07:30 PM"); + ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 12:07:30 PM"); } TEST(IntlDateTimePatternGenerator, GetBestPattern) @@ -114,7 +114,7 @@ TEST(IntlDateTimePatternGenerator, GetBestPattern) TestBuffer buffer; gen->GetBestPattern(MakeStringSpan(u"yMd"), buffer).unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"M/d/y"); + ASSERT_EQ(buffer.get_string_view(), u"M/d/y"); } TEST(IntlDateTimePatternGenerator, GetSkeleton) @@ -124,7 +124,7 @@ TEST(IntlDateTimePatternGenerator, GetSkeleton) DateTimePatternGenerator::GetSkeleton(MakeStringSpan(u"M/d/y"), buffer) .unwrap(); - ASSERT_EQ(buffer.get_string_view(), u"yMd"); + ASSERT_EQ(buffer.get_string_view(), u"yMd"); } } // namespace mozilla::intl diff --git a/intl/components/gtest/TestNumberFormat.cpp b/intl/components/gtest/TestNumberFormat.cpp index db1382db1584..70b3de34318d 100644 --- a/intl/components/gtest/TestNumberFormat.cpp +++ b/intl/components/gtest/TestNumberFormat.cpp @@ -4,7 +4,7 @@ #include "gtest/gtest.h" #include "mozilla/intl/NumberFormat.h" -#include "./TestBuffer.h" +#include "TestBuffer.h" namespace mozilla { namespace intl { @@ -14,12 +14,12 @@ TEST(IntlNumberFormat, Basic) NumberFormatOptions options; UniquePtr nf = NumberFormat::TryCreate("en-US", options).unwrap(); - TestBuffer buf8; + TestBuffer buf8; ASSERT_TRUE(nf->format(1234.56, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "1,234.56"); + ASSERT_EQ(buf8.get_string_view(), "1,234.56"); TestBuffer buf16; ASSERT_TRUE(nf->format(1234.56, buf16).isOk()); - ASSERT_EQ(buf16.get_string_view(), u"1,234.56"); + ASSERT_EQ(buf16.get_string_view(), u"1,234.56"); const char16_t* res16 = nf->format(1234.56).unwrap().data(); ASSERT_TRUE(res16 != nullptr); ASSERT_EQ(std::u16string_view(res16), u"1,234.56"); @@ -27,9 +27,9 @@ TEST(IntlNumberFormat, Basic) UniquePtr nfAr = NumberFormat::TryCreate("ar", options).unwrap(); ASSERT_TRUE(nfAr->format(1234.56, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "١٬٢٣٤٫٥٦"); + ASSERT_EQ(buf8.get_string_view(), "١٬٢٣٤٫٥٦"); ASSERT_TRUE(nfAr->format(1234.56, buf16).isOk()); - ASSERT_EQ(buf16.get_string_view(), u"١٬٢٣٤٫٥٦"); + ASSERT_EQ(buf16.get_string_view(), u"١٬٢٣٤٫٥٦"); res16 = nfAr->format(1234.56).unwrap().data(); ASSERT_TRUE(res16 != nullptr); ASSERT_EQ(std::u16string_view(res16), u"١٬٢٣٤٫٥٦"); @@ -40,12 +40,12 @@ TEST(IntlNumberFormat, Numbers) NumberFormatOptions options; UniquePtr nf = NumberFormat::TryCreate("es-ES", options).unwrap(); - TestBuffer buf8; + TestBuffer buf8; ASSERT_TRUE(nf->format(123456.789, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "123.456,789"); + ASSERT_EQ(buf8.get_string_view(), "123.456,789"); TestBuffer buf16; ASSERT_TRUE(nf->format(123456.789, buf16).isOk()); - ASSERT_EQ(buf16.get_string_view(), u"123.456,789"); + ASSERT_EQ(buf16.get_string_view(), u"123.456,789"); const char16_t* res = nf->format(123456.789).unwrap().data(); ASSERT_TRUE(res != nullptr); @@ -58,11 +58,11 @@ TEST(IntlNumberFormat, SignificantDigits) options.mSignificantDigits = Some(std::make_pair(3, 5)); UniquePtr nf = NumberFormat::TryCreate("es-ES", options).unwrap(); - TestBuffer buf8; + TestBuffer buf8; ASSERT_TRUE(nf->format(123456.789, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "123.460"); + ASSERT_EQ(buf8.get_string_view(), "123.460"); ASSERT_TRUE(nf->format(0.7, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "0,700"); + ASSERT_EQ(buf8.get_string_view(), "0,700"); } TEST(IntlNumberFormat, Currency) @@ -72,12 +72,12 @@ TEST(IntlNumberFormat, Currency) Some(std::make_pair("MXN", NumberFormatOptions::CurrencyDisplay::Symbol)); UniquePtr nf = NumberFormat::TryCreate("es-MX", options).unwrap(); - TestBuffer buf8; + TestBuffer buf8; ASSERT_TRUE(nf->format(123456.789, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "$123,456.79"); + ASSERT_EQ(buf8.get_string_view(), "$123,456.79"); TestBuffer buf16; ASSERT_TRUE(nf->format(123456.789, buf16).isOk()); - ASSERT_EQ(buf16.get_string_view(), u"$123,456.79"); + ASSERT_EQ(buf16.get_string_view(), u"$123,456.79"); const char16_t* res = nf->format(123456.789).unwrap().data(); ASSERT_TRUE(res != nullptr); ASSERT_EQ(std::u16string_view(res), u"$123,456.79"); @@ -90,12 +90,12 @@ TEST(IntlNumberFormat, Unit) NumberFormatOptions::UnitDisplay::Long)); UniquePtr nf = NumberFormat::TryCreate("es-MX", options).unwrap(); - TestBuffer buf8; + TestBuffer buf8; ASSERT_TRUE(nf->format(12.34, buf8).isOk()); - ASSERT_EQ(buf8.get_string_view(), "12.34 metros por segundo"); + ASSERT_EQ(buf8.get_string_view(), "12.34 metros por segundo"); TestBuffer buf16; ASSERT_TRUE(nf->format(12.34, buf16).isOk()); - ASSERT_EQ(buf16.get_string_view(), u"12.34 metros por segundo"); + ASSERT_EQ(buf16.get_string_view(), u"12.34 metros por segundo"); const char16_t* res = nf->format(12.34).unwrap().data(); ASSERT_TRUE(res != nullptr); ASSERT_EQ(std::u16string_view(res), u"12.34 metros por segundo"); diff --git a/intl/components/src/NumberFormat.h b/intl/components/src/NumberFormat.h index 4cd467a29a64..959ee24dec9f 100644 --- a/intl/components/src/NumberFormat.h +++ b/intl/components/src/NumberFormat.h @@ -383,13 +383,13 @@ class NumberFormat final { template Result formatResult(B& buffer) const { - // We only support buffers with uint8_t or char16_t for now. - static_assert(std::is_same::value || + // We only support buffers with char or char16_t. + static_assert(std::is_same::value || std::is_same::value); return formatResult().andThen([&buffer](std::u16string_view result) -> Result { - if constexpr (std::is_same::value) { + if constexpr (std::is_same::value) { if (!FillUTF8Buffer(Span(result.data(), result.size()), buffer)) { return Err(FormatError::OutOfMemory); } diff --git a/intl/l10n/FluentBundle.cpp b/intl/l10n/FluentBundle.cpp index 21a25a3368f8..fa6dc8e4b515 100644 --- a/intl/l10n/FluentBundle.cpp +++ b/intl/l10n/FluentBundle.cpp @@ -21,7 +21,7 @@ namespace intl { class SizeableUTF8Buffer { public: - using CharType = uint8_t; + using CharType = char; bool reserve(size_t size) { mBuffer.reset(reinterpret_cast(malloc(size))); @@ -303,7 +303,7 @@ uint8_t* FluentBuiltInNumberFormatterFormat( if (nf->format(input, buffer).isOk()) { *aOutCount = buffer.mWritten; *aOutCapacity = buffer.mCapacity; - return buffer.mBuffer.release(); + return reinterpret_cast(buffer.mBuffer.release()); } return nullptr; @@ -373,7 +373,7 @@ uint8_t* FluentBuiltInDateTimeFormatterFormat( *aOutCount = buffer.mWritten; - return buffer.mBuffer.release(); + return reinterpret_cast(buffer.mBuffer.release()); } void FluentBuiltInDateTimeFormatterDestroy(