Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-23 21:01:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1715595 - Use char rather than uint8_t for utf-8 in unified components r=platform-i18n-reviewers,gregtatum

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D120508
This commit is contained in:
Dan Minor 2021-07-23 14:58:00 +00:00
parent 4b66c66bb3
commit 26b1e33737
6 changed files with 46 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
intl/components/gtest/TestBuffer.h
View File

@ -60,10 +60,8 @@ class TestBuffer {
/**
* Get a string view into the buffer, which is useful for test assertions.
*/
template <typename C2>
std::basic_string_view<const C2> get_string_view() {
return std::basic_string_view<const C2>(reinterpret_cast<const C2*>(data()),
length());
std::basic_string_view<CharType> get_string_view() {
return std::basic_string_view<CharType>(data(), length());
}
Vector<C, inlineCapacity> mBuffer{};

6
intl/components/gtest/TestCalendar.cpp
View File

@ -5,7 +5,7 @@
#include "mozilla/intl/Calendar.h"
#include "mozilla/Span.h"
#include "./TestBuffer.h"
#include "TestBuffer.h"
namespace mozilla::intl {
@ -95,12 +95,12 @@ TEST(IntlCalendar, GetCanonicalTimeZoneID)
// Providing a canonical time zone results in the same string at the end.
Calendar::GetCanonicalTimeZoneID(MakeStringSpan(u"America/Chicago"), buffer)
.unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"America/Chicago");
ASSERT_EQ(buffer.get_string_view(), u"America/Chicago");
// Providing an alias will result in the canonical representation.
Calendar::GetCanonicalTimeZoneID(MakeStringSpan(u"Europe/Belfast"), buffer)
.unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"Europe/London");
ASSERT_EQ(buffer.get_string_view(), u"Europe/London");
// An unknown time zone results in an error.
ASSERT_TRUE(Calendar::GetCanonicalTimeZoneID(

34
intl/components/gtest/TestDateTimeFormat.cpp
View File

@ -7,7 +7,7 @@
#include "mozilla/intl/DateTimeFormat.h"
#include "mozilla/intl/DateTimePatternGenerator.h"
#include "mozilla/Span.h"
#include "./TestBuffer.h"
#include "TestBuffer.h"
namespace mozilla::intl {
@ -29,10 +29,10 @@ TEST(IntlDateTimeFormat, Style_enUS_utf8)
{
auto dtFormat =
testStyle("en-US", DateTimeStyle::Medium, DateTimeStyle::Medium);
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "Sep 23, 2002, 8:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM");
}
TEST(IntlDateTimeFormat, Style_enUS_utf16)
@ -42,16 +42,16 @@ TEST(IntlDateTimeFormat, Style_enUS_utf16)
TestBuffer<char16_t> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"Sep 23, 2002, 8:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), u"Sep 23, 2002, 8:07:30 PM");
}
TEST(IntlDateTimeFormat, Style_ar_utf8)
{
auto dtFormat = testStyle("ar", DateTimeStyle::Medium, DateTimeStyle::None);
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "٨:٠٧:٣٠ م");
ASSERT_EQ(buffer.get_string_view(), "٨:٠٧:٣٠ م");
}
TEST(IntlDateTimeFormat, Style_ar_utf16)
@ -60,16 +60,16 @@ TEST(IntlDateTimeFormat, Style_ar_utf16)
TestBuffer<char16_t> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"٨:٠٧:٣٠ م");
ASSERT_EQ(buffer.get_string_view(), u"٨:٠٧:٣٠ م");
}
TEST(IntlDateTimeFormat, Style_enUS_fallback_to_default_styles)
{
auto dtFormat = testStyle("en-US", DateTimeStyle::None, DateTimeStyle::None);
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "Sep 23, 2002, 8:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 8:07:30 PM");
}
TEST(IntlDateTimeFormat, Skeleton_enUS_utf8_in)
@ -78,10 +78,10 @@ TEST(IntlDateTimeFormat, Skeleton_enUS_utf8_in)
DateTimeFormat::TryCreateFromSkeleton(
"en-US", MakeStringSpan("yMdhhmmss"), Some(MakeStringSpan("GMT+3")))
.unwrap();
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "9/23/2002, 8:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM");
}
TEST(IntlDateTimeFormat, Skeleton_enUS_utf16_in)
@ -90,10 +90,10 @@ TEST(IntlDateTimeFormat, Skeleton_enUS_utf16_in)
DateTimeFormat::TryCreateFromSkeleton(
"en-US", MakeStringSpan(u"yMdhhmmss"), Some(MakeStringSpan(u"GMT+3")))
.unwrap();
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "9/23/2002, 8:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), "9/23/2002, 8:07:30 PM");
}
TEST(IntlDateTimeFormat, Time_zone_IANA_identifier)
@ -103,9 +103,9 @@ TEST(IntlDateTimeFormat, Time_zone_IANA_identifier)
MakeStringSpan("en-US"), DateTimeStyle::Medium, DateTimeStyle::Medium,
Some(MakeStringSpan(u"America/Chicago")))
.unwrap();
TestBuffer<uint8_t> buffer;
TestBuffer<char> buffer;
dtFormat->TryFormat(DATE, buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char>(), "Sep 23, 2002, 12:07:30 PM");
ASSERT_EQ(buffer.get_string_view(), "Sep 23, 2002, 12:07:30 PM");
}
TEST(IntlDateTimePatternGenerator, GetBestPattern)
@ -114,7 +114,7 @@ TEST(IntlDateTimePatternGenerator, GetBestPattern)
TestBuffer<char16_t> buffer;
gen->GetBestPattern(MakeStringSpan(u"yMd"), buffer).unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"M/d/y");
ASSERT_EQ(buffer.get_string_view(), u"M/d/y");
}
TEST(IntlDateTimePatternGenerator, GetSkeleton)
@ -124,7 +124,7 @@ TEST(IntlDateTimePatternGenerator, GetSkeleton)
DateTimePatternGenerator::GetSkeleton(MakeStringSpan(u"M/d/y"), buffer)
.unwrap();
ASSERT_EQ(buffer.get_string_view<char16_t>(), u"yMd");
ASSERT_EQ(buffer.get_string_view(), u"yMd");
}
} // namespace mozilla::intl

36
intl/components/gtest/TestNumberFormat.cpp
View File

@ -4,7 +4,7 @@
#include "gtest/gtest.h"
#include "mozilla/intl/NumberFormat.h"
#include "./TestBuffer.h"
#include "TestBuffer.h"
namespace mozilla {
namespace intl {
@ -14,12 +14,12 @@ TEST(IntlNumberFormat, Basic)
NumberFormatOptions options;
UniquePtr<NumberFormat> nf =
NumberFormat::TryCreate("en-US", options).unwrap();
TestBuffer<uint8_t> buf8;
TestBuffer<char> buf8;
ASSERT_TRUE(nf->format(1234.56, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "1,234.56");
ASSERT_EQ(buf8.get_string_view(), "1,234.56");
TestBuffer<char16_t> buf16;
ASSERT_TRUE(nf->format(1234.56, buf16).isOk());
ASSERT_EQ(buf16.get_string_view<char16_t>(), u"1,234.56");
ASSERT_EQ(buf16.get_string_view(), u"1,234.56");
const char16_t* res16 = nf->format(1234.56).unwrap().data();
ASSERT_TRUE(res16 != nullptr);
ASSERT_EQ(std::u16string_view(res16), u"1,234.56");
@ -27,9 +27,9 @@ TEST(IntlNumberFormat, Basic)
UniquePtr<NumberFormat> nfAr =
NumberFormat::TryCreate("ar", options).unwrap();
ASSERT_TRUE(nfAr->format(1234.56, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "١٬٢٣٤٫٥٦");
ASSERT_EQ(buf8.get_string_view(), "١٬٢٣٤٫٥٦");
ASSERT_TRUE(nfAr->format(1234.56, buf16).isOk());
ASSERT_EQ(buf16.get_string_view<char16_t>(), u"١٬٢٣٤٫٥٦");
ASSERT_EQ(buf16.get_string_view(), u"١٬٢٣٤٫٥٦");
res16 = nfAr->format(1234.56).unwrap().data();
ASSERT_TRUE(res16 != nullptr);
ASSERT_EQ(std::u16string_view(res16), u"١٬٢٣٤٫٥٦");
@ -40,12 +40,12 @@ TEST(IntlNumberFormat, Numbers)
NumberFormatOptions options;
UniquePtr<NumberFormat> nf =
NumberFormat::TryCreate("es-ES", options).unwrap();
TestBuffer<uint8_t> buf8;
TestBuffer<char> buf8;
ASSERT_TRUE(nf->format(123456.789, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "123.456,789");
ASSERT_EQ(buf8.get_string_view(), "123.456,789");
TestBuffer<char16_t> buf16;
ASSERT_TRUE(nf->format(123456.789, buf16).isOk());
ASSERT_EQ(buf16.get_string_view<char16_t>(), u"123.456,789");
ASSERT_EQ(buf16.get_string_view(), u"123.456,789");
const char16_t* res = nf->format(123456.789).unwrap().data();
ASSERT_TRUE(res != nullptr);
@ -58,11 +58,11 @@ TEST(IntlNumberFormat, SignificantDigits)
options.mSignificantDigits = Some(std::make_pair(3, 5));
UniquePtr<NumberFormat> nf =
NumberFormat::TryCreate("es-ES", options).unwrap();
TestBuffer<uint8_t> buf8;
TestBuffer<char> buf8;
ASSERT_TRUE(nf->format(123456.789, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "123.460");
ASSERT_EQ(buf8.get_string_view(), "123.460");
ASSERT_TRUE(nf->format(0.7, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "0,700");
ASSERT_EQ(buf8.get_string_view(), "0,700");
}
TEST(IntlNumberFormat, Currency)
@ -72,12 +72,12 @@ TEST(IntlNumberFormat, Currency)
Some(std::make_pair("MXN", NumberFormatOptions::CurrencyDisplay::Symbol));
UniquePtr<NumberFormat> nf =
NumberFormat::TryCreate("es-MX", options).unwrap();
TestBuffer<uint8_t> buf8;
TestBuffer<char> buf8;
ASSERT_TRUE(nf->format(123456.789, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "$123,456.79");
ASSERT_EQ(buf8.get_string_view(), "$123,456.79");
TestBuffer<char16_t> buf16;
ASSERT_TRUE(nf->format(123456.789, buf16).isOk());
ASSERT_EQ(buf16.get_string_view<char16_t>(), u"$123,456.79");
ASSERT_EQ(buf16.get_string_view(), u"$123,456.79");
const char16_t* res = nf->format(123456.789).unwrap().data();
ASSERT_TRUE(res != nullptr);
ASSERT_EQ(std::u16string_view(res), u"$123,456.79");
@ -90,12 +90,12 @@ TEST(IntlNumberFormat, Unit)
NumberFormatOptions::UnitDisplay::Long));
UniquePtr<NumberFormat> nf =
NumberFormat::TryCreate("es-MX", options).unwrap();
TestBuffer<uint8_t> buf8;
TestBuffer<char> buf8;
ASSERT_TRUE(nf->format(12.34, buf8).isOk());
ASSERT_EQ(buf8.get_string_view<char>(), "12.34 metros por segundo");
ASSERT_EQ(buf8.get_string_view(), "12.34 metros por segundo");
TestBuffer<char16_t> buf16;
ASSERT_TRUE(nf->format(12.34, buf16).isOk());
ASSERT_EQ(buf16.get_string_view<char16_t>(), u"12.34 metros por segundo");
ASSERT_EQ(buf16.get_string_view(), u"12.34 metros por segundo");
const char16_t* res = nf->format(12.34).unwrap().data();
ASSERT_TRUE(res != nullptr);
ASSERT_EQ(std::u16string_view(res), u"12.34 metros por segundo");

6
intl/components/src/NumberFormat.h
View File

@ -383,13 +383,13 @@ class NumberFormat final {
template <typename C, typename B>
Result<Ok, NumberFormat::FormatError> formatResult(B& buffer) const {
// We only support buffers with uint8_t or char16_t for now.
static_assert(std::is_same<C, uint8_t>::value ||
// We only support buffers with char or char16_t.
static_assert(std::is_same<C, char>::value ||
std::is_same<C, char16_t>::value);
return formatResult().andThen([&buffer](std::u16string_view result)
-> Result<Ok, NumberFormat::FormatError> {
if constexpr (std::is_same<C, uint8_t>::value) {
if constexpr (std::is_same<C, char>::value) {
if (!FillUTF8Buffer(Span(result.data(), result.size()), buffer)) {
return Err(FormatError::OutOfMemory);
}

6
intl/l10n/FluentBundle.cpp
View File

@ -21,7 +21,7 @@ namespace intl {
class SizeableUTF8Buffer {
public:
using CharType = uint8_t;
using CharType = char;
bool reserve(size_t size) {
mBuffer.reset(reinterpret_cast<CharType*>(malloc(size)));
@ -303,7 +303,7 @@ uint8_t* FluentBuiltInNumberFormatterFormat(
if (nf->format(input, buffer).isOk()) {
*aOutCount = buffer.mWritten;
*aOutCapacity = buffer.mCapacity;
return buffer.mBuffer.release();
return reinterpret_cast<uint8_t*>(buffer.mBuffer.release());
}
return nullptr;
@ -373,7 +373,7 @@ uint8_t* FluentBuiltInDateTimeFormatterFormat(
*aOutCount = buffer.mWritten;
return buffer.mBuffer.release();
return reinterpret_cast<uint8_t*>(buffer.mBuffer.release());
}
void FluentBuiltInDateTimeFormatterDestroy(