openharmony/third_party_rust_unicode-bidi
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_unicode-bidi.git synced 2026-06-30 21:27:57 -04:00
Code Issues Packages Projects Releases Wiki Activity

[lib] Implement L1 rule

Browse Source 
To be able to implement L1, we need access to more information from
`BidiInfo`, namely `original_classes` of the `text`, in `visual_runs()`,
which would mean it should pass through `reorder_line()`.

The fact that information from `BidiInfo` is needed for both steps of
the public API (generating `BidiInfo` and consuming it
per-paragraph/per-level) made me change the API design and move these
methods into `impl BidiInfo`.

Then, since we needed access to `text` for every `BidiInfo` consumption,
I added a reference to `text` to `BidiInfo`, which also enables more
compile-time checks for `BidiInfo` isntance not outliving the text in
the user code.

NOTE: We are already breaking API in version 0.3.0 and paving for full
spec support is a good reason to do so, IMHO.

The L1 rule works by one pass on the text of the line.

Conformance Test: this implementation reduces the number of failures
from 60494 to 23770 (out of total 256747 cases).

Fix #2
This commit is contained in:
Behnam Esfahbod
2017-05-11 23:33:09 -05:00
parent 19d985856d
commit 6427532127
8 changed files with 472 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AUTHORS
+1
View File
@@ -1,3 +1,4 @@
This software was written by the following people:
Matt Brubeck <mbrubeck@limpet.net>
Behnam Esfahbod <behnam@zwnj.org>
src/char_data/mod.rs
+8 -1
View File
@@ -24,6 +24,13 @@ pub fn bidi_class(c: char) -> BidiClass {
bsearch_range_value_table(c, bidi_class_table)
}
pub fn is_rtl(bidi_class: BidiClass) -> bool {
match bidi_class {
RLE | RLO | RLI => true,
_ => false,
}
}
fn bsearch_range_value_table(c: char, r: &'static [(char, char, BidiClass)]) -> BidiClass {
match r.binary_search_by(
|&(lo, hi, _)| if lo <= c && c <= hi {
@@ -45,7 +52,7 @@ fn bsearch_range_value_table(c: char, r: &'static [(char, char, BidiClass)]) ->
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
#[test]
src/explicit.rs
+11 -16
View File
@@ -11,21 +11,21 @@
//!
//! http://www.unicode.org/reports/tr9/#Explicit_Levels_and_Directions
use super::char_data::BidiClass;
use super::char_data::{BidiClass, is_rtl};
use super::level::Level;
use BidiClass::*;
/// Compute explicit embedding levels for one paragraph of text (X1-X8).
///
/// `classes[i]` must contain the BidiClass of the char at byte index `i`,
/// `processing_classes[i]` must contain the BidiClass of the char at byte index `i`,
/// for each char in `text`.
pub fn compute(
text: &str,
para_level: Level,
initial_classes: &[BidiClass],
levels: &mut [Level],
classes: &mut [BidiClass],
processing_classes: &mut [BidiClass],
) {
assert!(text.len() == initial_classes.len());
@@ -41,13 +41,8 @@ pub fn compute(
match initial_classes[i] {
// Rules X2-X5c
RLE | LRE | RLO | LRO | RLI | LRI | FSI => {
let char_is_rtl = match initial_classes[i] {
RLE | RLO | RLI => true,
_ => false,
};
let last_level = stack.last().level;
let new_level = if char_is_rtl {
let new_level = if is_rtl(initial_classes[i]) {
last_level.new_explicit_next_rtl()
} else {
last_level.new_explicit_next_ltr()
@@ -58,8 +53,8 @@ pub fn compute(
if is_isolate {
levels[i] = last_level;
match stack.last().status {
OverrideStatus::RTL => classes[i] = R,
OverrideStatus::LTR => classes[i] = L,
OverrideStatus::RTL => processing_classes[i] = R,
OverrideStatus::LTR => processing_classes[i] = L,
_ => {}
}
}
@@ -108,8 +103,8 @@ pub fn compute(
let last = stack.last();
levels[i] = last.level;
match last.status {
OverrideStatus::RTL => classes[i] = R,
OverrideStatus::LTR => classes[i] = L,
OverrideStatus::RTL => processing_classes[i] = R,
OverrideStatus::LTR => processing_classes[i] = L,
_ => {}
}
}
@@ -135,8 +130,8 @@ pub fn compute(
let last = stack.last();
levels[i] = last.level;
match last.status {
OverrideStatus::RTL => classes[i] = R,
OverrideStatus::LTR => classes[i] = L,
OverrideStatus::RTL => processing_classes[i] = R,
OverrideStatus::LTR => processing_classes[i] = L,
_ => {}
}
}
@@ -144,7 +139,7 @@ pub fn compute(
// Handle multi-byte characters.
for j in 1..c.len_utf8() {
levels[i + j] = levels[i];
classes[i + j] = classes[i];
processing_classes[i + j] = processing_classes[i];
}
}
}
src/implicit.rs
+22 -22
View File
@@ -20,7 +20,7 @@ use BidiClass::*;
/// 3.3.4 Resolving Weak Types
///
/// http://www.unicode.org/reports/tr9/#Resolving_Weak_Types
pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass]) {
pub fn resolve_weak(sequence: &IsolatingRunSequence, processing_classes: &mut [BidiClass]) {
// FIXME (#8): This function applies steps W1-W6 in a single pass. This can produce
// incorrect results in cases where a "later" rule changes the value of `prev_class` seen
// by an "earlier" rule. We should either split this into separate passes, or preserve
@@ -41,10 +41,10 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
.flat_map(id as fn(LevelRun) -> LevelRun);
while let Some(i) = indices.next() {
match classes[i] {
match processing_classes[i] {
// http://www.unicode.org/reports/tr9/#W1
NSM => {
classes[i] = match prev_class {
processing_classes[i] = match prev_class {
RLI | LRI | FSI | PDI => ON,
_ => prev_class,
};
@@ -52,27 +52,27 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
EN => {
if last_strong_is_al {
// W2. If previous strong char was AL, change EN to AN.
classes[i] = AN;
processing_classes[i] = AN;
} else {
// W5. If a run of ETs is adjacent to an EN, change the ETs to EN.
for j in &et_run_indices {
classes[*j] = EN;
processing_classes[*j] = EN;
}
et_run_indices.clear();
}
}
// http://www.unicode.org/reports/tr9/#W3
AL => classes[i] = R,
AL => processing_classes[i] = R,
// http://www.unicode.org/reports/tr9/#W4
ES | CS => {
let next_class = indices
.clone()
.map(|j| classes[j])
.map(|j| processing_classes[j])
.filter(not_removed_by_x9)
.next()
.unwrap_or(sequence.eos);
classes[i] = match (prev_class, classes[i], next_class) {
processing_classes[i] = match (prev_class, processing_classes[i], next_class) {
(EN, ES, EN) | (EN, CS, EN) => EN,
(AN, CS, AN) => AN,
(_, _, _) => ON,
@@ -81,7 +81,7 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
// http://www.unicode.org/reports/tr9/#W5
ET => {
match prev_class {
EN => classes[i] = EN,
EN => processing_classes[i] = EN,
_ => et_run_indices.push(i), // In case this is followed by an EN.
}
}
@@ -92,7 +92,7 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
}
}
prev_class = classes[i];
prev_class = processing_classes[i];
match prev_class {
L | R => {
last_strong_is_al = false;
@@ -105,7 +105,7 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
if prev_class != ET {
// W6. If we didn't find an adjacent EN, turn any ETs into ON instead.
for j in &et_run_indices {
classes[*j] = ON;
processing_classes[*j] = ON;
}
et_run_indices.clear();
}
@@ -115,9 +115,9 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
let mut last_strong_is_l = sequence.sos == L;
for run in &sequence.runs {
for i in run.clone() {
match classes[i] {
match processing_classes[i] {
EN if last_strong_is_l => {
classes[i] = L;
processing_classes[i] = L;
}
L => {
last_strong_is_l = true;
@@ -137,7 +137,7 @@ pub fn resolve_weak(sequence: &IsolatingRunSequence, classes: &mut [BidiClass])
pub fn resolve_neutral(
sequence: &IsolatingRunSequence,
levels: &[Level],
classes: &mut [BidiClass],
processing_classes: &mut [BidiClass],
) {
let mut indices = sequence.runs.iter().flat_map(Clone::clone);
let mut prev_class = sequence.sos;
@@ -154,7 +154,7 @@ pub fn resolve_neutral(
// Process sequences of NI characters.
let mut ni_run = Vec::new();
if ni(classes[i]) {
if ni(processing_classes[i]) {
// Consume a run of consecutive NI characters.
ni_run.push(i);
let mut next_class;
@@ -162,10 +162,10 @@ pub fn resolve_neutral(
match indices.next() {
Some(j) => {
i = j;
if removed_by_x9(classes[i]) {
if removed_by_x9(processing_classes[i]) {
continue;
}
next_class = classes[j];
next_class = processing_classes[j];
if ni(next_class) {
ni_run.push(i);
} else {
@@ -187,11 +187,11 @@ pub fn resolve_neutral(
(_, _) => levels[i].bidi_class(),
};
for j in &ni_run {
classes[*j] = new_class;
processing_classes[*j] = new_class;
}
ni_run.clear();
}
prev_class = classes[i];
prev_class = processing_classes[i];
}
}
@@ -200,12 +200,12 @@ pub fn resolve_neutral(
/// Returns the maximum embedding level in the paragraph.
///
/// http://www.unicode.org/reports/tr9/#Resolving_Implicit_Levels
pub fn resolve_levels(classes: &[BidiClass], levels: &mut [Level]) -> Level {
pub fn resolve_levels(original_classes: &[BidiClass], levels: &mut [Level]) -> Level {
let mut max_level = Level::ltr();
assert!(classes.len() == levels.len());
assert!(original_classes.len() == levels.len());
for i in 0..levels.len() {
match (levels[i].is_rtl(), classes[i]) {
match (levels[i].is_rtl(), original_classes[i]) {
// http://www.unicode.org/reports/tr9/#I1
(false, R) => levels[i].raise(1).expect("Level number error"),
(false, AN) | (false, EN) => levels[i].raise(2).expect("Level number error"),
src/level.rs
+1 -1
View File
@@ -228,7 +228,7 @@ impl<'a> PartialEq<&'a str> for Level {
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
#[test]
src/lib.rs
+414 -314
View File
@@ -14,39 +14,43 @@
//! ## Example
//!
//! ```rust
//! use unicode_bidi::{process_text, reorder_line};
//! use unicode_bidi::BidiInfo;
//!
//! // This example text is defined using `concat!` because some browsers
//! // and text editors have trouble displaying bidi strings.
//! let text = concat!["א",
//! "ב",
//! "ג",
//! "a",
//! "b",
//! "c"];
//! let text = concat![
//! "א",
//! "ב",
//! "ג",
//! "a",
//! "b",
//! "c",
//! ];
//!
//! // Resolve embedding levels within the text. Pass `None` to detect the
//! // paragraph level automatically.
//! let info = process_text(&text, None);
//! let bidi_info = BidiInfo::new(&text, None);
//!
//! // This paragraph has embedding level 1 because its first strong character is RTL.
//! assert_eq!(info.paragraphs.len(), 1);
//! let paragraph_info = &info.paragraphs[0];
//! assert_eq!(paragraph_info.level.number(), 1);
//! assert_eq!(paragraph_info.level.is_rtl(), true);
//! assert_eq!(bidi_info.paragraphs.len(), 1);
//! let para = &bidi_info.paragraphs[0];
//! assert_eq!(para.level.number(), 1);
//! assert_eq!(para.level.is_rtl(), true);
//!
//! // Re-ordering is done after wrapping each paragraph into a sequence of
//! // lines. For this example, I'll just use a single line that spans the
//! // entire paragraph.
//! let line = paragraph_info.range.clone();
//! let line = para.range.clone();
//!
//! let display = reorder_line(&text, line, &info.levels);
//! assert_eq!(display, concat!["a",
//! "b",
//! "c",
//! "ג",
//! "ב",
//! "א"]);
//! let display = bidi_info.reorder_line(para, line);
//! assert_eq!(display, concat![
//! "a",
//! "b",
//! "c",
//! "ג",
//! "ב",
//! "א",
//! ]);
//! ```
//!
//! [tr9]: http://www.unicode.org/reports/tr9/
@@ -76,36 +80,8 @@ use std::ops::Range;
use BidiClass::*;
use format_chars as chars;
/// Output of `process_text`
///
/// The `classes` and `levels` vectors are indexed by byte offsets into the text. If a character
/// is multiple bytes wide, then its class and level will appear multiple times in these vectors.
#[derive(Debug, PartialEq)]
pub struct BidiInfo {
/// The BidiClass of the character at each byte in the text.
pub classes: Vec<BidiClass>,
/// The directional embedding level of each byte in the text.
pub levels: Vec<Level>,
/// The boundaries and paragraph embedding level of each paragraph within the text.
///
/// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?
/// Or just don't include the first paragraph, which always starts at 0?
pub paragraphs: Vec<ParagraphInfo>,
}
impl BidiInfo {
/// If processed text has any RTL computed bidi levels
///
/// This information is usually used to skip re-ordering of text when no RTL level is present
#[inline]
pub fn has_rtl(&self) -> bool {
level::has_rtl(&self.levels)
}
}
/// Info about a single paragraph
/// Bidi information about a single paragraph
#[derive(Debug, PartialEq)]
pub struct ParagraphInfo {
/// The paragraphs boundaries within the text, as byte indices.
@@ -119,243 +95,336 @@ pub struct ParagraphInfo {
pub level: Level,
}
/// Split the text into paragraphs and determine the bidirectional embedding levels for each
/// paragraph.
/// Initial bidi information of the text
///
/// TODO: In early steps, check for special cases that allow later steps to be skipped. like text
/// that is entirely LTR. See the `nsBidi` class from Gecko for comparison.
///
/// TODO: Support auto-RTL base direction
pub fn process_text(text: &str, level: Option<Level>) -> BidiInfo {
let InitialProperties {
initial_classes,
paragraphs,
} = initial_scan(text, level);
let mut levels = Vec::<Level>::with_capacity(text.len());
let mut classes = initial_classes.clone();
for para in &paragraphs {
let text = &text[para.range.clone()];
let classes = &mut classes[para.range.clone()];
let initial_classes = &initial_classes[para.range.clone()];
// FIXME: Use `levels.resize(...)` when it becomes stable.
levels.extend(repeat(para.level).take(para.range.len()));
let levels = &mut levels[para.range.clone()];
explicit::compute(text, para.level, &initial_classes, levels, classes);
let sequences = prepare::isolating_run_sequences(para.level, &initial_classes, levels);
for sequence in &sequences {
implicit::resolve_weak(sequence, classes);
implicit::resolve_neutral(sequence, levels, classes);
}
implicit::resolve_levels(classes, levels);
assign_levels_to_removed_chars(para.level, &initial_classes, levels);
}
BidiInfo {
levels: levels,
classes: initial_classes,
paragraphs: paragraphs,
}
}
/// Re-order a line based on resolved levels.
///
/// `levels` are the embedding levels returned by `process_text`.
/// `line` is a range of bytes indices within `text`.
///
/// Returns the line in display order.
pub fn reorder_line<'a>(text: &'a str, line: Range<usize>, levels: &[Level]) -> Cow<'a, str> {
let runs = visual_runs(line.clone(), &levels);
if runs.len() == 1 && !levels[runs[0].start].is_rtl() {
return text.into();
}
let mut result = String::with_capacity(line.len());
for run in runs {
if levels[run.start].is_rtl() {
result.extend(text[run].chars().rev());
} else {
result.push_str(&text[run]);
}
}
result.into()
}
/// Find the level runs within a line and return them in visual order.
///
/// `line` is a range of bytes indices within `levels`.
///
/// http://www.unicode.org/reports/tr9/#Reordering_Resolved_Levels
pub fn visual_runs(line: Range<usize>, levels: &[Level]) -> Vec<LevelRun> {
assert!(line.start <= levels.len());
assert!(line.end <= levels.len());
// TODO: Whitespace handling.
// http://www.unicode.org/reports/tr9/#L1
let mut runs = Vec::new();
// Find consecutive level runs.
let mut start = line.start;
let mut level = levels[start];
let mut min_level = level;
let mut max_level = level;
for i in (start + 1)..line.end {
let new_level = levels[i];
if new_level != level {
// End of the previous run, start of a new one.
runs.push(start..i);
start = i;
level = new_level;
min_level = min(level, min_level);
max_level = max(level, max_level);
}
}
runs.push(start..line.end);
let run_count = runs.len();
// Re-order the odd runs.
// http://www.unicode.org/reports/tr9/#L2
// Stop at the lowest *odd* level.
min_level = min_level.new_lowest_ge_rtl().expect("Level error");
while max_level >= min_level {
// Look for the start of a sequence of consecutive runs of max_level or higher.
let mut seq_start = 0;
while seq_start < run_count {
if levels[runs[seq_start].start] < max_level {
seq_start += 1;
continue;
}
// Found the start of a sequence. Now find the end.
let mut seq_end = seq_start + 1;
while seq_end < run_count {
if levels[runs[seq_end].start] < max_level {
break;
}
seq_end += 1;
}
// Reverse the runs within this sequence.
runs[seq_start..seq_end].reverse();
seq_start = seq_end;
}
max_level
.lower(1)
.expect("Lowering embedding level below zero");
}
runs
}
/// Output of `initial_scan`
/// Contains the paragraphs and BidiClasses in a string of text.
#[derive(PartialEq, Debug)]
pub struct InitialProperties {
pub struct InitialInfo<'text> {
/// The text
pub text: &'text str,
/// The BidiClass of the character at each byte in the text.
/// If a character is multiple bytes, its class will appear multiple times in the vector.
pub initial_classes: Vec<BidiClass>,
pub original_classes: Vec<BidiClass>,
/// The boundaries and level of each paragraph within the text.
pub paragraphs: Vec<ParagraphInfo>,
}
/// Find the paragraphs and BidiClasses in a string of text.
///
/// http://www.unicode.org/reports/tr9/#The_Paragraph_Level
///
/// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong
/// character is found before the matching PDI. If no strong character is found, the class will
/// remain FSI, and it's up to later stages to treat these as LRI when needed.
pub fn initial_scan(text: &str, default_para_level: Option<Level>) -> InitialProperties {
let mut classes = Vec::with_capacity(text.len());
impl<'text> InitialInfo<'text> {
/// Find the paragraphs and BidiClasses in a string of text.
///
/// http://www.unicode.org/reports/tr9/#The_Paragraph_Level
///
/// Also sets the class for each First Strong Isolate initiator (FSI) to LRI or RLI if a strong
/// character is found before the matching PDI. If no strong character is found, the class will
/// remain FSI, and it's up to later stages to treat these as LRI when needed.
pub fn new(text: &str, default_para_level: Option<Level>) -> InitialInfo {
let mut original_classes = Vec::with_capacity(text.len());
// The stack contains the starting byte index for each nested isolate we're inside.
let mut isolate_stack = Vec::new();
let mut paragraphs = Vec::new();
// The stack contains the starting byte index for each nested isolate we're inside.
let mut isolate_stack = Vec::new();
let mut paragraphs = Vec::new();
let mut para_start = 0;
let mut para_level = default_para_level;
let mut para_start = 0;
let mut para_level = default_para_level;
for (i, c) in text.char_indices() {
let class = bidi_class(c);
classes.extend(repeat(class).take(c.len_utf8()));
match class {
B => {
// P1. Split the text into separate paragraphs. The paragraph separator is kept
// with the previous paragraph.
let para_end = i + c.len_utf8();
paragraphs.push(
ParagraphInfo {
range: para_start..para_end,
// P3. If no character is found in p2, set the paragraph level to zero.
level: para_level.unwrap_or(Level::ltr()),
},
);
// Reset state for the start of the next paragraph.
para_start = para_end;
// TODO: Support defaulting to direction of previous paragraph
//
// http://www.unicode.org/reports/tr9/#HL1
para_level = default_para_level;
isolate_stack.clear();
}
L | R | AL => {
match isolate_stack.last() {
Some(&start) => {
if classes[start] == FSI {
// X5c. If the first strong character between FSI and its matching PDI
// is R or AL, treat it as RLI. Otherwise, treat it as LRI.
for j in 0..chars::FSI.len_utf8() {
classes[start + j] = if class == L { LRI } else { RLI };
for (i, c) in text.char_indices() {
let class = bidi_class(c);
original_classes.extend(repeat(class).take(c.len_utf8()));
match class {
B => {
// P1. Split the text into separate paragraphs. The paragraph separator is kept
// with the previous paragraph.
let para_end = i + c.len_utf8();
paragraphs.push(
ParagraphInfo {
range: para_start..para_end,
// P3. If no character is found in p2, set the paragraph level to zero.
level: para_level.unwrap_or(Level::ltr()),
},
);
// Reset state for the start of the next paragraph.
para_start = para_end;
// TODO: Support defaulting to direction of previous paragraph
//
// http://www.unicode.org/reports/tr9/#HL1
para_level = default_para_level;
isolate_stack.clear();
}
L | R | AL => {
match isolate_stack.last() {
Some(&start) => {
if original_classes[start] == FSI {
// X5c. If the first strong character between FSI and its matching PDI
// is R or AL, treat it as RLI. Otherwise, treat it as LRI.
for j in 0..chars::FSI.len_utf8() {
original_classes[start + j] =
if class == L { LRI } else { RLI };
}
}
}
None => {
if para_level.is_none() {
// P2. Find the first character of type L, AL, or R, while skipping any
// characters between an isolate initiator and its matching PDI.
para_level = Some(
if class != L {
Level::rtl()
} else {
Level::ltr()
},
);
}
}
}
None => {
if para_level.is_none() {
// P2. Find the first character of type L, AL, or R, while skipping any
// characters between an isolate initiator and its matching PDI.
para_level = Some(
if class != L {
Level::rtl()
} else {
Level::ltr()
},
);
}
}
}
RLI | LRI | FSI => {
isolate_stack.push(i);
}
PDI => {
isolate_stack.pop();
}
_ => {}
}
RLI | LRI | FSI => {
isolate_stack.push(i);
}
PDI => {
isolate_stack.pop();
}
_ => {}
}
if para_start < text.len() {
paragraphs.push(
ParagraphInfo {
range: para_start..text.len(),
level: para_level.unwrap_or(Level::ltr()),
},
);
}
assert!(original_classes.len() == text.len());
InitialInfo {
text,
original_classes,
paragraphs,
}
}
if para_start < text.len() {
paragraphs.push(
ParagraphInfo {
range: para_start..text.len(),
level: para_level.unwrap_or(Level::ltr()),
},
);
}
assert!(classes.len() == text.len());
}
InitialProperties {
initial_classes: classes,
paragraphs: paragraphs,
/// Bidi information of the text
///
/// The `original_classes` and `levels` vectors are indexed by byte offsets into the text. If a character
/// is multiple bytes wide, then its class and level will appear multiple times in these vectors.
// TODO: Impl `struct StringProperty<T> { values: Vec<T> }` and use instead of Vec<T>
#[derive(Debug, PartialEq)]
pub struct BidiInfo<'text> {
/// The text
pub text: &'text str,
/// The BidiClass of the character at each byte in the text.
pub original_classes: Vec<BidiClass>,
/// The directional embedding level of each byte in the text.
pub levels: Vec<Level>,
/// The boundaries and paragraph embedding level of each paragraph within the text.
///
/// TODO: Use SmallVec or similar to avoid overhead when there are only one or two paragraphs?
/// Or just don't include the first paragraph, which always starts at 0?
pub paragraphs: Vec<ParagraphInfo>,
}
impl<'text> BidiInfo<'text> {
/// Split the text into paragraphs and determine the bidi embedding levels for each paragraph.
///
/// TODO: In early steps, check for special cases that allow later steps to be skipped. like text
/// that is entirely LTR. See the `nsBidi` class from Gecko for comparison.
///
/// TODO: Support auto-RTL base direction
pub fn new(text: &str, default_para_level: Option<Level>) -> BidiInfo {
let InitialInfo {
text: _,
original_classes,
paragraphs,
} = InitialInfo::new(text, default_para_level);
let mut levels = Vec::<Level>::with_capacity(text.len());
let mut processing_classes = original_classes.clone();
for para in &paragraphs {
let text = &text[para.range.clone()];
let original_classes = &original_classes[para.range.clone()];
let processing_classes = &mut processing_classes[para.range.clone()];
let new_len = levels.len() + para.range.len();
levels.resize(new_len, para.level);
let levels = &mut levels[para.range.clone()];
explicit::compute(
text,
para.level,
&original_classes,
levels,
processing_classes,
);
let sequences = prepare::isolating_run_sequences(para.level, &original_classes, levels);
for sequence in &sequences {
implicit::resolve_weak(sequence, processing_classes);
implicit::resolve_neutral(sequence, levels, processing_classes);
}
implicit::resolve_levels(processing_classes, levels);
assign_levels_to_removed_chars(para.level, &original_classes, levels);
}
BidiInfo {
text,
original_classes,
paragraphs,
levels,
}
}
/// Re-order a line based on resolved levels and return only the embedding levels.
pub fn reordered_levels(&self, para: &ParagraphInfo, line: Range<usize>) -> Vec<Level> {
let (levels, _) = self.visual_runs(para, line.clone());
levels
}
/// Re-order a line based on resolved levels and return the line in display order.
pub fn reorder_line(&self, para: &ParagraphInfo, line: Range<usize>) -> Cow<'text, str> {
let (levels, runs) = self.visual_runs(para, line.clone());
if runs.len() == 1 && levels[runs[0].start].is_ltr() {
return self.text.into();
}
let mut result = String::with_capacity(line.len());
for run in runs {
if levels[run.start].is_rtl() {
result.extend(self.text[run].chars().rev());
} else {
result.push_str(&self.text[run]);
}
}
result.into()
}
/// Find the level runs within a line and return them in visual order.
///
/// `line` is a range of bytes indices within `levels`.
///
/// http://www.unicode.org/reports/tr9/#Reordering_Resolved_Levels
pub fn visual_runs(
&self,
para: &ParagraphInfo,
line: Range<usize>,
) -> (Vec<Level>, Vec<LevelRun>) {
assert!(line.start <= self.levels.len());
assert!(line.end <= self.levels.len());
let mut levels = self.levels.clone();
// Reset some whitespace chars to paragraph level.
// http://www.unicode.org/reports/tr9/#L1
let line_str: &str = &self.text[line.clone()];
let mut reset_from: Option<usize> = None;
let mut reset_to: Option<usize> = None;
for (i, c) in line_str.char_indices() {
match self.original_classes[i] {
// Segment separator, Paragraph separator
B | S => {
assert!(reset_to == None);
reset_to = Some(i + c.len_utf8());
if reset_from == None {
reset_from = Some(i);
}
}
// Whitespace, isolate formatting
WS | FSI | LRI | RLI | PDI => {
if reset_from == None {
reset_from = Some(i);
}
}
_ => {
reset_from = None;
}
}
if let (Some(from), Some(to)) = (reset_from, reset_to) {
for j in from..to {
levels[j] = para.level;
}
reset_from = None;
reset_to = None;
}
}
if let Some(from) = reset_from {
for j in from..line_str.len() {
levels[j] = para.level;
}
}
let mut runs = Vec::new();
// Find consecutive level runs.
let mut start = line.start;
let mut level = levels[start];
let mut min_level = level;
let mut max_level = level;
for i in (start + 1)..line.end {
let new_level = levels[i];
if new_level != level {
// End of the previous run, start of a new one.
runs.push(start..i);
start = i;
level = new_level;
min_level = min(level, min_level);
max_level = max(level, max_level);
}
}
runs.push(start..line.end);
let run_count = runs.len();
// Re-order the odd runs.
// http://www.unicode.org/reports/tr9/#L2
// Stop at the lowest *odd* level.
min_level = min_level.new_lowest_ge_rtl().expect("Level error");
while max_level >= min_level {
// Look for the start of a sequence of consecutive runs of max_level or higher.
let mut seq_start = 0;
while seq_start < run_count {
if self.levels[runs[seq_start].start] < max_level {
seq_start += 1;
continue;
}
// Found the start of a sequence. Now find the end.
let mut seq_end = seq_start + 1;
while seq_end < run_count {
if self.levels[runs[seq_end].start] < max_level {
break;
}
seq_end += 1;
}
// Reverse the runs within this sequence.
runs[seq_start..seq_end].reverse();
seq_start = seq_end;
}
max_level
.lower(1)
.expect("Lowering embedding level below zero");
}
(levels, runs)
}
/// If processed text has any computed RTL levels
///
/// This information is usually used to skip re-ordering of text when no RTL level is present
#[inline]
pub fn has_rtl(&self) -> bool {
level::has_rtl(&self.levels)
}
}
@@ -372,15 +441,17 @@ fn assign_levels_to_removed_chars(para_level: Level, classes: &[BidiClass], leve
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
#[test]
fn test_initial_scan() {
fn test_initial_text_info() {
let text = "a1";
assert_eq!(
initial_scan("a1", None),
InitialProperties {
initial_classes: vec![L, EN],
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![L, EN],
paragraphs: vec![
ParagraphInfo {
range: 0..2,
@@ -390,10 +461,12 @@ mod test {
}
);
let text = "غ א";
assert_eq!(
initial_scan("غ א", None),
InitialProperties {
initial_classes: vec![AL, AL, WS, R, R],
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![AL, AL, WS, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..5,
@@ -403,10 +476,12 @@ mod test {
}
);
let text = "a\u{2029}b";
assert_eq!(
initial_scan("a\u{2029}b", None),
InitialProperties {
initial_classes: vec![L, B, B, B, L],
InitialInfo::new(text, None),
InitialInfo {
text,
original_classes: vec![L, B, B, B, L],
paragraphs: vec![
ParagraphInfo {
range: 0..4,
@@ -422,9 +497,10 @@ mod test {
let s = format!("{}א{}a", chars::FSI, chars::PDI);
assert_eq!(
initial_scan(&s, None),
InitialProperties {
initial_classes: vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],
InitialInfo::new(&s, None),
InitialInfo {
text: &s,
original_classes: vec![RLI, RLI, RLI, R, R, PDI, PDI, PDI, L],
paragraphs: vec![
ParagraphInfo {
range: 0..9,
@@ -437,11 +513,14 @@ mod test {
#[test]
fn test_process_text() {
let text = "abc123";
assert_eq!(
process_text("abc123", Some(Level::ltr())),
BidiInfo::new(text, Some(Level::ltr())),
BidiInfo {
text,
levels: Level::vec(&[0, 0, 0, 0, 0, 0]),
classes: vec![L, L, L, EN, EN, EN],
original_classes: vec![L, L, L, EN, EN, EN],
paragraphs: vec![
ParagraphInfo {
range: 0..6,
@@ -450,11 +529,14 @@ mod test {
],
}
);
let text = "abc אבג";
assert_eq!(
process_text("abc אבג", Some(Level::ltr())),
BidiInfo::new(text, Some(Level::ltr())),
BidiInfo {
text,
levels: Level::vec(&[0, 0, 0, 0, 1, 1, 1, 1, 1, 1]),
classes: vec![L, L, L, WS, R, R, R, R, R, R],
original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
@@ -464,10 +546,11 @@ mod test {
}
);
assert_eq!(
process_text("abc אבג", Some(Level::rtl())),
BidiInfo::new(text, Some(Level::rtl())),
BidiInfo {
text,
levels: Level::vec(&[2, 2, 2, 1, 1, 1, 1, 1, 1, 1]),
classes: vec![L, L, L, WS, R, R, R, R, R, R],
original_classes: vec![L, L, L, WS, R, R, R, R, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
@@ -476,11 +559,14 @@ mod test {
],
}
);
let text = "אבג abc";
assert_eq!(
process_text("אבג abc", Some(Level::ltr())),
BidiInfo::new(text, Some(Level::ltr())),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 1, 1, 1, 1, 0, 0, 0, 0]),
classes: vec![R, R, R, R, R, R, WS, L, L, L],
original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
@@ -490,10 +576,11 @@ mod test {
}
);
assert_eq!(
process_text("אבג abc", None),
BidiInfo::new(text, None),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 1, 1, 1, 1, 1, 2, 2, 2]),
classes: vec![R, R, R, R, R, R, WS, L, L, L],
original_classes: vec![R, R, R, R, R, R, WS, L, L, L],
paragraphs: vec![
ParagraphInfo {
range: 0..10,
@@ -502,11 +589,14 @@ mod test {
],
}
);
let text = "غ2ظ א2ג";
assert_eq!(
process_text("غ2ظ א2ג", Some(Level::ltr())),
BidiInfo::new(text, Some(Level::ltr())),
BidiInfo {
text,
levels: Level::vec(&[1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1]),
classes: vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],
original_classes: vec![AL, AL, EN, AL, AL, WS, R, R, EN, R, R],
paragraphs: vec![
ParagraphInfo {
range: 0..11,
@@ -515,10 +605,13 @@ mod test {
],
}
);
let text = "a א.\nג";
assert_eq!(
process_text("a א.\nג", None),
BidiInfo::new(text, None),
BidiInfo {
classes: vec![L, WS, R, R, CS, B, R, R],
text,
original_classes: vec![L, WS, R, R, CS, B, R, R],
levels: Level::vec(&[0, 0, 1, 1, 0, 0, 1, 1]),
paragraphs: vec![
ParagraphInfo {
@@ -537,33 +630,40 @@ mod test {
#[test]
fn test_bidi_info_has_rtl() {
// ASCII only
assert_eq!(process_text("123", None).has_rtl(), false);
assert_eq!(process_text("123", Some(Level::ltr())).has_rtl(), false);
assert_eq!(process_text("123", Some(Level::rtl())).has_rtl(), false);
assert_eq!(process_text("abc", None).has_rtl(), false);
assert_eq!(process_text("abc", Some(Level::ltr())).has_rtl(), false);
assert_eq!(process_text("abc", Some(Level::rtl())).has_rtl(), false);
assert_eq!(process_text("abc 123", None).has_rtl(), false);
assert_eq!(process_text("abc\n123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("123", Some(Level::ltr())).has_rtl(), false);
assert_eq!(BidiInfo::new("123", Some(Level::rtl())).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", None).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", Some(Level::ltr())).has_rtl(), false);
assert_eq!(BidiInfo::new("abc", Some(Level::rtl())).has_rtl(), false);
assert_eq!(BidiInfo::new("abc 123", None).has_rtl(), false);
assert_eq!(BidiInfo::new("abc\n123", None).has_rtl(), false);
// With Hebrew
assert_eq!(process_text("אבּג", None).has_rtl(), true);
assert_eq!(process_text("אבּג", Some(Level::ltr())).has_rtl(), true);
assert_eq!(process_text("אבּג", Some(Level::rtl())).has_rtl(), true);
assert_eq!(process_text("abc אבּג", None).has_rtl(), true);
assert_eq!(process_text("abc\nאבּג", None).has_rtl(), true);
assert_eq!(process_text("אבּג abc", None).has_rtl(), true);
assert_eq!(process_text("אבּג\nabc", None).has_rtl(), true);
assert_eq!(process_text("אבּג 123", None).has_rtl(), true);
assert_eq!(process_text("אבּג\n123", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג", None).has_rtl(), true);
assert_eq!(
BidiInfo::new("אבּג", Some(Level::ltr())).has_rtl(),
true
);
assert_eq!(
BidiInfo::new("אבּג", Some(Level::rtl())).has_rtl(),
true
);
assert_eq!(BidiInfo::new("abc אבּג", None).has_rtl(), true);
assert_eq!(BidiInfo::new("abc\nאבּג", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג abc", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג\nabc", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג 123", None).has_rtl(), true);
assert_eq!(BidiInfo::new("אבּג\n123", None).has_rtl(), true);
}
#[test]
fn test_reorder_line() {
fn reorder(s: &str) -> Cow<str> {
let info = process_text(s, None);
let para = &info.paragraphs[0];
reorder_line(s, para.range.clone(), &info.levels)
let bidi_info = BidiInfo::new(s, None);
let para = &bidi_info.paragraphs[0];
let line = para.range.clone();
bidi_info.reorder_line(para, line)
}
assert_eq!(reorder("abc123"), "abc123");
assert_eq!(reorder("1.-2"), "1.-2");
src/prepare.rs
+4 -4
View File
@@ -126,8 +126,8 @@ pub fn isolating_run_sequences(
/// Finds the level runs in a paragraph.
///
/// http://www.unicode.org/reports/tr9/#BD7
fn level_runs(levels: &[Level], classes: &[BidiClass]) -> Vec<LevelRun> {
assert!(levels.len() == classes.len());
fn level_runs(levels: &[Level], original_classes: &[BidiClass]) -> Vec<LevelRun> {
assert!(levels.len() == original_classes.len());
let mut runs = Vec::new();
if levels.len() == 0 {
@@ -138,7 +138,7 @@ fn level_runs(levels: &[Level], classes: &[BidiClass]) -> Vec<LevelRun> {
let mut current_run_start = 0;
for i in 1..levels.len() {
if !removed_by_x9(classes[i]) {
if !removed_by_x9(original_classes[i]) {
if levels[i] != current_run_level {
// End the last run and start a new one.
runs.push(current_run_start..i);
@@ -164,7 +164,7 @@ pub fn not_removed_by_x9(class: &BidiClass) -> bool {
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
#[test]
tests/conformance_tests.rs
+11 -9
View File
@@ -15,7 +15,7 @@ use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
use unicode_bidi::{bidi_class, BidiInfo, format_chars, Level, process_text};
use unicode_bidi::{bidi_class, BidiInfo, format_chars, Level};
const TEST_DATA_DIR: &str = "tests/data";
const BASE_TEST_FILE_NAME: &str = "BidiTest.txt";
@@ -30,13 +30,13 @@ fn open_test_file(filename: &str) -> File {
struct Fail {
pub levels: Vec<String>,
pub ordering: Vec<String>,
pub input: Vec<String>,
pub input_classes: Vec<String>,
pub input_chars: String,
pub para_level: Option<Level>,
}
#[test]
#[should_panic(expected = "60494 test cases failed! (196253 passed)")]
#[should_panic(expected = "12827 test cases failed! (243920 passed)")]
fn base_tests() {
let file = open_test_file(BASE_TEST_FILE_NAME);
let read = BufReader::new(file);
@@ -83,15 +83,15 @@ fn base_tests() {
assert!(set_ordering.len() <= set_levels.len());
let pieces: Vec<&str> = line.split(';').collect();
let input: Vec<&str> = pieces[0].split_whitespace().collect();
let input_classes: Vec<&str> = pieces[0].split_whitespace().collect();
let bitset: u8 = pieces[1].trim().parse().unwrap();
assert!(input.len() > 0);
assert!(input_classes.len() > 0);
assert!(bitset > 0);
let input_chars = get_sample_string_from_bidi_classes(input.to_owned());
let input_chars = get_sample_string_from_bidi_classes(input_classes.to_owned());
for para_level in gen_para_levels(bitset) {
let bidi_info = process_text(&input_chars, para_level);
let bidi_info = BidiInfo::new(&input_chars, para_level);
// Levels
let exp_levels: Vec<&str> = set_levels.iter().map(|x| x.as_ref()).collect();
@@ -101,7 +101,7 @@ fn base_tests() {
Fail {
levels: set_levels.to_owned(),
ordering: set_ordering.to_owned(),
input: input.iter().map(|x| x.to_string()).collect(),
input_classes: input_classes.iter().map(|x| x.to_string()).collect(),
input_chars: input_chars.to_owned(),
para_level,
},
@@ -154,9 +154,11 @@ fn gen_para_levels(bitset: u8) -> Vec<Option<Level>> {
///
/// TODO: Move to impl BidiInfo as pub api
fn gen_levels_list_from_bidi_info(input_chars: &str, bidi_info: &BidiInfo) -> Vec<Level> {
let para = &bidi_info.paragraphs[0];
let levels = bidi_info.reordered_levels(para, para.range.clone());
input_chars
.char_indices()
.map(|(i, _)| bidi_info.levels[i])
.map(|(i, _)| levels[i])
.collect()
}