mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
Fix invalid escape sequences in text normalization and models (#3709)
Three modules use invalid escapes in non-raw string literals: - fun_text_processing/text_normalization/normalize.py - \., \w - funasr/models/fsmn_kws/encoder.py - \[ - funasr/models/fun_asr_nano/tools/format5res.py - \. On Python 3.12+ each raises SyntaxWarning: invalid escape sequence on import, and FunASR runs the 3.10-3.12 CI matrix. Escaping the backslashes leaves every string value identical (ruff W605-style). Claude-Session: https://claude.ai/code/session_014HBQNzAf5C2E3MiJC48HQw Co-authored-by: Mehdi Boutayeb <[email protected]> Co-authored-by: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Mehdi Boutayeb
Claude Opus 4.8
parent
2d4566d4a4
commit
da12182b6b
@@ -351,7 +351,7 @@ class Normalizer:
|
||||
upper_case_unicode = "\u0410-\u042F"
|
||||
|
||||
# Read and split transcript by utterance (roughly, sentences)
|
||||
split_pattern = f"(?<!\w\.\w.)(?<![A-Z{upper_case_unicode}][a-z{lower_case_unicode}]+\.)(?<![A-Z{upper_case_unicode}]\.)(?<=\.|\?|\!|\.”|\?”\!”)\s(?![0-9]+[a-z]*\.)"
|
||||
split_pattern = f"(?<!\\w\\.\\w.)(?<![A-Z{upper_case_unicode}][a-z{lower_case_unicode}]+\\.)(?<![A-Z{upper_case_unicode}]\\.)(?<=\\.|\\?|\\!|\\.”|\\?”\\!”)\\s(?![0-9]+[a-z]*\\.)"
|
||||
|
||||
sentences = regex.split(split_pattern, text)
|
||||
return sentences
|
||||
|
||||
@@ -84,7 +84,7 @@ class LinearTransform(nn.Module):
|
||||
dtype=torch.float32)
|
||||
for i in range(self.output_dim):
|
||||
line = fread.readline()
|
||||
splits = line.strip().strip('\[\]').strip().split()
|
||||
splits = line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(splits) == self.input_dim
|
||||
cols = torch.tensor([float(item) for item in splits],
|
||||
dtype=torch.float32)
|
||||
@@ -160,7 +160,7 @@ class AffineTransform(nn.Module):
|
||||
dtype=torch.float32)
|
||||
for i in range(self.output_dim):
|
||||
line = fread.readline()
|
||||
splits = line.strip().strip('\[\]').strip().split()
|
||||
splits = line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(splits) == self.input_dim
|
||||
cols = torch.tensor([float(item) for item in splits],
|
||||
dtype=torch.float32)
|
||||
@@ -171,7 +171,7 @@ class AffineTransform(nn.Module):
|
||||
linear_bias = self.state_dict()['linear.bias']
|
||||
#print(linear_bias.shape)
|
||||
bias_line = fread.readline()
|
||||
splits = bias_line.strip().strip('\[\]').strip().split()
|
||||
splits = bias_line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(splits) == self.output_dim
|
||||
new_bias = torch.tensor([float(item) for item in splits],
|
||||
dtype=torch.float32)
|
||||
@@ -331,7 +331,7 @@ class FSMNBlock(nn.Module):
|
||||
self.dim = int(fsmn_split[1])
|
||||
|
||||
params_line = fread.readline()
|
||||
params_split = params_line.strip().strip('\[\]').strip().split()
|
||||
params_split = params_line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(params_split) == 12
|
||||
assert params_split[0] == '<LearnRateCoef>'
|
||||
assert params_split[2] == '<LOrder>'
|
||||
@@ -352,7 +352,7 @@ class FSMNBlock(nn.Module):
|
||||
for i in range(self.lorder):
|
||||
print('read conv_left weight -- %d' % i)
|
||||
line = fread.readline()
|
||||
splits = line.strip().strip('\[\]').strip().split()
|
||||
splits = line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(splits) == self.dim
|
||||
cols = torch.tensor([float(item) for item in splits],
|
||||
dtype=torch.float32)
|
||||
@@ -375,7 +375,7 @@ class FSMNBlock(nn.Module):
|
||||
for i in range(self.rorder):
|
||||
print('read conv_right weight -- %d' % i)
|
||||
line = fread.readline()
|
||||
splits = line.strip().strip('\[\]').strip().split()
|
||||
splits = line.strip().strip('\\[\\]').strip().split()
|
||||
assert len(splits) == self.dim
|
||||
cols = torch.tensor([float(item) for item in splits],
|
||||
dtype=torch.float32)
|
||||
|
||||
@@ -320,8 +320,8 @@ if __name__ == "__main__":
|
||||
continue
|
||||
name = tmp[0]
|
||||
content = tmp[1]
|
||||
name = re.sub("\.pcm", "", name)
|
||||
name = re.sub("\.wav", "", name)
|
||||
name = re.sub("\\.pcm", "", name)
|
||||
name = re.sub("\\.wav", "", name)
|
||||
content = recoformat(content)
|
||||
content = numbersingle(content)
|
||||
content = ch_number2digit(content)
|
||||
|
||||
Reference in New Issue
Block a user