From 837ccf451eb3dd237ac85b80f76206896d22c63f Mon Sep 17 00:00:00 2001 From: "shanjian%netscape.com" Date: Tue, 26 Nov 2002 20:56:08 +0000 Subject: [PATCH] #171813 Universal auto detector doesn't work well on sohu news page adjust the detecting parameter. r=ftang, sr=jst --- extensions/universalchardet/src/Big5Freq.tab | 4 ++-- extensions/universalchardet/src/EUCTWFreq.tab | 4 ++-- extensions/universalchardet/src/GB2312Freq.tab | 4 ++-- extensions/universalchardet/src/JISFreq.tab | 2 +- extensions/universalchardet/src/nsLatin1Prober.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extensions/universalchardet/src/Big5Freq.tab b/extensions/universalchardet/src/Big5Freq.tab index d4e5e39c754e..3d8448624a3f 100644 --- a/extensions/universalchardet/src/Big5Freq.tab +++ b/extensions/universalchardet/src/Big5Freq.tab @@ -50,10 +50,10 @@ * Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 * Random Distribution Ration = 512/(5401-512)=0.105 * - * Typical Distribution Ratio about 50% of Ideal one, + * Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR *****************************************************************************/ -#define BIG5_TYPICAL_DISTRIBUTION_RATIO (float)1.5 +#define BIG5_TYPICAL_DISTRIBUTION_RATIO (float)0.75 //Char to FreqOrder table , diff --git a/extensions/universalchardet/src/EUCTWFreq.tab b/extensions/universalchardet/src/EUCTWFreq.tab index 5b93b6515f34..92349675be05 100644 --- a/extensions/universalchardet/src/EUCTWFreq.tab +++ b/extensions/universalchardet/src/EUCTWFreq.tab @@ -52,10 +52,10 @@ * Idea Distribution Ratio = 0.74851/(1-0.74851) =2.98 * Random Distribution Ration = 512/(5401-512)=0.105 * - * Typical Distribution Ratio about 50% of Ideal one, + * Typical Distribution Ratio about 25% of Ideal one, still much higher than RDR *****************************************************************************/ -#define EUCTW_TYPICAL_DISTRIBUTION_RATIO (float)1.5 +#define EUCTW_TYPICAL_DISTRIBUTION_RATIO (float)0.75 //Char to FreqOrder table , #define EUCTW_TABLE_SIZE 8102 diff --git a/extensions/universalchardet/src/GB2312Freq.tab b/extensions/universalchardet/src/GB2312Freq.tab index 631474789967..9180ffcd0cdf 100644 --- a/extensions/universalchardet/src/GB2312Freq.tab +++ b/extensions/universalchardet/src/GB2312Freq.tab @@ -49,10 +49,10 @@ * Idea Distribution Ratio = 0.79135/(1-0.79135) = 3.79 * Random Distribution Ration = 512 / (3755 - 512) = 0.157 * - * Typical Distribution Ratio about 50% of Ideal one, + * Typical Distribution Ratio about 25% of Ideal one, still much higher that RDR *****************************************************************************/ -#define GB2312_TYPICAL_DISTRIBUTION_RATIO (float)1.8 +#define GB2312_TYPICAL_DISTRIBUTION_RATIO (float)0.9 #define GB2312_TABLE_SIZE 3760 diff --git a/extensions/universalchardet/src/JISFreq.tab b/extensions/universalchardet/src/JISFreq.tab index 2828f69f9b5a..d7b786432213 100644 --- a/extensions/universalchardet/src/JISFreq.tab +++ b/extensions/universalchardet/src/JISFreq.tab @@ -51,7 +51,7 @@ * Idea Distribution Ratio = 0.92635 / (1-0.92635) = 12.58 * Random Distribution Ration = 512 / (2965+62+83+86-512) = 0.191 * - * Typical Distribution Ratio + * Typical Distribution Ratio, 25% of IDR *****************************************************************************/ #define JIS_TYPICAL_DISTRIBUTION_RATIO (float) 3.0 diff --git a/extensions/universalchardet/src/nsLatin1Prober.cpp b/extensions/universalchardet/src/nsLatin1Prober.cpp index 1eeff105941d..3efc85bc08ce 100644 --- a/extensions/universalchardet/src/nsLatin1Prober.cpp +++ b/extensions/universalchardet/src/nsLatin1Prober.cpp @@ -199,7 +199,7 @@ float nsLatin1Prober::GetConfidence(void) // lower the confidence of latin1 so that other more accurate detector // can take priority. - confidence *= 0.60f; + confidence *= 0.50f; return confidence; }