fix for bug reported by ToR (unknown charset 'utf-8, text/html')

doc/THANKS

@@ -422,6 +422,9 @@ Stuffe <[email protected]>
 Sylphid <[email protected]>
     for suggesting some features
 
+ToR <[email protected]>
+    for reporting a minor bug
+
 == Organizations ==
 
 Black Hat team <[email protected]>

lib/request/basic.py

@@ -81,8 +81,9 @@ def checkCharEncoding(encoding):
     #http://www.destructor.de/charsets/index.htm
     translate = { 'windows-874':'iso-8859-11' }
 
-    if ';' in encoding:
-        encoding = encoding[:encoding.find(';')]
+    for delimiter in (';', ','):
+        if delimiter in encoding:
+            encoding = encoding[:encoding.find(delimiter)]
 
     # http://philip.html5.org/data/charsets-2.html
     if encoding in translate:
@@ -97,9 +98,9 @@ def checkCharEncoding(encoding):
     except LookupError:
         warnMsg  = "unknown charset '%s'. " % encoding
         warnMsg += "Please report by e-mail to [email protected]."
-
         logger.warn(warnMsg)
         encoding = conf.dataEncoding
+
     return encoding
 
 def decodePage(page, contentEncoding, contentType):