##// END OF EJS Templates
helpers: unicode/str add flag to control usage of chardet....
marcink -
r4109:19a6ab7e default
parent child Browse files
Show More
@@ -207,7 +207,7 b' def safe_int(val, default=None):'
207 return val
207 return val
208
208
209
209
210 def safe_unicode(str_, from_encoding=None):
210 def safe_unicode(str_, from_encoding=None, use_chardet=False):
211 """
211 """
212 safe unicode function. Does few trick to turn str_ into unicode
212 safe unicode function. Does few trick to turn str_ into unicode
213
213
@@ -240,6 +240,7 b' def safe_unicode(str_, from_encoding=Non'
240 except UnicodeDecodeError:
240 except UnicodeDecodeError:
241 pass
241 pass
242
242
243 if use_chardet:
243 try:
244 try:
244 import chardet
245 import chardet
245 encoding = chardet.detect(str_)['encoding']
246 encoding = chardet.detect(str_)['encoding']
@@ -248,9 +249,10 b' def safe_unicode(str_, from_encoding=Non'
248 return str_.decode(encoding)
249 return str_.decode(encoding)
249 except (ImportError, UnicodeDecodeError, Exception):
250 except (ImportError, UnicodeDecodeError, Exception):
250 return unicode(str_, from_encoding[0], 'replace')
251 return unicode(str_, from_encoding[0], 'replace')
251
252 else:
253 return unicode(str_, from_encoding[0], 'replace')
252
254
253 def safe_str(unicode_, to_encoding=None):
255 def safe_str(unicode_, to_encoding=None, use_chardet=False):
254 """
256 """
255 safe str function. Does few trick to turn unicode_ into string
257 safe str function. Does few trick to turn unicode_ into string
256
258
@@ -283,6 +285,7 b' def safe_str(unicode_, to_encoding=None)'
283 except UnicodeEncodeError:
285 except UnicodeEncodeError:
284 pass
286 pass
285
287
288 if use_chardet:
286 try:
289 try:
287 import chardet
290 import chardet
288 encoding = chardet.detect(unicode_)['encoding']
291 encoding = chardet.detect(unicode_)['encoding']
@@ -292,6 +295,8 b' def safe_str(unicode_, to_encoding=None)'
292 return unicode_.encode(encoding)
295 return unicode_.encode(encoding)
293 except (ImportError, UnicodeEncodeError):
296 except (ImportError, UnicodeEncodeError):
294 return unicode_.encode(to_encoding[0], 'replace')
297 return unicode_.encode(to_encoding[0], 'replace')
298 else:
299 return unicode_.encode(to_encoding[0], 'replace')
295
300
296
301
297 def remove_suffix(s, suffix):
302 def remove_suffix(s, suffix):
General Comments 0
You need to be logged in to leave comments. Login now