--- spambayes-1.0.4/spambayes/tokenizer.py.orig	2005-08-30 11:28:50.000000000 +0900
+++ spambayes-1.0.4/spambayes/tokenizer.py	2006-10-31 18:22:05.000000000 +0900
@@ -25,6 +25,9 @@
 
 from spambayes.mboxutils import get_message
 
+# 2005-0830: updated to support iso-2022-jp, by owa.
+from spambayes import SplitterForU
+
 try:
     True, False
 except NameError:
@@ -1209,6 +1212,10 @@
                     "%d %b %Y %H:%M (%Z)",
                     "%d %b %Y %H:%M %Z")
 
+    # 2005-0830: updated to support iso-2022-jp, by owa.
+    # 2006-1031: added shift_jis, by owa.
+    jp_encodes = ('iso-2022-jp', 'shift_jis')
+
     def __init__(self):
         if options["Tokenizer", "basic_header_tokenize"]:
             self.basic_skip = [re.compile(s)
@@ -1319,11 +1326,20 @@
         for x, subjcharset in subjcharsetlist:
             if subjcharset is not None:
                 yield 'subjectcharset:' + subjcharset
-            for w in subject_word_re.findall(x):
-                for t in tokenize_word(w):
-                    yield 'subject:' + t
-            for w in punctuation_run_re.findall(x):
-                yield 'subject:' + w
+            # 2005-0830: updated to support iso-2022-jp, by owa.
+            if subjcharset and subjcharset.lower() in self.jp_encodes:
+                try:
+                    u = unicode(x, subjcharset, errors='ignore')
+                    for w in SplitterForU.tokenize(u):
+                        yield 'subject:' + w
+                except:
+                    pass
+            else:
+                for w in subject_word_re.findall(x):
+                    for t in tokenize_word(w):
+                        yield 'subject:' + t
+                for w in punctuation_run_re.findall(x):
+                    yield 'subject:' + w
 
         # Dang -- I can't use Sender:.  If I do,
         #     'sender:email name:python-list-admin'
@@ -1349,9 +1365,16 @@
                     except (binascii.Error, email.Errors.HeaderParseError):
                         subjcharsetlist = [(name, 'invalid')]
                     for name, charset in subjcharsetlist:
-                        yield "%s:name:%s" % (field, name.lower())
+                        # 2005-0830: updated to support iso-2022-jp, by owa.
                         if charset is not None:
                             yield "%s:charset:%s" % (field, charset)
+                            if charset in self.jp_encodes:
+                                u = unicode(name, charset, errors='ignore')
+                                yield "%s:name:%s" % (field, u)
+                            else:
+                                yield "%s:name:%s" % (field, name.lower())
+                        else:
+                            yield "%s:name:%s" % (field, name.lower())
                 else:
                     noname_count += 1
                 if addr:
@@ -1574,6 +1597,17 @@
             # 'a').
             text = numeric_entity_re.sub(numeric_entity_replacer, text)
 
+            # 2005-0830: updated to support iso-2022-jp, by owa.
+            unicode_text = None
+            try:
+                content_type_charset = None
+        	for x in msg.get_charsets(None):
+                    if x.lower() in self.jp_encodes:
+                        content_type_charset = x.lower()
+                if content_type_charset:
+                    unicode_text = unicode(text, content_type_charset, 'ignore')
+            except:
+                unicode_text = None
             # Normalize case.
             text = text.lower()
 
@@ -1608,14 +1642,21 @@
             text = html_re.sub('', text)
 
             # Tokenize everything in the body.
-            for w in text.split():
-                n = len(w)
-                # Make sure this range matches in tokenize_word().
-                if 3 <= n <= maxword:
-                    yield w
-
-                elif n >= 3:
-                    for t in tokenize_word(w):
-                        yield t
+            # 2005-0830: updated to support iso-2022-jp, by owa.
+            if unicode_text is None:
+                for w in text.split():
+                    n = len(w)
+                    # Make sure this range matches in tokenize_word().
+                    if 3 <= n <= maxword:
+                        yield w
+                    elif n >= 3:
+                        for t in tokenize_word(w):
+                            yield t
+            else:
+                try:
+                    for w in SplitterForU.tokenize(unicode_text):
+                        yield w
+                except:
+                    pass
 
 tokenize = Tokenizer().tokenize