# vim: set ts=4 sw=4 sts=0 :
#
# SplitterForU.py
#
# 2005/09/11: 0.1 release new
#
# Copyright (C) 2005, Shu KONNO <owa@bg.wakwak.com>
# This program distributed under GPL2 (GNU General Public License).
#
# thanks for original code Unicode Block Tokenizer
# http://tokuhirom.dnsalias.org/~tokuhirom/wiki/UnicodeBlockTokenizer

from __future__ import generators   # only needed for Python 2.2

def tokenize(s):
	""" Tokenize the text by the unicode block name. """
	buf = u""
	befblockname = ''
	for c in s:
		if c.isspace():
			if buf:
				yield buf
				buf = u""
		elif c in u'!"#$%&()~-^=|{}[]*+;:,.<>?_/\'':
			if buf:
				yield buf
				buf = u""
		else:
			blname = getBlockName(c)
			if blname != befblockname and buf:
				yield buf
				buf = u""
			befblockname = blname
			buf += c
	if buf:
		yield buf

def getBlockName(c):
	"""
	Get the Unicode Block name of the character ``c''.
	c is unicode string. c's length is 1.
	If not implement, raise UnicodeError.
	"""
	ary = [
		(u"\u007F", "Basic Latin"),
		(u"\u00FF", "Latin-1 Supplement"),
		(u"\u017F", "Latin Extended-A"),
		(u"\u024F", "Latin Extended-B"),
		(u"\u02AF", "IPA Extensions"),
		(u"\u02FF", "Spacing Modifier Letters"),
		(u"\u036F", "Combining Diacritical Marks"),
		(u"\u03FF", "Greek and Coptic"),
		(u"\u04FF", "Cyrillic"),
		(u"\u052F", "Cyrillic Supplement"),
		(u"\u058F", "Armenian"),
		(u"\u05FF", "Hebrew"),
		(u"\u06FF", "Arabic"),
		(u"\u074F", "Syriac"),
		(u"\u07BF", "Thaana"),
		(u"\u097F", "Devanagari"),
		(u"\u09FF", "Bengali"),
		(u"\u0A7F", "Gurmukhi"),
		(u"\u0AFF", "Gujarati"),
		(u"\u0B7F", "Oriya"),
		(u"\u0BFF", "Tamil"),
		(u"\u0C7F", "Telugu"),
		(u"\u0CFF", "Kannada"),
		(u"\u0D7F", "Malayalam"),
		(u"\u0DFF", "Sinhala"),
		(u"\u0E7F", "Thai"),
		(u"\u0EFF", "Lao"),
		(u"\u0FFF", "Tibetan"),
		(u"\u109F", "Myanmar"),
		(u"\u10FF", "Georgian"),
		(u"\u11FF", "Hangul Jamo"),
		(u"\u137F", "Ethiopic"),
		(u"\u13FF", "Cherokee"),
		(u"\u167F", "Unified Canadian Aboriginal Syllabics"),
		(u"\u169F", "Ogham"),
		(u"\u16FF", "Runic"),
		(u"\u171F", "Tagalog"),
		(u"\u173F", "Hanunoo"),
		(u"\u175F", "Buhid"),
		(u"\u177F", "Tagbanwa"),
		(u"\u17FF", "Khmer"),
		(u"\u18AF", "Mongolian"),
		(u"\u194F", "Limbu"),
		(u"\u197F", "Tai Le"),
		(u"\u19FF", "Khmer Symbols"),
		(u"\u1D7F", "Phonetic Extensions"),
		(u"\u1EFF", "Latin Extended Additional"),
		(u"\u1FFF", "Greek Extended"),
		(u"\u206F", "General Punctuation"),
		(u"\u209F", "Superscripts and Subscripts"),
		(u"\u20CF", "Currency Symbols"),
		(u"\u20FF", "Combining Diacritical Marks for Symbols"),
		(u"\u214F", "Letterlike Symbols"),
		(u"\u218F", "Number Forms"),
		(u"\u21FF", "Arrows"),
		(u"\u22FF", "Mathematical Operators"),
		(u"\u23FF", "Miscellaneous Technical"),
		(u"\u243F", "Control Pictures"),
		(u"\u245F", "Optical Character Recognition"),
		(u"\u24FF", "Enclosed Alphanumerics"),
		(u"\u257F", "Box Drawing"),
		(u"\u259F", "Block Elements"),
		(u"\u25FF", "Geometric Shapes"),
		(u"\u26FF", "Miscellaneous Symbols"),
		(u"\u27BF", "Dingbats"),
		(u"\u27EF", "Miscellaneous Mathematical Symbols-A"),
		(u"\u27FF", "Supplemental Arrows-A"),
		(u"\u28FF", "Braille Patterns"),
		(u"\u297F", "Supplemental Arrows-B"),
		(u"\u29FF", "Miscellaneous Mathematical Symbols-B"),
		(u"\u2AFF", "Supplemental Mathematical Operators"),
		(u"\u2BFF", "Miscellaneous Symbols and Arrows"),
		(u"\u2EFF", "CJK Radicals Supplement"),
		(u"\u2FDF", "Kangxi Radicals"),
		(u"\u2FFF", "Ideographic Description Characters"),
		(u"\u303F", "CJK Symbols and Punctuation"),
		(u"\u309F", "Hiragana"),
		(u"\u30FF", "Katakana"),
		(u"\u312F", "Bopomofo"),
		(u"\u318F", "Hangul Compatibility Jamo"),
		(u"\u319F", "Kanbun"),
		(u"\u31BF", "Bopomofo Extended"),
		(u"\u31FF", "Katakana Phonetic Extensions"),
		(u"\u32FF", "Enclosed CJK Letters and Months"),
		(u"\u33FF", "CJK Compatibility"),
		(u"\u4DBF", "CJK Unified Ideographs Extension A"),
		(u"\u4DFF", "Yijing Hexagram Symbols"),
		(u"\u9FFF", "CJK Unified Ideographs"),
		(u"\uA48F", "Yi Syllables"),
		(u"\uA4CF", "Yi Radicals"),
		(u"\uD7AF", "Hangul Syllables"),
		(u"\uDB7F", "High Surrogates"),
		(u"\uDBFF", "High Private Use Surrogates"),
		(u"\uDFFF", "Low Surrogates"),
		(u"\uF8FF", "Private Use Area"),
		(u"\uFAFF", "CJK Compatibility Ideographs"),
		(u"\uFB4F", "Alphabetic Presentation Forms"),
		(u"\uFDFF", "Arabic Presentation Forms-A"),
		(u"\uFE0F", "Variation Selectors"),
		(u"\uFE2F", "Combining Half Marks"),
		(u"\uFE4F", "CJK Compatibility Forms"),
		(u"\uFE6F", "Small Form Variants"),
		(u"\uFEFF", "Arabic Presentation Forms-B"),
		(u"\uFFEF", "Halfwidth and Fullwidth Forms"),
		(u"\uFFFF", "Specials")]

	# FIXME: Binary search makes more faster.
	# see http://www.python.jp/doc/release/lib/module-bisect.html
	for endcode, blockname in ary:
		if c < endcode:
			return blockname
	raise UnicodeError