package morph

// Token is one IPADIC morpheme. Start and End are byte offsets into the
// tokenized string. Feature fields are "" when IPADIC has "*".
type Token struct {
	Surface    string
	Start, End int
	POS        [4]string // e.g. {"動詞", "自立", "", ""}
	InflType   string    // 活用型, e.g. "一段"
	InflForm   string    // 活用形, e.g. "連用形"
	Base       string    // 基本形; equals Surface for unknown words
	Reading    string    // katakana; "" for unknown words
	Pron       string    // katakana pronunciation with ー for long vowels; "" for unknown words
	Known      bool      // false for unknown-word (未知語) tokens
}