// Package analyze turns a selection into words with furigana and romaji
// and finds the dictionary entries for each word.
package analyze
import (
"slices"
"strings"
"unicode"
"unicode/utf8"
"chirami/internal/dict"
"chirami/internal/jp"
"chirami/internal/morph"
)
type Tokenizer interface {
Tokenize(s string) []morph.Token
}
type Dictionary interface {
Lookup(word string) []dict.Entry
LookupReading(word, reading string) []dict.Entry
LongestPrefix(s string, maxRunes int) (int, []dict.Entry)
Kanji(r rune) (dict.Kanji, bool)
Furigana(word, reading string) ([]dict.Ruby, bool)
}
// Seg is a run of surface text with the reading shown above it; Ruby is ""
// over kana or anything else that needs no reading.
type Seg struct{ Text, Ruby string }
// Word is a content word with the particles, endings and auxiliaries that
// attach to it, or a lone particle or symbol.
type Word struct {
Start, End int // byte offsets into the analysed text
Surface string
Segs []Seg
Reading string // hiragana
Romaji string
Key string // dictionary form of the head morpheme
KeyReading string // its hiragana reading, "" when unknown
Content bool // a word worth looking up first (not a particle or symbol)
Symbol bool
Parts []Word // constituent particles when merged from a compound particle
pos string // head morpheme's major POS (t.POS[0])
// katakana reading and pronunciation not yet turned into Romaji; romaji
// is made per run rather than per morpheme so a trailing っ doubles the
// next morpheme's consonant (なかっ+た -> nakatta)
kana, pron string
}
type Analyzer struct {
Morph Tokenizer // nil: every selection is one unanalysed word
Dict Dictionary // nil: no compound merging
Style jp.Style
}
// Words splits text into display words.
func (a *Analyzer) Words(text string) []Word {
toks := a.tokens(text)
var words []Word
var cur *Word
var prev *morph.Token
for i := range toks {
t := &toks[i]
if strings.TrimSpace(t.Surface) == "" {
cur, prev = nil, nil
continue
}
switch {
case cur == nil || !attaches(t, prev, cur):
words = append(words, Word{Start: t.Start})
cur = &words[len(words)-1]
a.head(cur, t)
case prev.POS[0] == "接頭詞":
a.head(cur, t) // お薬 is looked up as 薬 when お薬 is not in the dictionary
}
a.add(cur, t)
prev = t
}
for i := range words {
a.flush(&words[i])
}
return a.mergeCompounds(text, words)
}
func (a *Analyzer) tokens(text string) []morph.Token {
if a.Morph != nil {
return a.Morph.Tokenize(text)
}
tok := morph.Token{Surface: text, End: len(text), Base: text}
if jp.IsKanaOnly(text) {
tok.Reading = jp.ToKatakana(text)
tok.Known = true
}
return []morph.Token{tok}
}
// attaches reports whether t continues the current word, given the token
// before it, rather than starting a new one.
func attaches(t, prev *morph.Token, cur *Word) bool {
if prev.POS[0] == "接頭詞" {
return true
}
if cur.Symbol {
return false
}
inflecting := prev.POS[0] == "動詞" || prev.POS[0] == "形容詞" || prev.POS[0] == "助動詞"
switch t.POS[0] {
case "助動詞":
// endings after verbs and adjectives; the copula after nouns stands alone
return inflecting
case "動詞", "形容詞":
// 食べ+させる, 読んで+いる; not いけない after the は of 読んではいけない
return t.POS[1] == "接尾" || t.POS[1] == "非自立" && (inflecting || prev.POS[1] == "接続助詞")
case "名詞":
return t.POS[1] == "接尾" && cur.Content
case "助詞":
// て, ば, ながら, and the たり of 食べたり
return inflecting && (t.POS[1] == "接続助詞" || t.POS[1] == "並立助詞" && (t.Base == "たり" || t.Base == "だり"))
}
return false
}
// head sets the word's dictionary key from its first morpheme.
func (a *Analyzer) head(w *Word, t *morph.Token) {
w.pos = t.POS[0]
w.Symbol = t.POS[0] == "記号" || (!t.Known && !jp.HasJapanese(t.Surface))
w.Content = !w.Symbol && t.POS[0] != "助詞" && t.POS[0] != "助動詞"
if w.Symbol {
return
}
w.Key = t.Base
if w.Key == "" {
w.Key = t.Surface
}
w.KeyReading = baseReading(t.Surface, t.Base, jp.ToHiragana(t.Reading))
}
func (a *Analyzer) add(w *Word, t *morph.Token) {
w.End = t.End
w.Surface += t.Surface
reading := jp.ToHiragana(t.Reading)
if reading == "" && jp.IsKanaOnly(t.Surface) {
reading = jp.ToHiragana(t.Surface)
}
w.Reading += reading
w.Segs = appendSegs(w.Segs, a.furigana(t, reading))
switch {
case reading == "" && jp.HasJapanese(t.Surface):
a.flush(w) // kanji nobody knows the reading of: no romaji rather than a guess
case t.POS[0] == "記号" || !jp.HasJapanese(t.Surface) || reading == "":
a.flush(w)
w.Romaji += punct(t.Surface)
default:
pron := t.Pron
if pron == "" {
pron = t.Reading
}
if pron == "" {
pron = jp.ToKatakana(t.Surface)
}
w.kana += jp.ToKatakana(reading)
w.pron += pron
}
}
func (a *Analyzer) flush(w *Word) {
if w.kana != "" {
w.Romaji += jp.RomajiPron(w.kana, w.pron, a.Style)
w.kana, w.pron = "", ""
}
}
var punctMap = map[rune]string{
'。': ".", '、': ",", '！': "!", '？': "?", '「': "“", '」': "”", '『': "“", '』': "”",
'（': "(", '）': ")", '・': "·", '…': "…", '〜': "~", '～': "~", '：': ":", ' ': " ",
}
func punct(s string) string {
var b strings.Builder
for _, r := range s {
if p, ok := punctMap[r]; ok {
b.WriteString(p)
} else if r < utf8.RuneSelf || unicode.IsLetter(r) || unicode.IsDigit(r) {
b.WriteRune(r)
}
}
return strings.TrimSpace(b.String())
}
// baseReading derives the reading of a morpheme's dictionary form from its
// surface reading: 書か/かか with base 書く gives かく.
func baseReading(surface, base, reading string) string {
if base == "" || base == surface {
return reading
}
if base == "来る" { // the stem's reading changes: 来(き)ます, 来(こ)ない
return "くる"
}
sr, br := []rune(surface), []rune(base)
n := 0
for n < len(sr) && n < len(br) && sr[n] == br[n] {
n++
}
sTail := jp.ToHiragana(string(sr[n:]))
bTail := string(br[n:])
if !strings.HasSuffix(reading, sTail) || !jp.IsKanaOnly(bTail) && bTail != "" {
return ""
}
return strings.TrimSuffix(reading, sTail) + jp.ToHiragana(bTail)
}
// mergeCompounds joins runs of nouns or particles that the dictionary knows
// as one word (東京+大学 -> 東京大学, と+は -> とは), so the card shows the
// compound's entry.
func (a *Analyzer) mergeCompounds(text string, words []Word) []Word {
if a.Dict == nil || len(words) < 2 {
return words
}
out := words[:0:0]
for i := 0; i < len(words); i++ {
w := words[i]
switch {
case isNounish(w):
if n, es := a.Dict.LongestPrefix(text[w.Start:], 16); len(es) > 0 {
end := w.Start + n
j := i + 1
for j < len(words) && words[j].Start == words[j-1].End && words[j].End <= end && isNounish(words[j]) {
j++
}
if j-i > 1 && words[j-1].End == end {
w = a.join(words[i:j])
i = j - 1
}
}
case w.pos == "助詞":
j := i + 1
for j < len(words) && j-i < 3 && words[j].Start == words[j-1].End && words[j].pos == "助詞" {
j++
}
for k := j; k > i+1; k-- {
if a.isCompoundParticle(text[w.Start:words[k-1].End]) {
w = a.join(words[i:k])
w.Parts = slices.Clone(words[i:k])
i = k - 1
break
}
}
}
out = append(out, w)
}
return out
}
func (a *Analyzer) isCompoundParticle(s string) bool {
for _, e := range a.Dict.Lookup(s) {
if !dict.WrittenInKana(e) || len(e.Senses) == 0 {
continue
}
pos := e.Senses[0].POS
if slices.Contains(pos, "prt") || slices.Contains(pos, "exp") || slices.Contains(pos, "conj") {
return true
}
}
return false
}
// isNounish reports whether every rune is kanji or katakana, the shape of
// noun compounds; good enough without carrying POS along.
func isNounish(w Word) bool {
if w.Symbol || !w.Content {
return false
}
for _, r := range w.Surface {
if !jp.IsKanji(r) && !jp.IsKatakana(r) {
return false
}
}
return true
}
func appendSegs(segs, more []Seg) []Seg {
for _, s := range more {
if n := len(segs); n > 0 && s.Ruby == "" && segs[n-1].Ruby == "" {
segs[n-1].Text += s.Text
continue
}
segs = append(segs, s)
}
return segs
}
// join merges consecutive words into one looked up as a whole.
func (a *Analyzer) join(words []Word) Word {
m := words[0]
m.Segs = slices.Clone(m.Segs)
sep := " "
if m.pos == "助詞" {
sep = ""
}
for _, x := range words[1:] {
m.End = x.End
m.Surface += x.Surface
m.Reading += x.Reading
m.Romaji += sep + x.Romaji
m.Segs = appendSegs(m.Segs, x.Segs)
}
m.Key, m.KeyReading = m.Surface, m.Reading
if m.pos != "助詞" {
m.Content = true
}
if rs, ok := a.Dict.Furigana(m.Surface, m.Reading); ok {
m.Segs = segs(rs)
}
return m
}