package analyze
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"chirami/internal/dict"
"chirami/internal/morph"
"chirami/internal/morph/mkmorph"
)
var tokenizer *morph.Dict
func TestMain(m *testing.M) {
dir, err := os.MkdirTemp("", "analyze")
if err != nil {
panic(err)
}
path := filepath.Join(dir, "morph.bin")
if err := mkmorph.Build(path); err != nil {
panic(err)
}
if tokenizer, err = morph.Open(path); err != nil {
panic(err)
}
code := m.Run()
tokenizer.Close()
os.RemoveAll(dir)
os.Exit(code)
}
func TestWords(t *testing.T) {
a := &Analyzer{Morph: tokenizer}
cases := []struct {
text, words, romaji string
}{
{"今日は良い天気ですね。", "今日|は|良い|天気|です|ね|。", "kyō|wa|yoi|tenki|desu|ne|."},
{"食べさせられなかった", "食べさせられなかった", "tabesaserarenakatta"},
{"行ってきます", "行ってきます", "ittekimasu"},
{"東京に行った", "東京|に|行った", "tōkyō|ni|itta"},
{"コーヒーを飲みたい", "コーヒー|を|飲みたい", "kōhī|o|nomitai"},
{"学生です", "学生|です", "gakusei|desu"},
{"お茶を下さい", "お茶|を|下さい", "ocha|o|kudasai"},
{"本を読んでいる", "本|を|読んでいる", "hon|o|yondeiru"},
{"読んではいけない", "読んで|は|いけない", "yonde|wa|ikenai"},
{"食べたり飲んだり", "食べたり|飲んだり", "tabetari|nondari"},
}
for _, c := range cases {
var ws, rs []string
for _, w := range a.Words(c.text) {
ws, rs = append(ws, w.Surface), append(rs, w.Romaji)
}
if got := strings.Join(ws, "|"); got != c.words {
t.Errorf("%s: words %s, want %s", c.text, got, c.words)
}
if got := strings.Join(rs, "|"); got != c.romaji {
t.Errorf("%s: romaji %s, want %s", c.text, got, c.romaji)
}
}
}
func TestKeys(t *testing.T) {
a := &Analyzer{Morph: tokenizer}
cases := []struct{ text, key, reading string }{
{"書かなかった", "書く", "かく"},
{"お薬", "薬", "くすり"},
{"ご質問", "質問", "しつもん"},
}
for _, c := range cases {
ws := a.Words(c.text)
if len(ws) != 1 || ws[0].Key != c.key || ws[0].KeyReading != c.reading {
t.Errorf("%s: got %+v", c.text, ws)
}
}
}
func TestInflection(t *testing.T) {
a := &Analyzer{Morph: tokenizer}
cases := []struct{ text, parts, names, romaji string }{
{"食べさせられなかった", "食(た)べ|させ|られ|なかっ|た", "|causative|potential or passive|negative|-た", "tabe|sase|rare|nakat|ta"},
{"書かれた", "書(か)|かれ|た", "|passive|-た", "ka|kare|ta"},
{"見たくない", "見(み)|たく|ない", "|-たい|negative", "mi|taku|nai"},
{"読んでいる", "読(よ)|んで|いる", "|-て|-いる", "yo|nde|iru"},
}
for _, c := range cases {
ws := a.Words(c.text)
if len(ws) != 1 {
t.Fatalf("%s: %d words", c.text, len(ws))
}
ps, ok := a.Inflection(ws[0])
if !ok {
t.Errorf("%s: no inflection", c.text)
continue
}
var parts, names, romaji []string
for _, p := range ps {
var b strings.Builder
for _, s := range p.Segs {
b.WriteString(s.Text)
if s.Ruby != "" {
b.WriteString("(" + s.Ruby + ")")
}
}
parts, names, romaji = append(parts, b.String()), append(names, p.Name), append(romaji, p.Romaji)
}
if got := strings.Join(parts, "|"); got != c.parts {
t.Errorf("%s: parts %s, want %s", c.text, got, c.parts)
}
if got := strings.Join(names, "|"); got != c.names {
t.Errorf("%s: names %s, want %s", c.text, got, c.names)
}
if got := strings.Join(romaji, "|"); got != c.romaji {
t.Errorf("%s: romaji %s, want %s", c.text, got, c.romaji)
}
}
if _, ok := a.Inflection(a.Words("日本")[0]); ok {
t.Error("日本 is not inflected")
}
}
func TestPhrase(t *testing.T) {
// a fake dictionary that knows only the idiom as a whole
a := &Analyzer{Morph: tokenizer, Dict: phraseDict{"猫に小判"}}
ws := a.Words("猫に小判")
if len(ws) != 3 {
t.Fatalf("the idiom's words stay separate, got %d", len(ws))
}
p, ok := a.Phrase("猫に小判", ws)
if !ok || p.Surface != "猫に小判" || p.Reading != "ねこにこばん" {
t.Fatalf("got %+v %v", p, ok)
}
if _, ok := a.Phrase("猫に", a.Words("猫に")); ok {
t.Fatal("not a phrase the dictionary knows")
}
}
func TestCompoundParticlesAndLookup(t *testing.T) {
d := testDict{entries: map[string][]dict.Entry{
"とは": {{
Kana: []dict.Form{{Text: "とは", Common: true}},
Senses: []dict.Sense{{POS: []string{"exp", "prt"}, Gloss: []string{"indicates word or phrase being defined"}}},
}},
"と": {
{Kanji: []dict.Form{{Text: "十", Common: true}}, Kana: []dict.Form{{Text: "じゅう", Common: true}, {Text: "と"}}, Senses: []dict.Sense{{POS: []string{"num"}, Gloss: []string{"ten"}}}},
{Kana: []dict.Form{{Text: "と", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"if; when"}}}},
},
"は": {
{Kanji: []dict.Form{{Text: "葉", Common: true}}, Kana: []dict.Form{{Text: "は", Common: true}}, Senses: []dict.Sense{{POS: []string{"n"}, Gloss: []string{"leaf"}}}},
{Kana: []dict.Form{{Text: "は", Common: true}}, Senses: []dict.Sense{{POS: []string{"prt"}, Gloss: []string{"indicates sentence topic"}}}},
},
"した": {
{Kanji: []dict.Form{{Text: "舌", Common: true}}, Kana: []dict.Form{{Text: "した", Common: true}}, Senses: []dict.Sense{{POS: []string{"n"}, Gloss: []string{"tongue"}}}},
},
"する": {
{Kanji: []dict.Form{{Text: "為る", Common: true}}, Kana: []dict.Form{{Text: "する", Common: true}}, Senses: []dict.Sense{{POS: []string{"vs-i"}, Misc: []string{"uk"}, Gloss: []string{"to do"}}}},
},
}}
a := &Analyzer{Morph: tokenizer, Dict: d}
ws := a.Words("ミックスリストとは、YouTube が作成した")
var surfaces, romaji []string
for _, w := range ws {
surfaces = append(surfaces, w.Surface)
romaji = append(romaji, w.Romaji)
}
if got, want := strings.Join(surfaces, "|"), "ミックス|リスト|とは|、|YouTube|が|作成|した"; got != want {
t.Fatalf("words = %q, want %q", got, want)
}
if got, want := strings.Join(romaji, "|"), "mikkusu|risuto|towa|,|YouTube|ga|sakusei|shita"; got != want {
t.Fatalf("romaji = %q, want %q", got, want)
}
towa := ws[2]
if len(towa.Parts) != 2 || towa.Parts[0].Surface != "と" || towa.Parts[1].Surface != "は" {
t.Fatalf("とは parts = %+v, want [と は]", towa.Parts)
}
if es := a.Entries(towa.Parts[1]); len(es) == 0 || es[0].Senses[0].Gloss[0] != "indicates sentence topic" {
t.Fatalf("は entries = %+v, want topic particle first", es)
}
shita := ws[len(ws)-1]
if es := a.Entries(shita); len(es) == 0 || es[0].Senses[0].Gloss[0] != "to do" {
t.Fatalf("した entries = %+v, want する (to do) rather than 舌 (tongue)", es)
}
}
type testDict struct {
entries map[string][]dict.Entry
}
func (d testDict) Lookup(w string) []dict.Entry { return slices.Clone(d.entries[w]) }
func (d testDict) LookupReading(w, r string) []dict.Entry {
es := d.Lookup(w)
slices.SortStableFunc(es, func(a, b dict.Entry) int {
var ak, bk int
if dict.WrittenInKana(a) {
ak = 1
}
if dict.WrittenInKana(b) {
bk = 1
}
return bk - ak
})
return es
}
func (testDict) LongestPrefix(s string, n int) (int, []dict.Entry) { return 0, nil }
func (testDict) Kanji(r rune) (dict.Kanji, bool) { return dict.Kanji{}, false }
func (testDict) Furigana(w, r string) ([]dict.Ruby, bool) { return nil, false }
type phraseDict struct{ known string }
func (d phraseDict) Lookup(w string) []dict.Entry {
if w == d.known {
return []dict.Entry{{}}
}
return nil
}
func (phraseDict) LookupReading(w, r string) []dict.Entry { return nil }
func (phraseDict) LongestPrefix(s string, n int) (int, []dict.Entry) { return 0, nil }
func (phraseDict) Kanji(r rune) (dict.Kanji, bool) { return dict.Kanji{}, false }
func (phraseDict) Furigana(w, r string) ([]dict.Ruby, bool) { return nil, false }