package morph_test
import (
"bufio"
"os"
"path/filepath"
"reflect"
"testing"
"time"
"chirami/internal/morph"
"chirami/internal/morph/mkmorph"
"github.com/ikawaha/kagome-dict/ipa"
"github.com/ikawaha/kagome/v2/tokenizer"
)
func fromKagome(tok tokenizer.Token) morph.Token {
clean := func(s string) string {
if s == "*" {
return ""
}
return s
}
t := morph.Token{
Surface: tok.Surface,
Start: tok.Position,
End: tok.Position + len(tok.Surface),
Known: tok.Class == tokenizer.KNOWN,
}
f := tok.Features()
if t.Known {
if len(f) > 0 {
t.POS[0] = clean(f[0])
}
if len(f) > 1 {
t.POS[1] = clean(f[1])
}
if len(f) > 2 {
t.POS[2] = clean(f[2])
}
if len(f) > 3 {
t.POS[3] = clean(f[3])
}
if len(f) > 4 {
t.InflType = clean(f[4])
}
if len(f) > 5 {
t.InflForm = clean(f[5])
}
if len(f) > 6 {
t.Base = clean(f[6])
}
if len(f) > 7 {
t.Reading = clean(f[7])
}
if len(f) > 8 {
t.Pron = clean(f[8])
}
} else {
if len(f) > 0 {
t.POS[0] = clean(f[0])
}
if len(f) > 1 {
t.POS[1] = clean(f[1])
}
if len(f) > 2 {
t.POS[2] = clean(f[2])
}
if len(f) > 3 {
t.POS[3] = clean(f[3])
}
t.Base = tok.Surface
}
return t
}
func TestBuildAndParity(t *testing.T) {
tmpDir := t.TempDir()
binPath := filepath.Join(tmpDir, "morph.bin")
t0 := time.Now()
if err := mkmorph.Build(binPath); err != nil {
t.Fatalf("mkmorph.Build failed: %v", err)
}
buildDur := time.Since(t0)
fi, err := os.Stat(binPath)
if err != nil {
t.Fatalf("stat morph.bin: %v", err)
}
t.Logf("morph.bin built in %v, size: %d bytes (%.2f MB)", buildDur, fi.Size(), float64(fi.Size())/(1024*1024))
t0 = time.Now()
d, err := morph.Open(binPath)
if err != nil {
t.Fatalf("morph.Open failed: %v", err)
}
defer d.Close()
openDur := time.Since(t0)
t.Logf("morph.Open took %v", openDur)
if openDur > 5*time.Millisecond {
t.Logf("Warning: morph.Open took %v (> 5ms target)", openDur)
}
kt, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos())
if err != nil {
t.Fatalf("tokenizer.New failed: %v", err)
}
// Read corpus
corpusFile, err := os.Open(filepath.Join("testdata", "corpus.txt"))
if err != nil {
t.Fatalf("open testdata/corpus.txt: %v", err)
}
defer corpusFile.Close()
var lines []string
scanner := bufio.NewScanner(corpusFile)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err := scanner.Err(); err != nil {
t.Fatalf("read corpus: %v", err)
}
if len(lines) < 200 {
t.Fatalf("corpus must contain at least 200 sentences, got %d", len(lines))
}
t.Logf("Testing parity on %d corpus sentences...", len(lines))
for i, s := range lines {
got := d.Tokenize(s)
kToks := kt.Tokenize(s)
var want []morph.Token
for _, tok := range kToks {
want = append(want, fromKagome(tok))
}
if len(got) != len(want) {
t.Fatalf("[%d] len mismatch for %q: got %d, want %d", i, s, len(got), len(want))
}
for j := range got {
if !reflect.DeepEqual(got[j], want[j]) {
t.Fatalf("[%d] mismatch on %q token %d:\n got: %+v\n want: %+v",
i, s, j, got[j], want[j])
}
}
}
}
func TestValidation(t *testing.T) {
tmpDir := t.TempDir()
// Non-existent file
if _, err := morph.Open(filepath.Join(tmpDir, "nonexistent")); err == nil {
t.Errorf("expected error for nonexistent file")
}
// Too small file
smallPath := filepath.Join(tmpDir, "small.bin")
if err := os.WriteFile(smallPath, []byte("short"), 0644); err != nil {
t.Fatal(err)
}
if _, err := morph.Open(smallPath); err == nil {
t.Errorf("expected error for small file")
}
// Invalid magic
badMagic := filepath.Join(tmpDir, "badmagic.bin")
buf := make([]byte, 128)
copy(buf, "BADMAGIC")
if err := os.WriteFile(badMagic, buf, 0644); err != nil {
t.Fatal(err)
}
if _, err := morph.Open(badMagic); err == nil {
t.Errorf("expected error for bad magic")
}
// Bad version
badVer := filepath.Join(tmpDir, "badver.bin")
copy(buf, morph.Magic)
buf[8] = 99
if err := os.WriteFile(badVer, buf, 0644); err != nil {
t.Fatal(err)
}
if _, err := morph.Open(badVer); err == nil {
t.Errorf("expected error for bad version")
}
// Section out of bounds
oobPath := filepath.Join(tmpDir, "oob.bin")
copy(buf, morph.Magic)
buf[8] = 1 // version 1
// Set DaOffset = 1000000
buf[16] = 0x40
buf[17] = 0x42
buf[18] = 0x0f
buf[19] = 0x00
buf[20] = 10
if err := os.WriteFile(oobPath, buf, 0644); err != nil {
t.Fatal(err)
}
if _, err := morph.Open(oobPath); err == nil {
t.Errorf("expected error for out of bounds section")
}
}
func BenchmarkTokenize(b *testing.B) {
tmpDir := b.TempDir()
binPath := filepath.Join(tmpDir, "morph.bin")
if err := mkmorph.Build(binPath); err != nil {
b.Fatalf("build: %v", err)
}
d, err := morph.Open(binPath)
if err != nil {
b.Fatalf("open: %v", err)
}
defer d.Close()
cases := []struct {
name string
text string
}{
{"Short50Char", "すもももももももものうち。日本の夏は暑いですが、東京の空はとても綺麗です。"},
{"Okurigana", "昨日は全然眠れなくて、結局朝まで起きていさせられた。食べさせられなかった料理がたくさんある。"},
{"News", "政府は本日、新たな経済対策の基本方針を閣議決定した。日銀は金融政策決定会合で長短金利操作の修正を見送った。"},
{"MixedASCII", "Go言語とPython 3.12でAPIサーバーを構築し、GitHub ActionsとDockerで自動デプロイを行う。"},
}
for _, tc := range cases {
b.Run(tc.name, func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = d.Tokenize(tc.text)
}
})
}
}